@oakgreencc/cambium 0.0.14 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -4
- package/package.json +1 -1
- package/src/cli.mjs +54 -3
- package/src/config.mjs +14 -2
- package/src/daemon.mjs +12 -2
- package/src/dispatch-engine.mjs +13 -4
- package/src/graph-engine.mjs +11 -1
- package/src/graphs/dispatch.mjs +5 -4
- package/src/graphs/release.mjs +18 -5
- package/src/graphs/task.mjs +8 -6
- package/src/issue-state.mjs +4 -0
- package/src/store.mjs +24 -5
- package/src/tick.mjs +2 -0
- package/src/tracing.mjs +56 -0
- package/src/update.mjs +185 -0
- package/src/workers.mjs +6 -1
package/README.md
CHANGED
|
@@ -29,6 +29,7 @@ cambium status
|
|
|
29
29
|
| `cambium run task --issue <n> [--repo <owner/name>]` | Claim one open `q:task` issue, run its worker in the foreground, then make the same transition the daemon would. |
|
|
30
30
|
| `cambium run <graph> --issue <n> [--repo <owner/name>] [--follow]` | `engine: "graph"` only (`task` is the one graph today). Advance one issue through the graph runtime in the foreground: a `q:task` issue without a lease starts with a foreground worker; anything else resumes where its marker rests. Stops at the first wait that is not ready; `--follow` re-reads and polls every `tickSeconds` until the graph ends. |
|
|
31
31
|
| `cambium install [--dry-run]` | Write the launchd agent (`KeepAlive`, `RunAtLoad`) for `start --config <this config>`; `--dry-run` prints it. It does not call `launchctl`; it prints the command. |
|
|
32
|
+
| `cambium update [--check] [--version <v>]` | Install the latest version (or `<v>`, even an older one) from npm into this global install, then SIGTERM the launchd agent so it finishes its tick and KeepAlive starts the new version. `--check` only reports. No config file needed. See [Self-update](#self-update). |
|
|
32
33
|
|
|
33
34
|
Every command takes `--config <file>`; the default is `$CAMBIUM_CONFIG`, then `~/.cambium/config.json`.
|
|
34
35
|
|
|
@@ -80,7 +81,7 @@ A worker is `claude -p --output-format stream-json --verbose --max-turns <maxTur
|
|
|
80
81
|
- in its own worktree, by the target repo's [worktree convention](#target-repo-convention);
|
|
81
82
|
- behind the prepare step: `node src/prepare.mjs -- claude …` runs install-fresh, hydrate and the repo's `prepare` commands in the worktree, writing to the worker's stderr log, then becomes the worker. It runs in the detached process so a long install never blocks a tick;
|
|
82
83
|
- detached, leading its own process group (a kill takes its children), stdin from the prompt file;
|
|
83
|
-
- with `CAMBIUM_REPO`, `CAMBIUM_ISSUE`, `CAMBIUM_PHASE`, `CAMBIUM_BRANCH` set,
|
|
84
|
+
- with `CAMBIUM_REPO`, `CAMBIUM_ISSUE`, `CAMBIUM_PHASE`, `CAMBIUM_BRANCH` set, `GRAFT_APP=<worker.graftApp>` when configured, then `worker.env`, then `TRACEPARENT` (the starting node's span) when [tracing](#tracing) is on;
|
|
84
85
|
- with its prompt, stdout and stderr at `<paths.logs>/<owner>__<name>/issue-<n>/a<attempt>-<stamp>.{prompt.md,out.log,err.log}`, keeping the newest `logs.keepPerIssue` runs per issue.
|
|
85
86
|
|
|
86
87
|
The prompt states the four-point contract (task issue, `pnpm verify`, `graft publish <branch> … --closes <n>`, `graft ci wait`), the attempt, the existing PR if any (`graft advance` onto it), the last failure, the issue's history, the issue body, and the worktree's `AGENTS.md`.
|
|
@@ -111,10 +112,13 @@ One JSON file. Everything but `repos` is optional; these are the defaults:
|
|
|
111
112
|
},
|
|
112
113
|
"logs": { "keepPerIssue": 20, "daemonMaxBytes": 10485760 },
|
|
113
114
|
"graft": { "command": "graft", "app": null, "botLogin": null },
|
|
114
|
-
"worker": { "command": "claude", "args": [], "graftApp": null, "fetch": true }
|
|
115
|
+
"worker": { "command": "claude", "args": [], "graftApp": null, "fetch": true, "env": {} },
|
|
116
|
+
"update": { "auto": false, "checkMinutes": 360 }
|
|
115
117
|
}
|
|
116
118
|
```
|
|
117
119
|
|
|
120
|
+
- `worker.env` is extra environment for every worker (string values only), e.g. Claude Code's own telemetry ([Tracing](#claude-codes-own-spans-under-the-worker)).
|
|
121
|
+
|
|
118
122
|
- The daemon re-reads the file before every tick, so caps, limits, `tickSeconds`, `repos`, priority labels, `release` and `worker` take effect without a restart (logged as `config reloaded: <keys>`). `host`, `engine`, `paths`, `logs` and `graft` are baked in at start: a change to them logs `restart cambium to apply` once and is not applied. A file that does not load, or has no `repos`, keeps the running config and logs one warning until it is fixed.
|
|
119
123
|
- `repos[].path` is a local clone; worktrees are added from it and `graft` runs in it.
|
|
120
124
|
- `graft.app` is the App cambium acts as (`GRAFT_APP` for its own `graft` calls); `worker.graftApp` the one workers act as. Neither leaks into the other.
|
|
@@ -150,6 +154,14 @@ How a repo's worktrees are made is the repo's call: a `"cambium"` key in its roo
|
|
|
150
154
|
|
|
151
155
|
The key decides which commands run on this host, so list `package.json` as protected (CODEOWNERS) in the target repo, next to its `"graft"` policy.
|
|
152
156
|
|
|
157
|
+
## Self-update
|
|
158
|
+
|
|
159
|
+
cambium updates itself only when it is a **global npm install** (`<prefix>/lib/node_modules/@oakgreencc/cambium`, e.g. nvm's or Homebrew's prefix), and only into that same prefix, so the launchd agent's node and bin paths stay valid. A source checkout (`pnpm start`), an `npm link` or a copy inside another package's `node_modules` is left to whoever installed it. npm is the one next to the running Node, else `npm` on `PATH`; the version is the registry's `latest` dist-tag.
|
|
160
|
+
|
|
161
|
+
- **`cambium update`** installs, then `launchctl kill TERM gui/<uid>/com.oakgreencc.cambium`: the daemon finishes its tick and exits, and KeepAlive starts the new version. Workers keep running and are re-adopted. When the agent is not loaded it only installs.
|
|
162
|
+
- **`"update": { "auto": true }`** (off by default; reloaded with the config) makes the daemon check every `update.checkMinutes` (default 360, at least 5) between ticks. On a newer version it installs it, logs `updating cambium a → b`, and ends its loop; the process exits and launchd starts the new one. Only when it is the launchd agent (`XPC_SERVICE_NAME=com.oakgreencc.cambium`), since nothing else would bring it back; otherwise, or when it is not a global install, it logs why once and skips. A failed check or install is a warning and is retried at the next check. The last check time is `update.lastCheck` in the SQLite `control` table.
|
|
163
|
+
- The plist is never rewritten, so env added by hand (e.g. `OTEL_*`) survives an update. To pick up a plist change, `oakgreen reload cambium`.
|
|
164
|
+
|
|
153
165
|
## Heartbeat (`state.json`)
|
|
154
166
|
|
|
155
167
|
Written at the end of every tick to `paths.heartbeat` (default `~/.cambium/state.json`), whole and renamed into place, so a reader never sees a partial file. canopy reads it. `version` changes only on an incompatible change. All times are ISO 8601 UTC strings; durations are milliseconds.
|
|
@@ -452,6 +464,16 @@ Why not a shared package: cambium is published to npm, so it can only depend on
|
|
|
452
464
|
|
|
453
465
|
Node and wait spans carry `cambium.graph`, `cambium.graph.version`, `cambium.node`, `cambium.attempt`, `cambium.repo`, `cambium.issue.number`, `cambium.edge`, `cambium.outcome`.
|
|
454
466
|
|
|
467
|
+
### What is traced
|
|
468
|
+
|
|
469
|
+
With `engine: "graph"`, every graph run is traced: the `task` and `dispatch` graphs per issue and both release graphs (the legacy stage tick is not). [canopy](../canopy)'s Traces tab shows them from the local collector's file.
|
|
470
|
+
|
|
471
|
+
- **Lifecycle.** `cambium start` and `cambium run` create the telemetry from their environment (a no-op without OTLP config), flush it after every tick and shut it down on exit.
|
|
472
|
+
- **One trace per issue, across ticks.** Each turn (`drive`, `driveDispatch`, release `drive`) runs `advance` with `turnTracer`'s `instrument` (`advanceHooks` underneath). The first span of an issue starts its `cambium.issue` root, a child of mycelium's intake span when the body carries one; the marker written in that turn takes the root's traceparent (`setMarker` fills `traceparent` from the turn's subject when the marker has none), and every later tick parents on it. Requeues and later attempts stay in the trace, and the dispatch → task hand-off carries it, so one issue reads intake → triage → each task attempt → CI → close-out.
|
|
473
|
+
- **Release.** A release-prepare run is a root of its own; the promotion PR it opens or refreshes takes that trace into its marker when it has none, and its release-promote turns (`merge`, `merging`) continue it, with `cambium.issue.number` the PR's number.
|
|
474
|
+
- **Workers.** start-worker and start-triage start their worker with `TRACEPARENT` set to their own `cambium.node` span and record it on the worker row (`workers.traceparent` in SQLite; older databases gain the column on open). When the worker is found exited, its stdout log becomes `claude.*` spans under that span ([below](#worker-spans-from-stream-json)); a log that cannot be read is a warning, never a failed tick.
|
|
475
|
+
- **Off is unchanged.** Without OTLP config there is no `instrument`, no `TRACEPARENT`, and markers keep `traceparent: null`; the legacy/graph differential test covers that the moves are identical.
|
|
476
|
+
|
|
455
477
|
```js
|
|
456
478
|
import { advanceHooks, createTelemetry } from './tracing.mjs' // inside cambium
|
|
457
479
|
|
|
@@ -495,7 +517,40 @@ telemetry.workerSpans(stream, parent) // same, with cambium's tracer
|
|
|
495
517
|
- `claude.task`: each background task, `task_started` → `task_notification` (or `task_updated.patch.end_time`), under the tool that started it. These events have no timestamp, so their times are those of the latest timestamped event.
|
|
496
518
|
- Malformed lines are counted (`malformed`) and skipped; `system/thinking_tokens`, `tool_progress`, hooks and rate-limit events are ignored.
|
|
497
519
|
|
|
498
|
-
###
|
|
520
|
+
### Claude Code's own spans under the worker
|
|
521
|
+
|
|
522
|
+
**`claude -p` honours `TRACEPARENT`: verified live** (Claude Code 2.1.327, 2026-11-18). A `claude -p` run with a Bash call, given `TRACEPARENT=00-<trace>-<span>-01` and the variables below, exported into `<trace>`:
|
|
523
|
+
|
|
524
|
+
```
|
|
525
|
+
<the TRACEPARENT span>
|
|
526
|
+
└ claude_code.interaction parent.source=env
|
|
527
|
+
├ claude_code.llm_request model, input/output/cache tokens, ttft_ms, stop_reason
|
|
528
|
+
├ claude_code.tool tool_name=Bash
|
|
529
|
+
│ ├ claude_code.tool.blocked_on_user
|
|
530
|
+
│ └ claude_code.tool.execution
|
|
531
|
+
└ claude_code.llm_request
|
|
532
|
+
```
|
|
533
|
+
|
|
534
|
+
Workers already get `TRACEPARENT`, so turning these on is config only:
|
|
535
|
+
|
|
536
|
+
```jsonc
|
|
537
|
+
"worker": {
|
|
538
|
+
"env": {
|
|
539
|
+
"CLAUDE_CODE_ENABLE_TELEMETRY": "1",
|
|
540
|
+
"CLAUDE_CODE_ENHANCED_TELEMETRY_BETA": "1",
|
|
541
|
+
"OTEL_TRACES_EXPORTER": "otlp",
|
|
542
|
+
"OTEL_METRICS_EXPORTER": "none",
|
|
543
|
+
"OTEL_LOGS_EXPORTER": "none",
|
|
544
|
+
"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
|
|
545
|
+
"OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:4318"
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
```
|
|
549
|
+
|
|
550
|
+
- They complement the stream-json spans rather than replace them: they carry per-request timing and tokens but no cost, and need telemetry on in the worker; `claude.session` carries the session's `claude.cost_usd` and works without it. With both on, a worker has `claude.*` and `claude_code.*` spans side by side under start-worker.
|
|
551
|
+
- **Identity in spans.** Every `claude_code.*` span carries `user.email`, `user.account_uuid`, `user.account_id` and `organization.id`. Fine for the local collector file; mind it before exporting elsewhere (strip them in the collector with an `attributes` processor).
|
|
552
|
+
|
|
553
|
+
### Does `claude -p` export traces that honour `TRACEPARENT`? (the docs)
|
|
499
554
|
|
|
500
555
|
**Yes, opt-in and beta**; cambium does not depend on it (worker spans come from stream-json above). Per the [Claude Code monitoring docs](https://code.claude.com/docs/en/monitoring-usage) (checked with Claude Code 2.1.327; `claude --help` lists no telemetry flags, it is env-only):
|
|
501
556
|
|
|
@@ -504,7 +559,7 @@ telemetry.workerSpans(stream, parent) // same, with cambium's tracer
|
|
|
504
559
|
- Bash subprocesses it runs inherit a `TRACEPARENT` for the active tool span, so a `graft` call inside a worker could continue the trace.
|
|
505
560
|
- Prompts and tool inputs are redacted from spans unless `OTEL_LOG_USER_PROMPTS` / `OTEL_LOG_TOOL_DETAILS` / `OTEL_LOG_TOOL_CONTENT` are set.
|
|
506
561
|
|
|
507
|
-
|
|
562
|
+
Confirmed against a live run: see [Claude Code's own spans under the worker](#claude-codes-own-spans-under-the-worker).
|
|
508
563
|
|
|
509
564
|
### Local collector
|
|
510
565
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@oakgreencc/cambium",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.16",
|
|
4
4
|
"description": "The scheduler daemon: reads the q:* queues from GitHub every tick, runs headless claude workers in their own worktrees up to a cap, and makes every queue transition itself.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|
package/src/cli.mjs
CHANGED
|
@@ -14,8 +14,10 @@ import { plist, plistPath } from './launchd.mjs'
|
|
|
14
14
|
import { openStore } from './store.mjs'
|
|
15
15
|
import { finish, newCtx, startTask } from './tick.mjs'
|
|
16
16
|
import { runIssue } from './graph-engine.mjs'
|
|
17
|
+
import { createTelemetry } from './tracing.mjs'
|
|
18
|
+
import { defaultExec, realUpdater, runUpdate, VERSION } from './update.mjs'
|
|
17
19
|
|
|
18
|
-
export
|
|
20
|
+
export { VERSION }
|
|
19
21
|
|
|
20
22
|
export const HELP = `cambium ${VERSION} — the scheduler daemon
|
|
21
23
|
|
|
@@ -30,6 +32,9 @@ export const HELP = `cambium ${VERSION} — the scheduler daemon
|
|
|
30
32
|
engine "graph": advance the issue in the foreground to the first wait
|
|
31
33
|
that is not ready; --follow keeps polling until the graph ends
|
|
32
34
|
cambium install [--dry-run] write the launchd agent (KeepAlive); --dry-run prints it
|
|
35
|
+
cambium update [--check] [--version <v>]
|
|
36
|
+
install the latest (or <v>) from npm into this global install, then
|
|
37
|
+
restart the launchd agent onto it; --check only reports
|
|
33
38
|
|
|
34
39
|
Common: --config <file> (default $CAMBIUM_CONFIG, then ~/.cambium/config.json)`
|
|
35
40
|
|
|
@@ -41,6 +46,24 @@ const flag = (argv, name) => {
|
|
|
41
46
|
/** @param {string[]} argv @param {string} name */
|
|
42
47
|
const has = (argv, name) => argv.includes(name)
|
|
43
48
|
|
|
49
|
+
/**
|
|
50
|
+
* Run `fn` with `deps.telemetry` set (from the OTEL_* vars in `io.env`, a no-op without them, unless the deps
|
|
51
|
+
* already carry one), shutting it down afterwards so queued spans are exported before the process exits.
|
|
52
|
+
*
|
|
53
|
+
* @template T
|
|
54
|
+
* @param {import('./tick.mjs').Deps} deps @param {IO} io @param {() => Promise<T>} fn
|
|
55
|
+
* @returns {Promise<T>}
|
|
56
|
+
*/
|
|
57
|
+
async function withTelemetry(deps, io, fn) {
|
|
58
|
+
const own = !deps.telemetry
|
|
59
|
+
if (own) deps.telemetry = await createTelemetry({ env: io.env })
|
|
60
|
+
try {
|
|
61
|
+
return await fn()
|
|
62
|
+
} finally {
|
|
63
|
+
if (own) await deps.telemetry?.shutdown().catch(() => {})
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
44
67
|
/**
|
|
45
68
|
* @typedef {{
|
|
46
69
|
* out: (line: string) => void,
|
|
@@ -50,6 +73,9 @@ const has = (argv, name) => argv.includes(name)
|
|
|
50
73
|
* signal?: AbortSignal,
|
|
51
74
|
* now?: () => number,
|
|
52
75
|
* sleep?: (ms: number) => Promise<void>,
|
|
76
|
+
* updater?: import('./update.mjs').Updater,
|
|
77
|
+
* exec?: import('./update.mjs').Exec,
|
|
78
|
+
* uid?: number,
|
|
53
79
|
* }} IO
|
|
54
80
|
*/
|
|
55
81
|
|
|
@@ -64,6 +90,22 @@ export async function main(argv, io) {
|
|
|
64
90
|
io.out(VERSION)
|
|
65
91
|
return 0
|
|
66
92
|
}
|
|
93
|
+
if (verb === 'update') {
|
|
94
|
+
try {
|
|
95
|
+
const r = await runUpdate(
|
|
96
|
+
{ check: has(rest, '--check'), version: flag(rest, '--version') },
|
|
97
|
+
{
|
|
98
|
+
updater: io.updater ?? realUpdater(),
|
|
99
|
+
exec: io.exec ?? defaultExec,
|
|
100
|
+
uid: io.uid ?? process.getuid?.() ?? 0,
|
|
101
|
+
out: io.out,
|
|
102
|
+
},
|
|
103
|
+
)
|
|
104
|
+
return r.ok ? 0 : fail(io, /** @type {string} */ (r.error))
|
|
105
|
+
} catch (err) {
|
|
106
|
+
return fail(io, `update: ${/** @type {Error} */ (err)?.message ?? err}`)
|
|
107
|
+
}
|
|
108
|
+
}
|
|
67
109
|
const cfgPath = configPath(flag(rest, '--config'), io.env)
|
|
68
110
|
const config = loadConfig(cfgPath)
|
|
69
111
|
const depsFor = io.deps ?? ((c) => realDeps(c, { echo: io.out }))
|
|
@@ -73,7 +115,14 @@ export async function main(argv, io) {
|
|
|
73
115
|
if (!config.repos.length) return fail(io, `no repos in ${cfgPath}; see the README for the config file`)
|
|
74
116
|
const deps = depsFor(config)
|
|
75
117
|
acquireLock(deps.store, deps.procs, deps.pid ?? process.pid)
|
|
76
|
-
await
|
|
118
|
+
await withTelemetry(deps, io, () =>
|
|
119
|
+
runDaemon(deps, {
|
|
120
|
+
signal: io.signal,
|
|
121
|
+
once: has(rest, '--once'),
|
|
122
|
+
reload: () => loadConfig(cfgPath),
|
|
123
|
+
env: io.env,
|
|
124
|
+
}),
|
|
125
|
+
)
|
|
77
126
|
return 0
|
|
78
127
|
}
|
|
79
128
|
case 'status':
|
|
@@ -169,7 +218,9 @@ async function runOne(config, argv, io, depsFor) {
|
|
|
169
218
|
)
|
|
170
219
|
const deps = depsFor(config)
|
|
171
220
|
if (config.engine === 'graph') {
|
|
172
|
-
const r = await
|
|
221
|
+
const r = await withTelemetry(deps, io, () =>
|
|
222
|
+
runIssue(deps, { graph, slug, number: n, follow, out: io.out, sleep: io.sleep, signal: io.signal }),
|
|
223
|
+
)
|
|
173
224
|
return r.ok ? 0 : fail(io, /** @type {string} */ (r.error))
|
|
174
225
|
}
|
|
175
226
|
if (deps.store.worker(slug, n)) return fail(io, `${slug}#${n} already has a tracked worker`)
|
package/src/config.mjs
CHANGED
|
@@ -27,7 +27,8 @@ import { join, resolve } from 'node:path'
|
|
|
27
27
|
* paths: { home: string, db: string, heartbeat: string, logs: string },
|
|
28
28
|
* logs: { keepPerIssue: number, daemonMaxBytes: number },
|
|
29
29
|
* graft: { command: string, app: string | null, botLogin: string | null },
|
|
30
|
-
* worker: { command: string, args: string[], graftApp: string | null, fetch: boolean },
|
|
30
|
+
* worker: { command: string, args: string[], graftApp: string | null, fetch: boolean, env: Record<string, string> },
|
|
31
|
+
* update: { auto: boolean, checkMinutes: number },
|
|
31
32
|
* }} Config
|
|
32
33
|
*/
|
|
33
34
|
|
|
@@ -56,7 +57,8 @@ export function defaults(home = DEFAULT_HOME) {
|
|
|
56
57
|
},
|
|
57
58
|
logs: { keepPerIssue: 20, daemonMaxBytes: 10 * 1024 * 1024 },
|
|
58
59
|
graft: { command: 'graft', app: null, botLogin: null },
|
|
59
|
-
worker: { command: 'claude', args: [], graftApp: null, fetch: true },
|
|
60
|
+
worker: { command: 'claude', args: [], graftApp: null, fetch: true, env: {} },
|
|
61
|
+
update: { auto: false, checkMinutes: 360 },
|
|
60
62
|
}
|
|
61
63
|
}
|
|
62
64
|
|
|
@@ -103,6 +105,16 @@ export function resolveConfig(raw = {}) {
|
|
|
103
105
|
if (!/^\d{2}:\d{2}$/.test(release.time)) throw new Error(`config: repo ${r.slug} release.time must be HH:MM`)
|
|
104
106
|
return { slug: r.slug, path: untilde(r.path), baseBranch, release }
|
|
105
107
|
})
|
|
108
|
+
const wenv = config.worker.env
|
|
109
|
+
if (
|
|
110
|
+
!wenv ||
|
|
111
|
+
typeof wenv !== 'object' ||
|
|
112
|
+
Array.isArray(wenv) ||
|
|
113
|
+
Object.values(wenv).some((v) => typeof v !== 'string')
|
|
114
|
+
)
|
|
115
|
+
throw new Error('config: worker.env must be an object of string values')
|
|
116
|
+
if (typeof config.update.auto !== 'boolean') throw new Error('config: update.auto must be true or false')
|
|
117
|
+
if (!(Number(config.update.checkMinutes) >= 5)) throw new Error('config: update.checkMinutes must be at least 5')
|
|
106
118
|
if (config.engine !== 'legacy' && config.engine !== 'graph')
|
|
107
119
|
throw new Error(`config: engine must be "legacy" or "graph", got ${JSON.stringify(config.engine)}`)
|
|
108
120
|
return config
|
package/src/daemon.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { applyReload } from './config.mjs'
|
|
|
6
6
|
import { graftGitHub } from './github.mjs'
|
|
7
7
|
import { openStore } from './store.mjs'
|
|
8
8
|
import { tick } from './tick.mjs'
|
|
9
|
+
import { autoUpdate, realUpdater } from './update.mjs'
|
|
9
10
|
import { processSpawner, realProcs } from './workers.mjs'
|
|
10
11
|
|
|
11
12
|
/** @param {string} path @param {unknown} state */
|
|
@@ -80,6 +81,7 @@ export function realDeps(config, { echo } = {}) {
|
|
|
80
81
|
heartbeat: (state) => writeHeartbeat(config.paths.heartbeat, state),
|
|
81
82
|
daemonStartedAt: Date.now(),
|
|
82
83
|
pid: process.pid,
|
|
84
|
+
updater: realUpdater(),
|
|
83
85
|
}
|
|
84
86
|
}
|
|
85
87
|
|
|
@@ -101,12 +103,14 @@ export function acquireLock(store, procs, pid) {
|
|
|
101
103
|
* @param {import('./tick.mjs').Deps} deps
|
|
102
104
|
* @param {{
|
|
103
105
|
* signal?: AbortSignal, wait?: (ms: number, signal?: AbortSignal) => Promise<unknown>, once?: boolean,
|
|
104
|
-
* reload?: () => import('./config.mjs').Config,
|
|
106
|
+
* reload?: () => import('./config.mjs').Config, env?: Record<string, string | undefined>,
|
|
105
107
|
* }} [opts]
|
|
108
|
+
* After each tick, with `update.auto`, {@link autoUpdate} may install a newer cambium; the loop then ends so
|
|
109
|
+
* the process exits and launchd starts the new version.
|
|
106
110
|
*/
|
|
107
111
|
export async function runDaemon(
|
|
108
112
|
deps,
|
|
109
|
-
{ signal, wait = (ms, s) => sleep(ms, undefined, { signal: s }), once = false, reload } = {},
|
|
113
|
+
{ signal, wait = (ms, s) => sleep(ms, undefined, { signal: s }), once = false, reload, env = process.env } = {},
|
|
110
114
|
) {
|
|
111
115
|
let n = 0
|
|
112
116
|
/** @type {string | null} */
|
|
@@ -134,6 +138,12 @@ export async function runDaemon(
|
|
|
134
138
|
} catch (err) {
|
|
135
139
|
deps.log('error', `tick ${n}: ${/** @type {Error} */ (err)?.message ?? err}`)
|
|
136
140
|
}
|
|
141
|
+
// Every span is ended within its tick; export them now rather than when the batch fills.
|
|
142
|
+
await deps.telemetry?.flush().catch((err) => deps.log('warn', `telemetry flush: ${err?.message ?? err}`))
|
|
143
|
+
if (deps.updater && !signal?.aborted) {
|
|
144
|
+
const installed = await autoUpdate(deps, { updater: deps.updater, env, now: deps.clock.now() })
|
|
145
|
+
if (installed) break
|
|
146
|
+
}
|
|
137
147
|
if (once) break
|
|
138
148
|
try {
|
|
139
149
|
await wait(deps.config.tickSeconds * 1000, signal)
|
package/src/dispatch-engine.mjs
CHANGED
|
@@ -15,6 +15,7 @@ import { advance } from './graph/index.mjs'
|
|
|
15
15
|
import { dispatch, lostLeaseOf, nodeOf } from './graphs/dispatch.mjs'
|
|
16
16
|
import { byPriorityThenAge, Q, RUNNING } from './labels.mjs'
|
|
17
17
|
import { each, staleLeases } from './tick.mjs'
|
|
18
|
+
import { markerTrace, turnTracer } from './tracing.mjs'
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
21
|
* @typedef {import('./tick.mjs').Deps} Deps
|
|
@@ -33,15 +34,23 @@ export const PHASE = 'dispatch'
|
|
|
33
34
|
*/
|
|
34
35
|
export async function driveDispatch(deps, ctx, subject, at) {
|
|
35
36
|
const c = { ...deps, tick: ctx, subject }
|
|
37
|
+
// A new issue's cambium.issue root continues mycelium's intake span from the body, so these node and wait spans
|
|
38
|
+
// join the intake trace (tracing.mjs `issueTrace`); the marker carries it on into the task graph.
|
|
39
|
+
const traced = turnTracer(deps.telemetry, subject, {
|
|
40
|
+
graph: dispatch.name,
|
|
41
|
+
graphVersion: dispatch.version,
|
|
42
|
+
repo: subject.slug,
|
|
43
|
+
issue: subject.number,
|
|
44
|
+
issueBody: subject.issue?.body ?? null,
|
|
45
|
+
...markerTrace(subject.issue?.stateComment?.marker),
|
|
46
|
+
})
|
|
36
47
|
/** @type {Transition[]} */
|
|
37
48
|
const transitions = []
|
|
38
49
|
let pos = at
|
|
39
50
|
for (;;) {
|
|
40
|
-
|
|
41
|
-
// here; the issue's cambium.issue root continues mycelium's intake span from the body, so these node and wait
|
|
42
|
-
// spans join the intake trace (tracing.mjs `issueTrace`).
|
|
43
|
-
const r = await advance(dispatch, pos, c, { maxTransitions: 1 })
|
|
51
|
+
const r = await advance(dispatch, pos, c, { maxTransitions: 1, instrument: traced.instrument })
|
|
44
52
|
if (r.error) throw r.error.cause instanceof Error ? r.error.cause : r.error
|
|
53
|
+
traced.sync()
|
|
45
54
|
transitions.push(...r.transitions)
|
|
46
55
|
pos = { node: r.node, input: r.input }
|
|
47
56
|
if (r.reason !== 'limit' || r.node === dispatch.entry.node) break
|
package/src/graph-engine.mjs
CHANGED
|
@@ -22,6 +22,7 @@ import { iso, key, markerOf, setLabels, setMarker } from './issue-state.mjs'
|
|
|
22
22
|
import { effectivePosition, graphVersionDecision, withHistory } from './marker.mjs'
|
|
23
23
|
import { each, newCtx, staleLeases, taskCandidates, watched, watchPairs } from './tick.mjs'
|
|
24
24
|
import { move, Q } from './labels.mjs'
|
|
25
|
+
import { markerTrace, turnTracer } from './tracing.mjs'
|
|
25
26
|
import { DISPATCH_STEPS } from './dispatch-engine.mjs'
|
|
26
27
|
|
|
27
28
|
/**
|
|
@@ -45,16 +46,25 @@ const taskCtx = (deps, ctx, subject) => ({ ...deps, tick: ctx, subject })
|
|
|
45
46
|
*/
|
|
46
47
|
export async function drive(deps, ctx, subject, at) {
|
|
47
48
|
const c = taskCtx(deps, ctx, subject)
|
|
49
|
+
const traced = turnTracer(deps.telemetry, subject, {
|
|
50
|
+
graph: task.name,
|
|
51
|
+
graphVersion: task.version,
|
|
52
|
+
repo: subject.slug,
|
|
53
|
+
issue: subject.number,
|
|
54
|
+
issueBody: subject.issue?.body ?? null,
|
|
55
|
+
...markerTrace(subject.issue?.stateComment?.marker),
|
|
56
|
+
})
|
|
48
57
|
/** @type {Transition[]} */
|
|
49
58
|
const transitions = []
|
|
50
59
|
let pos = at
|
|
51
60
|
/** @type {Turn['reason']} */
|
|
52
61
|
let reason
|
|
53
62
|
for (;;) {
|
|
54
|
-
const r = await advance(task, pos, c, { maxTransitions: 1 })
|
|
63
|
+
const r = await advance(task, pos, c, { maxTransitions: 1, instrument: traced.instrument })
|
|
55
64
|
if (r.error) throw r.error.cause instanceof Error ? r.error.cause : r.error
|
|
56
65
|
transitions.push(...r.transitions)
|
|
57
66
|
const t = r.transitions[0]
|
|
67
|
+
traced.sync()
|
|
58
68
|
if (t && t.to !== END) await nodeOf(t.to).enter?.(t.next, c)
|
|
59
69
|
pos = { node: r.node, input: r.input }
|
|
60
70
|
if (r.reason !== 'limit') {
|
package/src/graphs/dispatch.mjs
CHANGED
|
@@ -14,6 +14,7 @@ import { iso, key, markerOf, MIN, setLabels, setMarker } from '../issue-state.mj
|
|
|
14
14
|
import { move, Q, WHY } from '../labels.mjs'
|
|
15
15
|
import { newMarker, withHistory } from '../marker.mjs'
|
|
16
16
|
import { readTriage, TriageResult, triageComment, triagePrompt } from '../phases/dispatch.mjs'
|
|
17
|
+
import { traceWorker } from '../tracing.mjs'
|
|
17
18
|
import { task } from './task.mjs'
|
|
18
19
|
|
|
19
20
|
/**
|
|
@@ -167,8 +168,8 @@ const startTriage = declare(
|
|
|
167
168
|
phase: 'dispatch',
|
|
168
169
|
maxTurns: caps.maxTurns,
|
|
169
170
|
readOnly: true,
|
|
170
|
-
//
|
|
171
|
-
|
|
171
|
+
// This node's cambium.node span, so Claude Code's own spans (when worker.env turns them on) join the trace.
|
|
172
|
+
env: ctx.traceparent ? { TRACEPARENT: ctx.traceparent } : undefined,
|
|
172
173
|
buildPrompt: (agentsMd) => triagePrompt({ repo: slug, baseBranch: repo.baseBranch, issue, queued, agentsMd }),
|
|
173
174
|
})
|
|
174
175
|
} catch (err) {
|
|
@@ -191,6 +192,7 @@ const startTriage = declare(
|
|
|
191
192
|
host: config.host,
|
|
192
193
|
stdout: s.stdout,
|
|
193
194
|
stderr: s.stderr,
|
|
195
|
+
traceparent: ctx.traceparent ?? null,
|
|
194
196
|
})
|
|
195
197
|
ctx.log('info', `started triage worker pid ${s.pid} (attempt ${marker.attempt})`, {
|
|
196
198
|
repo: slug,
|
|
@@ -241,8 +243,7 @@ const triaging = declare(
|
|
|
241
243
|
ctx.log('info', 'triage worker gone; issue is closed or left q:intake', { repo: w.repo, issue: w.issue })
|
|
242
244
|
return { ready: true, output: { exit: 'gone' } }
|
|
243
245
|
}
|
|
244
|
-
|
|
245
|
-
// `workerSpans(<w.stdout log>, { tracer, parent: <start-triage's traceparent> })`.
|
|
246
|
+
traceWorker(ctx, w)
|
|
246
247
|
const read = w.killedReason ? { ok: false, error: w.killedReason } : readTriage(spawner.readLog(w.stdout), number)
|
|
247
248
|
if (read.ok && 'result' in read)
|
|
248
249
|
return { ready: true, output: { exit: 'triaged', result: read.result, log: w.stdout } }
|
package/src/graphs/release.mjs
CHANGED
|
@@ -19,13 +19,17 @@ import { iso, setLabels, setMarker } from '../issue-state.mjs'
|
|
|
19
19
|
import { move, Q } from '../labels.mjs'
|
|
20
20
|
import { effectivePosition, newMarker, withHistory } from '../marker.mjs'
|
|
21
21
|
import { releaseDue } from '../phases/release.mjs'
|
|
22
|
+
import { markerTrace, turnTracer } from '../tracing.mjs'
|
|
22
23
|
|
|
23
24
|
/**
|
|
24
25
|
* @typedef {import('../github.mjs').PromotionPr} PromotionPr
|
|
25
26
|
* @typedef {import('../github.mjs').Release} Release
|
|
26
27
|
* @typedef {import('../marker.mjs').Marker} Marker
|
|
27
|
-
* @typedef {{
|
|
28
|
-
*
|
|
28
|
+
* @typedef {{
|
|
29
|
+
* slug: string, repo: import('../config.mjs').Repo, release: Release, pr?: PromotionPr, traceparent?: string | null,
|
|
30
|
+
* }} ReleaseSubject
|
|
31
|
+
* One repo's turn; `pr` is the promotion PR a `release-promote` turn is about; `traceparent` the turn's trace
|
|
32
|
+
* while it is traced (tracing.mjs `turnTracer`), which markers written in the turn take when they have none.
|
|
29
33
|
* @typedef {import('../tick.mjs').Deps & { tick: import('../tick.mjs').Ctx, subject: ReleaseSubject }} ReleaseCtx
|
|
30
34
|
* @typedef {{ queue: string | null, enter?: (input: any, ctx: ReleaseCtx) => Promise<void> }} Declared
|
|
31
35
|
* `queue`: the label a PR resting here carries. `enter`: the write that lands a PR on this wait.
|
|
@@ -248,7 +252,6 @@ const merge = declare(
|
|
|
248
252
|
*/
|
|
249
253
|
run: async (input, ctx) => {
|
|
250
254
|
const pr = prOf(ctx)
|
|
251
|
-
// TRACING (follow-up): a span per merge attempt here, parented on the PR marker's traceparent.
|
|
252
255
|
try {
|
|
253
256
|
await ctx.github.mergePull(ctx.subject.slug, input.pr)
|
|
254
257
|
} catch (err) {
|
|
@@ -364,11 +367,21 @@ export const RELEASE_QUEUES = {
|
|
|
364
367
|
* @param {{ node: string, input: unknown }} at
|
|
365
368
|
*/
|
|
366
369
|
async function drive(graph, ctx, at) {
|
|
370
|
+
const { pr } = ctx.subject
|
|
371
|
+
// A promote turn is traced on its PR (the marker's trace); a prepare run is a root of its own, which the PR it
|
|
372
|
+
// opens or updates takes into its marker when that has none.
|
|
373
|
+
const traced = turnTracer(ctx.telemetry, ctx.subject, {
|
|
374
|
+
graph: graph.name,
|
|
375
|
+
graphVersion: graph.version,
|
|
376
|
+
repo: ctx.subject.slug,
|
|
377
|
+
issue: pr?.number,
|
|
378
|
+
...markerTrace(pr?.stateComment?.marker),
|
|
379
|
+
})
|
|
367
380
|
let pos = at
|
|
368
381
|
for (;;) {
|
|
369
|
-
|
|
370
|
-
const r = await advance(graph, pos, ctx, { maxTransitions: 1 })
|
|
382
|
+
const r = await advance(graph, pos, ctx, { maxTransitions: 1, instrument: traced.instrument })
|
|
371
383
|
if (r.error) throw r.error.cause instanceof Error ? r.error.cause : r.error
|
|
384
|
+
traced.sync()
|
|
372
385
|
const t = r.transitions[0]
|
|
373
386
|
const next = /** @type {(import('../graph/define.mjs').GraphNode & Declared) | undefined} */ (
|
|
374
387
|
graph.nodes.find((n) => n.name === r.node)
|
package/src/graphs/task.mjs
CHANGED
|
@@ -15,6 +15,7 @@ import { move, Q, WHY } from '../labels.mjs'
|
|
|
15
15
|
import { withHistory } from '../marker.mjs'
|
|
16
16
|
import { prFor, taskExit, taskPrompt } from '../phases/task.mjs'
|
|
17
17
|
import { PREPARE_FAILED_LINE } from '../prepare.mjs'
|
|
18
|
+
import { traceWorker } from '../tracing.mjs'
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
21
|
* @typedef {import('../github.mjs').Issue} Issue
|
|
@@ -24,10 +25,11 @@ import { PREPARE_FAILED_LINE } from '../prepare.mjs'
|
|
|
24
25
|
* @typedef {{
|
|
25
26
|
* slug: string, number: number, issue?: Issue, snap?: Snapshot,
|
|
26
27
|
* pr?: PullRequest | null, deleted?: Set<string>, foreground?: boolean, reaped?: Set<number>,
|
|
27
|
-
* started?: import('../workers.mjs').Started & { attempt: number },
|
|
28
|
+
* started?: import('../workers.mjs').Started & { attempt: number }, traceparent?: string | null,
|
|
28
29
|
* }} Subject
|
|
29
30
|
* One issue's turn. `pr` is the PR Watch is looking at; `deleted` the branches already deleted this tick;
|
|
30
|
-
* `foreground`/`reaped`/`started` are for `cambium run
|
|
31
|
+
* `foreground`/`reaped`/`started` are for `cambium run`; `traceparent` is the issue trace while the turn is
|
|
32
|
+
* traced (tracing.mjs `turnTracer`), which markers written in the turn take.
|
|
31
33
|
* @typedef {import('../tick.mjs').Deps & {
|
|
32
34
|
* tick: import('../tick.mjs').Ctx, subject: Subject, traceparent?: string | null,
|
|
33
35
|
* }} TaskCtx
|
|
@@ -240,8 +242,8 @@ const startWorker = declare(
|
|
|
240
242
|
phase: 'task',
|
|
241
243
|
maxTurns: caps.maxTurns,
|
|
242
244
|
foreground,
|
|
243
|
-
//
|
|
244
|
-
|
|
245
|
+
// This node's cambium.node span, so Claude Code's own spans (when worker.env turns them on) join the trace.
|
|
246
|
+
env: ctx.traceparent ? { TRACEPARENT: ctx.traceparent } : undefined,
|
|
245
247
|
buildPrompt: (agentsMd) =>
|
|
246
248
|
taskPrompt({ repo: slug, issue, marker, branch, pr, agentsMd, maxAttempts: caps.maxAttempts }),
|
|
247
249
|
})
|
|
@@ -266,6 +268,7 @@ const startWorker = declare(
|
|
|
266
268
|
host: config.host,
|
|
267
269
|
stdout: s.stdout,
|
|
268
270
|
stderr: s.stderr,
|
|
271
|
+
traceparent: ctx.traceparent ?? null,
|
|
269
272
|
})
|
|
270
273
|
const where = { repo: slug, issue: issue.number }
|
|
271
274
|
if (s.reused) ctx.log('warn', `continuing in the previous worktree ${s.worktree}: ${s.reused}`, where)
|
|
@@ -314,8 +317,7 @@ const workerRunning = declare(
|
|
|
314
317
|
ctx.log('info', 'worker gone; issue is closed or no longer queued', { repo: w.repo, issue: w.issue })
|
|
315
318
|
return { ready: true, output: { exit: 'gone' } }
|
|
316
319
|
}
|
|
317
|
-
|
|
318
|
-
// `workerSpans(<w.stdout log>, { tracer, parent: <start-worker's traceparent> })`.
|
|
320
|
+
traceWorker(ctx, w)
|
|
319
321
|
const pr = prFor(snap, w)
|
|
320
322
|
const m = markerOf(issue)
|
|
321
323
|
const prepFailed = !w.killedReason && spawner.tail(w.stderr, 15).includes(PREPARE_FAILED_LINE)
|
package/src/issue-state.mjs
CHANGED
|
@@ -24,6 +24,10 @@ export async function setLabels(deps, slug, issue, change) {
|
|
|
24
24
|
|
|
25
25
|
/** @param {Writers} deps @param {string} slug @param {Issue} issue @param {import('./marker.mjs').Marker} marker */
|
|
26
26
|
export async function setMarker(deps, slug, issue, marker) {
|
|
27
|
+
// A traced graph turn keeps its issue trace on the subject (tracing.mjs `turnTracer`); a marker without one
|
|
28
|
+
// takes it, so later ticks parent on the same trace. One that already has a trace keeps it.
|
|
29
|
+
const tp = /** @type {any} */ (deps).subject?.traceparent
|
|
30
|
+
if (tp && !marker.traceparent) marker = { ...marker, traceparent: tp }
|
|
27
31
|
const maxAttempts = deps.config.phases[marker.phase]?.maxAttempts
|
|
28
32
|
const id = await deps.github.writeState(
|
|
29
33
|
slug,
|
package/src/store.mjs
CHANGED
|
@@ -13,7 +13,9 @@ import { DatabaseSync } from 'node:sqlite'
|
|
|
13
13
|
* @typedef {{
|
|
14
14
|
* repo: string, issue: number, phase: string, pid: number, worktree: string, branch: string,
|
|
15
15
|
* attempt: number, startedAt: number, host: string, stdout: string, stderr: string, killedReason: string | null,
|
|
16
|
+
* traceparent: string | null,
|
|
16
17
|
* }} Worker
|
|
18
|
+
* `traceparent`: the span of the step that started it, when traced; its `claude.*` spans go under it on exit.
|
|
17
19
|
*/
|
|
18
20
|
|
|
19
21
|
export const MODES = /** @type {const} */ (['running', 'paused', 'draining'])
|
|
@@ -22,7 +24,7 @@ const SCHEMA = `
|
|
|
22
24
|
CREATE TABLE IF NOT EXISTS workers (
|
|
23
25
|
repo TEXT NOT NULL, issue INTEGER NOT NULL, phase TEXT NOT NULL, pid INTEGER NOT NULL,
|
|
24
26
|
worktree TEXT NOT NULL, branch TEXT NOT NULL, attempt INTEGER NOT NULL, started_at INTEGER NOT NULL,
|
|
25
|
-
host TEXT NOT NULL, stdout TEXT NOT NULL, stderr TEXT NOT NULL, killed_reason TEXT,
|
|
27
|
+
host TEXT NOT NULL, stdout TEXT NOT NULL, stderr TEXT NOT NULL, killed_reason TEXT, traceparent TEXT,
|
|
26
28
|
PRIMARY KEY (repo, issue)
|
|
27
29
|
);
|
|
28
30
|
CREATE TABLE IF NOT EXISTS control (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
@@ -42,6 +44,7 @@ const toWorker = (r) => ({
|
|
|
42
44
|
stdout: r.stdout,
|
|
43
45
|
stderr: r.stderr,
|
|
44
46
|
killedReason: r.killed_reason ?? null,
|
|
47
|
+
traceparent: r.traceparent ?? null,
|
|
45
48
|
})
|
|
46
49
|
|
|
47
50
|
/** @param {string} path a file, or `:memory:` */
|
|
@@ -50,6 +53,9 @@ export function openStore(path) {
|
|
|
50
53
|
const db = new DatabaseSync(path)
|
|
51
54
|
db.exec('PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;')
|
|
52
55
|
db.exec(SCHEMA)
|
|
56
|
+
// Databases from before `traceparent` existed.
|
|
57
|
+
const cols = /** @type {{ name: string }[]} */ (db.prepare('PRAGMA table_info(workers)').all())
|
|
58
|
+
if (!cols.some((c) => c.name === 'traceparent')) db.exec('ALTER TABLE workers ADD COLUMN traceparent TEXT')
|
|
53
59
|
const get = (/** @type {string} */ key) =>
|
|
54
60
|
/** @type {any} */ (db.prepare('SELECT value FROM control WHERE key = ?').get(key))?.value ?? null
|
|
55
61
|
const set = (/** @type {string} */ key, /** @type {string} */ value) =>
|
|
@@ -65,12 +71,25 @@ export function openStore(path) {
|
|
|
65
71
|
const r = db.prepare('SELECT * FROM workers WHERE repo = ? AND issue = ?').get(repo, issue)
|
|
66
72
|
return r ? toWorker(r) : null
|
|
67
73
|
},
|
|
68
|
-
/** @param {Omit<Worker, 'killedReason'>} w */
|
|
74
|
+
/** @param {Omit<Worker, 'killedReason' | 'traceparent'> & { traceparent?: string | null }} w */
|
|
69
75
|
addWorker(w) {
|
|
70
76
|
db.prepare(
|
|
71
|
-
`INSERT OR REPLACE INTO workers (repo, issue, phase, pid, worktree, branch, attempt, started_at, host, stdout, stderr, killed_reason)
|
|
72
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`,
|
|
73
|
-
).run(
|
|
77
|
+
`INSERT OR REPLACE INTO workers (repo, issue, phase, pid, worktree, branch, attempt, started_at, host, stdout, stderr, killed_reason, traceparent)
|
|
78
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)`,
|
|
79
|
+
).run(
|
|
80
|
+
w.repo,
|
|
81
|
+
w.issue,
|
|
82
|
+
w.phase,
|
|
83
|
+
w.pid,
|
|
84
|
+
w.worktree,
|
|
85
|
+
w.branch,
|
|
86
|
+
w.attempt,
|
|
87
|
+
w.startedAt,
|
|
88
|
+
w.host,
|
|
89
|
+
w.stdout,
|
|
90
|
+
w.stderr,
|
|
91
|
+
w.traceparent ?? null,
|
|
92
|
+
)
|
|
74
93
|
},
|
|
75
94
|
/** @param {string} repo @param {number} issue @param {string} reason */
|
|
76
95
|
markKilled(repo, issue, reason) {
|
package/src/tick.mjs
CHANGED
|
@@ -35,6 +35,8 @@ import { DISPATCH_NOT_BUILT, dispatchNotBuilt } from './phases/dispatch.mjs'
|
|
|
35
35
|
* dispatch?: import('./phases/dispatch.mjs').Dispatch,
|
|
36
36
|
* release?: import('./phases/release.mjs').Release,
|
|
37
37
|
* tickNumber?: number,
|
|
38
|
+
* telemetry?: import('./tracing.mjs').Telemetry,
|
|
39
|
+
* updater?: import('./update.mjs').Updater,
|
|
38
40
|
* daemonStartedAt?: number,
|
|
39
41
|
* pid?: number,
|
|
40
42
|
* }} Deps
|
package/src/tracing.mjs
CHANGED
|
@@ -290,6 +290,62 @@ export function advanceHooks(telemetry, base) {
|
|
|
290
290
|
}
|
|
291
291
|
}
|
|
292
292
|
|
|
293
|
+
/**
|
|
294
|
+
* Tracing for one turn of a graph driver (graph-engine `drive`, dispatch-engine `driveDispatch`, release `drive`).
|
|
295
|
+
* Wraps {@link advanceHooks} and keeps `holder.traceparent` (the turn's subject) on the issue trace, so a marker
|
|
296
|
+
* written during the turn stores it (issue-state.mjs `setMarker`). Call `sync()` before writes made outside
|
|
297
|
+
* `advance` (a wait's `enter`). With telemetry off, or none, `instrument` is undefined and `advance` runs exactly
|
|
298
|
+
* as it does untraced.
|
|
299
|
+
*
|
|
300
|
+
* @param {Telemetry | undefined} telemetry
|
|
301
|
+
* @param {{ traceparent?: string | null }} holder
|
|
302
|
+
* @param {TraceInfo & { enteredAt?: string | null }} base as for {@link advanceHooks}
|
|
303
|
+
* @returns {{ instrument: import('./graph/advance.mjs').Instrument | undefined, sync: () => void }}
|
|
304
|
+
*/
|
|
305
|
+
export function turnTracer(telemetry, holder, base) {
|
|
306
|
+
if (!telemetry?.enabled) return { instrument: undefined, sync: () => {} }
|
|
307
|
+
const hooks = advanceHooks(telemetry, base)
|
|
308
|
+
const sync = () => {
|
|
309
|
+
if (hooks.traceparent) holder.traceparent = hooks.traceparent
|
|
310
|
+
}
|
|
311
|
+
return {
|
|
312
|
+
instrument: (visit, step) =>
|
|
313
|
+
hooks.instrument(visit, (extra) => {
|
|
314
|
+
sync()
|
|
315
|
+
return step(extra)
|
|
316
|
+
}),
|
|
317
|
+
sync,
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* The marker fields a turn's {@link turnTracer} starts from. No marker yet is a fresh issue: attempt 1, as
|
|
323
|
+
* marker.mjs `newMarker` has it.
|
|
324
|
+
* @param {{ traceparent?: string | null, enteredAt?: string | null, attempt?: number } | null | undefined} marker
|
|
325
|
+
*/
|
|
326
|
+
export const markerTrace = (marker) => ({
|
|
327
|
+
traceparent: marker?.traceparent ?? null,
|
|
328
|
+
enteredAt: marker?.enteredAt ?? null,
|
|
329
|
+
attempt: marker?.attempt ?? 1,
|
|
330
|
+
})
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* A finished worker's `claude.*` spans from its stream-json stdout log, under the step span that started it
|
|
334
|
+
* (`parent`, kept on the worker row). Never throws: tracing must not fail a tick.
|
|
335
|
+
*
|
|
336
|
+
* @param {{ telemetry?: Telemetry, spawner: { readLog(path: string): string },
|
|
337
|
+
* log: (level: 'info' | 'warn' | 'error', msg: string, where?: { repo?: string, issue?: number }) => void }} ctx
|
|
338
|
+
* @param {{ repo: string, issue: number, stdout: string, traceparent?: string | null }} w
|
|
339
|
+
*/
|
|
340
|
+
export function traceWorker(ctx, w) {
|
|
341
|
+
if (!ctx.telemetry?.enabled || !w.traceparent) return
|
|
342
|
+
try {
|
|
343
|
+
ctx.telemetry.workerSpans(ctx.spawner.readLog(w.stdout), w.traceparent)
|
|
344
|
+
} catch (err) {
|
|
345
|
+
ctx.log('warn', `worker spans: ${/** @type {Error} */ (err)?.message ?? err}`, { repo: w.repo, issue: w.issue })
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
293
349
|
/** A node output's own `outcome` string when it has one, else `ok`. @param {unknown} output */
|
|
294
350
|
const outcomeOf = (output) => {
|
|
295
351
|
const o = /** @type {any} */ (output)?.outcome
|
package/src/update.mjs
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* Self-update. cambium updates itself only when it is a global npm install (`<prefix>/lib/node_modules/
|
|
4
|
+
* @oakgreencc/cambium`), into that same prefix, so the launchd agent's paths stay valid. A source checkout, a
|
|
5
|
+
* workspace link or a copy inside another package's node_modules is left to whoever installed it.
|
|
6
|
+
*
|
|
7
|
+
* - `cambium update` installs the latest version, then has the launchd agent restart onto it (SIGTERM; it
|
|
8
|
+
* finishes its tick, exits, and KeepAlive starts it again).
|
|
9
|
+
* - With `update.auto`, the daemon checks every `update.checkMinutes`; on a newer version it installs it and
|
|
10
|
+
* exits between ticks, and launchd's KeepAlive starts it again on the new code. Only under launchd: anywhere
|
|
11
|
+
* else nothing would bring it back.
|
|
12
|
+
*/
|
|
13
|
+
import { execFile } from 'node:child_process'
|
|
14
|
+
import { existsSync, readFileSync, realpathSync } from 'node:fs'
|
|
15
|
+
import { dirname, join } from 'node:path'
|
|
16
|
+
import { fileURLToPath } from 'node:url'
|
|
17
|
+
import { LABEL } from './launchd.mjs'
|
|
18
|
+
|
|
19
|
+
export const PACKAGE = '@oakgreencc/cambium'
|
|
20
|
+
export const REGISTRY = 'https://registry.npmjs.org'
|
|
21
|
+
const MIN = 60_000
|
|
22
|
+
|
|
23
|
+
const PKG_DIR = fileURLToPath(new URL('..', import.meta.url))
|
|
24
|
+
|
|
25
|
+
/** The version of this copy of cambium, from its package.json. */
|
|
26
|
+
export const VERSION = /** @type {string} */ (JSON.parse(readFileSync(join(PKG_DIR, 'package.json'), 'utf8')).version)
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @typedef {{ code: number, stdout: string, stderr: string }} ExecResult
|
|
30
|
+
* @typedef {(cmd: string, args: string[]) => Promise<ExecResult>} Exec
|
|
31
|
+
* @typedef {{
|
|
32
|
+
* current: string,
|
|
33
|
+
* installOf: () => { prefix: string, dir: string } | null,
|
|
34
|
+
* latest: () => Promise<string>,
|
|
35
|
+
* install: (prefix: string, version: string) => Promise<void>,
|
|
36
|
+
* }} Updater
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
/** @type {Exec} */
|
|
40
|
+
export const defaultExec = (cmd, args) =>
|
|
41
|
+
new Promise((resolve) => {
|
|
42
|
+
execFile(cmd, args, { maxBuffer: 16 * 1024 * 1024 }, (err, stdout, stderr) => {
|
|
43
|
+
const code = err ? (typeof err.code === 'number' ? err.code : 1) : 0
|
|
44
|
+
resolve({ code, stdout: String(stdout), stderr: String(stderr || (err && !err.code ? err.message : '')) })
|
|
45
|
+
})
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Where this cambium is installed, when it is a global npm install: the npm prefix and the package directory.
|
|
50
|
+
* @param {string} [dir] the package directory (default: this one, symlinks resolved)
|
|
51
|
+
*/
|
|
52
|
+
export function installOf(dir = PKG_DIR) {
|
|
53
|
+
let real = dir
|
|
54
|
+
try {
|
|
55
|
+
real = realpathSync(dir)
|
|
56
|
+
} catch {}
|
|
57
|
+
const m = /^(.*)\/lib\/node_modules\/@oakgreencc\/cambium\/?$/.exec(real)
|
|
58
|
+
return m ? { prefix: m[1], dir: real } : null
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** `a` is a later x.y.z than `b` (pre-release tags are ignored). @param {string} a @param {string} b */
|
|
62
|
+
export function newer(a, b) {
|
|
63
|
+
const parts = (/** @type {string} */ v) => String(v).split('-')[0].split('.').map(Number)
|
|
64
|
+
const [x, y] = [parts(a), parts(b)]
|
|
65
|
+
for (let i = 0; i < 3; i++) {
|
|
66
|
+
const d = (x[i] || 0) - (y[i] || 0)
|
|
67
|
+
if (d) return d > 0
|
|
68
|
+
}
|
|
69
|
+
return false
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The `latest` dist-tag from the registry.
|
|
74
|
+
* @param {{ fetch?: typeof globalThis.fetch, registry?: string }} [o]
|
|
75
|
+
*/
|
|
76
|
+
export async function latestVersion({ fetch = globalThis.fetch, registry = REGISTRY } = {}) {
|
|
77
|
+
const res = await fetch(`${registry}/${PACKAGE.replace('/', '%2f')}/latest`, {
|
|
78
|
+
headers: { accept: 'application/json' },
|
|
79
|
+
})
|
|
80
|
+
if (!res.ok) throw new Error(`registry answered ${res.status} for ${PACKAGE}`)
|
|
81
|
+
const v = /** @type {any} */ (await res.json())?.version
|
|
82
|
+
if (typeof v !== 'string' || !/^\d+\.\d+\.\d+/.test(v)) throw new Error(`registry gave no version for ${PACKAGE}`)
|
|
83
|
+
return v
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** npm next to this Node (nvm, Homebrew and the official installer put it there), else npm on PATH. */
|
|
87
|
+
export function npmBin(node = process.execPath) {
|
|
88
|
+
const beside = join(dirname(node), 'npm')
|
|
89
|
+
return existsSync(beside) ? beside : 'npm'
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* `npm install -g --prefix <prefix> @oakgreencc/cambium@<version>`; throws with npm's output on failure.
|
|
94
|
+
* @param {Exec} exec @param {string} prefix @param {string} version
|
|
95
|
+
*/
|
|
96
|
+
export async function npmInstall(exec, prefix, version) {
|
|
97
|
+
const r = await exec(npmBin(), ['install', '--global', '--prefix', prefix, `${PACKAGE}@${version}`])
|
|
98
|
+
if (r.code !== 0) throw new Error(`npm install ${PACKAGE}@${version} failed: ${(r.stderr || r.stdout).trim()}`)
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** The real updater. @param {{ exec?: Exec, fetch?: typeof globalThis.fetch }} [o] @returns {Updater} */
|
|
102
|
+
export function realUpdater({ exec = defaultExec, fetch } = {}) {
|
|
103
|
+
return {
|
|
104
|
+
current: VERSION,
|
|
105
|
+
installOf: () => installOf(),
|
|
106
|
+
latest: () => latestVersion({ fetch }),
|
|
107
|
+
install: (prefix, version) => npmInstall(exec, prefix, version),
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Is this process the launchd agent (KeepAlive brings it back after an exit)? @param {Record<string, string | undefined>} env */
|
|
112
|
+
export const underLaunchd = (env) => env.XPC_SERVICE_NAME === LABEL
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The daemon's between-ticks check. Returns the version it installed (the caller then exits so launchd restarts
|
|
116
|
+
* it), or null. Never throws: a failed check or install is logged and retried at the next `checkMinutes`.
|
|
117
|
+
*
|
|
118
|
+
* @param {import('./tick.mjs').Deps} deps
|
|
119
|
+
* @param {{ updater: Updater, env: Record<string, string | undefined>, now: number }} o
|
|
120
|
+
* @returns {Promise<string | null>}
|
|
121
|
+
*/
|
|
122
|
+
export async function autoUpdate(deps, { updater, env, now }) {
|
|
123
|
+
const { auto, checkMinutes } = deps.config.update
|
|
124
|
+
if (!auto) return null
|
|
125
|
+
const last = Number(deps.store.get('update.lastCheck')) || 0
|
|
126
|
+
if (now - last < checkMinutes * MIN) return null
|
|
127
|
+
deps.store.set('update.lastCheck', String(now))
|
|
128
|
+
const skip = (/** @type {string} */ why) => {
|
|
129
|
+
if (deps.store.get('update.skipped') !== why) deps.log('warn', `auto-update skipped: ${why}`)
|
|
130
|
+
deps.store.set('update.skipped', why)
|
|
131
|
+
return null
|
|
132
|
+
}
|
|
133
|
+
if (!underLaunchd(env)) return skip('not running as the launchd agent, so nothing would restart it')
|
|
134
|
+
const where = updater.installOf()
|
|
135
|
+
if (!where) return skip('not a global npm install; update it the way it was installed')
|
|
136
|
+
try {
|
|
137
|
+
const latest = await updater.latest()
|
|
138
|
+
if (!newer(latest, updater.current)) return null
|
|
139
|
+
deps.log('info', `updating cambium ${updater.current} → ${latest} in ${where.prefix}`)
|
|
140
|
+
await updater.install(where.prefix, latest)
|
|
141
|
+
deps.log('info', `cambium ${latest} installed; exiting so launchd starts the new version`)
|
|
142
|
+
return latest
|
|
143
|
+
} catch (err) {
|
|
144
|
+
deps.log('warn', `auto-update: ${/** @type {Error} */ (err)?.message ?? err}`)
|
|
145
|
+
return null
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* `cambium update [--check] [--version <v>]`: report or install, then restart the launchd agent when it runs.
|
|
151
|
+
*
|
|
152
|
+
* @param {{ check: boolean, version?: string }} opts
|
|
153
|
+
* @param {{ updater: Updater, exec: Exec, uid: number, out: (s: string) => void }} io
|
|
154
|
+
* @returns {Promise<{ ok: boolean, error?: string }>}
|
|
155
|
+
*/
|
|
156
|
+
export async function runUpdate({ check, version }, { updater, exec, uid, out }) {
|
|
157
|
+
const target = version ?? (await updater.latest())
|
|
158
|
+
if (!newer(target, updater.current) && !version) {
|
|
159
|
+
out(`cambium ${updater.current} is up to date`)
|
|
160
|
+
return { ok: true }
|
|
161
|
+
}
|
|
162
|
+
if (check) {
|
|
163
|
+
out(`cambium ${updater.current} → ${target} available; run \`cambium update\``)
|
|
164
|
+
return { ok: true }
|
|
165
|
+
}
|
|
166
|
+
const where = updater.installOf()
|
|
167
|
+
if (!where)
|
|
168
|
+
return {
|
|
169
|
+
ok: false,
|
|
170
|
+
error: `this cambium is not a global npm install (${PACKAGE} under <prefix>/lib/node_modules); update it the way it was installed`,
|
|
171
|
+
}
|
|
172
|
+
out(`installing ${PACKAGE}@${target} into ${where.prefix}…`)
|
|
173
|
+
await updater.install(where.prefix, target)
|
|
174
|
+
out(`installed cambium ${target}`)
|
|
175
|
+
const service = `gui/${uid}/${LABEL}`
|
|
176
|
+
if ((await exec('launchctl', ['print', service])).code !== 0) {
|
|
177
|
+
out('the launchd agent is not loaded; start cambium yourself to run the new version')
|
|
178
|
+
return { ok: true }
|
|
179
|
+
}
|
|
180
|
+
// SIGTERM, not `kickstart -k`: the daemon finishes its current tick, exits, and KeepAlive starts the new version.
|
|
181
|
+
const r = await exec('launchctl', ['kill', 'TERM', service])
|
|
182
|
+
if (r.code !== 0) return { ok: false, error: `launchctl kill TERM ${service}: ${(r.stderr || r.stdout).trim()}` }
|
|
183
|
+
out(`stopping ${LABEL} after its current tick; launchd starts ${target} (workers keep running and are re-adopted)`)
|
|
184
|
+
return { ok: true }
|
|
185
|
+
}
|
package/src/workers.mjs
CHANGED
|
@@ -28,10 +28,11 @@ import { exec as defaultExec } from './github.mjs'
|
|
|
28
28
|
* @typedef {{
|
|
29
29
|
* repo: import('./config.mjs').Repo, convention: Convention, issue: number, attempt: number, branch: string,
|
|
30
30
|
* phase: string, maxTurns: number, buildPrompt: (agentsMd: string | null) => string, foreground?: boolean,
|
|
31
|
-
* readOnly?: boolean,
|
|
31
|
+
* readOnly?: boolean, env?: Record<string, string>,
|
|
32
32
|
* }} StartSpec
|
|
33
33
|
* `readOnly`: a disposable checkout for a worker that only reads (triage): detached at the base branch, no
|
|
34
34
|
* install, hydrate or prepare, no Edit/Write; `branch` only names its directory (see {@link topicOf}).
|
|
35
|
+
* `env`: added to the worker's environment after `config.worker.env` (the step's `TRACEPARENT`).
|
|
35
36
|
* @typedef {{
|
|
36
37
|
* pid: number, worktree: string, stdout: string, stderr: string, prompt: string, exited: Promise<number | null>,
|
|
37
38
|
* reused: string | null,
|
|
@@ -221,6 +222,7 @@ export function processSpawner({
|
|
|
221
222
|
buildPrompt,
|
|
222
223
|
foreground = false,
|
|
223
224
|
readOnly = false,
|
|
225
|
+
env: extraEnv = {},
|
|
224
226
|
}) {
|
|
225
227
|
const worktree = join(convention.root, topicOf(branch))
|
|
226
228
|
if (config.worker.fetch) {
|
|
@@ -280,6 +282,9 @@ export function processSpawner({
|
|
|
280
282
|
},
|
|
281
283
|
),
|
|
282
284
|
...(config.worker.graftApp ? { GRAFT_APP: config.worker.graftApp } : {}),
|
|
285
|
+
// config.worker.env (e.g. Claude Code's own telemetry), then this start's (TRACEPARENT).
|
|
286
|
+
...config.worker.env,
|
|
287
|
+
...extraEnv,
|
|
283
288
|
}
|
|
284
289
|
/** @type {import('node:child_process').ChildProcess} */
|
|
285
290
|
let child
|