@oakgreencc/cambium 0.0.21 → 0.0.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -8
- package/package.json +1 -1
- package/src/github.mjs +15 -0
- package/src/graphs/task.mjs +30 -8
- package/src/tracing.mjs +57 -1
- package/src/workers.mjs +3 -0
package/README.md
CHANGED
|
@@ -78,7 +78,7 @@ Set `graft.botLogin` to the Clock App's login and only markers written by it are
|
|
|
78
78
|
|
|
79
79
|
## Workers
|
|
80
80
|
|
|
81
|
-
A worker is `claude -p --output-format stream-json --verbose --max-turns <maxTurns> --allowedTools Read,Edit,Write,Glob,Grep,Bash --disallowedTools "Bash(graft issue label:*),Bash(graft issue edit:*),Bash(gh:*)" <worker.args>`:
|
|
81
|
+
A worker is `claude -p --output-format stream-json --verbose --thinking-display summarized --max-turns <maxTurns> --allowedTools Read,Edit,Write,Glob,Grep,Bash --disallowedTools "Bash(graft issue label:*),Bash(graft issue edit:*),Bash(gh:*)" <worker.args>`:
|
|
82
82
|
|
|
83
83
|
- in its own worktree, by the target repo's [worktree convention](#target-repo-convention);
|
|
84
84
|
- behind the prepare step: `node src/prepare.mjs -- claude …` runs install-fresh, hydrate and the repo's `prepare` commands in the worktree, writing to the worker's stderr log, then becomes the worker. It runs in the detached process so a long install never blocks a tick;
|
|
@@ -319,6 +319,7 @@ Neither of our repos runs a GitHub merge queue (it costs), so `merge` says who m
|
|
|
319
319
|
|
|
320
320
|
- `"none"` (default): not cambium. A merge queue, auto-merge or a human merges it; `mergeQueueMinutes` after submission an unmerged PR sends its issue to `q:human` (`stalled`).
|
|
321
321
|
- `"green"`: cambium merges it. Needs `engine: "graph"` (config refuses it on legacy). When `ci-running` sees the PR's checks `SUCCESS`, with no auto-merge armed and not in a merge queue, it goes to `merge`: `graft pr merge <n> --auto`, so graft's merge policy (bases, protected paths, a red base) still decides: a clean PR merges now, otherwise auto-merge is armed. Then `merging` waits for the merge. A refusal is recorded in the marker (`mergeRefusal`, history `merge-refused`) and asked again on a new head or after 10 minutes. Red CI still requeues, and `mergeQueueMinutes` still stalls to `q:human`.
|
|
322
|
+
- **Up to date first.** Before merging, `merge` asks GitHub live how far the PR's head is behind its base (`compare`, so a merge earlier in the same tick counts). Behind → `graft pr update <n>` instead of the merge (history `branch-updated`, held like a refusal on the old head), and the updated head merges once its CI is green; red there requeues like any red CI. This is a merge queue without the cost: two PRs green on the same base can be red together (sektorapp/monorepo#4624 + #4625 broke develop 8 s apart), and the base does not require up-to-date branches. An update GitHub refuses (a conflict) is a `merge-refused` and stalls to `q:human` like any other.
|
|
322
323
|
|
|
323
324
|
The worker prompt says which: "Green: stop; cambium merges the PR once CI is green" or "…the PR is merged outside this session".
|
|
324
325
|
|
|
@@ -498,13 +499,13 @@ if (polled.ready) out = await app.invoke(new Command({ resume: polled.output }),
|
|
|
498
499
|
|
|
499
500
|
Why not a shared package: cambium is published to npm, so it can only depend on published packages; a new publishable `@oakgreencc/telemetry` needs a protected `nx.json` release change. mycelium already publishes `createTracing`, so it gained a `serviceName` option and cambium depends on `@oakgreencc/mycelium` (which also gives it `parseTraceparent` for the intake trace).
|
|
500
501
|
|
|
501
|
-
| Span / metric
|
|
502
|
-
|
|
|
503
|
-
| `cambium.issue`
|
|
504
|
-
| `cambium.node`
|
|
505
|
-
| `cambium.wait`
|
|
506
|
-
| `cambium.wait.not_ready`
|
|
507
|
-
| `claude.session` / `.tool` / `.task` | From the worker's stream-json log, see below | The start-worker `cambium.node` span |
|
|
502
|
+
| Span / metric | When | Parent |
|
|
503
|
+
| ---------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- |
|
|
504
|
+
| `cambium.issue` | Once per issue, when the marker has no `traceparent`. Ended at once; its traceparent goes in the marker. | mycelium's intake span from the issue body, else none (a root) |
|
|
505
|
+
| `cambium.node` | Each step-node run | The marker's `traceparent` |
|
|
506
|
+
| `cambium.wait` | When a wait ends; `startTime` = the marker's `enteredAt`. No span stays open across ticks. | The marker's `traceparent` |
|
|
507
|
+
| `cambium.wait.not_ready` | Counter, +1 per not-ready poll (attributes graph, node, repo) | |
|
|
508
|
+
| `claude.session` / `.tool` / `.turn` / `.task` | From the worker's stream-json log, see below | The start-worker `cambium.node` span |
|
|
508
509
|
|
|
509
510
|
Node and wait spans carry `cambium.graph`, `cambium.graph.version`, `cambium.node`, `cambium.attempt`, `cambium.repo`, `cambium.issue.number`, `cambium.edge`, `cambium.outcome`.
|
|
510
511
|
|
|
@@ -558,6 +559,7 @@ telemetry.workerSpans(stream, parent) // same, with cambium's tracer
|
|
|
558
559
|
|
|
559
560
|
- `claude.session`: first to last `timestamp`. Attributes from `result` and `system/init`: `claude.session_id`, `claude.model`, `claude_code.version`, `claude.cost_usd`, `claude.tokens.{input,output,cache_read,cache_creation}`, `claude.turns`, `claude.duration_ms`, `claude.terminal_reason`, `claude.result`, `claude.permission_denials`. No `result` (killed, truncated) or an error result → ERROR.
|
|
560
561
|
- `claude.tool`: each `tool_use` → `tool_result` by id (a subagent's tools nest under its Agent tool span); `claude.tool.name`, `claude.tool.outcome` (`ok` / `error` / `unfinished`), `claude.tool.command` (Bash, clipped to 200 chars). Unpaired → ends at the last timestamp, ERROR.
|
|
562
|
+
- `claude.turn`: each assistant message (stream events grouped by `message.id`) with thinking or text; `claude.thinking` (the thinking summary), `claude.text`, each clipped to 4000 chars, and `claude.turn.tools` (tools it called). Runs from the event before the message to its last; a subagent's turns nest under its Agent tool span. Workers run with `--thinking-display summarized`, since stream-json otherwise carries thinking blocks with empty text; canopy shows a one-line preview in the span row and the full text in its detail.
|
|
561
563
|
- `claude.task`: each background task, `task_started` → `task_notification` (or `task_updated.patch.end_time`), under the tool that started it. These events have no timestamp, so their times are those of the latest timestamped event.
|
|
562
564
|
- Malformed lines are counted (`malformed`) and skipped; `system/thinking_tokens`, `tool_progress`, hooks and rate-limit events are ignored.
|
|
563
565
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@oakgreencc/cambium",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.23",
|
|
4
4
|
"description": "The scheduler daemon: reads the q:* queues from GitHub every tick, runs headless claude workers in their own worktrees up to a cap, and makes every queue transition itself.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|
package/src/github.mjs
CHANGED
|
@@ -47,8 +47,12 @@ import { Q, QUEUE_LABELS } from './labels.mjs'
|
|
|
47
47
|
* editPull(slug: string, number: number, change: { title: string, body: string }): Promise<void>,
|
|
48
48
|
* closePull(slug: string, number: number, comment: string): Promise<void>,
|
|
49
49
|
* mergePull(slug: string, number: number): Promise<void>,
|
|
50
|
+
* behindBase(slug: string, number: number): Promise<{ behind: number, base: string }>,
|
|
51
|
+
* updatePull(slug: string, number: number): Promise<void>,
|
|
50
52
|
* linkIssue(slug: string, pr: number, issue: number): Promise<void>,
|
|
51
53
|
* }} GitHub
|
|
54
|
+
* `behindBase`: how many commits of its base the PR's head lacks, read live (not from the tick's snapshot), so a
|
|
55
|
+
* merge earlier in the same tick counts. `updatePull`: merge the base into the PR (`graft pr update`).
|
|
52
56
|
* `linkIssue`: put `Closes #<issue>` at the top of the PR's body, so the PR names the issue it closes.
|
|
53
57
|
*/
|
|
54
58
|
|
|
@@ -368,6 +372,17 @@ export function graftGitHub({ config, exec: run = exec }) {
|
|
|
368
372
|
async mergePull(slug, number) {
|
|
369
373
|
await graft(slug, ['pr', 'merge', String(number), '--auto'])
|
|
370
374
|
},
|
|
375
|
+
async behindBase(slug, number) {
|
|
376
|
+
const pr = await json(graft(slug, ['api', 'GET', `/repos/${slug}/pulls/${number}`]))
|
|
377
|
+
const base = pr.base.ref
|
|
378
|
+
const cmp = await json(
|
|
379
|
+
graft(slug, ['api', 'GET', `/repos/${slug}/compare/${encodeURIComponent(base)}...${pr.head.sha}`]),
|
|
380
|
+
)
|
|
381
|
+
return { behind: Number(cmp.behind_by ?? 0), base }
|
|
382
|
+
},
|
|
383
|
+
async updatePull(slug, number) {
|
|
384
|
+
await graft(slug, ['pr', 'update', String(number)])
|
|
385
|
+
},
|
|
371
386
|
async linkIssue(slug, pr, issue) {
|
|
372
387
|
const path = `/repos/${slug}/pulls/${pr}`
|
|
373
388
|
const { body } = await json(graft(slug, ['api', 'GET', path]))
|
package/src/graphs/task.mjs
CHANGED
|
@@ -429,18 +429,36 @@ const merge = declare(
|
|
|
429
429
|
defineNode({
|
|
430
430
|
name: 'merge',
|
|
431
431
|
input: MergeIn,
|
|
432
|
-
output: z.object({
|
|
432
|
+
output: z.object({
|
|
433
|
+
pr: z.number().int(),
|
|
434
|
+
branch: z.string(),
|
|
435
|
+
refused: z.string().nullable(),
|
|
436
|
+
updated: z.string().nullable().optional(),
|
|
437
|
+
}),
|
|
433
438
|
/**
|
|
434
439
|
* Green: merge through `graft pr merge --auto` (a merge now when the PR is clean, else auto-merge), so graft's
|
|
435
440
|
* merge policy decides. A refusal is not a failure of the work: `merging` holds and asks again later.
|
|
441
|
+
*
|
|
442
|
+
* Behind its base (read live, so a merge earlier in this tick counts): update the branch instead. Its green
|
|
443
|
+
* was earned against an older base, and two PRs green on the same base can be red together (no merge queue,
|
|
444
|
+
* and the base does not require up-to-date branches). `merging` holds on the old head and merges once the
|
|
445
|
+
* updated head is green; red goes back to a worker like any red CI.
|
|
436
446
|
* @param {z.infer<typeof MergeIn>} input @param {TaskCtx} ctx
|
|
437
447
|
*/
|
|
438
448
|
run: async (input, ctx) => {
|
|
449
|
+
const { slug } = ctx.subject
|
|
439
450
|
try {
|
|
440
|
-
await ctx.github.
|
|
441
|
-
|
|
451
|
+
const { behind, base } = await ctx.github.behindBase(slug, input.pr)
|
|
452
|
+
if (behind > 0) {
|
|
453
|
+
await ctx.github.updatePull(slug, input.pr)
|
|
454
|
+
const commits = `${behind} commit${behind === 1 ? '' : 's'}`
|
|
455
|
+
const updated = `PR #${input.pr} was ${commits} behind ${base}; updated it, merging once CI is green on the new head`
|
|
456
|
+
return { ...input, refused: null, updated }
|
|
457
|
+
}
|
|
458
|
+
await ctx.github.mergePull(slug, input.pr)
|
|
459
|
+
return { ...input, refused: null, updated: null }
|
|
442
460
|
} catch (err) {
|
|
443
|
-
return { ...input, refused: `merge refused: ${/** @type {Error} */ (err)?.message ?? err}
|
|
461
|
+
return { ...input, refused: `merge refused: ${/** @type {Error} */ (err)?.message ?? err}`, updated: null }
|
|
444
462
|
}
|
|
445
463
|
},
|
|
446
464
|
}),
|
|
@@ -477,7 +495,7 @@ const merging = declare(
|
|
|
477
495
|
{
|
|
478
496
|
queue: null,
|
|
479
497
|
running: false,
|
|
480
|
-
/** @param {z.infer<typeof MergeIn> & { refused: string | null }} input @param {TaskCtx} ctx */
|
|
498
|
+
/** @param {z.infer<typeof MergeIn> & { refused: string | null, updated?: string | null }} input @param {TaskCtx} ctx */
|
|
481
499
|
enter: async (input, ctx) => {
|
|
482
500
|
const { slug } = ctx.subject
|
|
483
501
|
const issue = issueOf(ctx)
|
|
@@ -485,19 +503,23 @@ const merging = declare(
|
|
|
485
503
|
const pr = prOf(ctx, input)
|
|
486
504
|
const at = iso(ctx.tick.now)
|
|
487
505
|
const { refused } = input
|
|
506
|
+
const updated = input.updated ?? null
|
|
507
|
+
// An update holds like a refusal on the head it replaced: the new head is due as soon as it is green.
|
|
508
|
+
const hold = refused ?? updated
|
|
488
509
|
await land(ctx, 'merging', {
|
|
489
510
|
...withHistory(m, {
|
|
490
511
|
at,
|
|
491
512
|
attempt: m.attempt,
|
|
492
|
-
outcome: refused ? 'merge-refused' : 'merge-armed',
|
|
513
|
+
outcome: refused ? 'merge-refused' : updated ? 'branch-updated' : 'merge-armed',
|
|
493
514
|
pr: input.pr,
|
|
494
|
-
...(
|
|
515
|
+
...(hold ? { reason: hold } : {}),
|
|
495
516
|
}),
|
|
496
|
-
mergeRefusal:
|
|
517
|
+
mergeRefusal: hold ? { sha: pr?.headSha ?? null, at, reason: hold } : null,
|
|
497
518
|
lastFailure: refused ?? m.lastFailure ?? null,
|
|
498
519
|
input: { pr: input.pr, branch: input.branch },
|
|
499
520
|
})
|
|
500
521
|
if (refused) ctx.log('warn', refused.split('\n')[0], { repo: slug, issue: issue.number })
|
|
522
|
+
else if (updated) ctx.log('info', updated, { repo: slug, issue: issue.number })
|
|
501
523
|
else ctx.log('info', `merging PR #${input.pr}`, { repo: slug, issue: issue.number })
|
|
502
524
|
},
|
|
503
525
|
},
|
package/src/tracing.mjs
CHANGED
|
@@ -353,6 +353,13 @@ const outcomeOf = (output) => {
|
|
|
353
353
|
}
|
|
354
354
|
|
|
355
355
|
const COMMAND_MAX = 200
|
|
356
|
+
const TURN_TEXT_MAX = 4000
|
|
357
|
+
|
|
358
|
+
/** @param {string} a @param {string} b */
|
|
359
|
+
const join = (a, b) => (a && b ? `${a}\n\n${b}` : a || b)
|
|
360
|
+
|
|
361
|
+
/** A turn worth a span: it thought or said something. @param {Turn} u */
|
|
362
|
+
const said = (u) => Boolean(u.thinking.trim() || u.text.trim())
|
|
356
363
|
|
|
357
364
|
/** @param {unknown} s @param {number} [max] */
|
|
358
365
|
const clip = (s, max = COMMAND_MAX) => {
|
|
@@ -387,9 +394,10 @@ function* events(stream) {
|
|
|
387
394
|
/**
|
|
388
395
|
* @typedef {{ id: string, name: string, input: any, start: number, end: number | null, error: boolean, parentToolId: string | null }} ToolUse
|
|
389
396
|
* @typedef {{ id: string, toolUseId: string | null, type?: string, description?: string, start: number, end: number | null, status?: string }} BgTask
|
|
397
|
+
* @typedef {{ id: string, start: number, end: number, thinking: string, text: string, tools: string[], parentToolId: string | null }} Turn
|
|
390
398
|
* @typedef {{
|
|
391
399
|
* sessionId: string | null, complete: boolean, ok: boolean, start: number | null, end: number | null,
|
|
392
|
-
* tools: number, tasks: number, malformed: number, costUsd: number | null,
|
|
400
|
+
* tools: number, tasks: number, turns: number, malformed: number, costUsd: number | null,
|
|
393
401
|
* }} WorkerSummary
|
|
394
402
|
*/
|
|
395
403
|
|
|
@@ -403,6 +411,10 @@ function* events(stream) {
|
|
|
403
411
|
* - `claude.tool` per `tool_use` → matching `tool_result` (a subagent's tools nest under its Agent/Task tool span),
|
|
404
412
|
* with the tool name, ok/error and, for Bash, the command clipped to 200 chars. Unpaired → ends at the last
|
|
405
413
|
* timestamp, ERROR.
|
|
414
|
+
* - `claude.turn` per assistant message (its stream events grouped by `message.id`) that says or thinks anything:
|
|
415
|
+
* the thinking summary and text, each clipped to 4000 chars, and the tools it called. It runs from the event before
|
|
416
|
+
* the message (the model's input was ready) to its last event, under the subagent's tool span when it has one.
|
|
417
|
+
* Thinking is empty unless the worker runs with `--thinking-display summarized`.
|
|
406
418
|
* - `claude.task` per background task, `task_started` → `task_notification` (or `task_updated`'s `end_time`). These
|
|
407
419
|
* system events carry no timestamp, so their times are those of the latest timestamped event before them.
|
|
408
420
|
*
|
|
@@ -427,6 +439,8 @@ export function workerSpans(stream, { tracer, parent }) {
|
|
|
427
439
|
const tools = new Map()
|
|
428
440
|
/** @type {Map<string, BgTask>} */
|
|
429
441
|
const tasks = new Map()
|
|
442
|
+
/** @type {Map<string, Turn>} */
|
|
443
|
+
const turns = new Map()
|
|
430
444
|
|
|
431
445
|
for (const e of events(stream)) {
|
|
432
446
|
if (e.type === '__malformed') {
|
|
@@ -434,6 +448,7 @@ export function workerSpans(stream, { tracer, parent }) {
|
|
|
434
448
|
continue
|
|
435
449
|
}
|
|
436
450
|
const ts = typeof e.timestamp === 'string' ? Date.parse(e.timestamp) : Number.NaN
|
|
451
|
+
const before = last
|
|
437
452
|
if (!Number.isNaN(ts)) {
|
|
438
453
|
first ??= ts
|
|
439
454
|
last = Math.max(last ?? ts, ts)
|
|
@@ -443,6 +458,28 @@ export function workerSpans(stream, { tracer, parent }) {
|
|
|
443
458
|
if (e.type === 'system' && e.subtype === 'init') init ??= e
|
|
444
459
|
else if (e.type === 'result') result = e
|
|
445
460
|
else if (e.type === 'assistant' && at !== null) {
|
|
461
|
+
const id = typeof e.message?.id === 'string' ? e.message.id : typeof e.uuid === 'string' ? e.uuid : null
|
|
462
|
+
let turn = id ? turns.get(id) : undefined
|
|
463
|
+
if (id && !turn) {
|
|
464
|
+
turn = {
|
|
465
|
+
id,
|
|
466
|
+
start: Math.min(before ?? at, at),
|
|
467
|
+
end: at,
|
|
468
|
+
thinking: '',
|
|
469
|
+
text: '',
|
|
470
|
+
tools: [],
|
|
471
|
+
parentToolId: typeof e.parent_tool_use_id === 'string' ? e.parent_tool_use_id : null,
|
|
472
|
+
}
|
|
473
|
+
turns.set(id, turn)
|
|
474
|
+
}
|
|
475
|
+
if (turn) {
|
|
476
|
+
turn.end = Math.max(turn.end, at)
|
|
477
|
+
for (const c of content) {
|
|
478
|
+
if (c?.type === 'thinking' && typeof c.thinking === 'string') turn.thinking = join(turn.thinking, c.thinking)
|
|
479
|
+
else if (c?.type === 'text' && typeof c.text === 'string') turn.text = join(turn.text, c.text)
|
|
480
|
+
else if (c?.type === 'tool_use') turn.tools.push(String(c.name ?? 'unknown'))
|
|
481
|
+
}
|
|
482
|
+
}
|
|
446
483
|
for (const c of content)
|
|
447
484
|
if (c?.type === 'tool_use' && typeof c.id === 'string' && !tools.has(c.id))
|
|
448
485
|
tools.set(c.id, {
|
|
@@ -495,6 +532,7 @@ export function workerSpans(stream, { tracer, parent }) {
|
|
|
495
532
|
end: last,
|
|
496
533
|
tools: tools.size,
|
|
497
534
|
tasks: tasks.size,
|
|
535
|
+
turns: [...turns.values()].filter(said).length,
|
|
498
536
|
malformed,
|
|
499
537
|
costUsd: typeof result?.total_cost_usd === 'number' ? result.total_cost_usd : null,
|
|
500
538
|
}
|
|
@@ -555,6 +593,24 @@ export function workerSpans(stream, { tracer, parent }) {
|
|
|
555
593
|
span.end(t.end ?? end)
|
|
556
594
|
toolCtx.set(t.id, trace.setSpan(sessionCtx, span))
|
|
557
595
|
}
|
|
596
|
+
for (const u of turns.values()) {
|
|
597
|
+
if (!said(u)) continue
|
|
598
|
+
tracer
|
|
599
|
+
.startSpan(
|
|
600
|
+
'claude.turn',
|
|
601
|
+
{
|
|
602
|
+
startTime: u.start,
|
|
603
|
+
attributes: defined({
|
|
604
|
+
'claude.message_id': u.id,
|
|
605
|
+
'claude.thinking': clip(u.thinking, TURN_TEXT_MAX) || undefined,
|
|
606
|
+
'claude.text': clip(u.text, TURN_TEXT_MAX) || undefined,
|
|
607
|
+
'claude.turn.tools': u.tools.length ? u.tools.join(',') : undefined,
|
|
608
|
+
}),
|
|
609
|
+
},
|
|
610
|
+
(u.parentToolId && toolCtx.get(u.parentToolId)) || sessionCtx,
|
|
611
|
+
)
|
|
612
|
+
.end(u.end)
|
|
613
|
+
}
|
|
558
614
|
for (const k of tasks.values()) {
|
|
559
615
|
const span = tracer.startSpan(
|
|
560
616
|
'claude.task',
|
package/src/workers.mjs
CHANGED
|
@@ -78,6 +78,9 @@ export const workerArgs = (config, maxTurns, { readOnly = false } = {}) => [
|
|
|
78
78
|
'--output-format',
|
|
79
79
|
'stream-json',
|
|
80
80
|
'--verbose',
|
|
81
|
+
// Thinking is redacted in stream-json without this; summaries become `claude.turn` spans (tracing.mjs).
|
|
82
|
+
'--thinking-display',
|
|
83
|
+
'summarized',
|
|
81
84
|
'--max-turns',
|
|
82
85
|
String(maxTurns),
|
|
83
86
|
'--allowedTools',
|