@oakgreencc/cambium 0.0.21 → 0.0.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -78,7 +78,7 @@ Set `graft.botLogin` to the Clock App's login and only markers written by it are
78
78
 
79
79
  ## Workers
80
80
 
81
- A worker is `claude -p --output-format stream-json --verbose --max-turns <maxTurns> --allowedTools Read,Edit,Write,Glob,Grep,Bash --disallowedTools "Bash(graft issue label:*),Bash(graft issue edit:*),Bash(gh:*)" <worker.args>`:
81
+ A worker is `claude -p --output-format stream-json --verbose --thinking-display summarized --max-turns <maxTurns> --allowedTools Read,Edit,Write,Glob,Grep,Bash --disallowedTools "Bash(graft issue label:*),Bash(graft issue edit:*),Bash(gh:*)" <worker.args>`:
82
82
 
83
83
  - in its own worktree, by the target repo's [worktree convention](#target-repo-convention);
84
84
  - behind the prepare step: `node src/prepare.mjs -- claude …` runs install-fresh, hydrate and the repo's `prepare` commands in the worktree, writing to the worker's stderr log, then becomes the worker. It runs in the detached process so a long install never blocks a tick;
@@ -319,6 +319,7 @@ Neither of our repos runs a GitHub merge queue (it costs), so `merge` says who m
319
319
 
320
320
  - `"none"` (default): not cambium. A merge queue, auto-merge or a human merges it; `mergeQueueMinutes` after submission an unmerged PR sends its issue to `q:human` (`stalled`).
321
321
  - `"green"`: cambium merges it. Needs `engine: "graph"` (config refuses it on legacy). When `ci-running` sees the PR's checks `SUCCESS`, with no auto-merge armed and not in a merge queue, it goes to `merge`: `graft pr merge <n> --auto`, so graft's merge policy (bases, protected paths, a red base) still decides: a clean PR merges now, otherwise auto-merge is armed. Then `merging` waits for the merge. A refusal is recorded in the marker (`mergeRefusal`, history `merge-refused`) and asked again on a new head or after 10 minutes. Red CI still requeues, and `mergeQueueMinutes` still stalls to `q:human`.
322
+ - **Up to date first.** Before merging, `merge` asks GitHub live how far the PR's head is behind its base (`compare`, so a merge earlier in the same tick counts). Behind → `graft pr update <n>` instead of the merge (history `branch-updated`, held like a refusal on the old head), and the updated head merges once its CI is green; red there requeues like any red CI. This is a merge queue without the cost: two PRs green on the same base can be red together (sektorapp/monorepo#4624 + #4625 broke develop 8 s apart), and the base does not require up-to-date branches. An update GitHub refuses (a conflict) is a `merge-refused` and stalls to `q:human` like any other.
322
323
 
323
324
  The worker prompt says which: "Green: stop; cambium merges the PR once CI is green" or "…the PR is merged outside this session".
324
325
 
@@ -498,13 +499,13 @@ if (polled.ready) out = await app.invoke(new Command({ resume: polled.output }),
498
499
 
499
500
  Why not a shared package: cambium is published to npm, so it can only depend on published packages; a new publishable `@oakgreencc/telemetry` needs a protected `nx.json` release change. mycelium already publishes `createTracing`, so it gained a `serviceName` option and cambium depends on `@oakgreencc/mycelium` (which also gives it `parseTraceparent` for the intake trace).
500
501
 
501
- | Span / metric | When | Parent |
502
- | ------------------------------------ | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- |
503
- | `cambium.issue` | Once per issue, when the marker has no `traceparent`. Ended at once; its traceparent goes in the marker. | mycelium's intake span from the issue body, else none (a root) |
504
- | `cambium.node` | Each step-node run | The marker's `traceparent` |
505
- | `cambium.wait` | When a wait ends; `startTime` = the marker's `enteredAt`. No span stays open across ticks. | The marker's `traceparent` |
506
- | `cambium.wait.not_ready` | Counter, +1 per not-ready poll (attributes graph, node, repo) | |
507
- | `claude.session` / `.tool` / `.task` | From the worker's stream-json log, see below | The start-worker `cambium.node` span |
502
+ | Span / metric | When | Parent |
503
+ | ---------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- |
504
+ | `cambium.issue` | Once per issue, when the marker has no `traceparent`. Ended at once; its traceparent goes in the marker. | mycelium's intake span from the issue body, else none (a root) |
505
+ | `cambium.node` | Each step-node run | The marker's `traceparent` |
506
+ | `cambium.wait` | When a wait ends; `startTime` = the marker's `enteredAt`. No span stays open across ticks. | The marker's `traceparent` |
507
+ | `cambium.wait.not_ready` | Counter, +1 per not-ready poll (attributes graph, node, repo) | |
508
+ | `claude.session` / `.tool` / `.turn` / `.task` | From the worker's stream-json log, see below | The start-worker `cambium.node` span |
508
509
 
509
510
  Node and wait spans carry `cambium.graph`, `cambium.graph.version`, `cambium.node`, `cambium.attempt`, `cambium.repo`, `cambium.issue.number`, `cambium.edge`, `cambium.outcome`.
510
511
 
@@ -558,6 +559,7 @@ telemetry.workerSpans(stream, parent) // same, with cambium's tracer
558
559
 
559
560
  - `claude.session`: first to last `timestamp`. Attributes from `result` and `system/init`: `claude.session_id`, `claude.model`, `claude_code.version`, `claude.cost_usd`, `claude.tokens.{input,output,cache_read,cache_creation}`, `claude.turns`, `claude.duration_ms`, `claude.terminal_reason`, `claude.result`, `claude.permission_denials`. No `result` (killed, truncated) or an error result → ERROR.
560
561
  - `claude.tool`: each `tool_use` → `tool_result` by id (a subagent's tools nest under its Agent tool span); `claude.tool.name`, `claude.tool.outcome` (`ok` / `error` / `unfinished`), `claude.tool.command` (Bash, clipped to 200 chars). Unpaired → ends at the last timestamp, ERROR.
562
+ - `claude.turn`: each assistant message (stream events grouped by `message.id`) with thinking or text; `claude.thinking` (the thinking summary), `claude.text`, each clipped to 4000 chars, and `claude.turn.tools` (tools it called). Runs from the event before the message to its last; a subagent's turns nest under its Agent tool span. Workers run with `--thinking-display summarized`, since stream-json otherwise carries thinking blocks with empty text; canopy shows a one-line preview in the span row and the full text in its detail.
561
563
  - `claude.task`: each background task, `task_started` → `task_notification` (or `task_updated.patch.end_time`), under the tool that started it. These events have no timestamp, so their times are those of the latest timestamped event.
562
564
  - Malformed lines are counted (`malformed`) and skipped; `system/thinking_tokens`, `tool_progress`, hooks and rate-limit events are ignored.
563
565
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@oakgreencc/cambium",
3
- "version": "0.0.21",
3
+ "version": "0.0.23",
4
4
  "description": "The scheduler daemon: reads the q:* queues from GitHub every tick, runs headless claude workers in their own worktrees up to a cap, and makes every queue transition itself.",
5
5
  "license": "AGPL-3.0-only",
6
6
  "type": "module",
package/src/github.mjs CHANGED
@@ -47,8 +47,12 @@ import { Q, QUEUE_LABELS } from './labels.mjs'
47
47
  * editPull(slug: string, number: number, change: { title: string, body: string }): Promise<void>,
48
48
  * closePull(slug: string, number: number, comment: string): Promise<void>,
49
49
  * mergePull(slug: string, number: number): Promise<void>,
50
+ * behindBase(slug: string, number: number): Promise<{ behind: number, base: string }>,
51
+ * updatePull(slug: string, number: number): Promise<void>,
50
52
  * linkIssue(slug: string, pr: number, issue: number): Promise<void>,
51
53
  * }} GitHub
54
+ * `behindBase`: how many commits of its base the PR's head lacks, read live (not from the tick's snapshot), so a
55
+ * merge earlier in the same tick counts. `updatePull`: merge the base into the PR (`graft pr update`).
52
56
  * `linkIssue`: put `Closes #<issue>` at the top of the PR's body, so the PR names the issue it closes.
53
57
  */
54
58
 
@@ -368,6 +372,17 @@ export function graftGitHub({ config, exec: run = exec }) {
368
372
  async mergePull(slug, number) {
369
373
  await graft(slug, ['pr', 'merge', String(number), '--auto'])
370
374
  },
375
+ async behindBase(slug, number) {
376
+ const pr = await json(graft(slug, ['api', 'GET', `/repos/${slug}/pulls/${number}`]))
377
+ const base = pr.base.ref
378
+ const cmp = await json(
379
+ graft(slug, ['api', 'GET', `/repos/${slug}/compare/${encodeURIComponent(base)}...${pr.head.sha}`]),
380
+ )
381
+ return { behind: Number(cmp.behind_by ?? 0), base }
382
+ },
383
+ async updatePull(slug, number) {
384
+ await graft(slug, ['pr', 'update', String(number)])
385
+ },
371
386
  async linkIssue(slug, pr, issue) {
372
387
  const path = `/repos/${slug}/pulls/${pr}`
373
388
  const { body } = await json(graft(slug, ['api', 'GET', path]))
@@ -429,18 +429,36 @@ const merge = declare(
429
429
  defineNode({
430
430
  name: 'merge',
431
431
  input: MergeIn,
432
- output: z.object({ pr: z.number().int(), branch: z.string(), refused: z.string().nullable() }),
432
+ output: z.object({
433
+ pr: z.number().int(),
434
+ branch: z.string(),
435
+ refused: z.string().nullable(),
436
+ updated: z.string().nullable().optional(),
437
+ }),
433
438
  /**
434
439
  * Green: merge through `graft pr merge --auto` (a merge now when the PR is clean, else auto-merge), so graft's
435
440
  * merge policy decides. A refusal is not a failure of the work: `merging` holds and asks again later.
441
+ *
442
+ * Behind its base (read live, so a merge earlier in this tick counts): update the branch instead. Its green
443
+ * was earned against an older base, and two PRs green on the same base can be red together (no merge queue,
444
+ * and the base does not require up-to-date branches). `merging` holds on the old head and merges once the
445
+ * updated head is green; red goes back to a worker like any red CI.
436
446
  * @param {z.infer<typeof MergeIn>} input @param {TaskCtx} ctx
437
447
  */
438
448
  run: async (input, ctx) => {
449
+ const { slug } = ctx.subject
439
450
  try {
440
- await ctx.github.mergePull(ctx.subject.slug, input.pr)
441
- return { ...input, refused: null }
451
+ const { behind, base } = await ctx.github.behindBase(slug, input.pr)
452
+ if (behind > 0) {
453
+ await ctx.github.updatePull(slug, input.pr)
454
+ const commits = `${behind} commit${behind === 1 ? '' : 's'}`
455
+ const updated = `PR #${input.pr} was ${commits} behind ${base}; updated it, merging once CI is green on the new head`
456
+ return { ...input, refused: null, updated }
457
+ }
458
+ await ctx.github.mergePull(slug, input.pr)
459
+ return { ...input, refused: null, updated: null }
442
460
  } catch (err) {
443
- return { ...input, refused: `merge refused: ${/** @type {Error} */ (err)?.message ?? err}` }
461
+ return { ...input, refused: `merge refused: ${/** @type {Error} */ (err)?.message ?? err}`, updated: null }
444
462
  }
445
463
  },
446
464
  }),
@@ -477,7 +495,7 @@ const merging = declare(
477
495
  {
478
496
  queue: null,
479
497
  running: false,
480
- /** @param {z.infer<typeof MergeIn> & { refused: string | null }} input @param {TaskCtx} ctx */
498
+ /** @param {z.infer<typeof MergeIn> & { refused: string | null, updated?: string | null }} input @param {TaskCtx} ctx */
481
499
  enter: async (input, ctx) => {
482
500
  const { slug } = ctx.subject
483
501
  const issue = issueOf(ctx)
@@ -485,19 +503,23 @@ const merging = declare(
485
503
  const pr = prOf(ctx, input)
486
504
  const at = iso(ctx.tick.now)
487
505
  const { refused } = input
506
+ const updated = input.updated ?? null
507
+ // An update holds like a refusal on the head it replaced: the new head is due as soon as it is green.
508
+ const hold = refused ?? updated
488
509
  await land(ctx, 'merging', {
489
510
  ...withHistory(m, {
490
511
  at,
491
512
  attempt: m.attempt,
492
- outcome: refused ? 'merge-refused' : 'merge-armed',
513
+ outcome: refused ? 'merge-refused' : updated ? 'branch-updated' : 'merge-armed',
493
514
  pr: input.pr,
494
- ...(refused ? { reason: refused } : {}),
515
+ ...(hold ? { reason: hold } : {}),
495
516
  }),
496
- mergeRefusal: refused ? { sha: pr?.headSha ?? null, at, reason: refused } : null,
517
+ mergeRefusal: hold ? { sha: pr?.headSha ?? null, at, reason: hold } : null,
497
518
  lastFailure: refused ?? m.lastFailure ?? null,
498
519
  input: { pr: input.pr, branch: input.branch },
499
520
  })
500
521
  if (refused) ctx.log('warn', refused.split('\n')[0], { repo: slug, issue: issue.number })
522
+ else if (updated) ctx.log('info', updated, { repo: slug, issue: issue.number })
501
523
  else ctx.log('info', `merging PR #${input.pr}`, { repo: slug, issue: issue.number })
502
524
  },
503
525
  },
package/src/tracing.mjs CHANGED
@@ -353,6 +353,13 @@ const outcomeOf = (output) => {
353
353
  }
354
354
 
355
355
  const COMMAND_MAX = 200
356
+ const TURN_TEXT_MAX = 4000
357
+
358
+ /** @param {string} a @param {string} b */
359
+ const join = (a, b) => (a && b ? `${a}\n\n${b}` : a || b)
360
+
361
+ /** A turn worth a span: it thought or said something. @param {Turn} u */
362
+ const said = (u) => Boolean(u.thinking.trim() || u.text.trim())
356
363
 
357
364
  /** @param {unknown} s @param {number} [max] */
358
365
  const clip = (s, max = COMMAND_MAX) => {
@@ -387,9 +394,10 @@ function* events(stream) {
387
394
  /**
388
395
  * @typedef {{ id: string, name: string, input: any, start: number, end: number | null, error: boolean, parentToolId: string | null }} ToolUse
389
396
  * @typedef {{ id: string, toolUseId: string | null, type?: string, description?: string, start: number, end: number | null, status?: string }} BgTask
397
+ * @typedef {{ id: string, start: number, end: number, thinking: string, text: string, tools: string[], parentToolId: string | null }} Turn
390
398
  * @typedef {{
391
399
  * sessionId: string | null, complete: boolean, ok: boolean, start: number | null, end: number | null,
392
- * tools: number, tasks: number, malformed: number, costUsd: number | null,
400
+ * tools: number, tasks: number, turns: number, malformed: number, costUsd: number | null,
393
401
  * }} WorkerSummary
394
402
  */
395
403
 
@@ -403,6 +411,10 @@ function* events(stream) {
403
411
  * - `claude.tool` per `tool_use` → matching `tool_result` (a subagent's tools nest under its Agent/Task tool span),
404
412
  * with the tool name, ok/error and, for Bash, the command clipped to 200 chars. Unpaired → ends at the last
405
413
  * timestamp, ERROR.
414
+ * - `claude.turn` per assistant message (its stream events grouped by `message.id`) that says or thinks anything:
415
+ * the thinking summary and text, each clipped to 4000 chars, and the tools it called. It runs from the event before
416
+ * the message (the model's input was ready) to its last event, under the subagent's tool span when it has one.
417
+ * Thinking is empty unless the worker runs with `--thinking-display summarized`.
406
418
  * - `claude.task` per background task, `task_started` → `task_notification` (or `task_updated`'s `end_time`). These
407
419
  * system events carry no timestamp, so their times are those of the latest timestamped event before them.
408
420
  *
@@ -427,6 +439,8 @@ export function workerSpans(stream, { tracer, parent }) {
427
439
  const tools = new Map()
428
440
  /** @type {Map<string, BgTask>} */
429
441
  const tasks = new Map()
442
+ /** @type {Map<string, Turn>} */
443
+ const turns = new Map()
430
444
 
431
445
  for (const e of events(stream)) {
432
446
  if (e.type === '__malformed') {
@@ -434,6 +448,7 @@ export function workerSpans(stream, { tracer, parent }) {
434
448
  continue
435
449
  }
436
450
  const ts = typeof e.timestamp === 'string' ? Date.parse(e.timestamp) : Number.NaN
451
+ const before = last
437
452
  if (!Number.isNaN(ts)) {
438
453
  first ??= ts
439
454
  last = Math.max(last ?? ts, ts)
@@ -443,6 +458,28 @@ export function workerSpans(stream, { tracer, parent }) {
443
458
  if (e.type === 'system' && e.subtype === 'init') init ??= e
444
459
  else if (e.type === 'result') result = e
445
460
  else if (e.type === 'assistant' && at !== null) {
461
+ const id = typeof e.message?.id === 'string' ? e.message.id : typeof e.uuid === 'string' ? e.uuid : null
462
+ let turn = id ? turns.get(id) : undefined
463
+ if (id && !turn) {
464
+ turn = {
465
+ id,
466
+ start: Math.min(before ?? at, at),
467
+ end: at,
468
+ thinking: '',
469
+ text: '',
470
+ tools: [],
471
+ parentToolId: typeof e.parent_tool_use_id === 'string' ? e.parent_tool_use_id : null,
472
+ }
473
+ turns.set(id, turn)
474
+ }
475
+ if (turn) {
476
+ turn.end = Math.max(turn.end, at)
477
+ for (const c of content) {
478
+ if (c?.type === 'thinking' && typeof c.thinking === 'string') turn.thinking = join(turn.thinking, c.thinking)
479
+ else if (c?.type === 'text' && typeof c.text === 'string') turn.text = join(turn.text, c.text)
480
+ else if (c?.type === 'tool_use') turn.tools.push(String(c.name ?? 'unknown'))
481
+ }
482
+ }
446
483
  for (const c of content)
447
484
  if (c?.type === 'tool_use' && typeof c.id === 'string' && !tools.has(c.id))
448
485
  tools.set(c.id, {
@@ -495,6 +532,7 @@ export function workerSpans(stream, { tracer, parent }) {
495
532
  end: last,
496
533
  tools: tools.size,
497
534
  tasks: tasks.size,
535
+ turns: [...turns.values()].filter(said).length,
498
536
  malformed,
499
537
  costUsd: typeof result?.total_cost_usd === 'number' ? result.total_cost_usd : null,
500
538
  }
@@ -555,6 +593,24 @@ export function workerSpans(stream, { tracer, parent }) {
555
593
  span.end(t.end ?? end)
556
594
  toolCtx.set(t.id, trace.setSpan(sessionCtx, span))
557
595
  }
596
+ for (const u of turns.values()) {
597
+ if (!said(u)) continue
598
+ tracer
599
+ .startSpan(
600
+ 'claude.turn',
601
+ {
602
+ startTime: u.start,
603
+ attributes: defined({
604
+ 'claude.message_id': u.id,
605
+ 'claude.thinking': clip(u.thinking, TURN_TEXT_MAX) || undefined,
606
+ 'claude.text': clip(u.text, TURN_TEXT_MAX) || undefined,
607
+ 'claude.turn.tools': u.tools.length ? u.tools.join(',') : undefined,
608
+ }),
609
+ },
610
+ (u.parentToolId && toolCtx.get(u.parentToolId)) || sessionCtx,
611
+ )
612
+ .end(u.end)
613
+ }
558
614
  for (const k of tasks.values()) {
559
615
  const span = tracer.startSpan(
560
616
  'claude.task',
package/src/workers.mjs CHANGED
@@ -78,6 +78,9 @@ export const workerArgs = (config, maxTurns, { readOnly = false } = {}) => [
78
78
  '--output-format',
79
79
  'stream-json',
80
80
  '--verbose',
81
+ // Thinking is redacted in stream-json without this; summaries become `claude.turn` spans (tracing.mjs).
82
+ '--thinking-display',
83
+ 'summarized',
81
84
  '--max-turns',
82
85
  String(maxTurns),
83
86
  '--allowedTools',