@oakgreencc/cambium 0.0.24 → 0.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -4
- package/package.json +1 -1
- package/src/base-health.mjs +208 -0
- package/src/config.mjs +20 -4
- package/src/dispatch-engine.mjs +12 -2
- package/src/github.mjs +41 -6
- package/src/graph-engine.mjs +2 -1
- package/src/graphs/dispatch.mjs +113 -4
- package/src/graphs/task.mjs +20 -6
- package/src/marker.mjs +7 -1
- package/src/phases/dispatch.mjs +12 -8
- package/src/tick.mjs +27 -9
package/README.md
CHANGED
|
@@ -101,7 +101,15 @@ One JSON file. Everything but `repos` is optional; these are the defaults:
|
|
|
101
101
|
"repos": [{ "slug": "oakgreencc/monorepo", "path": "~/OGCC/monorepo", "baseBranch": "main" }],
|
|
102
102
|
"phases": {
|
|
103
103
|
"task": { "cap": 2, "wallClockMinutes": 60, "maxTurns": 200, "maxAttempts": 3 },
|
|
104
|
-
"dispatch": {
|
|
104
|
+
"dispatch": {
|
|
105
|
+
"cap": 0,
|
|
106
|
+
"wallClockMinutes": 15,
|
|
107
|
+
"maxTurns": 50,
|
|
108
|
+
"maxAttempts": 3,
|
|
109
|
+
"maxRounds": 3,
|
|
110
|
+
"deferHours": 6,
|
|
111
|
+
"maxDefers": 8
|
|
112
|
+
}
|
|
105
113
|
},
|
|
106
114
|
"mergeQueueMinutes": 120,
|
|
107
115
|
"merge": "none",
|
|
@@ -127,6 +135,7 @@ One JSON file. Everything but `repos` is optional; these are the defaults:
|
|
|
127
135
|
- `repos[].path` is a local clone; worktrees are added from it and `graft` runs in it.
|
|
128
136
|
- `graft.app` is the App cambium acts as (`GRAFT_APP` for its own `graft` calls); `worker.graftApp` the one workers act as. Neither leaks into the other.
|
|
129
137
|
- `merge` (top level, or per repo in `repos[]`): who merges a Task PR. See [Merging](#merging).
|
|
138
|
+
- `repos[].requiredCheck` (default `"ci"`): the check whose runs on `baseBranch` say whether the base is green; `null` turns the [red-base](#red-base) gate and repair off for that repo.
|
|
130
139
|
- `engine` picks how the Task phase runs: `"legacy"` (the stage tick, the default) or `"graph"` (the [`task` graph](#the-task-graph)). Both make the same moves; see [The task graph](#the-task-graph). Only `"graph"` runs the [`dispatch` graph](#the-dispatch-graph) (gated by `phases.dispatch.cap`); legacy leaves `q:intake` alone.
|
|
131
140
|
- Setting `paths.home` moves every default path under it.
|
|
132
141
|
- `release.time` is host-local `HH:MM`.
|
|
@@ -323,9 +332,17 @@ Neither of our repos runs a GitHub merge queue (it costs), so `merge` says who m
|
|
|
323
332
|
|
|
324
333
|
The worker prompt says which: "Green: stop; cambium merges the PR once CI is green" or "…the PR is merged outside this session".
|
|
325
334
|
|
|
335
|
+
## Red base
|
|
336
|
+
|
|
337
|
+
Every Task PR lands on `baseBranch`, so a red base fails every worker for a reason that is not its own. Without a gate each one spends its attempts and lands in `q:human` (sektorapp/monorepo#3972: three attempts killed at the cap while #4624 + #4625, green apart, had broken `develop` together). `src/base-health.mjs`, graph engine:
|
|
338
|
+
|
|
339
|
+
- **Observe.** The tick's snapshot reads `baseBranch`'s last 20 commits with their `requiredCheck` run. The newest commit with a verdict decides: `green`, `red` (with the last green commit and the merges since it, the suspects), or `unknown` (no completed run in the window; a pending or cancelled run says nothing, so a red base stays red until a run passes). The heartbeat carries it per repo (`repos[].base`). The base needs its check to run on push: a repo whose CI runs only on pull requests is always `unknown`.
|
|
340
|
+
- **Gate.** While red, Schedule starts no Task worker on that repo except the repair, which takes no slot. A failure during a red base requeues at the same attempt (history `base-red`, "not charged") and never goes to `q:human`; it waits in `q:task` for green. `unknown` gates nothing.
|
|
341
|
+
- **Resolve.** Red with no open repair issue → cambium opens one: labelled `red-base`, `q:task` and the first `priorityLabels` entry, its body the worker's spec (the failing run, the suspects, fix forward or revert the culprit, `pnpm verify`). It runs as an ordinary Task issue and is charged like one, so a repair that spends its attempts goes to `q:human`. Its PR merges through graft's red-base rescue (up to date with the base, its own check green). Green again → a repair issue with no open PR and no worker is closed with a comment.
|
|
342
|
+
|
|
326
343
|
## The dispatch graph
|
|
327
344
|
|
|
328
|
-
With `"engine": "graph"` and `phases.dispatch.cap > 0`, each `q:intake` issue is triaged by a one-shot `claude -p` worker (`src/graphs/dispatch.mjs`, steps in `src/dispatch-engine.mjs`, prompt and result in `src/phases/dispatch.mjs`), then moved to `q:task`, moved to `q:human`,
|
|
345
|
+
With `"engine": "graph"` and `phases.dispatch.cap > 0`, each `q:intake` issue is triaged by a one-shot `claude -p` worker (`src/graphs/dispatch.mjs`, steps in `src/dispatch-engine.mjs`, prompt and result in `src/phases/dispatch.mjs`), then moved to `q:task`, moved to `q:human`, closed as a duplicate or obsolete, or deferred to be triaged again later.
|
|
329
346
|
|
|
330
347
|
```mermaid
|
|
331
348
|
flowchart TD
|
|
@@ -336,6 +353,8 @@ flowchart TD
|
|
|
336
353
|
n3["to-task"]
|
|
337
354
|
n4["to-human"]
|
|
338
355
|
n5["close-duplicate"]
|
|
356
|
+
n6["deferred"]
|
|
357
|
+
n7["close-obsolete"]
|
|
339
358
|
end_(("end"))
|
|
340
359
|
start_ --> n0
|
|
341
360
|
n0 -->|"started"| n1
|
|
@@ -345,6 +364,8 @@ flowchart TD
|
|
|
345
364
|
n1 -->|"leaseLost"| n2
|
|
346
365
|
n1 -->|"actionable"| n3
|
|
347
366
|
n1 -->|"duplicate"| n5
|
|
367
|
+
n1 -->|"defer"| n6
|
|
368
|
+
n1 -->|"obsolete"| n7
|
|
348
369
|
n1 -->|"needsHuman"| n4
|
|
349
370
|
n1 --> n4
|
|
350
371
|
n2 -->|"attemptsLeft"| n0
|
|
@@ -352,6 +373,9 @@ flowchart TD
|
|
|
352
373
|
n3 --> end_
|
|
353
374
|
n4 --> end_
|
|
354
375
|
n5 --> end_
|
|
376
|
+
n6 -->|"budgetSpent"| n4
|
|
377
|
+
n6 --> end_
|
|
378
|
+
n7 --> end_
|
|
355
379
|
```
|
|
356
380
|
|
|
357
381
|
| Node | Kind | Queue label | Holds `running` |
|
|
@@ -362,11 +386,14 @@ flowchart TD
|
|
|
362
386
|
| `to-task` | step | `q:task` | no |
|
|
363
387
|
| `to-human` | step | `q:human` | no |
|
|
364
388
|
| `close-duplicate` | step | none | no |
|
|
389
|
+
| `deferred` | step | `q:intake` | no |
|
|
390
|
+
| `close-obsolete` | step | none | no |
|
|
365
391
|
|
|
366
|
-
- **The verdict.** The worker's final message must end with a JSON block, validated by zod (`TriageResult`): `{ verdict: 'actionable' | 'needs-human' | 'duplicate', duplicateOf?, reason, title?, acceptance? }` (`duplicateOf` is required on a duplicate and cannot be the issue itself). cambium reads it from the worker's stdout log: the last `result` event's `result` text, its last fenced block (else the bare JSON).
|
|
367
|
-
- **Outcomes.** `actionable` → a comment with the reason, the task title and acceptance checks (when given), then `q:task` with a task-graph marker at `start-worker`, attempt 1, history kept; the checks also ride in the marker (`triage`) into the task worker's prompt. The issue body is never rewritten: cambium's write path has no issue edit, and the author's text stays as written. `needs-human` → `q:human` + `why:triage`, the reason as the state comment's last failure. `duplicate` → closed with reason `duplicate` and `Duplicate of #n (<link>). <reason>`. A missing or invalid verdict (no `result` event, no JSON, a schema error) or a worker killed at `wallClockMinutes` (the unlabelled `triaging → to-human` edge) → `q:human` + `why:triage` with the error.
|
|
392
|
+
- **The verdict.** The worker's final message must end with a JSON block, validated by zod (`TriageResult`): `{ verdict: 'actionable' | 'needs-human' | 'duplicate' | 'obsolete' | 'defer', duplicateOf?, recheckHours?, reason, title?, acceptance? }` (`duplicateOf` is required on a duplicate and cannot be the issue itself). cambium reads it from the worker's stdout log: the last `result` event's `result` text, its last fenced block (else the bare JSON).
|
|
393
|
+
- **Outcomes.** `actionable` → a comment with the reason, the task title and acceptance checks (when given), then `q:task` with a task-graph marker at `start-worker`, attempt 1, history kept; the checks also ride in the marker (`triage`) into the task worker's prompt. The issue body is never rewritten: cambium's write path has no issue edit, and the author's text stays as written. `needs-human` → `q:human` + `why:triage`, the reason as the state comment's last failure. `duplicate` → closed with reason `duplicate` and `Duplicate of #n (<link>). <reason>`. `obsolete` (the work no longer applies, verified: the code path is gone, already fixed, its check already passed) → closed as `not_planned` with `Obsolete: <reason>`. `defer` → stays in `q:intake` at `deferred` (see below). A missing or invalid verdict (no `result` event, no JSON, a schema error) or a worker killed at `wallClockMinutes` (the unlabelled `triaging → to-human` edge) → `q:human` + `why:triage` with the error.
|
|
368
394
|
- **Read-only checkout.** Triage reads code to judge feasibility and duplicates, so it gets a checkout, but a disposable one: a detached worktree at `<root>/triage-<n>` on `origin/<base>` (the repo's worktree root), no install, hydrate or `prepare`, tools `Read,Glob,Grep,Bash` with Edit/Write and every graft/git write verb denied. It is removed outright after the verdict (no `graft status`: nothing in it is worth keeping).
|
|
369
395
|
- **Retries.** A worker that cannot start, or a lost `running` lease on a `q:intake` issue (Sweep), goes back to `q:intake` with attempt + 1, then to `q:human` once `phases.dispatch.maxAttempts` are spent. A marker from another graph or a finished triage starts over at attempt 1.
|
|
396
|
+
- **Defer, not a human.** A verdict that hangs on something a machine can check later (a CD run that has not happened, a release, a merge, a deploy) is `defer`, never `needs-human`: asking a human to verify what a machine can is what the factory exists to avoid. The issue rests at `deferred` with `deferUntil` (now + `recheckHours`, else `phases.dispatch.deferHours`, default 6) and `deferReason` in its marker; Schedule skips it until then, and the next triage gets the reason in its prompt ("check it now"). A defer is not a failed attempt. After `phases.dispatch.maxDefers` (default 8) it goes to `q:human` + `why:triage` with the last reason.
|
|
370
397
|
- **Leaving intake.** If the issue is closed or relabelled while triage runs, the verdict is dropped and the checkout removed.
|
|
371
398
|
- **Tracing.** Call sites are marked (`TRACING (follow-up)`) as in the task graph: the worker's `TRACEPARENT`, its stream-json spans, and `advanceHooks` in the driver, whose `cambium.issue` root continues mycelium's intake span from the issue body.
|
|
372
399
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@oakgreencc/cambium",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.26",
|
|
4
4
|
"description": "The scheduler daemon: reads the q:* queues from GitHub every tick, runs headless claude workers in their own worktrees up to a cap, and makes every queue transition itself.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"type": "module",
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* The base branch's health, and the repair of a red one.
|
|
4
|
+
*
|
|
5
|
+
* Every Task PR lands on the repo's `baseBranch`. When that branch is red, every worker's verify fails for a
|
|
6
|
+
* reason that is not its own, and without this module each one spends its attempts and lands in `q:human`
|
|
7
|
+
* (sektorapp/monorepo#3972: three attempts, three kills, on a break #4624 + #4625 made together). So:
|
|
8
|
+
*
|
|
9
|
+
* - **Observe.** The snapshot reads the base's last {@link HISTORY} commits with their `requiredCheck` run
|
|
10
|
+
* ({@link BASE_FIELDS}); {@link baseHealthOf} turns them into green / red / unknown, the last green commit and
|
|
11
|
+
* the merges since it.
|
|
12
|
+
* - **Gate.** While red, Schedule starts no Task worker for that repo except the repair (tick.mjs
|
|
13
|
+
* `taskCandidates`), and a failure is requeued without spending an attempt (graphs/task.mjs `requeue`).
|
|
14
|
+
* - **Resolve.** {@link trackBaseHealth} opens one repair issue per red base ({@link REPAIR_LABEL}, `q:task`,
|
|
15
|
+
* the top priority): its body is the worker's spec, and its PR merges through graft's red-base rescue. A base
|
|
16
|
+
* that goes green again closes a repair issue nobody is working.
|
|
17
|
+
*
|
|
18
|
+
* Unknown (no completed run in the window: the base has no `requiredCheck` on push, or every run is cancelled or
|
|
19
|
+
* pending) gates nothing.
|
|
20
|
+
*/
|
|
21
|
+
import { iso, key } from './issue-state.mjs'
|
|
22
|
+
import { Q } from './labels.mjs'
|
|
23
|
+
|
|
24
|
+
/** Marks the repair issue. Not a `why:` label: those leave with `q:human` (labels.mjs `move`). */
|
|
25
|
+
export const REPAIR_LABEL = 'red-base'
|
|
26
|
+
|
|
27
|
+
/** How many base commits the snapshot reads, newest first. */
|
|
28
|
+
export const HISTORY = 20
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The snapshot's base read, under `repository`. Needs `$baseRef`, `$check` and `$health`.
|
|
32
|
+
*/
|
|
33
|
+
export const BASE_FIELDS = `baseRef: ref(qualifiedName: $baseRef) @include(if: $health) {
|
|
34
|
+
target { ... on Commit { history(first: ${HISTORY}) { nodes {
|
|
35
|
+
oid messageHeadline
|
|
36
|
+
associatedPullRequests(first: 1) { nodes { number } }
|
|
37
|
+
checkSuites(first: 30) { nodes { branch { name } checkRuns(first: 3, filterBy: { checkName: $check }) { nodes { status conclusion detailsUrl } } } }
|
|
38
|
+
} } } }
|
|
39
|
+
}
|
|
40
|
+
repairs: issues(labels: ["${REPAIR_LABEL}"], states: OPEN, first: 5) @include(if: $health) { nodes { number } }`
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* @typedef {{ sha: string, headline: string, pr: number | null }} BaseCommit
|
|
44
|
+
* @typedef {{
|
|
45
|
+
* branch: string, check: string, state: 'green' | 'red' | 'unknown', head: string | null,
|
|
46
|
+
* red: (BaseCommit & { url: string | null }) | null, lastGreen: BaseCommit | null, suspects: BaseCommit[],
|
|
47
|
+
* repairs: number[],
|
|
48
|
+
* }} BaseHealth
|
|
49
|
+
* `red`: the newest commit whose run failed (the verdict); `lastGreen`: the newest passing commit older than
|
|
50
|
+
* it; `suspects`: the commits between them, newest first, `red` included, one per PR. `repairs`: open repair
|
|
51
|
+
* issues.
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
const PASSED = new Set(['SUCCESS', 'NEUTRAL', 'SKIPPED'])
|
|
55
|
+
const FAILED = new Set(['FAILURE', 'TIMED_OUT', 'STARTUP_FAILURE'])
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* A commit's verdict from its `check` runs on `branch`: the first completed run that says anything. Only a suite
|
|
59
|
+
* run for the base branch counts: the history also holds the merged PRs' own commits, whose runs judged the PR
|
|
60
|
+
* against an older base (PR CI on sektorapp is affected-only), not the base.
|
|
61
|
+
* @param {any} node @param {string} branch
|
|
62
|
+
*/
|
|
63
|
+
function verdictOf(node, branch) {
|
|
64
|
+
for (const suite of node?.checkSuites?.nodes ?? []) {
|
|
65
|
+
if (suite?.branch?.name !== branch) continue
|
|
66
|
+
for (const run of suite?.checkRuns?.nodes ?? []) {
|
|
67
|
+
if (run?.status !== 'COMPLETED') continue
|
|
68
|
+
if (PASSED.has(run.conclusion)) return { pass: true, url: run.detailsUrl ?? null }
|
|
69
|
+
if (FAILED.has(run.conclusion)) return { pass: false, url: run.detailsUrl ?? null }
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return null
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** @param {any} n @returns {BaseCommit} */
|
|
76
|
+
const commitOf = (n) => ({
|
|
77
|
+
sha: n.oid,
|
|
78
|
+
headline: n.messageHeadline ?? '',
|
|
79
|
+
pr: n.associatedPullRequests?.nodes?.[0]?.number ?? null,
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The base's health from its history, newest first. The newest commit with a verdict decides: a pending or
|
|
84
|
+
* cancelled run on the head says nothing, so a red base stays red until a run passes.
|
|
85
|
+
*
|
|
86
|
+
* @param {any} repo the snapshot's `repository` @param {{ branch: string, check: string }} at
|
|
87
|
+
* @returns {BaseHealth}
|
|
88
|
+
*/
|
|
89
|
+
export function baseHealthOf(repo, { branch, check }) {
|
|
90
|
+
const nodes = repo?.baseRef?.target?.history?.nodes ?? []
|
|
91
|
+
const repairs = (repo?.repairs?.nodes ?? []).map((/** @type {any} */ n) => n.number)
|
|
92
|
+
const base = { branch, check, head: nodes[0]?.oid ?? null, repairs }
|
|
93
|
+
const verdict = (/** @type {any} */ n) => verdictOf(n, branch)
|
|
94
|
+
const decided = nodes.findIndex(verdict)
|
|
95
|
+
if (decided === -1) return { ...base, state: 'unknown', red: null, lastGreen: null, suspects: [] }
|
|
96
|
+
const v = /** @type {{ pass: boolean, url: string | null }} */ (verdict(nodes[decided]))
|
|
97
|
+
if (v.pass) return { ...base, state: 'green', red: null, lastGreen: commitOf(nodes[decided]), suspects: [] }
|
|
98
|
+
const greenAt = nodes.findIndex((/** @type {any} */ n, /** @type {number} */ i) => i > decided && verdict(n)?.pass)
|
|
99
|
+
/** @type {Set<number>} */
|
|
100
|
+
const seen = new Set()
|
|
101
|
+
const suspects = nodes
|
|
102
|
+
.slice(decided, greenAt === -1 ? undefined : greenAt)
|
|
103
|
+
.map(commitOf)
|
|
104
|
+
.filter((c) => c.pr === null || (!seen.has(c.pr) && seen.add(c.pr)))
|
|
105
|
+
return {
|
|
106
|
+
...base,
|
|
107
|
+
state: 'red',
|
|
108
|
+
red: { ...commitOf(nodes[decided]), url: v.url },
|
|
109
|
+
lastGreen: greenAt === -1 ? null : commitOf(nodes[greenAt]),
|
|
110
|
+
suspects,
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Red, per the snapshot. @param {import('./github.mjs').Snapshot | undefined} snap */
|
|
115
|
+
export const baseRed = (snap) => snap?.base?.state === 'red'
|
|
116
|
+
|
|
117
|
+
/** @param {import('./github.mjs').Issue} issue */
|
|
118
|
+
export const isRepair = (issue) => issue.labels.includes(REPAIR_LABEL)
|
|
119
|
+
|
|
120
|
+
const short = (/** @type {string} */ sha) => sha.slice(0, 7)
|
|
121
|
+
|
|
122
|
+
/** @param {BaseHealth} h */
|
|
123
|
+
export const repairTitle = (h) => `Red base: ${h.branch} \`${h.check}\` failing since ${short(h.red?.sha ?? '')}`
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* The repair issue's body: the worker's spec.
|
|
127
|
+
* @param {BaseHealth} h
|
|
128
|
+
*/
|
|
129
|
+
export function repairBody(h) {
|
|
130
|
+
const red = /** @type {NonNullable<BaseHealth['red']>} */ (h.red)
|
|
131
|
+
const line = (/** @type {BaseCommit} */ c) => `- ${short(c.sha)}${c.pr ? ` #${c.pr}` : ''} ${c.headline}`
|
|
132
|
+
return [
|
|
133
|
+
`\`${h.branch}\` is red: its \`${h.check}\` check failed on ${short(red.sha)}${red.url ? ` (${red.url})` : ''}.`,
|
|
134
|
+
'',
|
|
135
|
+
'Opened by cambium. While this is open cambium starts no other Task worker on this repo, and their failures cost no attempts.',
|
|
136
|
+
'',
|
|
137
|
+
'## Suspects',
|
|
138
|
+
'',
|
|
139
|
+
h.lastGreen
|
|
140
|
+
? `Last green: ${short(h.lastGreen.sha)} ${h.lastGreen.headline}. Merged since, newest first:`
|
|
141
|
+
: `No green commit in the last ${HISTORY}. Newest first:`,
|
|
142
|
+
'',
|
|
143
|
+
...h.suspects.map(line),
|
|
144
|
+
'',
|
|
145
|
+
'## Work',
|
|
146
|
+
'',
|
|
147
|
+
`1. Read why it failed: \`graft jobs ${red.url ?? '<run>'}\`. Reproduce with \`pnpm verify\` in your worktree, which starts at \`${h.branch}\`.`,
|
|
148
|
+
'2. Two PRs that are each green can be red together, so the cause may span suspects.',
|
|
149
|
+
'3. Fix forward when the fix is clear and small. Otherwise revert the merge that broke it, and say on the PR which one and why.',
|
|
150
|
+
'4. Change only what makes the check green; do not fold other work into this PR.',
|
|
151
|
+
'',
|
|
152
|
+
`Your PR closes this issue. cambium merges it through graft's red-base rescue, once it is up to date with \`${h.branch}\` and its own \`${h.check}\` is green.`,
|
|
153
|
+
'',
|
|
154
|
+
'## Done when',
|
|
155
|
+
'',
|
|
156
|
+
`- \`pnpm verify\` passes on \`${h.branch}\` with your change.`,
|
|
157
|
+
`- \`${h.branch}\`'s \`${h.check}\` is green after the merge.`,
|
|
158
|
+
].join('\n')
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* The repair issue's lifecycle, once per tick before Schedule. Red with no open repair issue → open one
|
|
163
|
+
* (`q:task`, the top priority label, {@link REPAIR_LABEL}). Green, with an open repair issue that has no open PR
|
|
164
|
+
* and no worker → close it. Unknown → nothing.
|
|
165
|
+
*
|
|
166
|
+
* @param {import('./tick.mjs').Deps} deps @param {import('./tick.mjs').Ctx} ctx
|
|
167
|
+
*/
|
|
168
|
+
export async function trackBaseHealth(deps, ctx) {
|
|
169
|
+
for (const [slug, snap] of ctx.snaps) {
|
|
170
|
+
const h = snap.base
|
|
171
|
+
if (!h) continue
|
|
172
|
+
try {
|
|
173
|
+
if (h.state === 'red' && !h.repairs.length) {
|
|
174
|
+
const made = await deps.github.createIssue(slug, {
|
|
175
|
+
title: repairTitle(h),
|
|
176
|
+
body: repairBody(h),
|
|
177
|
+
labels: [Q.task, deps.config.priorityLabels[0], REPAIR_LABEL].filter(Boolean),
|
|
178
|
+
})
|
|
179
|
+
h.repairs = [made.number]
|
|
180
|
+
deps.log('warn', `${h.branch} is red at ${short(h.red?.sha ?? '')}; opened repair #${made.number}`, {
|
|
181
|
+
repo: slug,
|
|
182
|
+
issue: made.number,
|
|
183
|
+
})
|
|
184
|
+
continue
|
|
185
|
+
}
|
|
186
|
+
if (h.state !== 'green') continue
|
|
187
|
+
/** @type {number[]} */
|
|
188
|
+
const closed = []
|
|
189
|
+
for (const n of h.repairs) {
|
|
190
|
+
const worked = deps.store.worker(slug, n) || snap.prs.some((p) => p.state === 'OPEN' && p.closes.includes(n))
|
|
191
|
+
if (worked || ctx.touched.has(key(slug, n))) continue
|
|
192
|
+
await deps.github.close(
|
|
193
|
+
slug,
|
|
194
|
+
n,
|
|
195
|
+
`\`${h.branch}\` is green again at ${short(h.lastGreen?.sha ?? h.head ?? '')} (${iso(ctx.now)}); nothing left to repair.`,
|
|
196
|
+
{ reason: 'completed' },
|
|
197
|
+
)
|
|
198
|
+
const issue = snap.issues.get(n)
|
|
199
|
+
if (issue) issue.state = 'CLOSED'
|
|
200
|
+
closed.push(n)
|
|
201
|
+
deps.log('info', `${h.branch} is green again; closed repair #${n}`, { repo: slug, issue: n })
|
|
202
|
+
}
|
|
203
|
+
h.repairs = h.repairs.filter((n) => !closed.includes(n))
|
|
204
|
+
} catch (err) {
|
|
205
|
+
deps.log('error', `base health: ${/** @type {Error} */ (err)?.message ?? err}`, { repo: slug })
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
package/src/config.mjs
CHANGED
|
@@ -16,7 +16,12 @@ import { join, resolve } from 'node:path'
|
|
|
16
16
|
* Who merges a Task PR. `none`: not cambium (a merge queue, auto-merge, or a human; past `mergeQueueMinutes`
|
|
17
17
|
* the issue goes to `q:human`). `green`: cambium, once its checks are green, through `graft pr merge --auto`
|
|
18
18
|
* (so graft's merge policy still decides). `green` needs `engine: "graph"`.
|
|
19
|
-
* @typedef {{
|
|
19
|
+
* @typedef {{
|
|
20
|
+
* slug: string, path: string, baseBranch: string, release: RepoRelease, merge: MergePolicy,
|
|
21
|
+
* requiredCheck: string | null,
|
|
22
|
+
* }} Repo
|
|
23
|
+
* `requiredCheck` (default `ci`): the check whose runs on `baseBranch` say whether the base is green (see
|
|
24
|
+
* base-health.mjs); `null` turns the red-base gate and repair off for the repo.
|
|
20
25
|
* @typedef {{ cap: number, wallClockMinutes: number, maxTurns: number, maxAttempts: number }} PhaseCaps
|
|
21
26
|
* @typedef {{
|
|
22
27
|
* host: string,
|
|
@@ -24,7 +29,7 @@ import { join, resolve } from 'node:path'
|
|
|
24
29
|
* tickSeconds: number,
|
|
25
30
|
* stepTimeoutSeconds: number,
|
|
26
31
|
* repos: Repo[],
|
|
27
|
-
* phases: { task: PhaseCaps, dispatch: PhaseCaps & { maxRounds: number } },
|
|
32
|
+
* phases: { task: PhaseCaps, dispatch: PhaseCaps & { maxRounds: number, deferHours: number, maxDefers: number } },
|
|
28
33
|
* mergeQueueMinutes: number,
|
|
29
34
|
* merge: MergePolicy,
|
|
30
35
|
* priorityLabels: string[],
|
|
@@ -57,7 +62,15 @@ export function defaults(home = DEFAULT_HOME) {
|
|
|
57
62
|
repos: [],
|
|
58
63
|
phases: {
|
|
59
64
|
task: { cap: 2, wallClockMinutes: 60, maxTurns: 200, maxAttempts: 3 },
|
|
60
|
-
dispatch: {
|
|
65
|
+
dispatch: {
|
|
66
|
+
cap: 0,
|
|
67
|
+
wallClockMinutes: 15,
|
|
68
|
+
maxTurns: 50,
|
|
69
|
+
maxAttempts: 3,
|
|
70
|
+
maxRounds: 3,
|
|
71
|
+
deferHours: 6,
|
|
72
|
+
maxDefers: 8,
|
|
73
|
+
},
|
|
61
74
|
},
|
|
62
75
|
mergeQueueMinutes: 120,
|
|
63
76
|
merge: 'none',
|
|
@@ -121,7 +134,10 @@ export function resolveConfig(raw = {}) {
|
|
|
121
134
|
const merge = r.merge ?? config.merge
|
|
122
135
|
if (!MERGE_POLICIES.includes(merge))
|
|
123
136
|
throw new Error(`config: repo ${r.slug} merge must be "none" or "green", got ${JSON.stringify(merge)}`)
|
|
124
|
-
|
|
137
|
+
const requiredCheck = r.requiredCheck === undefined ? 'ci' : r.requiredCheck
|
|
138
|
+
if (requiredCheck !== null && (typeof requiredCheck !== 'string' || !requiredCheck))
|
|
139
|
+
throw new Error(`config: repo ${r.slug} requiredCheck must be a check name or null`)
|
|
140
|
+
return { slug: r.slug, path: untilde(r.path), baseBranch, release, merge, requiredCheck }
|
|
125
141
|
})
|
|
126
142
|
if (!MERGE_POLICIES.includes(config.merge))
|
|
127
143
|
throw new Error(`config: merge must be "none" or "green", got ${JSON.stringify(config.merge)}`)
|
package/src/dispatch-engine.mjs
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* - Reconcile: every tracked triage worker's issue at `triaging`.
|
|
7
7
|
* - Sweep: a stale `running` lease on a `q:intake` issue runs `retry-triage`.
|
|
8
8
|
* - Schedule: `q:intake` issues without a lease, by priority then age, up to `phases.dispatch.cap` counted from
|
|
9
|
-
* live triage workers; each starts at `start-triage`.
|
|
9
|
+
* live triage workers, skipping one deferred until later; each starts at `start-triage`.
|
|
10
10
|
*
|
|
11
11
|
* A turn advances one transition at a time and stops on entering a wait, at the entry node again (only Schedule
|
|
12
12
|
* admits entries), at a wait that is not ready, or at the end.
|
|
@@ -119,7 +119,8 @@ export function intakeCandidates(deps, ctx) {
|
|
|
119
119
|
issue.state === 'OPEN' &&
|
|
120
120
|
issue.labels.includes(Q.intake) &&
|
|
121
121
|
!issue.labels.includes(RUNNING) &&
|
|
122
|
-
!store.worker(slug, issue.number)
|
|
122
|
+
!store.worker(slug, issue.number) &&
|
|
123
|
+
!deferredPast(issue, ctx.now)
|
|
123
124
|
)
|
|
124
125
|
candidates.push({ slug, issue, snap })
|
|
125
126
|
const order = byPriorityThenAge(config.priorityLabels)
|
|
@@ -127,6 +128,15 @@ export function intakeCandidates(deps, ctx) {
|
|
|
127
128
|
return candidates.slice(0, free)
|
|
128
129
|
}
|
|
129
130
|
|
|
131
|
+
/**
|
|
132
|
+
* Resting at `deferred` with its recheck time still ahead: triage waits for it.
|
|
133
|
+
* @param {import('./github.mjs').Issue} issue @param {number} now
|
|
134
|
+
*/
|
|
135
|
+
const deferredPast = (issue, now) => {
|
|
136
|
+
const m = issue.stateComment?.marker
|
|
137
|
+
return m?.graph === dispatch.name && m.node === 'deferred' && !!m.deferUntil && Date.parse(m.deferUntil) > now
|
|
138
|
+
}
|
|
139
|
+
|
|
130
140
|
/** @param {Deps} deps @param {Ctx} ctx */
|
|
131
141
|
async function schedule(deps, ctx) {
|
|
132
142
|
await each(
|
package/src/github.mjs
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { spawn } from 'node:child_process'
|
|
9
9
|
import { parseMarker } from './marker.mjs'
|
|
10
10
|
import { Q, QUEUE_LABELS } from './labels.mjs'
|
|
11
|
+
import { BASE_FIELDS, baseHealthOf } from './base-health.mjs'
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
14
|
* @typedef {{
|
|
@@ -35,13 +36,18 @@ import { Q, QUEUE_LABELS } from './labels.mjs'
|
|
|
35
36
|
* The repo's promotion state (when its `release` is on): how far `from` is ahead of `to` (null: `to` does not
|
|
36
37
|
* exist), the PRs merged into `from` since `to` (oldest first, from the last 100 commits ahead), and the
|
|
37
38
|
* promotion PRs: open ones from `from` to `to` plus any (open, merged or closed) still labelled `q:promote`.
|
|
38
|
-
* @typedef {{
|
|
39
|
+
* @typedef {{
|
|
40
|
+
* issues: Map<number, Issue>, prs: PullRequest[], release?: Release | null,
|
|
41
|
+
* base?: import('./base-health.mjs').BaseHealth,
|
|
42
|
+
* }} Snapshot
|
|
43
|
+
* `base`: the base branch's health, when the repo has a `requiredCheck`.
|
|
39
44
|
* @typedef {{
|
|
40
45
|
* read(slug: string): Promise<Snapshot>,
|
|
41
46
|
* label(slug: string, number: number, change: { add: string[], remove: string[] }): Promise<void>,
|
|
42
47
|
* writeState(slug: string, number: number, commentId: number | null, body: string): Promise<number>,
|
|
43
48
|
* close(slug: string, number: number, body: string, opts?: { reason?: 'completed' | 'not_planned' | 'duplicate' }): Promise<void>,
|
|
44
49
|
* comment(slug: string, number: number, body: string): Promise<void>,
|
|
50
|
+
* createIssue(slug: string, issue: { title: string, body: string, labels: string[] }): Promise<{ number: number, url: string }>,
|
|
45
51
|
* deleteBranch(slug: string, branch: string): Promise<void>,
|
|
46
52
|
* openPull(slug: string, pr: { head: string, base: string, title: string, body: string }): Promise<{ number: number, url: string }>,
|
|
47
53
|
* editPull(slug: string, number: number, change: { title: string, body: string }): Promise<void>,
|
|
@@ -70,7 +76,7 @@ const PR_FIELDS = `number url headRefName state headRefOid isInMergeQueue body
|
|
|
70
76
|
* One query per repo per tick: the queued issues and the agent PRs with their checks; with `$release`, also the
|
|
71
77
|
* promotion PRs and how far `$from` is ahead of `$to` (see {@link Release}).
|
|
72
78
|
*/
|
|
73
|
-
export const SNAPSHOT_QUERY = `query($owner: String!, $name: String!, $labels: [String!], $release: Boolean!, $from: String!, $to: String!) {
|
|
79
|
+
export const SNAPSHOT_QUERY = `query($owner: String!, $name: String!, $labels: [String!], $release: Boolean!, $from: String!, $to: String!, $health: Boolean!, $baseRef: String!, $check: String!) {
|
|
74
80
|
repository(owner: $owner, name: $name) {
|
|
75
81
|
issues(labels: $labels, states: OPEN, first: 100, orderBy: { field: CREATED_AT, direction: ASC }) { nodes { ...I } }
|
|
76
82
|
open: pullRequests(states: OPEN, first: 100, orderBy: { field: UPDATED_AT, direction: DESC }) { nodes { ${PR_FIELDS} } }
|
|
@@ -84,6 +90,7 @@ export const SNAPSHOT_QUERY = `query($owner: String!, $name: String!, $labels: [
|
|
|
84
90
|
commits(last: 100) { nodes { associatedPullRequests(first: 3) { nodes { number title baseRefName state } } } }
|
|
85
91
|
}
|
|
86
92
|
}
|
|
93
|
+
${BASE_FIELDS}
|
|
87
94
|
}
|
|
88
95
|
}
|
|
89
96
|
fragment I on Issue {
|
|
@@ -222,7 +229,7 @@ export function addIssues(snap, data, asked, botLogin = null) {
|
|
|
222
229
|
* `botLogin`: when set, only that login's markers count. `release`: the branches the query asked about.
|
|
223
230
|
* @returns {Snapshot}
|
|
224
231
|
*/
|
|
225
|
-
export function normalize(data, { botLogin = null, release = null } = {}) {
|
|
232
|
+
export function normalize(data, { botLogin = null, release = null, health = null } = {}) {
|
|
226
233
|
const repo = (data?.data ?? data)?.repository
|
|
227
234
|
if (!repo) throw new Error('snapshot: no repository in the answer')
|
|
228
235
|
/** @type {Map<number, Issue>} */
|
|
@@ -258,7 +265,12 @@ export function normalize(data, { botLogin = null, release = null } = {}) {
|
|
|
258
265
|
closes: [...new Set([...(p.closingIssuesReferences?.nodes ?? []).map(addIssue), ...bodyCloses(p.body)])],
|
|
259
266
|
})
|
|
260
267
|
}
|
|
261
|
-
return {
|
|
268
|
+
return {
|
|
269
|
+
issues,
|
|
270
|
+
prs,
|
|
271
|
+
...(release ? { release: normalizeRelease(repo, release, botLogin) } : {}),
|
|
272
|
+
...(health ? { base: baseHealthOf(repo, health) } : {}),
|
|
273
|
+
}
|
|
262
274
|
}
|
|
263
275
|
|
|
264
276
|
/**
|
|
@@ -309,9 +321,20 @@ export function graftGitHub({ config, exec: run = exec }) {
|
|
|
309
321
|
// (promotion.mjs); the release graphs check `enabled` themselves.
|
|
310
322
|
const release = r && r.from !== r.to ? { from: r.from, to: r.to } : null
|
|
311
323
|
const branches = release ?? { from: '', to: '' }
|
|
312
|
-
const
|
|
324
|
+
const repo = config.repos.find((x) => x.slug === slug)
|
|
325
|
+
const health = repo?.requiredCheck ? { branch: repo.baseBranch, check: repo.requiredCheck } : null
|
|
326
|
+
const vars = JSON.stringify({
|
|
327
|
+
owner,
|
|
328
|
+
name,
|
|
329
|
+
labels: QUEUE_LABELS,
|
|
330
|
+
release: Boolean(release),
|
|
331
|
+
...branches,
|
|
332
|
+
health: Boolean(health),
|
|
333
|
+
baseRef: health ? `refs/heads/${health.branch}` : '',
|
|
334
|
+
check: health?.check ?? '',
|
|
335
|
+
})
|
|
313
336
|
const data = await json(graft(slug, ['graphql', SNAPSHOT_QUERY, '--vars', vars]))
|
|
314
|
-
const snap = normalize(data, { botLogin: config.graft.botLogin, release })
|
|
337
|
+
const snap = normalize(data, { botLogin: config.graft.botLogin, release, health })
|
|
315
338
|
const unread = unreadCloses(snap)
|
|
316
339
|
if (!unread.length) return snap
|
|
317
340
|
// Issues named only in PR bodies (in no queue): one more query, for those.
|
|
@@ -353,6 +376,18 @@ export function graftGitHub({ config, exec: run = exec }) {
|
|
|
353
376
|
async comment(slug, number, body) {
|
|
354
377
|
await graft(slug, ['issue', 'comment', String(number), '--body-file', '-'], body)
|
|
355
378
|
},
|
|
379
|
+
async createIssue(slug, { title, body, labels }) {
|
|
380
|
+
const args = [
|
|
381
|
+
'issue',
|
|
382
|
+
'create',
|
|
383
|
+
'--title',
|
|
384
|
+
title,
|
|
385
|
+
'--body-file',
|
|
386
|
+
'-',
|
|
387
|
+
...(labels.length ? ['--labels', labels.join(',')] : []),
|
|
388
|
+
]
|
|
389
|
+
return json(graft(slug, args, body))
|
|
390
|
+
},
|
|
356
391
|
async deleteBranch(slug, branch) {
|
|
357
392
|
await graft(slug, ['api', 'DELETE', `/repos/${slug}/git/refs/heads/${branch}`])
|
|
358
393
|
},
|
package/src/graph-engine.mjs
CHANGED
|
@@ -25,6 +25,7 @@ import { move, Q } from './labels.mjs'
|
|
|
25
25
|
import { markerTrace, turnTracer } from './tracing.mjs'
|
|
26
26
|
import { DISPATCH_STEPS } from './dispatch-engine.mjs'
|
|
27
27
|
import { trackPromotion } from './promotion.mjs'
|
|
28
|
+
import { trackBaseHealth } from './base-health.mjs'
|
|
28
29
|
|
|
29
30
|
/**
|
|
30
31
|
* @typedef {import('./tick.mjs').Deps} Deps
|
|
@@ -237,7 +238,7 @@ const inOrder =
|
|
|
237
238
|
/** @type {import('./tick.mjs').Engine} */
|
|
238
239
|
export const GRAPH_ENGINE = {
|
|
239
240
|
reconcile: inOrder(reconcile, DISPATCH_STEPS.reconcile),
|
|
240
|
-
watch: inOrder(watch, trackPromotion),
|
|
241
|
+
watch: inOrder(watch, trackPromotion, trackBaseHealth),
|
|
241
242
|
sweep: inOrder(sweep, DISPATCH_STEPS.sweep),
|
|
242
243
|
scheduleTasks,
|
|
243
244
|
scheduleDispatch: DISPATCH_STEPS.schedule,
|
package/src/graphs/dispatch.mjs
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* The Dispatch phase as a graph: a `q:intake` issue is triaged by a one-shot `claude -p` worker whose final
|
|
4
4
|
* message carries a typed verdict ({@link TriageResult}), then moved to `q:task`, moved to `q:human`, or closed as
|
|
5
|
-
* a duplicate. Same conventions as the task graph (graphs/task.mjs): each node declares its queue label and
|
|
5
|
+
* a duplicate or obsolete, or deferred to be triaged again later. Same conventions as the task graph (graphs/task.mjs): each node declares its queue label and
|
|
6
6
|
* whether it holds `running`, and a node's write is the marker first, then the labels via `move()`.
|
|
7
7
|
*
|
|
8
8
|
* The triage worker runs in a read-only checkout (workers.mjs `readOnly`): detached at the base branch, no install,
|
|
@@ -28,7 +28,7 @@ import { task } from './task.mjs'
|
|
|
28
28
|
*/
|
|
29
29
|
|
|
30
30
|
/** The nodes an issue in `q:intake` can rest at; a marker elsewhere starts triage over at attempt 1. */
|
|
31
|
-
const INTAKE_NODES = new Set(['start-triage', 'triaging', 'retry-triage'])
|
|
31
|
+
const INTAKE_NODES = new Set(['start-triage', 'triaging', 'retry-triage', 'deferred'])
|
|
32
32
|
|
|
33
33
|
/** The pseudo-branch that names an issue's triage checkout (`<root>/triage-<n>`); never created. @param {number} n */
|
|
34
34
|
export const triageBranch = (n) => `cambium/triage-${n}`
|
|
@@ -107,6 +107,8 @@ const verdictOf = (/** @type {Triaged} */ o) => (o.exit === 'triaged' ? o.result
|
|
|
107
107
|
const actionable = (/** @type {Triaged} */ o) => verdictOf(o) === 'actionable'
|
|
108
108
|
const needsHuman = (/** @type {Triaged} */ o) => verdictOf(o) === 'needs-human'
|
|
109
109
|
const duplicate = (/** @type {Triaged} */ o) => verdictOf(o) === 'duplicate'
|
|
110
|
+
const obsolete = (/** @type {Triaged} */ o) => verdictOf(o) === 'obsolete'
|
|
111
|
+
const defer = (/** @type {Triaged} */ o) => verdictOf(o) === 'defer'
|
|
110
112
|
|
|
111
113
|
/** @param {DispatchCtx} ctx */
|
|
112
114
|
export const lostLeaseOf = (ctx) => ({
|
|
@@ -148,6 +150,7 @@ const startTriage = declare(
|
|
|
148
150
|
}
|
|
149
151
|
await land(ctx, 'triaging', {
|
|
150
152
|
...m0,
|
|
153
|
+
deferUntil: null,
|
|
151
154
|
host: config.host,
|
|
152
155
|
start: iso(ctx.tick.now),
|
|
153
156
|
input: { attempt: m0.attempt },
|
|
@@ -170,7 +173,16 @@ const startTriage = declare(
|
|
|
170
173
|
readOnly: true,
|
|
171
174
|
// This node's cambium.node span, so Claude Code's own spans (when worker.env turns them on) join the trace.
|
|
172
175
|
env: ctx.traceparent ? { TRACEPARENT: ctx.traceparent } : undefined,
|
|
173
|
-
buildPrompt: (agentsMd) =>
|
|
176
|
+
buildPrompt: (agentsMd) =>
|
|
177
|
+
triagePrompt({
|
|
178
|
+
repo: slug,
|
|
179
|
+
baseBranch: repo.baseBranch,
|
|
180
|
+
issue,
|
|
181
|
+
queued,
|
|
182
|
+
agentsMd,
|
|
183
|
+
deferHours: caps.deferHours,
|
|
184
|
+
deferred: m0.deferReason ? { count: m0.defers ?? 0, reason: m0.deferReason } : null,
|
|
185
|
+
}),
|
|
174
186
|
})
|
|
175
187
|
} catch (err) {
|
|
176
188
|
return {
|
|
@@ -416,6 +428,98 @@ const closeDuplicate = declare(
|
|
|
416
428
|
{ queue: null, running: false },
|
|
417
429
|
)
|
|
418
430
|
|
|
431
|
+
const DeferIn = z.object({ reason: z.string(), recheckHours: z.number().optional(), log: z.string() })
|
|
432
|
+
const deferred = declare(
|
|
433
|
+
defineNode({
|
|
434
|
+
name: 'deferred',
|
|
435
|
+
input: DeferIn,
|
|
436
|
+
output: z.discriminatedUnion('to', [
|
|
437
|
+
z.object({ to: z.literal('wait'), until: z.string() }),
|
|
438
|
+
z.object({ to: z.literal('human'), ...failure }),
|
|
439
|
+
]),
|
|
440
|
+
/**
|
|
441
|
+
* Defer: the verdict hangs on something a machine can check later (a CD run, a release, a merge). The issue stays
|
|
442
|
+
* in `q:intake`, resting here until `deferUntil`; Schedule skips it until then, then triage runs again with this
|
|
443
|
+
* reason in its prompt. A defer is not a failed attempt; `phases.dispatch.maxDefers` bounds them.
|
|
444
|
+
* @param {z.infer<typeof DeferIn>} d @param {DispatchCtx} ctx
|
|
445
|
+
*/
|
|
446
|
+
run: async (d, ctx) => {
|
|
447
|
+
const issue = issueOf(ctx)
|
|
448
|
+
const m = markerOf(issue)
|
|
449
|
+
const { deferHours, maxDefers } = ctx.config.phases.dispatch
|
|
450
|
+
const defers = (m.defers ?? 0) + 1
|
|
451
|
+
if (defers > maxDefers)
|
|
452
|
+
return {
|
|
453
|
+
to: 'human',
|
|
454
|
+
reason: `deferred ${maxDefers} times without an answer. Last: ${d.reason}`,
|
|
455
|
+
outcome: 'budget',
|
|
456
|
+
}
|
|
457
|
+
const hours = d.recheckHours ?? deferHours
|
|
458
|
+
const until = iso(ctx.tick.now + hours * 60 * MIN)
|
|
459
|
+
await land(ctx, 'deferred', {
|
|
460
|
+
...withHistory(m, {
|
|
461
|
+
at: iso(ctx.tick.now),
|
|
462
|
+
attempt: m.attempt,
|
|
463
|
+
outcome: 'defer',
|
|
464
|
+
reason: d.reason,
|
|
465
|
+
log: d.log,
|
|
466
|
+
}),
|
|
467
|
+
defers,
|
|
468
|
+
deferUntil: until,
|
|
469
|
+
deferReason: d.reason,
|
|
470
|
+
host: null,
|
|
471
|
+
start: null,
|
|
472
|
+
input: null,
|
|
473
|
+
})
|
|
474
|
+
ctx.log('info', `triage deferred until ${until} (${defers}/${maxDefers}): ${d.reason.split('\n')[0]}`, {
|
|
475
|
+
repo: ctx.subject.slug,
|
|
476
|
+
issue: issue.number,
|
|
477
|
+
})
|
|
478
|
+
return { to: 'wait', until }
|
|
479
|
+
},
|
|
480
|
+
}),
|
|
481
|
+
{ queue: Q.intake, running: false },
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
const ObsIn = z.object({ reason: z.string(), log: z.string() })
|
|
485
|
+
const closeObsolete = declare(
|
|
486
|
+
defineNode({
|
|
487
|
+
name: 'close-obsolete',
|
|
488
|
+
input: ObsIn,
|
|
489
|
+
output: Empty,
|
|
490
|
+
/** Out of every queue, closed as not planned with triage's evidence. @param {z.infer<typeof ObsIn>} d @param {DispatchCtx} ctx */
|
|
491
|
+
run: async (d, ctx) => {
|
|
492
|
+
const { slug } = ctx.subject
|
|
493
|
+
const issue = issueOf(ctx)
|
|
494
|
+
const m = markerOf(issue)
|
|
495
|
+
await land(
|
|
496
|
+
ctx,
|
|
497
|
+
'close-obsolete',
|
|
498
|
+
{
|
|
499
|
+
...withHistory(m, {
|
|
500
|
+
at: iso(ctx.tick.now),
|
|
501
|
+
attempt: m.attempt,
|
|
502
|
+
outcome: 'obsolete',
|
|
503
|
+
reason: d.reason,
|
|
504
|
+
log: d.log,
|
|
505
|
+
}),
|
|
506
|
+
host: null,
|
|
507
|
+
start: null,
|
|
508
|
+
input: null,
|
|
509
|
+
},
|
|
510
|
+
{ status: 'done' },
|
|
511
|
+
)
|
|
512
|
+
await ctx.github.close(slug, issue.number, `Obsolete: ${d.reason}\n\nClosed by cambium.`, {
|
|
513
|
+
reason: 'not_planned',
|
|
514
|
+
})
|
|
515
|
+
issue.state = 'CLOSED'
|
|
516
|
+
ctx.log('info', 'obsolete; closed as not planned', { repo: slug, issue: issue.number })
|
|
517
|
+
return {}
|
|
518
|
+
},
|
|
519
|
+
}),
|
|
520
|
+
{ queue: null, running: false },
|
|
521
|
+
)
|
|
522
|
+
|
|
419
523
|
/** @param {unknown} err */
|
|
420
524
|
const errText = (err) => /** @type {Error} */ (err)?.message ?? String(err)
|
|
421
525
|
|
|
@@ -424,7 +528,7 @@ export const dispatch = defineGraph({
|
|
|
424
528
|
name: 'dispatch',
|
|
425
529
|
version: 1,
|
|
426
530
|
entry: { queue: Q.intake, node: 'start-triage', input: () => ({}) },
|
|
427
|
-
nodes: [startTriage, triaging, retryTriage, toTask, toHuman, closeDuplicate],
|
|
531
|
+
nodes: [startTriage, triaging, retryTriage, toTask, toHuman, closeDuplicate, deferred, closeObsolete],
|
|
428
532
|
edges: [
|
|
429
533
|
['start-triage', 'triaging', started, (o) => ({ attempt: o.attempt })],
|
|
430
534
|
['start-triage', 'to-human', budgetSpent],
|
|
@@ -433,6 +537,8 @@ export const dispatch = defineGraph({
|
|
|
433
537
|
['triaging', 'retry-triage', leaseLost],
|
|
434
538
|
['triaging', 'to-task', actionable, (o) => ({ ...pick(o.result, ['reason', 'title', 'acceptance']), log: o.log })],
|
|
435
539
|
['triaging', 'close-duplicate', duplicate, (o) => ({ ...pick(o.result, ['duplicateOf', 'reason']), log: o.log })],
|
|
540
|
+
['triaging', 'deferred', defer, (o) => ({ ...pick(o.result, ['reason', 'recheckHours']), log: o.log })],
|
|
541
|
+
['triaging', 'close-obsolete', obsolete, (o) => ({ reason: o.result.reason, log: o.log })],
|
|
436
542
|
['triaging', 'to-human', needsHuman, (o) => ({ reason: o.result.reason, outcome: 'needs-human', log: o.log })],
|
|
437
543
|
['triaging', 'to-human'],
|
|
438
544
|
['retry-triage', 'start-triage', attemptsLeft, () => ({})],
|
|
@@ -440,6 +546,9 @@ export const dispatch = defineGraph({
|
|
|
440
546
|
['to-task', END],
|
|
441
547
|
['to-human', END],
|
|
442
548
|
['close-duplicate', END],
|
|
549
|
+
['deferred', 'to-human', budgetSpent],
|
|
550
|
+
['deferred', END],
|
|
551
|
+
['close-obsolete', END],
|
|
443
552
|
],
|
|
444
553
|
})
|
|
445
554
|
|
package/src/graphs/task.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import { withHistory } from '../marker.mjs'
|
|
|
16
16
|
import { linkIssue, prFor, taskExit, taskPrompt } from '../phases/task.mjs'
|
|
17
17
|
import { PREPARE_FAILED_LINE } from '../prepare.mjs'
|
|
18
18
|
import { traceWorker } from '../tracing.mjs'
|
|
19
|
+
import { baseRed, isRepair } from '../base-health.mjs'
|
|
19
20
|
|
|
20
21
|
/**
|
|
21
22
|
* @typedef {import('../github.mjs').Issue} Issue
|
|
@@ -607,27 +608,40 @@ const requeue = declare(
|
|
|
607
608
|
z.object({ to: z.literal('start'), attempt: z.number().int() }),
|
|
608
609
|
z.object({ to: z.literal('human'), ...failure }),
|
|
609
610
|
]),
|
|
610
|
-
/**
|
|
611
|
+
/**
|
|
612
|
+
* Back to `q:task` with attempt + 1, or to the Human Queue once the attempts are spent. While the base is red
|
|
613
|
+
* (base-health.mjs) a failure is not the work's own: it requeues at the same attempt, never to the Human
|
|
614
|
+
* Queue, and Schedule holds it until the base is green. The repair issue itself is charged as usual.
|
|
615
|
+
* @param {z.infer<typeof Failure>} what @param {TaskCtx} ctx
|
|
616
|
+
*/
|
|
611
617
|
run: async (what, ctx) => {
|
|
612
618
|
const issue = issueOf(ctx)
|
|
613
619
|
const m = markerOf(issue)
|
|
614
620
|
const max = ctx.config.phases.task.maxAttempts
|
|
615
|
-
|
|
616
|
-
|
|
621
|
+
const free = baseRed(ctx.subject.snap) && !isRepair(issue)
|
|
622
|
+
if (m.attempt >= max && !free)
|
|
623
|
+
return { ...what, to: 'human', reason: `${max} attempts spent. Last: ${what.reason}` }
|
|
624
|
+
const { pr, log, failedSha } = what
|
|
625
|
+
const base = ctx.subject.snap?.base
|
|
626
|
+
const outcome = free ? 'base-red' : what.outcome
|
|
627
|
+
const reason = free
|
|
628
|
+
? `${base?.branch} is red at ${(base?.red?.sha ?? '').slice(0, 7)}; not charged. ${what.reason}`
|
|
629
|
+
: what.reason
|
|
630
|
+
const next = free ? m.attempt : m.attempt + 1
|
|
617
631
|
await land(ctx, 'start-worker', {
|
|
618
632
|
...withHistory(m, { ...nodeAt(what), at: iso(ctx.tick.now), attempt: m.attempt, outcome, reason, pr, log }),
|
|
619
|
-
attempt:
|
|
633
|
+
attempt: next,
|
|
620
634
|
lastFailure: reason,
|
|
621
635
|
host: null,
|
|
622
636
|
start: null,
|
|
623
637
|
input: null,
|
|
624
638
|
...(failedSha !== undefined ? { failedSha } : {}),
|
|
625
639
|
})
|
|
626
|
-
ctx.log('info', `requeued (attempt ${
|
|
640
|
+
ctx.log('info', `requeued (attempt ${next}/${max}): ${reason.split('\n')[0]}`, {
|
|
627
641
|
repo: ctx.subject.slug,
|
|
628
642
|
issue: issue.number,
|
|
629
643
|
})
|
|
630
|
-
return { to: 'start', attempt:
|
|
644
|
+
return { to: 'start', attempt: next }
|
|
631
645
|
},
|
|
632
646
|
}),
|
|
633
647
|
{ queue: Q.task, running: false },
|
package/src/marker.mjs
CHANGED
|
@@ -43,8 +43,13 @@
|
|
|
43
43
|
* lastFailure?: string | null,
|
|
44
44
|
* mergeRefusal?: { sha: string | null, at: string, reason: string } | null,
|
|
45
45
|
* triage?: { title?: string, acceptance?: string[] } | null,
|
|
46
|
+
* defers?: number,
|
|
47
|
+
* deferUntil?: string | null,
|
|
48
|
+
* deferReason?: string | null,
|
|
46
49
|
* }} Marker
|
|
47
50
|
* `mergeRefusal`: the last `graft pr merge` cambium was refused (task graph `merge`), and on which head.
|
|
51
|
+
* `defers`, `deferUntil`, `deferReason`: the dispatch graph's `deferred` node: how many times triage deferred, until
|
|
52
|
+
* when, and what the next triage should check.
|
|
48
53
|
* `triage`: what the dispatch graph's triage added when it handed the issue to `q:task`; the task prompt shows it.
|
|
49
54
|
* @typedef {{ graph: string, entryNode: string, nodes?: Iterable<string> }} QueueOwner
|
|
50
55
|
*/
|
|
@@ -172,7 +177,8 @@ export function renderMarker(input, { maxAttempts } = {}) {
|
|
|
172
177
|
` · attempt ${marker.attempt}${of}` +
|
|
173
178
|
(marker.pr ? ` · PR #${marker.pr}` : '') +
|
|
174
179
|
(marker.host ? ` · host \`${marker.host}\`` : '') +
|
|
175
|
-
(marker.start ? ` · started ${marker.start}` : '')
|
|
180
|
+
(marker.start ? ` · started ${marker.start}` : '') +
|
|
181
|
+
(marker.node === 'deferred' && marker.deferUntil ? ` · re-triage after ${marker.deferUntil}` : ''),
|
|
176
182
|
]
|
|
177
183
|
if (marker.lastFailure)
|
|
178
184
|
lines.push(
|
package/src/phases/dispatch.mjs
CHANGED
|
@@ -21,8 +21,9 @@ export const DISPATCH_NOT_BUILT = 'dispatch phase is not built yet; q:intake iss
|
|
|
21
21
|
/** The verdict a triage worker ends with, as the last JSON block of its final message. */
|
|
22
22
|
export const TriageResult = z
|
|
23
23
|
.object({
|
|
24
|
-
verdict: z.enum(['actionable', 'needs-human', 'duplicate']),
|
|
24
|
+
verdict: z.enum(['actionable', 'needs-human', 'duplicate', 'defer', 'obsolete']),
|
|
25
25
|
duplicateOf: z.number().int().positive().optional(),
|
|
26
|
+
recheckHours: z.number().positive().max(168).optional(),
|
|
26
27
|
reason: z.string().trim().min(1),
|
|
27
28
|
title: z.string().trim().min(1).optional(),
|
|
28
29
|
acceptance: z.array(z.string().trim().min(1)).optional(),
|
|
@@ -109,34 +110,37 @@ export function readTriage(log, issue) {
|
|
|
109
110
|
* @param {{
|
|
110
111
|
* repo: string, baseBranch: string, issue: import('../github.mjs').Issue,
|
|
111
112
|
* queued: { number: number, title: string, labels: string[] }[], agentsMd: string | null,
|
|
113
|
+
* deferred?: { count: number, reason: string } | null, deferHours?: number,
|
|
112
114
|
* }} input
|
|
113
115
|
*/
|
|
114
|
-
export function triagePrompt({ repo, baseBranch, issue, queued, agentsMd }) {
|
|
116
|
+
export function triagePrompt({ repo, baseBranch, issue, queued, agentsMd, deferred = null, deferHours = 6 }) {
|
|
115
117
|
const others = queued
|
|
116
118
|
.filter((q) => q.number !== issue.number)
|
|
117
119
|
.map((q) => `- #${q.number} ${q.title} (${q.labels.filter((l) => l.startsWith('q:')).join(', ') || 'no queue'})`)
|
|
118
120
|
.join('\n')
|
|
119
121
|
return `You are a triage worker started by cambium, the scheduler, in a read-only checkout of ${repo} at origin/${baseBranch}.
|
|
120
|
-
Nobody is watching this session and nobody will answer a question: never ask one and wait. Where you would ask a human, that is your verdict (needs-human).
|
|
122
|
+
Nobody is watching this session and nobody will answer a question: never ask one and wait. Where you would ask a human about a decision only a human can make, that is your verdict (needs-human). Where you would ask a human to check something, check it yourself, or defer until it can be checked: a human is never the way to verify what a machine can.
|
|
121
123
|
|
|
122
124
|
Your one job: decide what happens to GitHub issue #${issue.number} (below), which is waiting in the intake queue. You do not do the work. You do not edit files, commit, publish, comment, label or close anything; cambium makes every move on your verdict.
|
|
123
125
|
|
|
124
126
|
Verdicts:
|
|
125
127
|
- actionable: a worker could do it unattended from what is written (plus what you add), and would know when it is done. It goes to the task queue. When the issue lacks a clear task or checkable acceptance criteria, give \`title\` (an imperative task title) and \`acceptance\` (checks a reviewer can verify; CI green is always required, so do not list only that). cambium posts them on the issue and hands them to the task worker.
|
|
126
|
-
- needs-human: too foggy to ticket; a decision only a human can make (direction, priority, money, access, policy); information
|
|
128
|
+
- needs-human: too foggy to ticket; a decision only a human can make (direction, priority, money, access, policy); information no one but a human has; or work that is not a code change in this repository. \`reason\` says exactly what the human must decide or supply. Never for something a machine can check, now or later.
|
|
127
129
|
- duplicate: the same work as another issue, open or closed. \`duplicateOf\` is that issue's number (never #${issue.number}); \`reason\` says why they are the same.
|
|
130
|
+
- obsolete: the work no longer applies, and you verified it: the code path is gone, the bug is already fixed on ${baseBranch}, or its verification already passed. cambium closes it as not planned. \`reason\` gives the evidence (commit, file, run).
|
|
131
|
+
- defer: the verdict hangs on something a machine can check later but not yet: a CI or CD run that has not happened, a release, a PR still merging, a deploy, an upstream fix. cambium triages it again after \`recheckHours\` (default ${deferHours}). \`reason\` says exactly what to check next time and what each answer means (e.g. "next main cd run green → obsolete; red → actionable").
|
|
128
132
|
|
|
129
133
|
How to look:
|
|
130
134
|
- Read the code here (Read, Glob, Grep) to judge what the change touches and whether it already exists.
|
|
131
135
|
- Read an issue with \`graft issue read <n>\`; search with \`graft api GET '/search/issues?q=repo:${repo}+is:issue+<words>'\`. Never \`gh\`.
|
|
132
|
-
- Be quick: a verdict, not an investigation. When in doubt
|
|
136
|
+
- Be quick: a verdict, not an investigation. When in doubt whether the work is clear enough to do, choose needs-human; when in doubt whether something is true, check it, or defer until it can be checked.
|
|
133
137
|
|
|
134
138
|
End your final message with exactly one fenced JSON block, and nothing after it:
|
|
135
139
|
\`\`\`json
|
|
136
|
-
{ "verdict": "actionable" | "needs-human" | "duplicate", "duplicateOf": 123, "reason": "…", "title": "…", "acceptance": ["…"] }
|
|
140
|
+
{ "verdict": "actionable" | "needs-human" | "duplicate" | "obsolete" | "defer", "duplicateOf": 123, "recheckHours": 6, "reason": "…", "title": "…", "acceptance": ["…"] }
|
|
137
141
|
\`\`\`
|
|
138
|
-
\`duplicateOf\` only for duplicate; \`title\` and \`acceptance\` only for actionable, and optional. A missing or malformed block sends the issue to a human with the error.
|
|
139
|
-
${others ? `\nOther issues in cambium's queues right now:\n${others}\n` : ''}
|
|
142
|
+
\`duplicateOf\` only for duplicate; \`recheckHours\` only for defer, and optional; \`title\` and \`acceptance\` only for actionable, and optional. A missing or malformed block sends the issue to a human with the error.
|
|
143
|
+
${deferred ? `\nThis is re-triage ${deferred.count + 1}: an earlier triage deferred it to check this:\n${deferred.reason}\nCheck it now. If it still cannot be checked, defer again.\n` : ''}${others ? `\nOther issues in cambium's queues right now:\n${others}\n` : ''}
|
|
140
144
|
--- Issue #${issue.number}: ${issue.title} (${issue.url})
|
|
141
145
|
${issue.body}
|
|
142
146
|
${agentsMd ? `\n--- AGENTS.md (the contract a task worker will follow)\n${agentsMd}` : ''}`
|
package/src/tick.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import { PREPARE_FAILED_LINE } from './prepare.mjs'
|
|
|
16
16
|
import { releaseDue, releaseNotBuilt } from './phases/release.mjs'
|
|
17
17
|
import { releaseGraphs } from './graphs/release.mjs'
|
|
18
18
|
import { DISPATCH_NOT_BUILT, dispatchNotBuilt } from './phases/dispatch.mjs'
|
|
19
|
+
import { isRepair } from './base-health.mjs'
|
|
19
20
|
|
|
20
21
|
/**
|
|
21
22
|
* @typedef {import('./github.mjs').Issue} Issue
|
|
@@ -438,6 +439,10 @@ async function scheduleTasks(deps, ctx) {
|
|
|
438
439
|
/**
|
|
439
440
|
* The `q:task` issues to start this tick: no `running`, no tracked worker, by priority then age, up to the free
|
|
440
441
|
* slots counted from live task workers.
|
|
442
|
+
*
|
|
443
|
+
* The base's health gates it (base-health.mjs): a repo whose base is red starts only its repair issue, which
|
|
444
|
+
* takes no slot (it is what frees the others); a repair issue on a green base is left for `trackBaseHealth` to
|
|
445
|
+
* close.
|
|
441
446
|
* @param {Deps} deps @param {Ctx} ctx
|
|
442
447
|
*/
|
|
443
448
|
export function taskCandidates(deps, ctx) {
|
|
@@ -445,18 +450,27 @@ export function taskCandidates(deps, ctx) {
|
|
|
445
450
|
const free = config.phases.task.cap - store.workers().filter((w) => w.phase === 'task').length
|
|
446
451
|
/** @type {{ slug: string, issue: Issue, snap: Snapshot }[]} */
|
|
447
452
|
const candidates = []
|
|
448
|
-
|
|
449
|
-
|
|
453
|
+
/** @type {{ slug: string, issue: Issue, snap: Snapshot }[]} */
|
|
454
|
+
const repairs = []
|
|
455
|
+
for (const [slug, snap] of ctx.snaps) {
|
|
456
|
+
const health = snap.base?.state ?? 'unknown'
|
|
457
|
+
for (const issue of snap.issues.values()) {
|
|
450
458
|
if (
|
|
451
|
-
issue.state
|
|
452
|
-
issue.labels.includes(Q.task)
|
|
453
|
-
|
|
454
|
-
|
|
459
|
+
issue.state !== 'OPEN' ||
|
|
460
|
+
!issue.labels.includes(Q.task) ||
|
|
461
|
+
issue.labels.includes(RUNNING) ||
|
|
462
|
+
store.worker(slug, issue.number)
|
|
455
463
|
)
|
|
456
|
-
|
|
464
|
+
continue
|
|
465
|
+
if (isRepair(issue)) {
|
|
466
|
+
if (health !== 'green') repairs.push({ slug, issue, snap })
|
|
467
|
+
} else if (health !== 'red') candidates.push({ slug, issue, snap })
|
|
468
|
+
}
|
|
469
|
+
}
|
|
457
470
|
const order = byPriorityThenAge(config.priorityLabels)
|
|
458
471
|
candidates.sort((a, b) => order(a.issue, b.issue))
|
|
459
|
-
|
|
472
|
+
repairs.sort((a, b) => order(a.issue, b.issue))
|
|
473
|
+
return [...repairs, ...candidates.slice(0, Math.max(0, free))]
|
|
460
474
|
}
|
|
461
475
|
|
|
462
476
|
/** @param {Deps} deps @param {Ctx} ctx */
|
|
@@ -624,7 +638,10 @@ async function release(deps, ctx) {
|
|
|
624
638
|
* steps: StepResult[],
|
|
625
639
|
* counts: Counts & { workers: number },
|
|
626
640
|
* caps: { task: number, dispatch: number },
|
|
627
|
-
* repos: {
|
|
641
|
+
* repos: {
|
|
642
|
+
* slug: string, ok: boolean, error: string | null, counts: Counts | null,
|
|
643
|
+
* base: import('./base-health.mjs').BaseHealth | null,
|
|
644
|
+
* }[],
|
|
628
645
|
* workers: {
|
|
629
646
|
* repo: string, issue: number, phase: string, pid: number, alive: boolean, attempt: number, branch: string,
|
|
630
647
|
* worktree: string, startedAt: string, elapsedMs: number, wallClockMs: number, killedReason: string | null,
|
|
@@ -665,6 +682,7 @@ export function heartbeatState(deps, ctx) {
|
|
|
665
682
|
ok: Boolean(snap),
|
|
666
683
|
error: ctx.readErrors.get(r.slug) ?? null,
|
|
667
684
|
counts: snap ? countsOf(snap) : null,
|
|
685
|
+
base: snap?.base ?? null,
|
|
668
686
|
}
|
|
669
687
|
})
|
|
670
688
|
const total = { intake: 0, task: 0, human: 0, promote: 0, running: 0, review: 0 }
|