@oakgreencc/cambium 0.0.39 → 0.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -2
- package/package.json +1 -1
- package/src/cli.mjs +5 -1
- package/src/config.mjs +22 -1
- package/src/github.mjs +32 -0
- package/src/graph/define.mjs +11 -3
- package/src/graph/describe.mjs +50 -0
- package/src/graph/index.mjs +2 -1
- package/src/graph-engine.mjs +9 -2
- package/src/graphs/dispatch.mjs +33 -10
- package/src/graphs/pr.mjs +23 -6
- package/src/graphs/release.mjs +14 -5
- package/src/graphs/rescue.mjs +504 -0
- package/src/graphs/task.mjs +20 -5
- package/src/labels.mjs +4 -0
- package/src/marker.mjs +1 -1
- package/src/phases/dispatch.mjs +23 -14
- package/src/phases/rescue.mjs +444 -0
- package/src/rescue-engine.mjs +280 -0
- package/src/tick.mjs +17 -4
- package/src/upkeep.mjs +8 -1
package/src/phases/dispatch.mjs
CHANGED
|
@@ -40,9 +40,6 @@ export const TriageResult = z
|
|
|
40
40
|
|
|
41
41
|
/** @typedef {z.infer<typeof TriageResult>} Triage */
|
|
42
42
|
|
|
43
|
-
/** @param {z.core.$ZodIssue} i */
|
|
44
|
-
const formatIssue = (i) => (i.path.length ? `${i.path.join('.')}: ${i.message}` : i.message)
|
|
45
|
-
|
|
46
43
|
/**
|
|
47
44
|
* The JSON in a final message: its last fenced block, else the whole text, else its outermost `{…}`.
|
|
48
45
|
* @param {string} text @returns {unknown} throws a SyntaxError-like Error when there is none
|
|
@@ -73,14 +70,12 @@ const clip = (s, n = 300) => {
|
|
|
73
70
|
}
|
|
74
71
|
|
|
75
72
|
/**
|
|
76
|
-
* The
|
|
77
|
-
* `result` text, its JSON
|
|
78
|
-
*
|
|
73
|
+
* The JSON a one-shot worker ends with, from its stdout (`claude -p --output-format stream-json`): the last `result`
|
|
74
|
+
* event's `result` text, its JSON ({@link jsonIn}). Anything else is an error saying what was wrong.
|
|
79
75
|
* @param {string} log the stdout log text
|
|
80
|
-
* @
|
|
81
|
-
* @returns {{ ok: true, result: Triage } | { ok: false, error: string }}
|
|
76
|
+
* @returns {{ ok: true, json: unknown } | { ok: false, error: string }}
|
|
82
77
|
*/
|
|
83
|
-
export function
|
|
78
|
+
export function finalJson(log) {
|
|
84
79
|
/** @type {any} */
|
|
85
80
|
let result = null
|
|
86
81
|
for (const line of log.split('\n')) {
|
|
@@ -93,16 +88,30 @@ export function readTriage(log, issue) {
|
|
|
93
88
|
if (!result) return { ok: false, error: 'no result event in the worker log (killed, crashed or truncated)' }
|
|
94
89
|
if (typeof result.result !== 'string')
|
|
95
90
|
return { ok: false, error: `the worker ended without a final message (${result.subtype ?? 'no subtype'})` }
|
|
96
|
-
/** @type {unknown} */
|
|
97
|
-
let json
|
|
98
91
|
try {
|
|
99
|
-
json
|
|
92
|
+
return { ok: true, json: jsonIn(result.result) }
|
|
100
93
|
} catch (err) {
|
|
101
94
|
return { ok: false, error: /** @type {Error} */ (err).message }
|
|
102
95
|
}
|
|
103
|
-
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** @param {z.core.$ZodIssue} i */
|
|
99
|
+
export const formatZodIssue = (i) => (i.path.length ? `${i.path.join('.')}: ${i.message}` : i.message)
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The triage verdict from a worker's stdout (`claude -p --output-format stream-json`): the last `result` event's
|
|
103
|
+
* `result` text, its JSON, validated by {@link TriageResult}. Anything else is an error saying what was wrong.
|
|
104
|
+
*
|
|
105
|
+
* @param {string} log the stdout log text
|
|
106
|
+
* @param {number} issue the triaged issue, which cannot be its own duplicate
|
|
107
|
+
* @returns {{ ok: true, result: Triage } | { ok: false, error: string }}
|
|
108
|
+
*/
|
|
109
|
+
export function readTriage(log, issue) {
|
|
110
|
+
const read = finalJson(log)
|
|
111
|
+
if (!read.ok) return read
|
|
112
|
+
const parsed = TriageResult.safeParse(read.json)
|
|
104
113
|
if (!parsed.success)
|
|
105
|
-
return { ok: false, error: `invalid triage result: ${parsed.error.issues.map(
|
|
114
|
+
return { ok: false, error: `invalid triage result: ${parsed.error.issues.map(formatZodIssue).join('; ')}` }
|
|
106
115
|
if (parsed.data.verdict === 'duplicate' && parsed.data.duplicateOf === issue)
|
|
107
116
|
return { ok: false, error: `invalid triage result: duplicateOf: #${issue} cannot duplicate itself` }
|
|
108
117
|
if (parsed.data.verdict === 'blocked' && parsed.data.blockedBy?.includes(issue))
|
|
@@ -0,0 +1,444 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* The rescue graph's parts (graphs/rescue.mjs): what counts as stuck ({@link findStuck}), the rescue worker's prompt,
|
|
4
|
+
* its typed result and the checks cambium runs on it before acting, and the comments and root-cause tickets it
|
|
5
|
+
* writes.
|
|
6
|
+
*
|
|
7
|
+
* An item is stuck when its labels break the queue rules (more than one `q:*`, `running` without a queue, `why:*`
|
|
8
|
+
* outside `q:human`, `wait:*` outside `q:intake`, cambium labels left on a closed issue), when its agent PR closed
|
|
9
|
+
* or merged and left it open in no queue, or when it has waited in a queue, or its PR has sat untouched, far longer
|
|
10
|
+
* than cambium ever takes to move it.
|
|
11
|
+
*/
|
|
12
|
+
import { fileURLToPath } from 'node:url'
|
|
13
|
+
import { z } from 'zod'
|
|
14
|
+
import { branchIssue } from '../github.mjs'
|
|
15
|
+
import { MIN } from '../issue-state.mjs'
|
|
16
|
+
import { Q, RUNNING } from '../labels.mjs'
|
|
17
|
+
import { finalJson, formatZodIssue } from './dispatch.mjs'
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @typedef {import('../github.mjs').Issue} Issue
|
|
21
|
+
* @typedef {import('../github.mjs').PullRequest} PullRequest
|
|
22
|
+
* @typedef {import('../github.mjs').Snapshot} Snapshot
|
|
23
|
+
* @typedef {{ code: string, detail: string }} Symptom
|
|
24
|
+
* @typedef {{ slug: string, number: number, pr: number | null, symptoms: Symptom[] }} Stuck
|
|
25
|
+
* One stuck item: always an issue (a stuck PR is reported on the issue it closes); `pr` is the PR involved, if any.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
const HOUR = 60 * MIN
|
|
29
|
+
|
|
30
|
+
/** The labels cambium owns, which only cambium moves. @param {string} l */
|
|
31
|
+
export const cambiumLabel = (l) => l.startsWith('q:') || l === RUNNING || l.startsWith('why:') || l.startsWith('wait:')
|
|
32
|
+
|
|
33
|
+
/** The issue an agent PR is for: the first one it closes that the snapshot has, else the one its branch names. @param {PullRequest} pr */
|
|
34
|
+
const issueOfPr = (pr) => pr.closes[0] ?? branchIssue(pr.branch)
|
|
35
|
+
|
|
36
|
+
/** @param {PullRequest} pr @param {number} n */
|
|
37
|
+
const isFor = (pr, n) => pr.closes.includes(n) || (!pr.closes.length && branchIssue(pr.branch) === n)
|
|
38
|
+
|
|
39
|
+
/** @param {number} ms */
|
|
40
|
+
const hours = (ms) => `${Math.round(ms / HOUR)}h`
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Every stuck item in one repo's snapshot. Pure: the caller reads, filters and acts.
|
|
44
|
+
*
|
|
45
|
+
* `recheck` asks only whether an item is still in its stuck condition (a diagnosis is about to act on it): the
|
|
46
|
+
* time and capacity gates are skipped.
|
|
47
|
+
*
|
|
48
|
+
* @param {Snapshot} snap
|
|
49
|
+
* @param {{
|
|
50
|
+
* slug: string, now: number, host: string, running: boolean, staleHours: number, graceMinutes: number,
|
|
51
|
+
* free: { task: number, dispatch: number }, busy: (n: number) => boolean, closed?: PullRequest[], recheck?: boolean,
|
|
52
|
+
* }} o
|
|
53
|
+
* `running`: the daemon's mode is `running` (Schedule ran this tick). `free`: the slots Schedule left open this tick;
|
|
54
|
+
* a waiting issue counts as stale only when one was free for it. `busy`: a live worker holds it, or the tick already
|
|
55
|
+
* moved it. `closed`: the newest closed-unmerged agent PRs ({@link import('../github.mjs').GitHub} `closedPulls`).
|
|
56
|
+
* @returns {Stuck[]}
|
|
57
|
+
*/
|
|
58
|
+
export function findStuck(snap, o) {
|
|
59
|
+
const { now, recheck = false } = o
|
|
60
|
+
const open = snap.prs.filter((p) => p.state === 'OPEN')
|
|
61
|
+
const merged = snap.prs.filter((p) => p.state === 'MERGED')
|
|
62
|
+
const closed = [...(o.closed ?? []), ...snap.prs.filter((p) => p.state === 'CLOSED')]
|
|
63
|
+
/** @type {Map<number, Stuck>} */
|
|
64
|
+
const out = new Map()
|
|
65
|
+
const add = (/** @type {number} */ n, /** @type {Symptom} */ s, pr = /** @type {number | null} */ (null)) => {
|
|
66
|
+
const item = out.get(n) ?? { slug: o.slug, number: n, pr: null, symptoms: [] }
|
|
67
|
+
item.symptoms.push(s)
|
|
68
|
+
if (pr !== null && item.pr === null) item.pr = pr
|
|
69
|
+
out.set(n, item)
|
|
70
|
+
}
|
|
71
|
+
for (const issue of snap.issues.values()) {
|
|
72
|
+
if (o.busy(issue.number)) continue
|
|
73
|
+
const n = issue.number
|
|
74
|
+
const labels = issue.labels
|
|
75
|
+
const m = issue.stateComment?.marker
|
|
76
|
+
const age = now - (Date.parse(m?.enteredAt ?? issue.createdAt) || now)
|
|
77
|
+
const graced = recheck || age >= o.graceMinutes * MIN
|
|
78
|
+
const stale = recheck || age >= o.staleHours * HOUR
|
|
79
|
+
const queues = labels.filter((l) => l.startsWith('q:'))
|
|
80
|
+
const running = labels.includes(RUNNING)
|
|
81
|
+
if (issue.state !== 'OPEN') {
|
|
82
|
+
const left = labels.filter((l) => cambiumLabel(l) && l !== Q.promote)
|
|
83
|
+
if (left.length && graced)
|
|
84
|
+
add(n, { code: 'closed-labelled', detail: `closed, still labelled ${left.join(', ')}` })
|
|
85
|
+
continue
|
|
86
|
+
}
|
|
87
|
+
if (!graced) continue
|
|
88
|
+
// Another host's lease may be live: its own sweep owns it.
|
|
89
|
+
const foreign = running && !!m?.host && m.host !== o.host
|
|
90
|
+
if (queues.length > 1)
|
|
91
|
+
add(n, { code: 'many-queues', detail: `in ${queues.length} queues at once: ${queues.join(', ')}` })
|
|
92
|
+
if (running && !foreign && !queues.some((q) => q === Q.task || q === Q.intake))
|
|
93
|
+
add(n, {
|
|
94
|
+
code: 'running-unqueued',
|
|
95
|
+
detail: `labelled \`${RUNNING}\` ${queues.length ? `in ${queues.join(', ')}` : 'in no queue'}`,
|
|
96
|
+
})
|
|
97
|
+
const why = labels.filter((l) => l.startsWith('why:'))
|
|
98
|
+
if (why.length && !labels.includes(Q.human))
|
|
99
|
+
add(n, { code: 'why-outside-human', detail: `${why.join(', ')} without ${Q.human}` })
|
|
100
|
+
const wait = labels.filter((l) => l.startsWith('wait:'))
|
|
101
|
+
if (wait.length && !labels.includes(Q.intake))
|
|
102
|
+
add(n, { code: 'wait-outside-intake', detail: `${wait.join(', ')} without ${Q.intake}` })
|
|
103
|
+
if (!queues.length && !running) {
|
|
104
|
+
if (open.some((p) => isFor(p, n))) continue
|
|
105
|
+
const done = merged.find((p) => isFor(p, n))
|
|
106
|
+
const gone = closed.find((p) => isFor(p, n))
|
|
107
|
+
// Watch closes out a merged PR's issue, and a rework requeues a closed one's: give them the grace too.
|
|
108
|
+
const last = done ?? gone
|
|
109
|
+
if (!recheck && last && now - (Date.parse(last.updatedAt ?? '') || 0) < o.graceMinutes * MIN) continue
|
|
110
|
+
if (done)
|
|
111
|
+
add(
|
|
112
|
+
n,
|
|
113
|
+
{ code: 'merged-open', detail: `its PR #${done.number} merged, but the issue is open in no queue` },
|
|
114
|
+
done.number,
|
|
115
|
+
)
|
|
116
|
+
else if (gone || m?.pr)
|
|
117
|
+
add(
|
|
118
|
+
n,
|
|
119
|
+
{
|
|
120
|
+
code: 'stranded',
|
|
121
|
+
detail: `open in no queue with no open agent PR${gone ? `: its PR #${gone.number} closed unmerged` : m?.pr ? ` (its marker names PR #${m.pr})` : ''}`,
|
|
122
|
+
},
|
|
123
|
+
gone?.number ?? m?.pr ?? null,
|
|
124
|
+
)
|
|
125
|
+
continue
|
|
126
|
+
}
|
|
127
|
+
if (!stale || running || queues.length !== 1) continue
|
|
128
|
+
const since = `waiting ${hours(age)} since ${m?.enteredAt ? `it entered ${m.node}` : 'it was opened'}`
|
|
129
|
+
if (queues[0] === Q.task && (recheck || (o.running && o.free.task > 0 && snap.base?.state !== 'red')))
|
|
130
|
+
add(n, { code: 'stale-task', detail: `${Q.task}, ${since}, with a task slot free` })
|
|
131
|
+
if (queues[0] === Q.intake && (recheck || (o.running && o.free.dispatch > 0)) && !parked(issue, now))
|
|
132
|
+
add(n, { code: 'stale-intake', detail: `${Q.intake}, ${since}, with a triage slot free` })
|
|
133
|
+
}
|
|
134
|
+
for (const pr of open) {
|
|
135
|
+
const touched = Date.parse(pr.updatedAt ?? pr.createdAt ?? '') || now
|
|
136
|
+
if (!recheck && now - touched < o.staleHours * HOUR) continue
|
|
137
|
+
const n = issueOfPr(pr)
|
|
138
|
+
const issue = n === null ? undefined : snap.issues.get(n)
|
|
139
|
+
// No issue, or a closed one: the reaper's. The Human Queue's PRs are the human's. A queued issue (a rework
|
|
140
|
+
// waiting for a slot, a re-triage) is the queue's: a long wait there is `stale-task` / `stale-intake`.
|
|
141
|
+
if (issue?.state !== 'OPEN' || o.busy(issue.number)) continue
|
|
142
|
+
if (issue.labels.some((l) => l === Q.human || l === Q.task || l === Q.intake)) continue
|
|
143
|
+
const state = [
|
|
144
|
+
`checks ${pr.checks ?? 'none'}`,
|
|
145
|
+
`mergeable ${pr.mergeable ?? 'unknown'}`,
|
|
146
|
+
...(pr.reviewDecision ? [`review ${pr.reviewDecision}`] : []),
|
|
147
|
+
...(pr.inMergeQueue ? ['in the merge queue'] : []),
|
|
148
|
+
].join(', ')
|
|
149
|
+
add(
|
|
150
|
+
issue.number,
|
|
151
|
+
{ code: 'pr-stale', detail: `PR #${pr.number} untouched for ${hours(now - touched)}: ${state}` },
|
|
152
|
+
pr.number,
|
|
153
|
+
)
|
|
154
|
+
}
|
|
155
|
+
return [...out.values()].sort((a, b) => a.number - b.number)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Parked by triage: deferred until later, or blocked by an open issue. @param {Issue} issue @param {number} now */
|
|
159
|
+
const parked = (issue, now) => {
|
|
160
|
+
const m = issue.stateComment?.marker
|
|
161
|
+
const deferred = m?.graph === 'dispatch' && m.node === 'deferred' && !!m.deferUntil && Date.parse(m.deferUntil) > now
|
|
162
|
+
return deferred || (issue.blockedBy ?? []).some((b) => b.state === 'OPEN')
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// ---- Rescue history (local: when cambium last rescued an item, to bound how often it tries)
|
|
166
|
+
|
|
167
|
+
const WEEK = 7 * 24 * HOUR
|
|
168
|
+
|
|
169
|
+
/** @param {string} slug @param {number} n */
|
|
170
|
+
const rescueKey = (slug, n) => `rescue:${slug}#${n}`
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* When cambium started a rescue of this item, newest last, within the last week.
|
|
174
|
+
* @param {{ get: (key: string) => string | null }} store @param {string} slug @param {number} n @param {number} now
|
|
175
|
+
* @returns {number[]}
|
|
176
|
+
*/
|
|
177
|
+
export function rescuesOf(store, slug, n, now) {
|
|
178
|
+
try {
|
|
179
|
+
const at = JSON.parse(store.get(rescueKey(slug, n)) ?? '[]')
|
|
180
|
+
return Array.isArray(at) ? at.filter((t) => typeof t === 'number' && now - t < WEEK) : []
|
|
181
|
+
} catch {
|
|
182
|
+
return []
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** @param {{ get: (key: string) => string | null, set: (key: string, value: string) => void }} store @param {string} slug @param {number} n @param {number} now */
|
|
187
|
+
export function recordRescue(store, slug, n, now) {
|
|
188
|
+
store.set(rescueKey(slug, n), JSON.stringify([...rescuesOf(store, slug, n, now), now]))
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** @param {string} slug @param {number} n */
|
|
192
|
+
const codesKey = (slug, n) => `rescue-codes:${slug}#${n}`
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Remember which symptoms a diagnosis started from, so its verdict is applied only while one of them still holds.
|
|
196
|
+
* @param {{ set: (key: string, value: string) => void }} store @param {Stuck} stuck
|
|
197
|
+
*/
|
|
198
|
+
export function recordSymptoms(store, stuck) {
|
|
199
|
+
store.set(codesKey(stuck.slug, stuck.number), JSON.stringify(stuck.symptoms.map((s) => s.code)))
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* The item as it is now, narrowed to the symptoms its diagnosis started from: null when none of them holds any more
|
|
204
|
+
* (a recheck finds every condition, so a fresh one, e.g. a stale wait after the labels were fixed, does not count).
|
|
205
|
+
* @param {{ get: (key: string) => string | null }} store @param {Stuck | undefined} now
|
|
206
|
+
* @returns {Stuck | null}
|
|
207
|
+
*/
|
|
208
|
+
export function stillStuck(store, now) {
|
|
209
|
+
if (!now) return null
|
|
210
|
+
/** @type {unknown} */
|
|
211
|
+
let codes = null
|
|
212
|
+
try {
|
|
213
|
+
codes = JSON.parse(store.get(codesKey(now.slug, now.number)) ?? 'null')
|
|
214
|
+
} catch {}
|
|
215
|
+
if (!Array.isArray(codes)) return now
|
|
216
|
+
const symptoms = now.symptoms.filter((s) => codes.includes(s.code))
|
|
217
|
+
return symptoms.length ? { ...now, symptoms } : null
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ---- The rescue worker's result
|
|
221
|
+
|
|
222
|
+
const text = z.string().trim().min(1)
|
|
223
|
+
|
|
224
|
+
export const RescueAction = z.discriminatedUnion('do', [
|
|
225
|
+
z.object({ do: z.literal('requeue'), queue: z.enum(['task', 'intake']), reason: text }),
|
|
226
|
+
z.object({ do: z.literal('to-human'), reason: text }),
|
|
227
|
+
z.object({
|
|
228
|
+
do: z.literal('close-issue'),
|
|
229
|
+
as: z.enum(['completed', 'not_planned']).default('completed'),
|
|
230
|
+
reason: text,
|
|
231
|
+
}),
|
|
232
|
+
z.object({ do: z.literal('adopt-pr'), pr: z.number().int().positive(), reason: text.optional() }),
|
|
233
|
+
z.object({ do: z.literal('close-pr'), pr: z.number().int().positive(), reason: text }),
|
|
234
|
+
z.object({ do: z.literal('clear-labels'), labels: z.array(text).min(1), reason: text.optional() }),
|
|
235
|
+
])
|
|
236
|
+
|
|
237
|
+
/** A ticket for a root cause: `fingerprint` names the cause, so the same cause found again is one ticket. */
|
|
238
|
+
export const RootCause = z.object({
|
|
239
|
+
title: text,
|
|
240
|
+
body: text,
|
|
241
|
+
fingerprint: z.string().regex(/^[a-z0-9][a-z0-9-]{2,63}$/, 'kebab-case, 3-64 characters'),
|
|
242
|
+
repo: z
|
|
243
|
+
.string()
|
|
244
|
+
.regex(/^[^/\s]+\/[^/\s]+$/)
|
|
245
|
+
.optional(),
|
|
246
|
+
})
|
|
247
|
+
|
|
248
|
+
/** What a rescue worker ends with, as the last JSON block of its final message. */
|
|
249
|
+
export const RescueResult = z.object({
|
|
250
|
+
stuck: z.boolean(),
|
|
251
|
+
how: text,
|
|
252
|
+
actions: z.array(RescueAction).max(4).default([]),
|
|
253
|
+
rootCauses: z.array(RootCause).max(3).default([]),
|
|
254
|
+
})
|
|
255
|
+
|
|
256
|
+
/** @typedef {z.infer<typeof RescueResult>} Rescue @typedef {z.infer<typeof RescueAction>} Action */
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* The rescue verdict from a worker's stdout log, validated by {@link RescueResult}.
|
|
260
|
+
* @param {string} log @returns {{ ok: true, result: Rescue } | { ok: false, error: string }}
|
|
261
|
+
*/
|
|
262
|
+
export function readRescue(log) {
|
|
263
|
+
const read = finalJson(log)
|
|
264
|
+
if (!read.ok) return read
|
|
265
|
+
const parsed = RescueResult.safeParse(read.json)
|
|
266
|
+
if (!parsed.success)
|
|
267
|
+
return { ok: false, error: `invalid rescue result: ${parsed.error.issues.map(formatZodIssue).join('; ')}` }
|
|
268
|
+
return { ok: true, result: parsed.data }
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** Actions that put the issue somewhere: at most one, and nothing after it. */
|
|
272
|
+
const MOVES = new Set(['requeue', 'to-human', 'close-issue', 'adopt-pr'])
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Whether cambium can carry out this verdict on this item, before it touches anything: an error naming the first
|
|
276
|
+
* problem, or null.
|
|
277
|
+
* @param {Rescue} r @param {{ issue: Issue, snap: Snapshot, repos: string[] }} at
|
|
278
|
+
* `repos`: where root-cause tickets may be filed.
|
|
279
|
+
* @returns {string | null}
|
|
280
|
+
*/
|
|
281
|
+
export function checkPlan(r, { issue, snap, repos }) {
|
|
282
|
+
const moves = r.actions.filter((a) => MOVES.has(a.do))
|
|
283
|
+
if (moves.length > 1) return `more than one move (${moves.map((a) => a.do).join(', ')}); give at most one`
|
|
284
|
+
if (moves.length && r.actions.at(-1) !== moves[0]) return `${moves[0].do} must be the last action`
|
|
285
|
+
if (!r.stuck && r.actions.length) return 'actions on an item that is not stuck'
|
|
286
|
+
for (const a of r.actions) {
|
|
287
|
+
if ((a.do === 'adopt-pr' || a.do === 'close-pr') && !snap.prs.some((p) => p.number === a.pr && p.state === 'OPEN'))
|
|
288
|
+
return `${a.do}: #${a.pr} is not an open agent PR`
|
|
289
|
+
if (a.do === 'adopt-pr' && r.actions.some((b) => b.do === 'close-pr' && b.pr === a.pr))
|
|
290
|
+
return `adopt-pr and close-pr name the same PR #${a.pr}`
|
|
291
|
+
if (a.do === 'clear-labels') {
|
|
292
|
+
const foreign = a.labels.filter((l) => !cambiumLabel(l))
|
|
293
|
+
if (foreign.length) return `clear-labels: ${foreign.join(', ')} are not cambium labels`
|
|
294
|
+
}
|
|
295
|
+
if (issue.state !== 'OPEN' && a.do !== 'clear-labels') return `${a.do}: #${issue.number} is closed`
|
|
296
|
+
}
|
|
297
|
+
for (const c of r.rootCauses)
|
|
298
|
+
if (c.repo && !repos.includes(c.repo))
|
|
299
|
+
return `root cause "${c.fingerprint}": ${c.repo} is not one of ${repos.join(', ')}`
|
|
300
|
+
return null
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** What an action did, for the report. @param {Action} a */
|
|
304
|
+
export function describe(a) {
|
|
305
|
+
switch (a.do) {
|
|
306
|
+
case 'requeue':
|
|
307
|
+
return `Requeued to \`${a.queue === 'task' ? Q.task : Q.intake}\` at attempt 1: ${a.reason}`
|
|
308
|
+
case 'to-human':
|
|
309
|
+
return `Handed to \`${Q.human}\`: ${a.reason}`
|
|
310
|
+
case 'close-issue':
|
|
311
|
+
return `Closed as ${a.as === 'completed' ? 'completed' : 'not planned'}: ${a.reason}`
|
|
312
|
+
case 'adopt-pr':
|
|
313
|
+
return `Handed PR #${a.pr} to the PR graph, which merges it when green or reworks it${a.reason ? `: ${a.reason}` : ''}`
|
|
314
|
+
case 'close-pr':
|
|
315
|
+
return `Closed PR #${a.pr}: ${a.reason}`
|
|
316
|
+
case 'clear-labels':
|
|
317
|
+
return `Removed ${a.labels.map((l) => `\`${l}\``).join(', ')}${a.reason ? `: ${a.reason}` : ''}`
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// ---- Root-cause tickets
|
|
322
|
+
|
|
323
|
+
/** The hidden line that names a root cause in its ticket's body. @param {string} fingerprint */
|
|
324
|
+
export const causeMarker = (fingerprint) => `<!-- cambium:rescue-cause ${fingerprint} -->`
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* A root-cause ticket's body: the worker's, then where it was found.
|
|
328
|
+
* @param {z.infer<typeof RootCause>} c @param {{ slug: string, number: number, how: string }} from
|
|
329
|
+
*/
|
|
330
|
+
export function causeBody(c, from) {
|
|
331
|
+
return [
|
|
332
|
+
c.body,
|
|
333
|
+
'',
|
|
334
|
+
'## Found by',
|
|
335
|
+
'',
|
|
336
|
+
`cambium's rescue graph, unsticking ${from.slug}#${from.number}. How that item got stuck:`,
|
|
337
|
+
'',
|
|
338
|
+
...from.how.split('\n').map((l) => `> ${l}`),
|
|
339
|
+
'',
|
|
340
|
+
causeMarker(c.fingerprint),
|
|
341
|
+
].join('\n')
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/** The comment on an existing root-cause ticket when the cause is found again. @param {{ slug: string, number: number, how: string }} from */
|
|
345
|
+
export const seenAgain = (from) =>
|
|
346
|
+
`**Rescue:** this cause struck again, on ${from.slug}#${from.number}.\n\n${from.how
|
|
347
|
+
.split('\n')
|
|
348
|
+
.map((l) => `> ${l}`)
|
|
349
|
+
.join('\n')}`
|
|
350
|
+
|
|
351
|
+
// ---- The rescue worker's prompt and the reports
|
|
352
|
+
|
|
353
|
+
/** @param {Symptom[]} symptoms */
|
|
354
|
+
const symptomList = (symptoms) => symptoms.map((s) => `- \`${s.code}\`: ${s.detail}`).join('\n')
|
|
355
|
+
|
|
356
|
+
/** cambium's own source, for the worker to read when the cause is cambium's. */
|
|
357
|
+
export const CAMBIUM_SRC = fileURLToPath(new URL('..', import.meta.url))
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* The rescue worker's prompt: one stuck item, one verdict.
|
|
361
|
+
* @param {{
|
|
362
|
+
* repo: string, baseBranch: string, stuck: Stuck, issue: Issue, pr: PullRequest | null, agentsMd: string | null,
|
|
363
|
+
* logs: string, repos: string[], previous: number,
|
|
364
|
+
* }} input
|
|
365
|
+
* `logs`: cambium's log directory. `repos`: where root-cause tickets may go. `previous`: rescues of this item this week.
|
|
366
|
+
*/
|
|
367
|
+
export function rescuePrompt({ repo, baseBranch, stuck, issue, pr, agentsMd, logs, repos, previous }) {
|
|
368
|
+
const marker = issue.stateComment?.marker
|
|
369
|
+
return `You are a rescue worker started by cambium, the scheduler, in a read-only checkout of ${repo} at origin/${baseBranch}.
|
|
370
|
+
Nobody is watching this session and nobody will answer a question: never ask one and wait.
|
|
371
|
+
|
|
372
|
+
GitHub issue #${issue.number} looks stuck: cambium's queues are not moving it. Your job has three parts:
|
|
373
|
+
1. How did it get stuck? Reconstruct the sequence of events with evidence: who or what changed its labels and when, what cambium did, what failed.
|
|
374
|
+
2. What gets it moving again? Choose the actions cambium should take now.
|
|
375
|
+
3. What would stop this happening again? Each root cause that needs a change (to code, config or process) becomes a ticket.
|
|
376
|
+
You do not do any of it yourself. You do not edit files, commit, publish, comment, label or close anything; cambium carries out your verdict.
|
|
377
|
+
|
|
378
|
+
What cambium found:
|
|
379
|
+
${symptomList(stuck.symptoms)}${previous ? `\n\ncambium has rescued this item ${previous} time${previous === 1 ? '' : 's'} this week already. If you cannot see a fix that lasts, hand it to a human.` : ''}
|
|
380
|
+
|
|
381
|
+
How cambium's queues work (labels are the queue; only cambium moves them):
|
|
382
|
+
- Exactly one queue label: \`${Q.intake}\` (triage decides what it is), \`${Q.task}\` (a task worker turns it into a PR), \`${Q.human}\` (waits for a person, with a \`why:*\` label saying why), \`${Q.promote}\` (merged, awaiting the release promotion).
|
|
383
|
+
- \`${RUNNING}\`: a worker holds a lease on it right now. Without a live worker it is a lost lease.
|
|
384
|
+
- \`why:*\` only with \`${Q.human}\`; \`wait:deferred\` / \`wait:blocked\` only with \`${Q.intake}\` (triage parked it).
|
|
385
|
+
- Once a task worker publishes its PR, the issue leaves every queue and the PR graph owns it: it merges a green PR (or waits for a merge queue or a human), reworks a red or conflicting one back through \`${Q.task}\`, and closes the issue when the PR merges.
|
|
386
|
+
- cambium's state comment on the issue holds a hidden JSON marker: the graph and node the issue rests at, and its history. It is below.
|
|
387
|
+
|
|
388
|
+
How to look:
|
|
389
|
+
- The issue's timeline: \`graft api GET '/repos/${repo}/issues/${issue.number}/timeline?per_page=100'\`: labeled/unlabeled events with their actor and time, cross-referenced and closed PRs. Comments: \`graft api GET /repos/${repo}/issues/${issue.number}/comments\`.
|
|
390
|
+
- A PR: \`graft api GET /repos/${repo}/pulls/<n>\`, its checks: \`graft api GET /repos/${repo}/commits/<sha>/check-runs\`.
|
|
391
|
+
- cambium's logs are under ${logs}: the daemon's log, and each worker's \`<owner>__<repo>/issue-<n>/a<attempt>-<time>.{prompt.md,out.log,err.log}\`. cambium's source is at ${CAMBIUM_SRC}: read it when the cause may be cambium's own.
|
|
392
|
+
- Never \`gh\`. Be quick: a verdict with evidence, not an essay.
|
|
393
|
+
|
|
394
|
+
Actions, in order (cambium checks them before it does anything; an invalid list sends the issue to a human):
|
|
395
|
+
- \`{ "do": "clear-labels", "labels": ["…"], "reason": "…" }\`: remove stray cambium labels (\`q:*\`, \`${RUNNING}\`, \`why:*\`, \`wait:*\`) and nothing else.
|
|
396
|
+
- \`{ "do": "close-pr", "pr": 123, "reason": "…" }\`: close an open agent PR that is superseded or abandoned.
|
|
397
|
+
- At most one of these, last:
|
|
398
|
+
- \`{ "do": "requeue", "queue": "task" | "intake", "reason": "…" }\`: back to a queue at its start, attempt 1: \`task\` when the work is still clear, \`intake\` when it needs triage again.
|
|
399
|
+
- \`{ "do": "adopt-pr", "pr": 123 }\`: hand an open agent PR that holds the work to the PR graph.
|
|
400
|
+
- \`{ "do": "close-issue", "as": "completed" | "not_planned", "reason": "…" }\`: the work is done (say where) or no longer applies.
|
|
401
|
+
- \`{ "do": "to-human", "reason": "…" }\`: only a person can unstick it; \`reason\` says exactly what they must decide or do.
|
|
402
|
+
If it is not actually stuck (cambium's check was wrong), say so with \`"stuck": false\` and no actions; a wrong check is itself a root cause worth a ticket.
|
|
403
|
+
|
|
404
|
+
Root causes: only ones that need a change and that you can name with evidence (a cambium bug, a missing guard, a config gap, a process that keeps mislabelling). A one-off human relabel needs none. Each ticket's \`body\` is a task a worker could do unattended: the evidence, the change, and acceptance checks. \`fingerprint\` names the cause in a few kebab-case words, the same every time it recurs (e.g. \`sweep-skips-foreign-host-lease\`). \`repo\` is where the change belongs, one of: ${repos.join(', ')} (default ${repo}).
|
|
405
|
+
|
|
406
|
+
End your final message with exactly one fenced JSON block, and nothing after it:
|
|
407
|
+
\`\`\`json
|
|
408
|
+
{ "stuck": true, "how": "…", "actions": [ { "do": "…" } ], "rootCauses": [ { "title": "…", "body": "…", "fingerprint": "…", "repo": "${repo}" } ] }
|
|
409
|
+
\`\`\`
|
|
410
|
+
\`how\` is the sequence of events, with times and evidence; cambium posts it on the issue.
|
|
411
|
+
|
|
412
|
+
--- Issue #${issue.number}: ${issue.title} (${issue.url})
|
|
413
|
+
State: ${issue.state} · labels: ${issue.labels.join(', ') || '(none)'}
|
|
414
|
+
${issue.body}
|
|
415
|
+
${marker ? `\n--- cambium's marker\n${JSON.stringify(marker, null, 2)}\n` : '\n(no cambium state comment)\n'}${pr ? `\n--- PR #${pr.number} (${pr.url})\n${JSON.stringify({ state: pr.state, branch: pr.branch, headSha: pr.headSha, checks: pr.checks, failing: pr.failing, mergeable: pr.mergeable, reviewDecision: pr.reviewDecision, inMergeQueue: pr.inMergeQueue, autoMerge: pr.autoMerge, updatedAt: pr.updatedAt ?? null }, null, 2)}\n` : ''}${agentsMd ? `\n--- AGENTS.md\n${agentsMd}` : ''}`
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* The comment that documents a rescue on the item.
|
|
420
|
+
* @param {{ stuck: Stuck, result: Rescue, done: string[], filed: { slug: string, number: number, title: string, again: boolean }[] }} r
|
|
421
|
+
*/
|
|
422
|
+
export function rescueReport({ stuck, result, done, filed }) {
|
|
423
|
+
return [
|
|
424
|
+
result.stuck ? '**Rescue: this issue was stuck.**' : '**Rescue: checked; not stuck after all.**',
|
|
425
|
+
'',
|
|
426
|
+
'**What cambium saw**',
|
|
427
|
+
symptomList(stuck.symptoms),
|
|
428
|
+
'',
|
|
429
|
+
`**${result.stuck ? 'How it got stuck' : 'Why it looked stuck'}**`,
|
|
430
|
+
result.how,
|
|
431
|
+
...(done.length ? ['', '**What rescue did**', ...done.map((d) => `- ${d}`)] : []),
|
|
432
|
+
...(filed.length
|
|
433
|
+
? [
|
|
434
|
+
'',
|
|
435
|
+
'**Root causes**',
|
|
436
|
+
...filed.map(
|
|
437
|
+
(f) => `- ${f.slug}#${f.number} ${f.title}${f.again ? ' (already open; noted this recurrence)' : ''}`,
|
|
438
|
+
),
|
|
439
|
+
]
|
|
440
|
+
: []),
|
|
441
|
+
'',
|
|
442
|
+
"_Rescued by cambium's rescue graph._",
|
|
443
|
+
].join('\n')
|
|
444
|
+
}
|