@oakgreencc/cambium 0.0.38 → 0.0.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -2
- package/package.json +2 -2
- package/src/cli.mjs +5 -1
- package/src/config.mjs +22 -1
- package/src/github.mjs +32 -0
- package/src/graph-engine.mjs +9 -2
- package/src/graphs/rescue.mjs +487 -0
- package/src/labels.mjs +4 -0
- package/src/marker.mjs +1 -1
- package/src/phases/dispatch.mjs +23 -14
- package/src/phases/rescue.mjs +444 -0
- package/src/rescue-engine.mjs +280 -0
- package/src/tick.mjs +3 -3
- package/src/upkeep.mjs +8 -1
|
@@ -0,0 +1,487 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* The rescue graph: one stuck item (phases/rescue.mjs `findStuck`) is diagnosed by a one-shot, read-only `claude -p`
|
|
4
|
+
* worker whose final message carries a typed verdict ({@link RescueResult}): how the item got stuck, the actions that
|
|
5
|
+
* unstick it, and the root causes worth a ticket. cambium checks the verdict, carries out the actions, files or
|
|
6
|
+
* re-notes each root-cause ticket in `q:intake` (labelled `rescue`, so triage takes it from there), and documents it
|
|
7
|
+
* all in a comment on the item. A missing or invalid verdict, or an item rescued `rescue.maxRescues` times this week,
|
|
8
|
+
* goes to `q:human` with `why:rescue`.
|
|
9
|
+
*
|
|
10
|
+
* The item is not claimed while it is diagnosed: its labels are the evidence. The rescue worker's row (phase
|
|
11
|
+
* `rescue`) keeps Schedule and Sweep off it meanwhile, and the verdict is acted on only while the item is still
|
|
12
|
+
* stuck.
|
|
13
|
+
*/
|
|
14
|
+
import { z } from 'zod'
|
|
15
|
+
import { advance, defineGraph, defineNode, defineWait, END } from '../graph/index.mjs'
|
|
16
|
+
import { iso, key, markerOf, setLabels, setMarker } from '../issue-state.mjs'
|
|
17
|
+
import { move, Q, RESCUE_LABEL, WHY } from '../labels.mjs'
|
|
18
|
+
import { newMarker, withHistory } from '../marker.mjs'
|
|
19
|
+
import {
|
|
20
|
+
causeBody,
|
|
21
|
+
causeMarker,
|
|
22
|
+
checkPlan,
|
|
23
|
+
describe,
|
|
24
|
+
readRescue,
|
|
25
|
+
recordRescue,
|
|
26
|
+
recordSymptoms,
|
|
27
|
+
RescueResult,
|
|
28
|
+
rescueReport,
|
|
29
|
+
rescuePrompt,
|
|
30
|
+
rescuesOf,
|
|
31
|
+
seenAgain,
|
|
32
|
+
} from '../phases/rescue.mjs'
|
|
33
|
+
import { traceWorker } from '../tracing.mjs'
|
|
34
|
+
import { stopOverdue } from '../workers.mjs'
|
|
35
|
+
import { dispatch } from './dispatch.mjs'
|
|
36
|
+
import { issueOf, task } from './task.mjs'
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* @typedef {import('../github.mjs').Issue} Issue
|
|
40
|
+
* @typedef {import('../marker.mjs').Marker} Marker
|
|
41
|
+
* @typedef {import('../phases/rescue.mjs').Stuck} Stuck
|
|
42
|
+
* @typedef {import('../phases/rescue.mjs').Rescue} Rescue
|
|
43
|
+
* @typedef {import('../phases/rescue.mjs').Action} Action
|
|
44
|
+
* @typedef {import('./task.mjs').Subject & { stuck?: Stuck | null }} RescueSubject
|
|
45
|
+
* `stuck`: the item as `findStuck` sees it now; null once it is no longer stuck.
|
|
46
|
+
* @typedef {import('./task.mjs').TaskCtx & { subject: RescueSubject }} RescueCtx
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
/** The pseudo-branch that names an item's rescue checkout; never created. @param {number} n */
|
|
50
|
+
export const rescueBranch = (n) => `cambium/rescue-${n}`
|
|
51
|
+
|
|
52
|
+
/** Where root-cause tickets may go: the item's repo, then `rescue.causeRepo`. @param {RescueCtx} ctx */
|
|
53
|
+
export const causeRepos = (ctx) => [
|
|
54
|
+
...new Set([ctx.subject.slug, ...(ctx.config.rescue.causeRepo ? [ctx.config.rescue.causeRepo] : [])]),
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
/** @param {RescueCtx} ctx */
|
|
58
|
+
function stuckOf(ctx) {
|
|
59
|
+
const { stuck } = ctx.subject
|
|
60
|
+
if (!stuck) throw new Error(`${ctx.subject.slug}#${ctx.subject.number} is not stuck`)
|
|
61
|
+
return stuck
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** @param {Stuck} stuck */
|
|
65
|
+
const symptomsLine = (stuck) => stuck.symptoms.map((s) => s.code).join(', ')
|
|
66
|
+
|
|
67
|
+
/** @param {unknown} err */
|
|
68
|
+
const errText = (err) => /** @type {Error} */ (err)?.message ?? String(err)
|
|
69
|
+
|
|
70
|
+
/** A marker as cambium writes it now: no stage-tick `stage`. @param {Marker} m */
|
|
71
|
+
const graphMarker = (m) => /** @type {Marker} */ (Object.fromEntries(Object.entries(m).filter(([k]) => k !== 'stage')))
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The marker, then the labels; the issue counts as moved this tick.
|
|
75
|
+
* @param {RescueCtx} ctx @param {Issue} issue @param {Marker} marker @param {{ add: string[], remove: string[] }} labels
|
|
76
|
+
*/
|
|
77
|
+
async function place(ctx, issue, marker, labels) {
|
|
78
|
+
await setMarker(ctx, ctx.subject.slug, issue, marker)
|
|
79
|
+
await setLabels(ctx, ctx.subject.slug, issue, labels)
|
|
80
|
+
ctx.tick.touched.add(key(ctx.subject.slug, issue.number))
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The history entry a rescue move leaves. @param {RescueCtx} ctx @param {Marker} m @param {string} reason */
|
|
84
|
+
const rescued = (ctx, m, reason) =>
|
|
85
|
+
withHistory(m, { at: iso(ctx.tick.now), attempt: m.attempt, outcome: 'rescued', reason })
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Into the Human Queue with `why:rescue`, resting at this graph's `to-human`.
|
|
89
|
+
* @param {RescueCtx} ctx @param {Issue} issue @param {string} reason @param {string} outcome
|
|
90
|
+
*/
|
|
91
|
+
async function handToHuman(ctx, issue, reason, outcome) {
|
|
92
|
+
const m = markerOf(issue)
|
|
93
|
+
await place(
|
|
94
|
+
ctx,
|
|
95
|
+
issue,
|
|
96
|
+
{
|
|
97
|
+
...graphMarker(withHistory(m, { at: iso(ctx.tick.now), attempt: m.attempt, outcome, reason })),
|
|
98
|
+
graph: rescue.name,
|
|
99
|
+
graphVersion: rescue.version,
|
|
100
|
+
phase: 'rescue',
|
|
101
|
+
node: 'to-human',
|
|
102
|
+
status: 'waiting',
|
|
103
|
+
enteredAt: iso(ctx.tick.now),
|
|
104
|
+
lastFailure: reason,
|
|
105
|
+
host: null,
|
|
106
|
+
start: null,
|
|
107
|
+
input: null,
|
|
108
|
+
},
|
|
109
|
+
move(issue.labels, Q.human, { extra: [WHY.rescue] }),
|
|
110
|
+
)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Back to a queue at its graph's entry, attempt 1, the history kept.
|
|
115
|
+
* @param {RescueCtx} ctx @param {Issue} issue @param {'task' | 'intake'} queue @param {string} reason
|
|
116
|
+
*/
|
|
117
|
+
async function requeue(ctx, issue, queue, reason) {
|
|
118
|
+
const m = markerOf(issue)
|
|
119
|
+
const g = queue === 'task' ? task : dispatch
|
|
120
|
+
await place(
|
|
121
|
+
ctx,
|
|
122
|
+
issue,
|
|
123
|
+
newMarker({
|
|
124
|
+
graph: g.name,
|
|
125
|
+
graphVersion: g.version,
|
|
126
|
+
node: g.entry.node,
|
|
127
|
+
phase: queue === 'task' ? 'task' : 'dispatch',
|
|
128
|
+
enteredAt: iso(ctx.tick.now),
|
|
129
|
+
traceparent: m.traceparent,
|
|
130
|
+
history: rescued(ctx, m, reason).history,
|
|
131
|
+
}),
|
|
132
|
+
move(issue.labels, queue === 'task' ? Q.task : Q.intake),
|
|
133
|
+
)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* One action of a checked verdict.
|
|
138
|
+
* @param {RescueCtx} ctx @param {Issue} issue @param {Action} a
|
|
139
|
+
*/
|
|
140
|
+
async function act(ctx, issue, a) {
|
|
141
|
+
const { slug, snap } = ctx.subject
|
|
142
|
+
switch (a.do) {
|
|
143
|
+
case 'clear-labels':
|
|
144
|
+
return setLabels(ctx, slug, issue, { add: [], remove: a.labels.filter((l) => issue.labels.includes(l)) })
|
|
145
|
+
case 'close-pr': {
|
|
146
|
+
await ctx.github.closePull(slug, a.pr, `${a.reason}\n\nClosed by cambium's rescue (#${issue.number}).`)
|
|
147
|
+
const pr = snap?.prs.find((p) => p.number === a.pr)
|
|
148
|
+
if (pr) pr.state = 'CLOSED'
|
|
149
|
+
return
|
|
150
|
+
}
|
|
151
|
+
case 'requeue':
|
|
152
|
+
return requeue(ctx, issue, a.queue, a.reason)
|
|
153
|
+
case 'to-human':
|
|
154
|
+
return handToHuman(ctx, issue, a.reason, 'rescued')
|
|
155
|
+
case 'close-issue': {
|
|
156
|
+
const m = markerOf(issue)
|
|
157
|
+
await place(
|
|
158
|
+
ctx,
|
|
159
|
+
issue,
|
|
160
|
+
{ ...graphMarker(rescued(ctx, m, a.reason)), status: 'done', host: null, input: null },
|
|
161
|
+
move(issue.labels, null),
|
|
162
|
+
)
|
|
163
|
+
await ctx.github.close(slug, issue.number, `${a.reason}\n\nClosed by cambium's rescue.`, { reason: a.as })
|
|
164
|
+
issue.state = 'CLOSED'
|
|
165
|
+
return
|
|
166
|
+
}
|
|
167
|
+
case 'adopt-pr': {
|
|
168
|
+
const pr = snap?.prs.find((p) => p.number === a.pr)
|
|
169
|
+
if (!pr) throw new Error(`PR #${a.pr} is not in the snapshot`)
|
|
170
|
+
// The task graph's `submit`, as a worker's publish or the reaper's adoption: the PR graph takes it next tick.
|
|
171
|
+
const r = await advance(task, { node: 'submit', input: { pr: pr.number, branch: pr.branch } }, ctx, {
|
|
172
|
+
maxTransitions: 1,
|
|
173
|
+
})
|
|
174
|
+
if (r.error) throw r.error.cause instanceof Error ? r.error.cause : r.error
|
|
175
|
+
return
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// ---- Schemas and edge predicates
|
|
181
|
+
|
|
182
|
+
const Empty = z.object({})
|
|
183
|
+
const failure = { reason: z.string(), outcome: z.string() }
|
|
184
|
+
const Verdict = z.object({ result: RescueResult, log: z.string() })
|
|
185
|
+
const Filed = z.object({ slug: z.string(), number: z.number().int(), title: z.string(), again: z.boolean() })
|
|
186
|
+
|
|
187
|
+
const started = (/** @type {{ to: string }} */ o) => o.to === 'running'
|
|
188
|
+
const budgetSpent = (/** @type {{ to: string }} */ o) => o.to === 'human'
|
|
189
|
+
const resolved = (/** @type {{ exit: string }} */ o) => o.exit === 'resolved' || o.exit === 'lost'
|
|
190
|
+
const diagnosed = (/** @type {{ exit: string }} */ o) => o.exit === 'diagnosed'
|
|
191
|
+
|
|
192
|
+
// ---- Nodes
|
|
193
|
+
|
|
194
|
+
const startRescue = defineNode({
|
|
195
|
+
name: 'start-rescue',
|
|
196
|
+
input: Empty,
|
|
197
|
+
output: z.discriminatedUnion('to', [
|
|
198
|
+
z.object({ to: z.literal('running'), pid: z.number().int(), attempt: z.number().int() }),
|
|
199
|
+
z.object({ to: z.literal('human'), ...failure }),
|
|
200
|
+
z.object({ to: z.literal('failed'), reason: z.string() }),
|
|
201
|
+
]),
|
|
202
|
+
/**
|
|
203
|
+
* Over budget (`rescue.maxRescues` this week): to a human without another diagnosis. Else count the rescue and
|
|
204
|
+
* start the rescue worker in a read-only checkout. A start that fails ends here; the next sweep tries again after
|
|
205
|
+
* `rescue.cooldownHours`, and counts it.
|
|
206
|
+
* @param {{}} _ @param {RescueCtx} ctx
|
|
207
|
+
*/
|
|
208
|
+
run: async (_, ctx) => {
|
|
209
|
+
const { config, spawner, store } = ctx
|
|
210
|
+
const { slug, number } = ctx.subject
|
|
211
|
+
const issue = issueOf(ctx)
|
|
212
|
+
const stuck = stuckOf(ctx)
|
|
213
|
+
const repo = config.repos.find((r) => r.slug === slug)
|
|
214
|
+
if (!repo) throw new Error(`${slug} is not in the config`)
|
|
215
|
+
const previous = rescuesOf(store, slug, number, ctx.tick.now).length
|
|
216
|
+
const { maxRescues } = config.rescue
|
|
217
|
+
if (previous >= maxRescues)
|
|
218
|
+
return {
|
|
219
|
+
to: 'human',
|
|
220
|
+
reason: `rescued ${previous} time${previous === 1 ? '' : 's'} this week and still stuck (${symptomsLine(stuck)})`,
|
|
221
|
+
outcome: 'budget',
|
|
222
|
+
}
|
|
223
|
+
recordRescue(store, slug, number, ctx.tick.now)
|
|
224
|
+
recordSymptoms(store, stuck)
|
|
225
|
+
const caps = config.phases.rescue
|
|
226
|
+
const pr = ctx.subject.pr ?? (stuck.pr ? (ctx.subject.snap?.prs.find((p) => p.number === stuck.pr) ?? null) : null)
|
|
227
|
+
/** @type {import('../workers.mjs').Started} */
|
|
228
|
+
let s
|
|
229
|
+
try {
|
|
230
|
+
const convention = spawner.convention(repo)
|
|
231
|
+
s = await spawner.start({
|
|
232
|
+
repo,
|
|
233
|
+
convention,
|
|
234
|
+
issue: number,
|
|
235
|
+
attempt: previous + 1,
|
|
236
|
+
branch: rescueBranch(number),
|
|
237
|
+
phase: 'rescue',
|
|
238
|
+
maxTurns: caps.maxTurns,
|
|
239
|
+
readOnly: true,
|
|
240
|
+
foreground: ctx.subject.foreground ?? false,
|
|
241
|
+
env: ctx.traceparent ? { TRACEPARENT: ctx.traceparent } : undefined,
|
|
242
|
+
buildPrompt: (agentsMd) =>
|
|
243
|
+
rescuePrompt({
|
|
244
|
+
repo: slug,
|
|
245
|
+
baseBranch: repo.baseBranch,
|
|
246
|
+
stuck,
|
|
247
|
+
issue,
|
|
248
|
+
pr,
|
|
249
|
+
agentsMd,
|
|
250
|
+
logs: config.paths.logs,
|
|
251
|
+
repos: causeRepos(ctx),
|
|
252
|
+
previous,
|
|
253
|
+
}),
|
|
254
|
+
})
|
|
255
|
+
} catch (err) {
|
|
256
|
+
ctx.log('warn', `rescue: could not start a worker: ${errText(err)}`, { repo: slug, issue: number })
|
|
257
|
+
return { to: 'failed', reason: errText(err) }
|
|
258
|
+
}
|
|
259
|
+
store.addWorker({
|
|
260
|
+
repo: slug,
|
|
261
|
+
issue: number,
|
|
262
|
+
phase: 'rescue',
|
|
263
|
+
pid: s.pid,
|
|
264
|
+
worktree: s.worktree,
|
|
265
|
+
branch: rescueBranch(number),
|
|
266
|
+
attempt: previous + 1,
|
|
267
|
+
startedAt: ctx.tick.now,
|
|
268
|
+
host: config.host,
|
|
269
|
+
stdout: s.stdout,
|
|
270
|
+
stderr: s.stderr,
|
|
271
|
+
traceparent: ctx.traceparent ?? null,
|
|
272
|
+
})
|
|
273
|
+
ctx.log('info', `rescue: started worker pid ${s.pid} (${symptomsLine(stuck)})`, { repo: slug, issue: number })
|
|
274
|
+
ctx.subject.started = { ...s, attempt: previous + 1 }
|
|
275
|
+
return { to: 'running', pid: s.pid, attempt: previous + 1 }
|
|
276
|
+
},
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
const diagnosing = defineWait({
|
|
280
|
+
name: 'diagnosing',
|
|
281
|
+
input: z.object({ attempt: z.number().int() }),
|
|
282
|
+
output: z.discriminatedUnion('exit', [
|
|
283
|
+
z.object({ exit: z.literal('diagnosed'), ...Verdict.shape }),
|
|
284
|
+
z.object({ exit: z.literal('invalid'), ...failure }),
|
|
285
|
+
z.object({ exit: z.literal('resolved') }),
|
|
286
|
+
z.object({ exit: z.literal('lost') }),
|
|
287
|
+
]),
|
|
288
|
+
/**
|
|
289
|
+
* Alive and under the wall-clock cap: hold. Over it: kill. Gone: unstuck meanwhile (someone or something moved it)
|
|
290
|
+
* ends the graph; otherwise the verdict, read from its stdout log and checked against the item (phases/rescue.mjs
|
|
291
|
+
* `checkPlan`). A missing, invalid or unworkable one is `invalid`, with the error.
|
|
292
|
+
* @param {unknown} _ @param {RescueCtx} ctx
|
|
293
|
+
*/
|
|
294
|
+
poll: async (_, ctx) => {
|
|
295
|
+
const { config, store, procs, spawner } = ctx
|
|
296
|
+
const { slug, number, snap, issue, reaped, stuck } = ctx.subject
|
|
297
|
+
const w = store.worker(slug, number)
|
|
298
|
+
if (w?.phase !== 'rescue') return { ready: true, output: { exit: 'lost' } }
|
|
299
|
+
const caps = config.phases.rescue
|
|
300
|
+
if (
|
|
301
|
+
!reaped?.has(w.pid) &&
|
|
302
|
+
stopOverdue({ procs, store, log: ctx.log }, w, caps.wallClockMinutes, ctx.tick.now) !== 'gone'
|
|
303
|
+
)
|
|
304
|
+
return { ready: false }
|
|
305
|
+
if (!snap || !issue) return { ready: false }
|
|
306
|
+
traceWorker(ctx, w)
|
|
307
|
+
if (!stuck) {
|
|
308
|
+
ctx.log('info', 'rescue: no longer stuck; verdict not applied', { repo: slug, issue: number })
|
|
309
|
+
return { ready: true, output: { exit: 'resolved' } }
|
|
310
|
+
}
|
|
311
|
+
const read = w.killedReason ? { ok: false, error: w.killedReason } : readRescue(spawner.readLog(w.stdout))
|
|
312
|
+
if (!read.ok || !('result' in read)) {
|
|
313
|
+
const error = 'error' in read ? read.error : 'no verdict'
|
|
314
|
+
const tail = error.startsWith('invalid') ? '' : spawner.tail(w.stderr, 15)
|
|
315
|
+
return {
|
|
316
|
+
ready: true,
|
|
317
|
+
output: {
|
|
318
|
+
exit: 'invalid',
|
|
319
|
+
reason: `${error}${tail ? `\n--- stderr (last lines, ${w.stderr})\n${tail}` : ''}`,
|
|
320
|
+
outcome: w.killedReason ? 'killed' : 'invalid',
|
|
321
|
+
},
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
const problem = checkPlan(read.result, { issue, snap, repos: causeRepos(ctx) })
|
|
325
|
+
if (problem)
|
|
326
|
+
return {
|
|
327
|
+
ready: true,
|
|
328
|
+
output: { exit: 'invalid', reason: `unworkable rescue verdict: ${problem}`, outcome: 'invalid' },
|
|
329
|
+
}
|
|
330
|
+
return { ready: true, output: { exit: 'diagnosed', result: read.result, log: w.stdout } }
|
|
331
|
+
},
|
|
332
|
+
})
|
|
333
|
+
|
|
334
|
+
const Unstuck = z.object({ ...Verdict.shape, done: z.array(z.string()) })
|
|
335
|
+
const unstick = defineNode({
|
|
336
|
+
name: 'unstick',
|
|
337
|
+
input: Verdict,
|
|
338
|
+
output: Unstuck,
|
|
339
|
+
/**
|
|
340
|
+
* The verdict's actions, in order. A failed write throws: the worker row stays, so the next tick reads the same
|
|
341
|
+
* verdict and tries again.
|
|
342
|
+
* @param {z.infer<typeof Verdict>} v @param {RescueCtx} ctx
|
|
343
|
+
*/
|
|
344
|
+
run: async (v, ctx) => {
|
|
345
|
+
const issue = issueOf(ctx)
|
|
346
|
+
/** @type {string[]} */
|
|
347
|
+
const done = []
|
|
348
|
+
for (const a of v.result.actions) {
|
|
349
|
+
await act(ctx, issue, a)
|
|
350
|
+
done.push(describe(a))
|
|
351
|
+
}
|
|
352
|
+
if (done.length)
|
|
353
|
+
ctx.log('info', `rescue: ${v.result.actions.map((a) => a.do).join(', ')}`, {
|
|
354
|
+
repo: ctx.subject.slug,
|
|
355
|
+
issue: issue.number,
|
|
356
|
+
})
|
|
357
|
+
return { ...v, done }
|
|
358
|
+
},
|
|
359
|
+
})
|
|
360
|
+
|
|
361
|
+
const Causes = z.object({ ...Unstuck.shape, filed: z.array(Filed) })
|
|
362
|
+
const fileCauses = defineNode({
|
|
363
|
+
name: 'file-causes',
|
|
364
|
+
input: Unstuck,
|
|
365
|
+
output: Causes,
|
|
366
|
+
/**
|
|
367
|
+
* Each root cause: when an open issue in the target repo already carries its fingerprint, a comment there notes the
|
|
368
|
+
* recurrence; otherwise a new ticket in `q:intake`, labelled `rescue`. A repo cambium does not read (only
|
|
369
|
+
* `rescue.causeRepo` can be one) is de-duplicated by what this host filed there.
|
|
370
|
+
* @param {z.infer<typeof Unstuck>} u @param {RescueCtx} ctx
|
|
371
|
+
*/
|
|
372
|
+
run: async (u, ctx) => {
|
|
373
|
+
const { slug, number } = ctx.subject
|
|
374
|
+
const from = { slug, number, how: u.result.how }
|
|
375
|
+
/** @type {z.infer<typeof Filed>[]} */
|
|
376
|
+
const filed = []
|
|
377
|
+
for (const c of u.result.rootCauses) {
|
|
378
|
+
const target = c.repo ?? slug
|
|
379
|
+
const snap = ctx.tick.snaps.get(target)
|
|
380
|
+
const kvKey = `rescue-cause:${target}:${c.fingerprint}`
|
|
381
|
+
const known = snap
|
|
382
|
+
? [...snap.issues.values()].find((i) => i.state === 'OPEN' && i.body.includes(causeMarker(c.fingerprint)))
|
|
383
|
+
?.number
|
|
384
|
+
: Number(ctx.store.get(kvKey)) || undefined
|
|
385
|
+
if (known) {
|
|
386
|
+
await ctx.github.comment(target, known, seenAgain(from))
|
|
387
|
+
filed.push({ slug: target, number: known, title: snap?.issues.get(known)?.title ?? c.title, again: true })
|
|
388
|
+
continue
|
|
389
|
+
}
|
|
390
|
+
const body = causeBody(c, from)
|
|
391
|
+
const labels = [Q.intake, RESCUE_LABEL]
|
|
392
|
+
const made = await ctx.github.createIssue(target, { title: c.title, body, labels })
|
|
393
|
+
ctx.store.set(kvKey, String(made.number))
|
|
394
|
+
snap?.issues.set(made.number, {
|
|
395
|
+
number: made.number,
|
|
396
|
+
title: c.title,
|
|
397
|
+
url: made.url,
|
|
398
|
+
state: 'OPEN',
|
|
399
|
+
createdAt: iso(ctx.tick.now),
|
|
400
|
+
body,
|
|
401
|
+
labels,
|
|
402
|
+
stateComment: null,
|
|
403
|
+
blockedBy: [],
|
|
404
|
+
})
|
|
405
|
+
ctx.log('info', `rescue: filed root cause ${target}#${made.number} (${c.fingerprint})`, {
|
|
406
|
+
repo: slug,
|
|
407
|
+
issue: number,
|
|
408
|
+
})
|
|
409
|
+
filed.push({ slug: target, number: made.number, title: c.title, again: false })
|
|
410
|
+
}
|
|
411
|
+
return { ...u, filed }
|
|
412
|
+
},
|
|
413
|
+
})
|
|
414
|
+
|
|
415
|
+
const report = defineNode({
|
|
416
|
+
name: 'report',
|
|
417
|
+
input: Causes,
|
|
418
|
+
output: Empty,
|
|
419
|
+
/** The comment that documents the rescue: what was seen, how it got stuck, what was done, the root causes. @param {z.infer<typeof Causes>} c @param {RescueCtx} ctx */
|
|
420
|
+
run: async (c, ctx) => {
|
|
421
|
+
const { slug, number } = ctx.subject
|
|
422
|
+
await ctx.github.comment(
|
|
423
|
+
slug,
|
|
424
|
+
number,
|
|
425
|
+
rescueReport({ stuck: stuckOf(ctx), result: c.result, done: c.done, filed: c.filed }),
|
|
426
|
+
)
|
|
427
|
+
ctx.log('info', `rescue: ${c.result.stuck ? 'unstuck' : 'not stuck'}; ${c.filed.length} root cause(s)`, {
|
|
428
|
+
repo: slug,
|
|
429
|
+
issue: number,
|
|
430
|
+
})
|
|
431
|
+
return {}
|
|
432
|
+
},
|
|
433
|
+
})
|
|
434
|
+
|
|
435
|
+
const HumanIn = z.object(failure)
|
|
436
|
+
const toHuman = defineNode({
|
|
437
|
+
name: 'to-human',
|
|
438
|
+
input: HumanIn,
|
|
439
|
+
output: Empty,
|
|
440
|
+
/** Into the Human Queue with `why:rescue`, and a comment saying what cambium saw and why it stopped. @param {z.infer<typeof HumanIn>} what @param {RescueCtx} ctx */
|
|
441
|
+
run: async (what, ctx) => {
|
|
442
|
+
const issue = issueOf(ctx)
|
|
443
|
+
const stuck = stuckOf(ctx)
|
|
444
|
+
// A closed issue is not queued again: the comment is enough for whoever looks.
|
|
445
|
+
if (issue.state === 'OPEN') await handToHuman(ctx, issue, what.reason, what.outcome)
|
|
446
|
+
await ctx.github.comment(
|
|
447
|
+
ctx.subject.slug,
|
|
448
|
+
issue.number,
|
|
449
|
+
[
|
|
450
|
+
'**Rescue: this issue is stuck, and needs a person.**',
|
|
451
|
+
'',
|
|
452
|
+
'**What cambium saw**',
|
|
453
|
+
...stuck.symptoms.map((s) => `- \`${s.code}\`: ${s.detail}`),
|
|
454
|
+
'',
|
|
455
|
+
'**Why rescue stopped**',
|
|
456
|
+
'```',
|
|
457
|
+
what.reason.replace(/```/g, "'''"),
|
|
458
|
+
'```',
|
|
459
|
+
].join('\n'),
|
|
460
|
+
)
|
|
461
|
+
ctx.log('warn', `rescue → q:human (${WHY.rescue}): ${what.reason.split('\n')[0]}`, {
|
|
462
|
+
repo: ctx.subject.slug,
|
|
463
|
+
issue: issue.number,
|
|
464
|
+
})
|
|
465
|
+
return {}
|
|
466
|
+
},
|
|
467
|
+
})
|
|
468
|
+
|
|
469
|
+
/** The rescue graph. Bump `version` when a node is removed or renamed. */
|
|
470
|
+
export const rescue = defineGraph({
|
|
471
|
+
name: 'rescue',
|
|
472
|
+
version: 1,
|
|
473
|
+
entry: { node: 'start-rescue', input: () => ({}) },
|
|
474
|
+
nodes: [startRescue, diagnosing, unstick, fileCauses, report, toHuman],
|
|
475
|
+
edges: [
|
|
476
|
+
['start-rescue', 'diagnosing', started, (o) => ({ attempt: o.attempt })],
|
|
477
|
+
['start-rescue', 'to-human', budgetSpent],
|
|
478
|
+
['start-rescue', END],
|
|
479
|
+
['diagnosing', END, resolved],
|
|
480
|
+
['diagnosing', 'unstick', diagnosed, (o) => ({ result: o.result, log: o.log })],
|
|
481
|
+
['diagnosing', 'to-human'],
|
|
482
|
+
['unstick', 'file-causes'],
|
|
483
|
+
['file-causes', 'report'],
|
|
484
|
+
['report', END],
|
|
485
|
+
['to-human', END],
|
|
486
|
+
],
|
|
487
|
+
})
|
package/src/labels.mjs
CHANGED
|
@@ -18,8 +18,12 @@ export const WHY = /** @type {const} */ ({
|
|
|
18
18
|
reject: 'why:reject',
|
|
19
19
|
approve: 'why:approve',
|
|
20
20
|
triage: 'why:triage',
|
|
21
|
+
rescue: 'why:rescue',
|
|
21
22
|
})
|
|
22
23
|
|
|
24
|
+
/** Marks a root-cause ticket the rescue graph filed (graphs/rescue.mjs). Not a `why:` label: those leave with `q:human`. */
|
|
25
|
+
export const RESCUE_LABEL = 'rescue'
|
|
26
|
+
|
|
23
27
|
/** @param {string[]} labels */
|
|
24
28
|
export const queueOf = (labels) => labels.find((l) => l.startsWith('q:')) ?? null
|
|
25
29
|
|
package/src/marker.mjs
CHANGED
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
* traceparent: string | null,
|
|
32
32
|
* enteredAt: string | null,
|
|
33
33
|
* history: HistoryEntry[],
|
|
34
|
-
* phase: 'task' | 'dispatch' | 'release',
|
|
34
|
+
* phase: 'task' | 'dispatch' | 'release' | 'rescue',
|
|
35
35
|
* stage?: Stage,
|
|
36
36
|
* pr: number | null,
|
|
37
37
|
* branch: string | null,
|
package/src/phases/dispatch.mjs
CHANGED
|
@@ -40,9 +40,6 @@ export const TriageResult = z
|
|
|
40
40
|
|
|
41
41
|
/** @typedef {z.infer<typeof TriageResult>} Triage */
|
|
42
42
|
|
|
43
|
-
/** @param {z.core.$ZodIssue} i */
|
|
44
|
-
const formatIssue = (i) => (i.path.length ? `${i.path.join('.')}: ${i.message}` : i.message)
|
|
45
|
-
|
|
46
43
|
/**
|
|
47
44
|
* The JSON in a final message: its last fenced block, else the whole text, else its outermost `{…}`.
|
|
48
45
|
* @param {string} text @returns {unknown} throws a SyntaxError-like Error when there is none
|
|
@@ -73,14 +70,12 @@ const clip = (s, n = 300) => {
|
|
|
73
70
|
}
|
|
74
71
|
|
|
75
72
|
/**
|
|
76
|
-
* The
|
|
77
|
-
* `result` text, its JSON
|
|
78
|
-
*
|
|
73
|
+
* The JSON a one-shot worker ends with, from its stdout (`claude -p --output-format stream-json`): the last `result`
|
|
74
|
+
* event's `result` text, its JSON ({@link jsonIn}). Anything else is an error saying what was wrong.
|
|
79
75
|
* @param {string} log the stdout log text
|
|
80
|
-
* @
|
|
81
|
-
* @returns {{ ok: true, result: Triage } | { ok: false, error: string }}
|
|
76
|
+
* @returns {{ ok: true, json: unknown } | { ok: false, error: string }}
|
|
82
77
|
*/
|
|
83
|
-
export function
|
|
78
|
+
export function finalJson(log) {
|
|
84
79
|
/** @type {any} */
|
|
85
80
|
let result = null
|
|
86
81
|
for (const line of log.split('\n')) {
|
|
@@ -93,16 +88,30 @@ export function readTriage(log, issue) {
|
|
|
93
88
|
if (!result) return { ok: false, error: 'no result event in the worker log (killed, crashed or truncated)' }
|
|
94
89
|
if (typeof result.result !== 'string')
|
|
95
90
|
return { ok: false, error: `the worker ended without a final message (${result.subtype ?? 'no subtype'})` }
|
|
96
|
-
/** @type {unknown} */
|
|
97
|
-
let json
|
|
98
91
|
try {
|
|
99
|
-
json
|
|
92
|
+
return { ok: true, json: jsonIn(result.result) }
|
|
100
93
|
} catch (err) {
|
|
101
94
|
return { ok: false, error: /** @type {Error} */ (err).message }
|
|
102
95
|
}
|
|
103
|
-
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** @param {z.core.$ZodIssue} i */
|
|
99
|
+
export const formatZodIssue = (i) => (i.path.length ? `${i.path.join('.')}: ${i.message}` : i.message)
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The triage verdict from a worker's stdout (`claude -p --output-format stream-json`): the last `result` event's
|
|
103
|
+
* `result` text, its JSON, validated by {@link TriageResult}. Anything else is an error saying what was wrong.
|
|
104
|
+
*
|
|
105
|
+
* @param {string} log the stdout log text
|
|
106
|
+
* @param {number} issue the triaged issue, which cannot be its own duplicate
|
|
107
|
+
* @returns {{ ok: true, result: Triage } | { ok: false, error: string }}
|
|
108
|
+
*/
|
|
109
|
+
export function readTriage(log, issue) {
|
|
110
|
+
const read = finalJson(log)
|
|
111
|
+
if (!read.ok) return read
|
|
112
|
+
const parsed = TriageResult.safeParse(read.json)
|
|
104
113
|
if (!parsed.success)
|
|
105
|
-
return { ok: false, error: `invalid triage result: ${parsed.error.issues.map(
|
|
114
|
+
return { ok: false, error: `invalid triage result: ${parsed.error.issues.map(formatZodIssue).join('; ')}` }
|
|
106
115
|
if (parsed.data.verdict === 'duplicate' && parsed.data.duplicateOf === issue)
|
|
107
116
|
return { ok: false, error: `invalid triage result: duplicateOf: #${issue} cannot duplicate itself` }
|
|
108
117
|
if (parsed.data.verdict === 'blocked' && parsed.data.blockedBy?.includes(issue))
|