@oakgreencc/cambium 0.0.39 → 0.0.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -2
- package/package.json +1 -1
- package/src/cli.mjs +5 -1
- package/src/config.mjs +22 -1
- package/src/github.mjs +32 -0
- package/src/graph/define.mjs +11 -3
- package/src/graph/describe.mjs +50 -0
- package/src/graph/index.mjs +2 -1
- package/src/graph-engine.mjs +9 -2
- package/src/graphs/dispatch.mjs +33 -10
- package/src/graphs/pr.mjs +23 -6
- package/src/graphs/release.mjs +14 -5
- package/src/graphs/rescue.mjs +504 -0
- package/src/graphs/task.mjs +20 -5
- package/src/labels.mjs +4 -0
- package/src/marker.mjs +1 -1
- package/src/phases/dispatch.mjs +23 -14
- package/src/phases/rescue.mjs +444 -0
- package/src/rescue-engine.mjs +280 -0
- package/src/tick.mjs +17 -4
- package/src/upkeep.mjs +8 -1
package/src/graphs/release.mjs
CHANGED
|
@@ -31,7 +31,7 @@ import { markerTrace, turnTracer } from '../tracing.mjs'
|
|
|
31
31
|
* One repo's turn; `pr` is the promotion PR a `release-promote` turn is about; `traceparent` the turn's trace
|
|
32
32
|
* while it is traced (tracing.mjs `turnTracer`), which markers written in the turn take when they have none.
|
|
33
33
|
* @typedef {import('../tick.mjs').Deps & { tick: import('../tick.mjs').Ctx, subject: ReleaseSubject }} ReleaseCtx
|
|
34
|
-
* @typedef {{ queue: string | null, enter?: (input: any, ctx: ReleaseCtx) => Promise<void> }} Declared
|
|
34
|
+
* @typedef {{ queue: string | null, about?: string, enter?: (input: any, ctx: ReleaseCtx) => Promise<void> }} Declared
|
|
35
35
|
* `queue`: the label a PR resting here carries. `enter`: the write that lands a PR on this wait.
|
|
36
36
|
*/
|
|
37
37
|
|
|
@@ -141,7 +141,7 @@ const cleanup = declare(
|
|
|
141
141
|
return { closed }
|
|
142
142
|
},
|
|
143
143
|
}),
|
|
144
|
-
{ queue: null },
|
|
144
|
+
{ queue: null, about: 'Closes every open promotion PR older than 24 h, with a comment.' },
|
|
145
145
|
)
|
|
146
146
|
|
|
147
147
|
const prepare = declare(
|
|
@@ -215,12 +215,16 @@ const prepare = declare(
|
|
|
215
215
|
},
|
|
216
216
|
}),
|
|
217
217
|
// The PR it opens waits in `q:promote`, release-promote's queue.
|
|
218
|
-
{
|
|
218
|
+
{
|
|
219
|
+
queue: Q.promote,
|
|
220
|
+
about: 'When from is ahead of to, opens the promotion PR (or refreshes the open one) and labels it q:promote.',
|
|
221
|
+
},
|
|
219
222
|
)
|
|
220
223
|
|
|
221
224
|
/** Bump `version` when a node is removed or renamed. */
|
|
222
225
|
export const releasePrepare = defineGraph({
|
|
223
226
|
name: 'release-prepare',
|
|
227
|
+
about: 'Once a day, opens or refreshes the promotion PR from the integration branch to the release branch.',
|
|
224
228
|
version: 1,
|
|
225
229
|
entry: { node: 'cleanup', input: () => ({}) },
|
|
226
230
|
nodes: [cleanup, prepare],
|
|
@@ -269,7 +273,10 @@ const merge = declare(
|
|
|
269
273
|
return { to: 'armed', pr: input.pr }
|
|
270
274
|
},
|
|
271
275
|
}),
|
|
272
|
-
{
|
|
276
|
+
{
|
|
277
|
+
queue: Q.promote,
|
|
278
|
+
about: 'The promotion PR is approved: merges it through graft pr merge --auto. A refused head is not retried.',
|
|
279
|
+
},
|
|
273
280
|
)
|
|
274
281
|
|
|
275
282
|
const merging = declare(
|
|
@@ -287,6 +294,7 @@ const merging = declare(
|
|
|
287
294
|
}),
|
|
288
295
|
{
|
|
289
296
|
queue: Q.promote,
|
|
297
|
+
about: 'Holds while the promotion PR is open; moves on once it merges or closes unmerged.',
|
|
290
298
|
/** @param {{ pr: number }} input @param {ReleaseCtx} ctx */
|
|
291
299
|
enter: async (input, ctx) => {
|
|
292
300
|
const pr = prOf(ctx)
|
|
@@ -324,12 +332,13 @@ const exit = (name, outcome, said) =>
|
|
|
324
332
|
return {}
|
|
325
333
|
},
|
|
326
334
|
}),
|
|
327
|
-
{ queue: null },
|
|
335
|
+
{ queue: null, about: `The promotion PR ${said}.` },
|
|
328
336
|
)
|
|
329
337
|
|
|
330
338
|
/** Bump `version` when a node is removed or renamed. */
|
|
331
339
|
export const releasePromote = defineGraph({
|
|
332
340
|
name: 'release-promote',
|
|
341
|
+
about: 'Merges an approved q:promote promotion PR and closes it out.',
|
|
333
342
|
version: 1,
|
|
334
343
|
entry: { queue: Q.promote, node: 'merge' },
|
|
335
344
|
nodes: [
|
|
@@ -0,0 +1,504 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* The rescue graph: one stuck item (phases/rescue.mjs `findStuck`) is diagnosed by a one-shot, read-only `claude -p`
|
|
4
|
+
* worker whose final message carries a typed verdict ({@link RescueResult}): how the item got stuck, the actions that
|
|
5
|
+
* unstick it, and the root causes worth a ticket. cambium checks the verdict, carries out the actions, files or
|
|
6
|
+
* re-notes each root-cause ticket in `q:intake` (labelled `rescue`, so triage takes it from there), and documents it
|
|
7
|
+
* all in a comment on the item. A missing or invalid verdict, or an item rescued `rescue.maxRescues` times this week,
|
|
8
|
+
* goes to `q:human` with `why:rescue`.
|
|
9
|
+
*
|
|
10
|
+
* The item is not claimed while it is diagnosed: its labels are the evidence. The rescue worker's row (phase
|
|
11
|
+
* `rescue`) keeps Schedule and Sweep off it meanwhile, and the verdict is acted on only while the item is still
|
|
12
|
+
* stuck.
|
|
13
|
+
*/
|
|
14
|
+
import { z } from 'zod'
|
|
15
|
+
import { advance, defineGraph, defineNode, defineWait, END } from '../graph/index.mjs'
|
|
16
|
+
import { iso, key, markerOf, setLabels, setMarker } from '../issue-state.mjs'
|
|
17
|
+
import { move, Q, RESCUE_LABEL, WHY } from '../labels.mjs'
|
|
18
|
+
import { newMarker, withHistory } from '../marker.mjs'
|
|
19
|
+
import {
|
|
20
|
+
causeBody,
|
|
21
|
+
causeMarker,
|
|
22
|
+
checkPlan,
|
|
23
|
+
describe,
|
|
24
|
+
readRescue,
|
|
25
|
+
recordRescue,
|
|
26
|
+
recordSymptoms,
|
|
27
|
+
RescueResult,
|
|
28
|
+
rescueReport,
|
|
29
|
+
rescuePrompt,
|
|
30
|
+
rescuesOf,
|
|
31
|
+
seenAgain,
|
|
32
|
+
} from '../phases/rescue.mjs'
|
|
33
|
+
import { traceWorker } from '../tracing.mjs'
|
|
34
|
+
import { stopOverdue } from '../workers.mjs'
|
|
35
|
+
import { dispatch } from './dispatch.mjs'
|
|
36
|
+
import { issueOf, task } from './task.mjs'
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* @typedef {import('../github.mjs').Issue} Issue
|
|
40
|
+
* @typedef {import('../marker.mjs').Marker} Marker
|
|
41
|
+
* @typedef {import('../phases/rescue.mjs').Stuck} Stuck
|
|
42
|
+
* @typedef {import('../phases/rescue.mjs').Rescue} Rescue
|
|
43
|
+
* @typedef {import('../phases/rescue.mjs').Action} Action
|
|
44
|
+
* @typedef {import('./task.mjs').Subject & { stuck?: Stuck | null }} RescueSubject
|
|
45
|
+
* `stuck`: the item as `findStuck` sees it now; null once it is no longer stuck.
|
|
46
|
+
* @typedef {import('./task.mjs').TaskCtx & { subject: RescueSubject }} RescueCtx
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
/** The pseudo-branch that names an item's rescue checkout; never created. @param {number} n */
|
|
50
|
+
export const rescueBranch = (n) => `cambium/rescue-${n}`
|
|
51
|
+
|
|
52
|
+
/** Where root-cause tickets may go: the item's repo, then `rescue.causeRepo`. @param {RescueCtx} ctx */
|
|
53
|
+
export const causeRepos = (ctx) => [
|
|
54
|
+
...new Set([ctx.subject.slug, ...(ctx.config.rescue.causeRepo ? [ctx.config.rescue.causeRepo] : [])]),
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
/** @param {RescueCtx} ctx */
|
|
58
|
+
function stuckOf(ctx) {
|
|
59
|
+
const { stuck } = ctx.subject
|
|
60
|
+
if (!stuck) throw new Error(`${ctx.subject.slug}#${ctx.subject.number} is not stuck`)
|
|
61
|
+
return stuck
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** @param {Stuck} stuck */
|
|
65
|
+
const symptomsLine = (stuck) => stuck.symptoms.map((s) => s.code).join(', ')
|
|
66
|
+
|
|
67
|
+
/** @param {unknown} err */
|
|
68
|
+
const errText = (err) => /** @type {Error} */ (err)?.message ?? String(err)
|
|
69
|
+
|
|
70
|
+
/** A marker as cambium writes it now: no stage-tick `stage`. @param {Marker} m */
|
|
71
|
+
const graphMarker = (m) => /** @type {Marker} */ (Object.fromEntries(Object.entries(m).filter(([k]) => k !== 'stage')))
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The marker, then the labels; the issue counts as moved this tick.
|
|
75
|
+
* @param {RescueCtx} ctx @param {Issue} issue @param {Marker} marker @param {{ add: string[], remove: string[] }} labels
|
|
76
|
+
*/
|
|
77
|
+
async function place(ctx, issue, marker, labels) {
|
|
78
|
+
await setMarker(ctx, ctx.subject.slug, issue, marker)
|
|
79
|
+
await setLabels(ctx, ctx.subject.slug, issue, labels)
|
|
80
|
+
ctx.tick.touched.add(key(ctx.subject.slug, issue.number))
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The history entry a rescue move leaves. @param {RescueCtx} ctx @param {Marker} m @param {string} reason */
|
|
84
|
+
const rescued = (ctx, m, reason) =>
|
|
85
|
+
withHistory(m, { at: iso(ctx.tick.now), attempt: m.attempt, outcome: 'rescued', reason })
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Into the Human Queue with `why:rescue`, resting at this graph's `to-human`.
|
|
89
|
+
* @param {RescueCtx} ctx @param {Issue} issue @param {string} reason @param {string} outcome
|
|
90
|
+
*/
|
|
91
|
+
async function handToHuman(ctx, issue, reason, outcome) {
|
|
92
|
+
const m = markerOf(issue)
|
|
93
|
+
await place(
|
|
94
|
+
ctx,
|
|
95
|
+
issue,
|
|
96
|
+
{
|
|
97
|
+
...graphMarker(withHistory(m, { at: iso(ctx.tick.now), attempt: m.attempt, outcome, reason })),
|
|
98
|
+
graph: rescue.name,
|
|
99
|
+
graphVersion: rescue.version,
|
|
100
|
+
phase: 'rescue',
|
|
101
|
+
node: 'to-human',
|
|
102
|
+
status: 'waiting',
|
|
103
|
+
enteredAt: iso(ctx.tick.now),
|
|
104
|
+
lastFailure: reason,
|
|
105
|
+
host: null,
|
|
106
|
+
start: null,
|
|
107
|
+
input: null,
|
|
108
|
+
},
|
|
109
|
+
move(issue.labels, Q.human, { extra: [WHY.rescue] }),
|
|
110
|
+
)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Back to a queue at its graph's entry, attempt 1, the history kept.
|
|
115
|
+
* @param {RescueCtx} ctx @param {Issue} issue @param {'task' | 'intake'} queue @param {string} reason
|
|
116
|
+
*/
|
|
117
|
+
async function requeue(ctx, issue, queue, reason) {
|
|
118
|
+
const m = markerOf(issue)
|
|
119
|
+
const g = queue === 'task' ? task : dispatch
|
|
120
|
+
await place(
|
|
121
|
+
ctx,
|
|
122
|
+
issue,
|
|
123
|
+
newMarker({
|
|
124
|
+
graph: g.name,
|
|
125
|
+
graphVersion: g.version,
|
|
126
|
+
node: g.entry.node,
|
|
127
|
+
phase: queue === 'task' ? 'task' : 'dispatch',
|
|
128
|
+
enteredAt: iso(ctx.tick.now),
|
|
129
|
+
traceparent: m.traceparent,
|
|
130
|
+
history: rescued(ctx, m, reason).history,
|
|
131
|
+
}),
|
|
132
|
+
move(issue.labels, queue === 'task' ? Q.task : Q.intake),
|
|
133
|
+
)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* One action of a checked verdict.
|
|
138
|
+
* @param {RescueCtx} ctx @param {Issue} issue @param {Action} a
|
|
139
|
+
*/
|
|
140
|
+
async function act(ctx, issue, a) {
|
|
141
|
+
const { slug, snap } = ctx.subject
|
|
142
|
+
switch (a.do) {
|
|
143
|
+
case 'clear-labels':
|
|
144
|
+
return setLabels(ctx, slug, issue, { add: [], remove: a.labels.filter((l) => issue.labels.includes(l)) })
|
|
145
|
+
case 'close-pr': {
|
|
146
|
+
await ctx.github.closePull(slug, a.pr, `${a.reason}\n\nClosed by cambium's rescue (#${issue.number}).`)
|
|
147
|
+
const pr = snap?.prs.find((p) => p.number === a.pr)
|
|
148
|
+
if (pr) pr.state = 'CLOSED'
|
|
149
|
+
return
|
|
150
|
+
}
|
|
151
|
+
case 'requeue':
|
|
152
|
+
return requeue(ctx, issue, a.queue, a.reason)
|
|
153
|
+
case 'to-human':
|
|
154
|
+
return handToHuman(ctx, issue, a.reason, 'rescued')
|
|
155
|
+
case 'close-issue': {
|
|
156
|
+
const m = markerOf(issue)
|
|
157
|
+
await place(
|
|
158
|
+
ctx,
|
|
159
|
+
issue,
|
|
160
|
+
{ ...graphMarker(rescued(ctx, m, a.reason)), status: 'done', host: null, input: null },
|
|
161
|
+
move(issue.labels, null),
|
|
162
|
+
)
|
|
163
|
+
await ctx.github.close(slug, issue.number, `${a.reason}\n\nClosed by cambium's rescue.`, { reason: a.as })
|
|
164
|
+
issue.state = 'CLOSED'
|
|
165
|
+
return
|
|
166
|
+
}
|
|
167
|
+
case 'adopt-pr': {
|
|
168
|
+
const pr = snap?.prs.find((p) => p.number === a.pr)
|
|
169
|
+
if (!pr) throw new Error(`PR #${a.pr} is not in the snapshot`)
|
|
170
|
+
// The task graph's `submit`, as a worker's publish or the reaper's adoption: the PR graph takes it next tick.
|
|
171
|
+
const r = await advance(task, { node: 'submit', input: { pr: pr.number, branch: pr.branch } }, ctx, {
|
|
172
|
+
maxTransitions: 1,
|
|
173
|
+
})
|
|
174
|
+
if (r.error) throw r.error.cause instanceof Error ? r.error.cause : r.error
|
|
175
|
+
return
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// ---- Schemas and edge predicates
|
|
181
|
+
|
|
182
|
+
const Empty = z.object({})
|
|
183
|
+
const failure = { reason: z.string(), outcome: z.string() }
|
|
184
|
+
const Verdict = z.object({ result: RescueResult, log: z.string() })
|
|
185
|
+
const Filed = z.object({ slug: z.string(), number: z.number().int(), title: z.string(), again: z.boolean() })
|
|
186
|
+
|
|
187
|
+
const started = (/** @type {{ to: string }} */ o) => o.to === 'running'
|
|
188
|
+
const budgetSpent = (/** @type {{ to: string }} */ o) => o.to === 'human'
|
|
189
|
+
const resolved = (/** @type {{ exit: string }} */ o) => o.exit === 'resolved' || o.exit === 'lost'
|
|
190
|
+
const diagnosed = (/** @type {{ exit: string }} */ o) => o.exit === 'diagnosed'
|
|
191
|
+
|
|
192
|
+
// ---- Nodes
|
|
193
|
+
|
|
194
|
+
const startRescue = defineNode({
|
|
195
|
+
name: 'start-rescue',
|
|
196
|
+
input: Empty,
|
|
197
|
+
output: z.discriminatedUnion('to', [
|
|
198
|
+
z.object({ to: z.literal('running'), pid: z.number().int(), attempt: z.number().int() }),
|
|
199
|
+
z.object({ to: z.literal('human'), ...failure }),
|
|
200
|
+
z.object({ to: z.literal('failed'), reason: z.string() }),
|
|
201
|
+
]),
|
|
202
|
+
/**
|
|
203
|
+
* Over budget (`rescue.maxRescues` this week): to a human without another diagnosis. Else count the rescue and
|
|
204
|
+
* start the rescue worker in a read-only checkout. A start that fails ends here; the next sweep tries again after
|
|
205
|
+
* `rescue.cooldownHours`, and counts it.
|
|
206
|
+
* @param {{}} _ @param {RescueCtx} ctx
|
|
207
|
+
*/
|
|
208
|
+
run: async (_, ctx) => {
|
|
209
|
+
const { config, spawner, store } = ctx
|
|
210
|
+
const { slug, number } = ctx.subject
|
|
211
|
+
const issue = issueOf(ctx)
|
|
212
|
+
const stuck = stuckOf(ctx)
|
|
213
|
+
const repo = config.repos.find((r) => r.slug === slug)
|
|
214
|
+
if (!repo) throw new Error(`${slug} is not in the config`)
|
|
215
|
+
const previous = rescuesOf(store, slug, number, ctx.tick.now).length
|
|
216
|
+
const { maxRescues } = config.rescue
|
|
217
|
+
if (previous >= maxRescues)
|
|
218
|
+
return {
|
|
219
|
+
to: 'human',
|
|
220
|
+
reason: `rescued ${previous} time${previous === 1 ? '' : 's'} this week and still stuck (${symptomsLine(stuck)})`,
|
|
221
|
+
outcome: 'budget',
|
|
222
|
+
}
|
|
223
|
+
recordRescue(store, slug, number, ctx.tick.now)
|
|
224
|
+
recordSymptoms(store, stuck)
|
|
225
|
+
const caps = config.phases.rescue
|
|
226
|
+
const pr = ctx.subject.pr ?? (stuck.pr ? (ctx.subject.snap?.prs.find((p) => p.number === stuck.pr) ?? null) : null)
|
|
227
|
+
/** @type {import('../workers.mjs').Started} */
|
|
228
|
+
let s
|
|
229
|
+
try {
|
|
230
|
+
const convention = spawner.convention(repo)
|
|
231
|
+
s = await spawner.start({
|
|
232
|
+
repo,
|
|
233
|
+
convention,
|
|
234
|
+
issue: number,
|
|
235
|
+
attempt: previous + 1,
|
|
236
|
+
branch: rescueBranch(number),
|
|
237
|
+
phase: 'rescue',
|
|
238
|
+
maxTurns: caps.maxTurns,
|
|
239
|
+
readOnly: true,
|
|
240
|
+
foreground: ctx.subject.foreground ?? false,
|
|
241
|
+
env: ctx.traceparent ? { TRACEPARENT: ctx.traceparent } : undefined,
|
|
242
|
+
buildPrompt: (agentsMd) =>
|
|
243
|
+
rescuePrompt({
|
|
244
|
+
repo: slug,
|
|
245
|
+
baseBranch: repo.baseBranch,
|
|
246
|
+
stuck,
|
|
247
|
+
issue,
|
|
248
|
+
pr,
|
|
249
|
+
agentsMd,
|
|
250
|
+
logs: config.paths.logs,
|
|
251
|
+
repos: causeRepos(ctx),
|
|
252
|
+
previous,
|
|
253
|
+
}),
|
|
254
|
+
})
|
|
255
|
+
} catch (err) {
|
|
256
|
+
ctx.log('warn', `rescue: could not start a worker: ${errText(err)}`, { repo: slug, issue: number })
|
|
257
|
+
return { to: 'failed', reason: errText(err) }
|
|
258
|
+
}
|
|
259
|
+
store.addWorker({
|
|
260
|
+
repo: slug,
|
|
261
|
+
issue: number,
|
|
262
|
+
phase: 'rescue',
|
|
263
|
+
pid: s.pid,
|
|
264
|
+
worktree: s.worktree,
|
|
265
|
+
branch: rescueBranch(number),
|
|
266
|
+
attempt: previous + 1,
|
|
267
|
+
startedAt: ctx.tick.now,
|
|
268
|
+
host: config.host,
|
|
269
|
+
stdout: s.stdout,
|
|
270
|
+
stderr: s.stderr,
|
|
271
|
+
traceparent: ctx.traceparent ?? null,
|
|
272
|
+
})
|
|
273
|
+
ctx.log('info', `rescue: started worker pid ${s.pid} (${symptomsLine(stuck)})`, { repo: slug, issue: number })
|
|
274
|
+
ctx.subject.started = { ...s, attempt: previous + 1 }
|
|
275
|
+
return { to: 'running', pid: s.pid, attempt: previous + 1 }
|
|
276
|
+
},
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
const diagnosing = defineWait({
|
|
280
|
+
name: 'diagnosing',
|
|
281
|
+
input: z.object({ attempt: z.number().int() }),
|
|
282
|
+
output: z.discriminatedUnion('exit', [
|
|
283
|
+
z.object({ exit: z.literal('diagnosed'), ...Verdict.shape }),
|
|
284
|
+
z.object({ exit: z.literal('invalid'), ...failure }),
|
|
285
|
+
z.object({ exit: z.literal('resolved') }),
|
|
286
|
+
z.object({ exit: z.literal('lost') }),
|
|
287
|
+
]),
|
|
288
|
+
/**
|
|
289
|
+
* Alive and under the wall-clock cap: hold. Over it: kill. Gone: unstuck meanwhile (someone or something moved it)
|
|
290
|
+
* ends the graph; otherwise the verdict, read from its stdout log and checked against the item (phases/rescue.mjs
|
|
291
|
+
* `checkPlan`). A missing, invalid or unworkable one is `invalid`, with the error.
|
|
292
|
+
* @param {unknown} _ @param {RescueCtx} ctx
|
|
293
|
+
*/
|
|
294
|
+
poll: async (_, ctx) => {
|
|
295
|
+
const { config, store, procs, spawner } = ctx
|
|
296
|
+
const { slug, number, snap, issue, reaped, stuck } = ctx.subject
|
|
297
|
+
const w = store.worker(slug, number)
|
|
298
|
+
if (w?.phase !== 'rescue') return { ready: true, output: { exit: 'lost' } }
|
|
299
|
+
const caps = config.phases.rescue
|
|
300
|
+
if (
|
|
301
|
+
!reaped?.has(w.pid) &&
|
|
302
|
+
stopOverdue({ procs, store, log: ctx.log }, w, caps.wallClockMinutes, ctx.tick.now) !== 'gone'
|
|
303
|
+
)
|
|
304
|
+
return { ready: false }
|
|
305
|
+
if (!snap || !issue) return { ready: false }
|
|
306
|
+
traceWorker(ctx, w)
|
|
307
|
+
if (!stuck) {
|
|
308
|
+
ctx.log('info', 'rescue: no longer stuck; verdict not applied', { repo: slug, issue: number })
|
|
309
|
+
return { ready: true, output: { exit: 'resolved' } }
|
|
310
|
+
}
|
|
311
|
+
const read = w.killedReason ? { ok: false, error: w.killedReason } : readRescue(spawner.readLog(w.stdout))
|
|
312
|
+
if (!read.ok || !('result' in read)) {
|
|
313
|
+
const error = 'error' in read ? read.error : 'no verdict'
|
|
314
|
+
const tail = error.startsWith('invalid') ? '' : spawner.tail(w.stderr, 15)
|
|
315
|
+
return {
|
|
316
|
+
ready: true,
|
|
317
|
+
output: {
|
|
318
|
+
exit: 'invalid',
|
|
319
|
+
reason: `${error}${tail ? `\n--- stderr (last lines, ${w.stderr})\n${tail}` : ''}`,
|
|
320
|
+
outcome: w.killedReason ? 'killed' : 'invalid',
|
|
321
|
+
},
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
const problem = checkPlan(read.result, { issue, snap, repos: causeRepos(ctx) })
|
|
325
|
+
if (problem)
|
|
326
|
+
return {
|
|
327
|
+
ready: true,
|
|
328
|
+
output: { exit: 'invalid', reason: `unworkable rescue verdict: ${problem}`, outcome: 'invalid' },
|
|
329
|
+
}
|
|
330
|
+
return { ready: true, output: { exit: 'diagnosed', result: read.result, log: w.stdout } }
|
|
331
|
+
},
|
|
332
|
+
})
|
|
333
|
+
|
|
334
|
+
const Unstuck = z.object({ ...Verdict.shape, done: z.array(z.string()) })
|
|
335
|
+
const unstick = defineNode({
|
|
336
|
+
name: 'unstick',
|
|
337
|
+
input: Verdict,
|
|
338
|
+
output: Unstuck,
|
|
339
|
+
/**
|
|
340
|
+
* The verdict's actions, in order. A failed write throws: the worker row stays, so the next tick reads the same
|
|
341
|
+
* verdict and tries again.
|
|
342
|
+
* @param {z.infer<typeof Verdict>} v @param {RescueCtx} ctx
|
|
343
|
+
*/
|
|
344
|
+
run: async (v, ctx) => {
|
|
345
|
+
const issue = issueOf(ctx)
|
|
346
|
+
/** @type {string[]} */
|
|
347
|
+
const done = []
|
|
348
|
+
for (const a of v.result.actions) {
|
|
349
|
+
await act(ctx, issue, a)
|
|
350
|
+
done.push(describe(a))
|
|
351
|
+
}
|
|
352
|
+
if (done.length)
|
|
353
|
+
ctx.log('info', `rescue: ${v.result.actions.map((a) => a.do).join(', ')}`, {
|
|
354
|
+
repo: ctx.subject.slug,
|
|
355
|
+
issue: issue.number,
|
|
356
|
+
})
|
|
357
|
+
return { ...v, done }
|
|
358
|
+
},
|
|
359
|
+
})
|
|
360
|
+
|
|
361
|
+
const Causes = z.object({ ...Unstuck.shape, filed: z.array(Filed) })
|
|
362
|
+
const fileCauses = defineNode({
|
|
363
|
+
name: 'file-causes',
|
|
364
|
+
input: Unstuck,
|
|
365
|
+
output: Causes,
|
|
366
|
+
/**
|
|
367
|
+
* Each root cause: when an open issue in the target repo already carries its fingerprint, a comment there notes the
|
|
368
|
+
* recurrence; otherwise a new ticket in `q:intake`, labelled `rescue`. A repo cambium does not read (only
|
|
369
|
+
* `rescue.causeRepo` can be one) is de-duplicated by what this host filed there.
|
|
370
|
+
* @param {z.infer<typeof Unstuck>} u @param {RescueCtx} ctx
|
|
371
|
+
*/
|
|
372
|
+
run: async (u, ctx) => {
|
|
373
|
+
const { slug, number } = ctx.subject
|
|
374
|
+
const from = { slug, number, how: u.result.how }
|
|
375
|
+
/** @type {z.infer<typeof Filed>[]} */
|
|
376
|
+
const filed = []
|
|
377
|
+
for (const c of u.result.rootCauses) {
|
|
378
|
+
const target = c.repo ?? slug
|
|
379
|
+
const snap = ctx.tick.snaps.get(target)
|
|
380
|
+
const kvKey = `rescue-cause:${target}:${c.fingerprint}`
|
|
381
|
+
const known = snap
|
|
382
|
+
? [...snap.issues.values()].find((i) => i.state === 'OPEN' && i.body.includes(causeMarker(c.fingerprint)))
|
|
383
|
+
?.number
|
|
384
|
+
: Number(ctx.store.get(kvKey)) || undefined
|
|
385
|
+
if (known) {
|
|
386
|
+
await ctx.github.comment(target, known, seenAgain(from))
|
|
387
|
+
filed.push({ slug: target, number: known, title: snap?.issues.get(known)?.title ?? c.title, again: true })
|
|
388
|
+
continue
|
|
389
|
+
}
|
|
390
|
+
const body = causeBody(c, from)
|
|
391
|
+
const labels = [Q.intake, RESCUE_LABEL]
|
|
392
|
+
const made = await ctx.github.createIssue(target, { title: c.title, body, labels })
|
|
393
|
+
ctx.store.set(kvKey, String(made.number))
|
|
394
|
+
snap?.issues.set(made.number, {
|
|
395
|
+
number: made.number,
|
|
396
|
+
title: c.title,
|
|
397
|
+
url: made.url,
|
|
398
|
+
state: 'OPEN',
|
|
399
|
+
createdAt: iso(ctx.tick.now),
|
|
400
|
+
body,
|
|
401
|
+
labels,
|
|
402
|
+
stateComment: null,
|
|
403
|
+
blockedBy: [],
|
|
404
|
+
})
|
|
405
|
+
ctx.log('info', `rescue: filed root cause ${target}#${made.number} (${c.fingerprint})`, {
|
|
406
|
+
repo: slug,
|
|
407
|
+
issue: number,
|
|
408
|
+
})
|
|
409
|
+
filed.push({ slug: target, number: made.number, title: c.title, again: false })
|
|
410
|
+
}
|
|
411
|
+
return { ...u, filed }
|
|
412
|
+
},
|
|
413
|
+
})
|
|
414
|
+
|
|
415
|
+
const report = defineNode({
|
|
416
|
+
name: 'report',
|
|
417
|
+
input: Causes,
|
|
418
|
+
output: Empty,
|
|
419
|
+
/** The comment that documents the rescue: what was seen, how it got stuck, what was done, the root causes. @param {z.infer<typeof Causes>} c @param {RescueCtx} ctx */
|
|
420
|
+
run: async (c, ctx) => {
|
|
421
|
+
const { slug, number } = ctx.subject
|
|
422
|
+
await ctx.github.comment(
|
|
423
|
+
slug,
|
|
424
|
+
number,
|
|
425
|
+
rescueReport({ stuck: stuckOf(ctx), result: c.result, done: c.done, filed: c.filed }),
|
|
426
|
+
)
|
|
427
|
+
ctx.log('info', `rescue: ${c.result.stuck ? 'unstuck' : 'not stuck'}; ${c.filed.length} root cause(s)`, {
|
|
428
|
+
repo: slug,
|
|
429
|
+
issue: number,
|
|
430
|
+
})
|
|
431
|
+
return {}
|
|
432
|
+
},
|
|
433
|
+
})
|
|
434
|
+
|
|
435
|
+
const HumanIn = z.object(failure)
|
|
436
|
+
const toHuman = defineNode({
|
|
437
|
+
name: 'to-human',
|
|
438
|
+
input: HumanIn,
|
|
439
|
+
output: Empty,
|
|
440
|
+
/** Into the Human Queue with `why:rescue`, and a comment saying what cambium saw and why it stopped. @param {z.infer<typeof HumanIn>} what @param {RescueCtx} ctx */
|
|
441
|
+
run: async (what, ctx) => {
|
|
442
|
+
const issue = issueOf(ctx)
|
|
443
|
+
const stuck = stuckOf(ctx)
|
|
444
|
+
// A closed issue is not queued again: the comment is enough for whoever looks.
|
|
445
|
+
if (issue.state === 'OPEN') await handToHuman(ctx, issue, what.reason, what.outcome)
|
|
446
|
+
await ctx.github.comment(
|
|
447
|
+
ctx.subject.slug,
|
|
448
|
+
issue.number,
|
|
449
|
+
[
|
|
450
|
+
'**Rescue: this issue is stuck, and needs a person.**',
|
|
451
|
+
'',
|
|
452
|
+
'**What cambium saw**',
|
|
453
|
+
...stuck.symptoms.map((s) => `- \`${s.code}\`: ${s.detail}`),
|
|
454
|
+
'',
|
|
455
|
+
'**Why rescue stopped**',
|
|
456
|
+
'```',
|
|
457
|
+
what.reason.replace(/```/g, "'''"),
|
|
458
|
+
'```',
|
|
459
|
+
].join('\n'),
|
|
460
|
+
)
|
|
461
|
+
ctx.log('warn', `rescue → q:human (${WHY.rescue}): ${what.reason.split('\n')[0]}`, {
|
|
462
|
+
repo: ctx.subject.slug,
|
|
463
|
+
issue: issue.number,
|
|
464
|
+
})
|
|
465
|
+
return {}
|
|
466
|
+
},
|
|
467
|
+
})
|
|
468
|
+
|
|
469
|
+
/** A node with its one-line description, which the heartbeat carries (graph/describe.mjs). @template N @param {N} node @param {string} text */
|
|
470
|
+
const about = (node, text) => Object.freeze({ ...node, about: text })
|
|
471
|
+
|
|
472
|
+
/** The rescue graph. Bump `version` when a node is removed or renamed. */
|
|
473
|
+
export const rescue = defineGraph({
|
|
474
|
+
name: 'rescue',
|
|
475
|
+
version: 1,
|
|
476
|
+
about: 'Diagnoses a stuck item with a one-shot read-only worker, unsticks it and files its root causes in q:intake.',
|
|
477
|
+
entry: { node: 'start-rescue', input: () => ({}) },
|
|
478
|
+
nodes: [
|
|
479
|
+
about(
|
|
480
|
+
startRescue,
|
|
481
|
+
'Starts the rescue worker in a read-only checkout; over the weekly budget, the item goes to a human.',
|
|
482
|
+
),
|
|
483
|
+
about(diagnosing, 'Holds while the worker diagnoses the item; on exit, checks its verdict against the item.'),
|
|
484
|
+
about(unstick, "Carries out the verdict's actions, in order."),
|
|
485
|
+
about(
|
|
486
|
+
fileCauses,
|
|
487
|
+
'Files each root cause as a q:intake ticket labelled rescue, or notes a recurrence on the open one.',
|
|
488
|
+
),
|
|
489
|
+
about(report, 'Comments on the item: what was seen, how it got stuck, what was done, the root causes.'),
|
|
490
|
+
about(toHuman, 'Into q:human with why:rescue and what cambium saw.'),
|
|
491
|
+
],
|
|
492
|
+
edges: [
|
|
493
|
+
['start-rescue', 'diagnosing', started, (o) => ({ attempt: o.attempt })],
|
|
494
|
+
['start-rescue', 'to-human', budgetSpent],
|
|
495
|
+
['start-rescue', END],
|
|
496
|
+
['diagnosing', END, resolved],
|
|
497
|
+
['diagnosing', 'unstick', diagnosed, (o) => ({ result: o.result, log: o.log })],
|
|
498
|
+
['diagnosing', 'to-human'],
|
|
499
|
+
['unstick', 'file-causes'],
|
|
500
|
+
['file-causes', 'report'],
|
|
501
|
+
['report', END],
|
|
502
|
+
['to-human', END],
|
|
503
|
+
],
|
|
504
|
+
})
|
package/src/graphs/task.mjs
CHANGED
|
@@ -39,7 +39,7 @@ import { stopOverdue } from '../workers.mjs'
|
|
|
39
39
|
* `traceparent`: the running step node's span, when advance is instrumented (tracing.mjs).
|
|
40
40
|
* @typedef {{ node: string, input: unknown }} Position
|
|
41
41
|
* @typedef {{
|
|
42
|
-
* queue: string | null, running: boolean, pollOnEntry?: boolean,
|
|
42
|
+
* queue: string | null, running: boolean, about?: string, pollOnEntry?: boolean,
|
|
43
43
|
* enter?: (input: any, ctx: TaskCtx) => Promise<void>,
|
|
44
44
|
* recover?: (ctx: TaskCtx) => Position,
|
|
45
45
|
* }} Declared
|
|
@@ -240,7 +240,11 @@ const startWorker = declare(
|
|
|
240
240
|
return { to: 'running', pid: s.pid, branch, worktree: s.worktree, attempt: marker.attempt }
|
|
241
241
|
},
|
|
242
242
|
}),
|
|
243
|
-
{
|
|
243
|
+
{
|
|
244
|
+
queue: Q.task,
|
|
245
|
+
running: false,
|
|
246
|
+
about: 'Claims the issue and starts a Claude worker on its branch; out of attempts, it goes to a human.',
|
|
247
|
+
},
|
|
244
248
|
)
|
|
245
249
|
|
|
246
250
|
const workerRunning = declare(
|
|
@@ -303,6 +307,7 @@ const workerRunning = declare(
|
|
|
303
307
|
}),
|
|
304
308
|
{
|
|
305
309
|
queue: Q.task,
|
|
310
|
+
about: 'Holds while the worker runs under its wall-clock cap; on exit, looks for the PR it published.',
|
|
306
311
|
running: true,
|
|
307
312
|
recover: (ctx) => ({ node: 'requeue', input: lostLeaseOf(ctx) }),
|
|
308
313
|
},
|
|
@@ -350,7 +355,11 @@ const submit = declare(
|
|
|
350
355
|
return {}
|
|
351
356
|
},
|
|
352
357
|
}),
|
|
353
|
-
{
|
|
358
|
+
{
|
|
359
|
+
queue: null,
|
|
360
|
+
running: false,
|
|
361
|
+
about: 'The worker published its PR: the issue leaves every queue and the PR graph takes over.',
|
|
362
|
+
},
|
|
354
363
|
)
|
|
355
364
|
|
|
356
365
|
const requeue = declare(
|
|
@@ -397,7 +406,12 @@ const requeue = declare(
|
|
|
397
406
|
return { to: 'start', attempt: next }
|
|
398
407
|
},
|
|
399
408
|
}),
|
|
400
|
-
{
|
|
409
|
+
{
|
|
410
|
+
queue: Q.task,
|
|
411
|
+
running: false,
|
|
412
|
+
about:
|
|
413
|
+
'Back to q:task with attempt + 1, or to a human once the attempts are spent. Not charged while the base is red.',
|
|
414
|
+
},
|
|
401
415
|
)
|
|
402
416
|
|
|
403
417
|
const HumanIn = z.object({ ...failure, why: z.string().optional() })
|
|
@@ -428,7 +442,7 @@ const toHuman = declare(
|
|
|
428
442
|
return {}
|
|
429
443
|
},
|
|
430
444
|
}),
|
|
431
|
-
{ queue: Q.human, running: false },
|
|
445
|
+
{ queue: Q.human, running: false, about: 'Into q:human with a why:* reason label.' },
|
|
432
446
|
)
|
|
433
447
|
|
|
434
448
|
/**
|
|
@@ -438,6 +452,7 @@ const toHuman = declare(
|
|
|
438
452
|
*/
|
|
439
453
|
export const task = defineGraph({
|
|
440
454
|
name: 'task',
|
|
455
|
+
about: 'Turns a q:task issue into a published PR; a failed attempt is retried until the budget is spent.',
|
|
441
456
|
version: 2,
|
|
442
457
|
entry: { queue: Q.task, node: 'start-worker', input: () => ({}) },
|
|
443
458
|
nodes: [startWorker, workerRunning, submit, requeue, toHuman],
|
package/src/labels.mjs
CHANGED
|
@@ -18,8 +18,12 @@ export const WHY = /** @type {const} */ ({
|
|
|
18
18
|
reject: 'why:reject',
|
|
19
19
|
approve: 'why:approve',
|
|
20
20
|
triage: 'why:triage',
|
|
21
|
+
rescue: 'why:rescue',
|
|
21
22
|
})
|
|
22
23
|
|
|
24
|
+
/** Marks a root-cause ticket the rescue graph filed (graphs/rescue.mjs). Not a `why:` label: those leave with `q:human`. */
|
|
25
|
+
export const RESCUE_LABEL = 'rescue'
|
|
26
|
+
|
|
23
27
|
/** @param {string[]} labels */
|
|
24
28
|
export const queueOf = (labels) => labels.find((l) => l.startsWith('q:')) ?? null
|
|
25
29
|
|
package/src/marker.mjs
CHANGED
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
* traceparent: string | null,
|
|
32
32
|
* enteredAt: string | null,
|
|
33
33
|
* history: HistoryEntry[],
|
|
34
|
-
* phase: 'task' | 'dispatch' | 'release',
|
|
34
|
+
* phase: 'task' | 'dispatch' | 'release' | 'rescue',
|
|
35
35
|
* stage?: Stage,
|
|
36
36
|
* pr: number | null,
|
|
37
37
|
* branch: string | null,
|