@oakgreencc/cambium 0.0.39 → 0.0.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +78 -2
- package/package.json +1 -1
- package/src/cli.mjs +5 -1
- package/src/config.mjs +22 -1
- package/src/github.mjs +32 -0
- package/src/graph-engine.mjs +9 -2
- package/src/graphs/rescue.mjs +487 -0
- package/src/labels.mjs +4 -0
- package/src/marker.mjs +1 -1
- package/src/phases/dispatch.mjs +23 -14
- package/src/phases/rescue.mjs +444 -0
- package/src/rescue-engine.mjs +280 -0
- package/src/tick.mjs +3 -3
- package/src/upkeep.mjs +8 -1
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* `engine: "graph"`: the rescue graph's steps (graphs/rescue.mjs), run in upkeep while `phases.rescue.cap > 0`.
|
|
4
|
+
*
|
|
5
|
+
* - Reconcile (every tick): every tracked rescue worker's item at `diagnosing`, with the item as `findStuck` sees it
|
|
6
|
+
* now (null once it is no longer stuck, so a stale verdict is never applied).
|
|
7
|
+
* - Sweep (every `rescue.everyMinutes`): read each repo's closed-unmerged agent PRs and the open issues they left in no
|
|
8
|
+
* queue, find the stuck items (phases/rescue.mjs `findStuck`), skip any rescued in the last `rescue.cooldownHours`,
|
|
9
|
+
* and start each at `start-rescue`: those over `rescue.maxRescues` go to a human whatever the cap, the rest up to
|
|
10
|
+
* the free rescue slots, oldest issue first.
|
|
11
|
+
*/
|
|
12
|
+
import { advance, END } from './graph/index.mjs'
|
|
13
|
+
import { rescue } from './graphs/rescue.mjs'
|
|
14
|
+
import { key, MIN } from './issue-state.mjs'
|
|
15
|
+
import { findStuck, rescuesOf, stillStuck } from './phases/rescue.mjs'
|
|
16
|
+
import { branchIssue } from './github.mjs'
|
|
17
|
+
import { each } from './tick.mjs'
|
|
18
|
+
import { markerTrace, turnTracer } from './tracing.mjs'
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* @typedef {import('./tick.mjs').Deps} Deps
|
|
22
|
+
* @typedef {import('./tick.mjs').Ctx} Ctx
|
|
23
|
+
* @typedef {import('./github.mjs').Snapshot} Snapshot
|
|
24
|
+
* @typedef {import('./github.mjs').PullRequest} PullRequest
|
|
25
|
+
* @typedef {import('./graphs/rescue.mjs').RescueSubject} RescueSubject
|
|
26
|
+
* @typedef {import('./graph/advance.mjs').Transition} Transition
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
export const PHASE = 'rescue'
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* One item's turn through the rescue graph from `at`, until it waits or ends. A node error is rethrown (the step logs
|
|
33
|
+
* it); a rescue worker whose `diagnosing` was left is torn down only after the turn's writes succeeded.
|
|
34
|
+
* @param {Deps} deps @param {Ctx} ctx @param {RescueSubject} subject @param {{ node: string, input: unknown }} at
|
|
35
|
+
*/
|
|
36
|
+
export async function driveRescue(deps, ctx, subject, at) {
|
|
37
|
+
const c = { ...deps, tick: ctx, subject }
|
|
38
|
+
const traced = turnTracer(deps.telemetry, subject, {
|
|
39
|
+
graph: rescue.name,
|
|
40
|
+
graphVersion: rescue.version,
|
|
41
|
+
repo: subject.slug,
|
|
42
|
+
issue: subject.number,
|
|
43
|
+
issueBody: subject.issue?.body ?? null,
|
|
44
|
+
...markerTrace(subject.issue?.stateComment?.marker),
|
|
45
|
+
})
|
|
46
|
+
/** @type {Transition[]} */
|
|
47
|
+
const transitions = []
|
|
48
|
+
let pos = at
|
|
49
|
+
for (;;) {
|
|
50
|
+
const r = await advance(rescue, pos, c, { maxTransitions: 1, instrument: traced.instrument })
|
|
51
|
+
if (r.error) throw r.error.cause instanceof Error ? r.error.cause : r.error
|
|
52
|
+
traced.sync()
|
|
53
|
+
transitions.push(...r.transitions)
|
|
54
|
+
pos = { node: r.node, input: r.input }
|
|
55
|
+
if (r.reason !== 'limit') break
|
|
56
|
+
if (rescue.nodes.find((n) => n.name === r.node)?.kind === 'wait') break
|
|
57
|
+
}
|
|
58
|
+
if (transitions.some((t) => t.from === 'diagnosing')) await releaseWorker(deps, subject)
|
|
59
|
+
return { transitions, ended: transitions.some((t) => t.to === END), ...pos }
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The rescue worker's row goes, then its read-only checkout. @param {Deps} deps @param {RescueSubject} s */
|
|
63
|
+
async function releaseWorker(deps, s) {
|
|
64
|
+
const w = deps.store.worker(s.slug, s.number)
|
|
65
|
+
if (!w || w.phase !== PHASE) return
|
|
66
|
+
deps.store.removeWorker(s.slug, s.number)
|
|
67
|
+
const repo = deps.config.repos.find((r) => r.slug === s.slug)
|
|
68
|
+
if (!repo) return
|
|
69
|
+
const t = await deps.spawner.cleanup({ repo, worktree: w.worktree, branch: w.branch, readOnly: true })
|
|
70
|
+
if (!t.removed) deps.log('warn', `kept worktree ${w.worktree}: ${t.reason}`, { repo: s.slug, issue: s.number })
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Live workers per phase. @param {Deps} deps @param {string} phase */
|
|
74
|
+
const live = (deps, phase) => deps.store.workers().filter((w) => w.phase === phase).length
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The repo's snapshot with the issues `numbers` read in where it lacks them. A copy: the tick's own snapshot is left as
|
|
78
|
+
* Read made it; the issue objects are shared, so a write through either is seen by both.
|
|
79
|
+
* @param {Deps} deps @param {string} slug @param {Snapshot} snap @param {number[]} numbers
|
|
80
|
+
* @returns {Promise<Snapshot>}
|
|
81
|
+
*/
|
|
82
|
+
async function withIssues(deps, slug, snap, numbers) {
|
|
83
|
+
const issues = new Map(snap.issues)
|
|
84
|
+
const unread = [...new Set(numbers)].filter((n) => !issues.has(n))
|
|
85
|
+
if (unread.length) for (const [n, i] of await deps.github.readIssues(slug, unread)) issues.set(n, i)
|
|
86
|
+
return { ...snap, issues }
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* The repo's snapshot as the sweep sees it: plus its closed-unmerged agent PRs, and the issues they name.
|
|
91
|
+
* @param {Deps} deps @param {string} slug @param {Snapshot} snap
|
|
92
|
+
* @returns {Promise<{ view: Snapshot, closed: PullRequest[] }>}
|
|
93
|
+
*/
|
|
94
|
+
export async function sweepView(deps, slug, snap) {
|
|
95
|
+
const closed = (await deps.github.closedPulls(slug)).filter((p) => p.branch.startsWith('agent/'))
|
|
96
|
+
const named = closed.flatMap((p) => (p.closes.length ? p.closes : [branchIssue(p.branch)].filter((n) => n !== null)))
|
|
97
|
+
const view = await withIssues(deps, slug, snap, /** @type {number[]} */ (named))
|
|
98
|
+
// Only an open issue left in no queue is stranded; a closed one is no concern of the rescue.
|
|
99
|
+
for (const n of named)
|
|
100
|
+
if (!snap.issues.has(/** @type {number} */ (n))) {
|
|
101
|
+
const i = view.issues.get(/** @type {number} */ (n))
|
|
102
|
+
if (i && (i.state !== 'OPEN' || i.labels.some((l) => l.startsWith('q:')))) view.issues.delete(i.number)
|
|
103
|
+
}
|
|
104
|
+
return { view, closed }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The stuck items in one repo's view.
|
|
109
|
+
* @param {Deps} deps @param {Ctx} ctx @param {string} slug @param {Snapshot} view
|
|
110
|
+
* @param {{ closed?: PullRequest[], recheck?: boolean, busy?: (n: number) => boolean }} [o]
|
|
111
|
+
*/
|
|
112
|
+
export function stuckIn(deps, ctx, slug, view, { closed = [], recheck = false, busy } = {}) {
|
|
113
|
+
const { config } = deps
|
|
114
|
+
return findStuck(view, {
|
|
115
|
+
slug,
|
|
116
|
+
now: ctx.now,
|
|
117
|
+
host: config.host,
|
|
118
|
+
running: ctx.mode === 'running',
|
|
119
|
+
staleHours: config.rescue.staleHours,
|
|
120
|
+
graceMinutes: config.rescue.graceMinutes,
|
|
121
|
+
free: {
|
|
122
|
+
task: config.phases.task.cap - live(deps, 'task'),
|
|
123
|
+
dispatch: config.phases.dispatch.cap - live(deps, 'dispatch'),
|
|
124
|
+
},
|
|
125
|
+
busy: busy ?? ((n) => ctx.touched.has(key(slug, n)) || Boolean(deps.store.worker(slug, n))),
|
|
126
|
+
closed,
|
|
127
|
+
recheck,
|
|
128
|
+
})
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** @param {Deps} deps @param {Ctx} ctx */
|
|
132
|
+
async function reconcile(deps, ctx) {
|
|
133
|
+
await each(
|
|
134
|
+
deps,
|
|
135
|
+
deps.store.workers().filter((w) => w.phase === PHASE),
|
|
136
|
+
async (w) => {
|
|
137
|
+
const base = ctx.snaps.get(w.repo)
|
|
138
|
+
const snap = base ? await withIssues(deps, w.repo, base, [w.issue]) : undefined
|
|
139
|
+
const issue = snap?.issues.get(w.issue)
|
|
140
|
+
const stuck = snap
|
|
141
|
+
? stillStuck(
|
|
142
|
+
deps.store,
|
|
143
|
+
stuckIn(deps, ctx, w.repo, snap, { recheck: true, busy: (n) => ctx.touched.has(key(w.repo, n)) }).find(
|
|
144
|
+
(s) => s.number === w.issue,
|
|
145
|
+
),
|
|
146
|
+
)
|
|
147
|
+
: null
|
|
148
|
+
await driveRescue(
|
|
149
|
+
deps,
|
|
150
|
+
ctx,
|
|
151
|
+
{ slug: w.repo, number: w.issue, snap, issue, stuck },
|
|
152
|
+
{
|
|
153
|
+
node: 'diagnosing',
|
|
154
|
+
input: { attempt: w.attempt },
|
|
155
|
+
},
|
|
156
|
+
)
|
|
157
|
+
},
|
|
158
|
+
(w) => ({ repo: w.repo, issue: w.issue }),
|
|
159
|
+
)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** @param {Deps} deps @param {Ctx} ctx */
|
|
163
|
+
async function sweep(deps, ctx) {
|
|
164
|
+
const { config, store } = deps
|
|
165
|
+
const last = Number(store.get('rescue.lastRun') ?? 0)
|
|
166
|
+
if (!config.rescue.everyMinutes || ctx.now - last < config.rescue.everyMinutes * MIN) return
|
|
167
|
+
store.set('rescue.lastRun', String(ctx.now))
|
|
168
|
+
let free = config.phases.rescue.cap - live(deps, PHASE)
|
|
169
|
+
for (const [slug, snap] of ctx.snaps) {
|
|
170
|
+
/** @type {{ view: Snapshot, closed: PullRequest[] }} */
|
|
171
|
+
let read
|
|
172
|
+
try {
|
|
173
|
+
read = await sweepView(deps, slug, snap)
|
|
174
|
+
} catch (err) {
|
|
175
|
+
deps.log('error', `rescue: reading closed PRs: ${/** @type {Error} */ (err)?.message ?? err}`, { repo: slug })
|
|
176
|
+
read = { view: snap, closed: [] }
|
|
177
|
+
}
|
|
178
|
+
const found = stuckIn(deps, ctx, slug, read.view, { closed: read.closed })
|
|
179
|
+
const cooled = found.filter((s) => {
|
|
180
|
+
const at = rescuesOf(store, slug, s.number, ctx.now)
|
|
181
|
+
return !at.length || ctx.now - /** @type {number} */ (at.at(-1)) >= config.rescue.cooldownHours * 60 * MIN
|
|
182
|
+
})
|
|
183
|
+
if (found.length)
|
|
184
|
+
deps.log(
|
|
185
|
+
'info',
|
|
186
|
+
`rescue: ${found.length} stuck (${found.map((s) => `#${s.number} ${s.symptoms.map((x) => x.code).join('+')}`).join(', ')}); ${found.length - cooled.length} cooling down`,
|
|
187
|
+
{ repo: slug },
|
|
188
|
+
)
|
|
189
|
+
const spent = cooled.filter((s) => rescuesOf(store, slug, s.number, ctx.now).length >= config.rescue.maxRescues)
|
|
190
|
+
const fresh = cooled.filter((s) => !spent.includes(s)).slice(0, Math.max(0, free))
|
|
191
|
+
free -= fresh.length
|
|
192
|
+
await each(
|
|
193
|
+
deps,
|
|
194
|
+
[...spent, ...fresh],
|
|
195
|
+
async (stuck) => {
|
|
196
|
+
const issue = read.view.issues.get(stuck.number)
|
|
197
|
+
const pr = stuck.pr ? ([...read.view.prs, ...read.closed].find((p) => p.number === stuck.pr) ?? null) : null
|
|
198
|
+
await driveRescue(
|
|
199
|
+
deps,
|
|
200
|
+
ctx,
|
|
201
|
+
{ slug, number: stuck.number, snap: read.view, issue, pr, stuck },
|
|
202
|
+
{
|
|
203
|
+
node: rescue.entry.node,
|
|
204
|
+
input: {},
|
|
205
|
+
},
|
|
206
|
+
)
|
|
207
|
+
},
|
|
208
|
+
(s) => ({ repo: slug, issue: s.number }),
|
|
209
|
+
)
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* The rescue steps, in upkeep: tracked rescue workers every tick (even with the cap since set to 0), the sweep on its
|
|
215
|
+
* clock while `phases.rescue.cap > 0`.
|
|
216
|
+
* @param {Deps} deps @param {Ctx} ctx
|
|
217
|
+
*/
|
|
218
|
+
export async function rescueStep(deps, ctx) {
|
|
219
|
+
await reconcile(deps, ctx)
|
|
220
|
+
if (deps.config.phases.rescue.cap > 0) await sweep(deps, ctx)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* `cambium run rescue [--repo <slug>] [--issue <n>]`. Without `--issue`: list what the sweep would find now (the
|
|
225
|
+
* cooldown aside), and change nothing. With it: rescue that item in the foreground if the sweep would find it stuck
|
|
226
|
+
* (the cooldown aside; one over `rescue.maxRescues` goes to a human): a foreground rescue worker, then its verdict
|
|
227
|
+
* carried out.
|
|
228
|
+
* @param {Deps} deps
|
|
229
|
+
* @param {{ slug: string, number?: number, out: (line: string) => void, newCtx: (deps: Deps) => Ctx }} o
|
|
230
|
+
* @returns {Promise<{ ok: boolean, error?: string }>}
|
|
231
|
+
*/
|
|
232
|
+
export async function runRescue(deps, { slug, number, out, newCtx }) {
|
|
233
|
+
const ctx = newCtx(deps)
|
|
234
|
+
const snap = await deps.github.read(slug)
|
|
235
|
+
ctx.snaps.set(slug, snap)
|
|
236
|
+
const { view, closed } = await sweepView(deps, slug, snap)
|
|
237
|
+
if (number === undefined) {
|
|
238
|
+
const found = stuckIn(deps, ctx, slug, view, { closed })
|
|
239
|
+
if (!found.length) out(`${slug}: nothing stuck`)
|
|
240
|
+
for (const s of found) {
|
|
241
|
+
const n = rescuesOf(deps.store, slug, s.number, ctx.now).length
|
|
242
|
+
out(`${slug}#${s.number}${n ? ` (rescued ${n}× this week)` : ''}`)
|
|
243
|
+
for (const x of s.symptoms) out(` ${x.code}: ${x.detail}`)
|
|
244
|
+
}
|
|
245
|
+
return { ok: true }
|
|
246
|
+
}
|
|
247
|
+
if (deps.store.worker(slug, number)) return { ok: false, error: `${slug}#${number} has a tracked worker` }
|
|
248
|
+
const pre = await withIssues(deps, slug, view, [number])
|
|
249
|
+
const stuck = stuckIn(deps, ctx, slug, pre, { closed }).find((s) => s.number === number)
|
|
250
|
+
if (!stuck) return { ok: false, error: `${slug}#${number} is not stuck` }
|
|
251
|
+
const issue = pre.issues.get(number)
|
|
252
|
+
const pr = stuck.pr ? ([...pre.prs, ...closed].find((p) => p.number === stuck.pr) ?? null) : null
|
|
253
|
+
/** @type {RescueSubject} */
|
|
254
|
+
const subject = { slug, number, snap: pre, issue, pr, stuck, foreground: true }
|
|
255
|
+
const first = await driveRescue(deps, ctx, subject, { node: rescue.entry.node, input: {} })
|
|
256
|
+
if (subject.started) {
|
|
257
|
+
const s = subject.started
|
|
258
|
+
out(`rescue worker pid ${s.pid} in ${s.worktree}\n stdout ${s.stdout}\n stderr ${s.stderr}`)
|
|
259
|
+
out(`rescue worker exited ${await s.exited}`)
|
|
260
|
+
const after = newCtx(deps)
|
|
261
|
+
const snap2 = await withIssues(deps, slug, await deps.github.read(slug), [number])
|
|
262
|
+
after.snaps.set(slug, snap2)
|
|
263
|
+
const now = stillStuck(
|
|
264
|
+
deps.store,
|
|
265
|
+
stuckIn(deps, after, slug, snap2, { closed, recheck: true, busy: () => false }).find(
|
|
266
|
+
(s2) => s2.number === number,
|
|
267
|
+
),
|
|
268
|
+
)
|
|
269
|
+
const turn = await driveRescue(
|
|
270
|
+
deps,
|
|
271
|
+
after,
|
|
272
|
+
{ slug, number, snap: snap2, issue: snap2.issues.get(number), stuck: now, reaped: new Set([s.pid]) },
|
|
273
|
+
{ node: 'diagnosing', input: { attempt: s.attempt } },
|
|
274
|
+
)
|
|
275
|
+
out(`${slug}#${number}: ${turn.transitions.map((t) => String(t.to)).join(' → ') || 'waiting'}`)
|
|
276
|
+
return { ok: true }
|
|
277
|
+
}
|
|
278
|
+
out(`${slug}#${number}: ${first.transitions.map((t) => String(t.to)).join(' → ')}`)
|
|
279
|
+
return { ok: true }
|
|
280
|
+
}
|
package/src/tick.mjs
CHANGED
|
@@ -230,8 +230,8 @@ export async function requeue(deps, ctx, slug, issue, what) {
|
|
|
230
230
|
// ---- Reconcile
|
|
231
231
|
|
|
232
232
|
/**
|
|
233
|
-
* A triage worker (phase `dispatch`, started by the graph engine before a switch to `legacy`)
|
|
234
|
-
* wall-clock cap: its row stays, so its `running` lease is not swept, and the graph engine reads its verdict once
|
|
233
|
+
* A triage or rescue worker (phase `dispatch` or `rescue`, started by the graph engine before a switch to `legacy`)
|
|
234
|
+
* is only held to its wall-clock cap: its row stays, so its `running` lease is not swept, and the graph engine reads its verdict once
|
|
235
235
|
* switched back. Legacy never finishes it as a Task.
|
|
236
236
|
* @param {Deps} deps @param {Ctx} ctx
|
|
237
237
|
*/
|
|
@@ -243,7 +243,7 @@ async function reconcile(deps, ctx) {
|
|
|
243
243
|
async (w) => {
|
|
244
244
|
const caps = config.phases[/** @type {'task'} */ (w.phase)] ?? config.phases.task
|
|
245
245
|
if (stopOverdue({ procs, store, log: deps.log }, w, caps.wallClockMinutes, ctx.now) !== 'gone') return
|
|
246
|
-
if (w.phase === 'dispatch') return
|
|
246
|
+
if (w.phase === 'dispatch' || w.phase === 'rescue') return
|
|
247
247
|
const snap = ctx.snaps.get(w.repo)
|
|
248
248
|
if (snap) await finish(deps, ctx, w, snap)
|
|
249
249
|
},
|
package/src/upkeep.mjs
CHANGED
|
@@ -12,12 +12,15 @@
|
|
|
12
12
|
* so triage looks again under its current rubric: done since (obsolete), now checkable (defer), or now a task.
|
|
13
13
|
* Not `why:approve` (a merge only a human may make: a re-triage would start a duplicate Task) and not one holding
|
|
14
14
|
* `running`.
|
|
15
|
+
* - **Rescue** (rescue-engine.mjs, while `phases.rescue.cap > 0`): rescue workers every tick, and every
|
|
16
|
+
* `rescue.everyMinutes` a sweep for stuck items, each diagnosed and unstuck by the rescue graph.
|
|
15
17
|
*/
|
|
16
18
|
import { drive } from './graph-engine.mjs'
|
|
17
19
|
import { branchIssue } from './github.mjs'
|
|
18
20
|
import { iso, key, MIN, setLabels } from './issue-state.mjs'
|
|
19
21
|
import { move, Q, RUNNING, WHY } from './labels.mjs'
|
|
20
22
|
import { releaseDue } from './phases/release.mjs'
|
|
23
|
+
import { rescueStep } from './rescue-engine.mjs'
|
|
21
24
|
import { each } from './tick.mjs'
|
|
22
25
|
|
|
23
26
|
/**
|
|
@@ -61,10 +64,14 @@ export function reapVerdict(pr, snap, { now, staleMinutes, orphanDays, busy }) {
|
|
|
61
64
|
return { act: 'adopt', issue, link: linked === undefined }
|
|
62
65
|
}
|
|
63
66
|
|
|
64
|
-
/**
|
|
67
|
+
/**
|
|
68
|
+
* The clocks, in order: adopt first, so an issue whose PR the reaper takes is not re-triaged; rescue last, so it sees
|
|
69
|
+
* what the others left. @param {Deps} deps @param {Ctx} ctx
|
|
70
|
+
*/
|
|
65
71
|
export async function upkeep(deps, ctx) {
|
|
66
72
|
await reap(deps, ctx)
|
|
67
73
|
await retriage(deps, ctx)
|
|
74
|
+
await rescueStep(deps, ctx)
|
|
68
75
|
}
|
|
69
76
|
|
|
70
77
|
/** @param {Deps} deps @param {Ctx} ctx */
|