@oakgreencc/cambium 0.0.39 → 0.0.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,280 @@
1
+ // @ts-check
2
+ /**
3
+ * `engine: "graph"`: the rescue graph's steps (graphs/rescue.mjs), run in upkeep while `phases.rescue.cap > 0`.
4
+ *
5
+ * - Reconcile (every tick): every tracked rescue worker's item at `diagnosing`, with the item as `findStuck` sees it
6
+ * now (null once it is no longer stuck, so a stale verdict is never applied).
7
+ * - Sweep (every `rescue.everyMinutes`): read each repo's closed-unmerged agent PRs and the open issues they left in no
8
+ * queue, find the stuck items (phases/rescue.mjs `findStuck`), skip any rescued in the last `rescue.cooldownHours`,
9
+ * and start each at `start-rescue`: those over `rescue.maxRescues` go to a human whatever the cap, the rest up to
10
+ * the free rescue slots, oldest issue first.
11
+ */
12
+ import { advance, END } from './graph/index.mjs'
13
+ import { rescue } from './graphs/rescue.mjs'
14
+ import { key, MIN } from './issue-state.mjs'
15
+ import { findStuck, rescuesOf, stillStuck } from './phases/rescue.mjs'
16
+ import { branchIssue } from './github.mjs'
17
+ import { each } from './tick.mjs'
18
+ import { markerTrace, turnTracer } from './tracing.mjs'
19
+
20
+ /**
21
+ * @typedef {import('./tick.mjs').Deps} Deps
22
+ * @typedef {import('./tick.mjs').Ctx} Ctx
23
+ * @typedef {import('./github.mjs').Snapshot} Snapshot
24
+ * @typedef {import('./github.mjs').PullRequest} PullRequest
25
+ * @typedef {import('./graphs/rescue.mjs').RescueSubject} RescueSubject
26
+ * @typedef {import('./graph/advance.mjs').Transition} Transition
27
+ */
28
+
29
+ export const PHASE = 'rescue'
30
+
31
+ /**
32
+ * One item's turn through the rescue graph from `at`, until it waits or ends. A node error is rethrown (the step logs
33
+ * it); a rescue worker whose `diagnosing` was left is torn down only after the turn's writes succeeded.
34
+ * @param {Deps} deps @param {Ctx} ctx @param {RescueSubject} subject @param {{ node: string, input: unknown }} at
35
+ */
36
+ export async function driveRescue(deps, ctx, subject, at) {
37
+ const c = { ...deps, tick: ctx, subject }
38
+ const traced = turnTracer(deps.telemetry, subject, {
39
+ graph: rescue.name,
40
+ graphVersion: rescue.version,
41
+ repo: subject.slug,
42
+ issue: subject.number,
43
+ issueBody: subject.issue?.body ?? null,
44
+ ...markerTrace(subject.issue?.stateComment?.marker),
45
+ })
46
+ /** @type {Transition[]} */
47
+ const transitions = []
48
+ let pos = at
49
+ for (;;) {
50
+ const r = await advance(rescue, pos, c, { maxTransitions: 1, instrument: traced.instrument })
51
+ if (r.error) throw r.error.cause instanceof Error ? r.error.cause : r.error
52
+ traced.sync()
53
+ transitions.push(...r.transitions)
54
+ pos = { node: r.node, input: r.input }
55
+ if (r.reason !== 'limit') break
56
+ if (rescue.nodes.find((n) => n.name === r.node)?.kind === 'wait') break
57
+ }
58
+ if (transitions.some((t) => t.from === 'diagnosing')) await releaseWorker(deps, subject)
59
+ return { transitions, ended: transitions.some((t) => t.to === END), ...pos }
60
+ }
61
+
62
+ /** The rescue worker's row goes, then its read-only checkout. @param {Deps} deps @param {RescueSubject} s */
63
+ async function releaseWorker(deps, s) {
64
+ const w = deps.store.worker(s.slug, s.number)
65
+ if (!w || w.phase !== PHASE) return
66
+ deps.store.removeWorker(s.slug, s.number)
67
+ const repo = deps.config.repos.find((r) => r.slug === s.slug)
68
+ if (!repo) return
69
+ const t = await deps.spawner.cleanup({ repo, worktree: w.worktree, branch: w.branch, readOnly: true })
70
+ if (!t.removed) deps.log('warn', `kept worktree ${w.worktree}: ${t.reason}`, { repo: s.slug, issue: s.number })
71
+ }
72
+
73
+ /** Live workers per phase. @param {Deps} deps @param {string} phase */
74
+ const live = (deps, phase) => deps.store.workers().filter((w) => w.phase === phase).length
75
+
76
+ /**
77
+ * The repo's snapshot with the issues `numbers` read in where it lacks them. A copy: the tick's own snapshot is left as
78
+ * Read made it; the issue objects are shared, so a write through either is seen by both.
79
+ * @param {Deps} deps @param {string} slug @param {Snapshot} snap @param {number[]} numbers
80
+ * @returns {Promise<Snapshot>}
81
+ */
82
+ async function withIssues(deps, slug, snap, numbers) {
83
+ const issues = new Map(snap.issues)
84
+ const unread = [...new Set(numbers)].filter((n) => !issues.has(n))
85
+ if (unread.length) for (const [n, i] of await deps.github.readIssues(slug, unread)) issues.set(n, i)
86
+ return { ...snap, issues }
87
+ }
88
+
89
+ /**
90
+ * The repo's snapshot as the sweep sees it: plus its closed-unmerged agent PRs, and the issues they name.
91
+ * @param {Deps} deps @param {string} slug @param {Snapshot} snap
92
+ * @returns {Promise<{ view: Snapshot, closed: PullRequest[] }>}
93
+ */
94
+ export async function sweepView(deps, slug, snap) {
95
+ const closed = (await deps.github.closedPulls(slug)).filter((p) => p.branch.startsWith('agent/'))
96
+ const named = closed.flatMap((p) => (p.closes.length ? p.closes : [branchIssue(p.branch)].filter((n) => n !== null)))
97
+ const view = await withIssues(deps, slug, snap, /** @type {number[]} */ (named))
98
+ // Only an open issue left in no queue is stranded; a closed one is no concern of the rescue.
99
+ for (const n of named)
100
+ if (!snap.issues.has(/** @type {number} */ (n))) {
101
+ const i = view.issues.get(/** @type {number} */ (n))
102
+ if (i && (i.state !== 'OPEN' || i.labels.some((l) => l.startsWith('q:')))) view.issues.delete(i.number)
103
+ }
104
+ return { view, closed }
105
+ }
106
+
107
+ /**
108
+ * The stuck items in one repo's view.
109
+ * @param {Deps} deps @param {Ctx} ctx @param {string} slug @param {Snapshot} view
110
+ * @param {{ closed?: PullRequest[], recheck?: boolean, busy?: (n: number) => boolean }} [o]
111
+ */
112
+ export function stuckIn(deps, ctx, slug, view, { closed = [], recheck = false, busy } = {}) {
113
+ const { config } = deps
114
+ return findStuck(view, {
115
+ slug,
116
+ now: ctx.now,
117
+ host: config.host,
118
+ running: ctx.mode === 'running',
119
+ staleHours: config.rescue.staleHours,
120
+ graceMinutes: config.rescue.graceMinutes,
121
+ free: {
122
+ task: config.phases.task.cap - live(deps, 'task'),
123
+ dispatch: config.phases.dispatch.cap - live(deps, 'dispatch'),
124
+ },
125
+ busy: busy ?? ((n) => ctx.touched.has(key(slug, n)) || Boolean(deps.store.worker(slug, n))),
126
+ closed,
127
+ recheck,
128
+ })
129
+ }
130
+
131
+ /** @param {Deps} deps @param {Ctx} ctx */
132
+ async function reconcile(deps, ctx) {
133
+ await each(
134
+ deps,
135
+ deps.store.workers().filter((w) => w.phase === PHASE),
136
+ async (w) => {
137
+ const base = ctx.snaps.get(w.repo)
138
+ const snap = base ? await withIssues(deps, w.repo, base, [w.issue]) : undefined
139
+ const issue = snap?.issues.get(w.issue)
140
+ const stuck = snap
141
+ ? stillStuck(
142
+ deps.store,
143
+ stuckIn(deps, ctx, w.repo, snap, { recheck: true, busy: (n) => ctx.touched.has(key(w.repo, n)) }).find(
144
+ (s) => s.number === w.issue,
145
+ ),
146
+ )
147
+ : null
148
+ await driveRescue(
149
+ deps,
150
+ ctx,
151
+ { slug: w.repo, number: w.issue, snap, issue, stuck },
152
+ {
153
+ node: 'diagnosing',
154
+ input: { attempt: w.attempt },
155
+ },
156
+ )
157
+ },
158
+ (w) => ({ repo: w.repo, issue: w.issue }),
159
+ )
160
+ }
161
+
162
+ /** @param {Deps} deps @param {Ctx} ctx */
163
+ async function sweep(deps, ctx) {
164
+ const { config, store } = deps
165
+ const last = Number(store.get('rescue.lastRun') ?? 0)
166
+ if (!config.rescue.everyMinutes || ctx.now - last < config.rescue.everyMinutes * MIN) return
167
+ store.set('rescue.lastRun', String(ctx.now))
168
+ let free = config.phases.rescue.cap - live(deps, PHASE)
169
+ for (const [slug, snap] of ctx.snaps) {
170
+ /** @type {{ view: Snapshot, closed: PullRequest[] }} */
171
+ let read
172
+ try {
173
+ read = await sweepView(deps, slug, snap)
174
+ } catch (err) {
175
+ deps.log('error', `rescue: reading closed PRs: ${/** @type {Error} */ (err)?.message ?? err}`, { repo: slug })
176
+ read = { view: snap, closed: [] }
177
+ }
178
+ const found = stuckIn(deps, ctx, slug, read.view, { closed: read.closed })
179
+ const cooled = found.filter((s) => {
180
+ const at = rescuesOf(store, slug, s.number, ctx.now)
181
+ return !at.length || ctx.now - /** @type {number} */ (at.at(-1)) >= config.rescue.cooldownHours * 60 * MIN
182
+ })
183
+ if (found.length)
184
+ deps.log(
185
+ 'info',
186
+ `rescue: ${found.length} stuck (${found.map((s) => `#${s.number} ${s.symptoms.map((x) => x.code).join('+')}`).join(', ')}); ${found.length - cooled.length} cooling down`,
187
+ { repo: slug },
188
+ )
189
+ const spent = cooled.filter((s) => rescuesOf(store, slug, s.number, ctx.now).length >= config.rescue.maxRescues)
190
+ const fresh = cooled.filter((s) => !spent.includes(s)).slice(0, Math.max(0, free))
191
+ free -= fresh.length
192
+ await each(
193
+ deps,
194
+ [...spent, ...fresh],
195
+ async (stuck) => {
196
+ const issue = read.view.issues.get(stuck.number)
197
+ const pr = stuck.pr ? ([...read.view.prs, ...read.closed].find((p) => p.number === stuck.pr) ?? null) : null
198
+ await driveRescue(
199
+ deps,
200
+ ctx,
201
+ { slug, number: stuck.number, snap: read.view, issue, pr, stuck },
202
+ {
203
+ node: rescue.entry.node,
204
+ input: {},
205
+ },
206
+ )
207
+ },
208
+ (s) => ({ repo: slug, issue: s.number }),
209
+ )
210
+ }
211
+ }
212
+
213
+ /**
214
+ * The rescue steps, in upkeep: tracked rescue workers every tick (even with the cap since set to 0), the sweep on its
215
+ * clock while `phases.rescue.cap > 0`.
216
+ * @param {Deps} deps @param {Ctx} ctx
217
+ */
218
+ export async function rescueStep(deps, ctx) {
219
+ await reconcile(deps, ctx)
220
+ if (deps.config.phases.rescue.cap > 0) await sweep(deps, ctx)
221
+ }
222
+
223
+ /**
224
+ * `cambium run rescue [--repo <slug>] [--issue <n>]`. Without `--issue`: list what the sweep would find now (the
225
+ * cooldown aside), and change nothing. With it: rescue that item in the foreground if the sweep would find it stuck
226
+ * (the cooldown aside; one over `rescue.maxRescues` goes to a human): a foreground rescue worker, then its verdict
227
+ * carried out.
228
+ * @param {Deps} deps
229
+ * @param {{ slug: string, number?: number, out: (line: string) => void, newCtx: (deps: Deps) => Ctx }} o
230
+ * @returns {Promise<{ ok: boolean, error?: string }>}
231
+ */
232
+ export async function runRescue(deps, { slug, number, out, newCtx }) {
233
+ const ctx = newCtx(deps)
234
+ const snap = await deps.github.read(slug)
235
+ ctx.snaps.set(slug, snap)
236
+ const { view, closed } = await sweepView(deps, slug, snap)
237
+ if (number === undefined) {
238
+ const found = stuckIn(deps, ctx, slug, view, { closed })
239
+ if (!found.length) out(`${slug}: nothing stuck`)
240
+ for (const s of found) {
241
+ const n = rescuesOf(deps.store, slug, s.number, ctx.now).length
242
+ out(`${slug}#${s.number}${n ? ` (rescued ${n}× this week)` : ''}`)
243
+ for (const x of s.symptoms) out(` ${x.code}: ${x.detail}`)
244
+ }
245
+ return { ok: true }
246
+ }
247
+ if (deps.store.worker(slug, number)) return { ok: false, error: `${slug}#${number} has a tracked worker` }
248
+ const pre = await withIssues(deps, slug, view, [number])
249
+ const stuck = stuckIn(deps, ctx, slug, pre, { closed }).find((s) => s.number === number)
250
+ if (!stuck) return { ok: false, error: `${slug}#${number} is not stuck` }
251
+ const issue = pre.issues.get(number)
252
+ const pr = stuck.pr ? ([...pre.prs, ...closed].find((p) => p.number === stuck.pr) ?? null) : null
253
+ /** @type {RescueSubject} */
254
+ const subject = { slug, number, snap: pre, issue, pr, stuck, foreground: true }
255
+ const first = await driveRescue(deps, ctx, subject, { node: rescue.entry.node, input: {} })
256
+ if (subject.started) {
257
+ const s = subject.started
258
+ out(`rescue worker pid ${s.pid} in ${s.worktree}\n stdout ${s.stdout}\n stderr ${s.stderr}`)
259
+ out(`rescue worker exited ${await s.exited}`)
260
+ const after = newCtx(deps)
261
+ const snap2 = await withIssues(deps, slug, await deps.github.read(slug), [number])
262
+ after.snaps.set(slug, snap2)
263
+ const now = stillStuck(
264
+ deps.store,
265
+ stuckIn(deps, after, slug, snap2, { closed, recheck: true, busy: () => false }).find(
266
+ (s2) => s2.number === number,
267
+ ),
268
+ )
269
+ const turn = await driveRescue(
270
+ deps,
271
+ after,
272
+ { slug, number, snap: snap2, issue: snap2.issues.get(number), stuck: now, reaped: new Set([s.pid]) },
273
+ { node: 'diagnosing', input: { attempt: s.attempt } },
274
+ )
275
+ out(`${slug}#${number}: ${turn.transitions.map((t) => String(t.to)).join(' → ') || 'waiting'}`)
276
+ return { ok: true }
277
+ }
278
+ out(`${slug}#${number}: ${first.transitions.map((t) => String(t.to)).join(' → ')}`)
279
+ return { ok: true }
280
+ }
package/src/tick.mjs CHANGED
@@ -230,8 +230,8 @@ export async function requeue(deps, ctx, slug, issue, what) {
230
230
  // ---- Reconcile
231
231
 
232
232
  /**
233
- * A triage worker (phase `dispatch`, started by the graph engine before a switch to `legacy`) is only held to its
234
- * wall-clock cap: its row stays, so its `running` lease is not swept, and the graph engine reads its verdict once
233
+ * A triage or rescue worker (phase `dispatch` or `rescue`, started by the graph engine before a switch to `legacy`)
234
+ * is only held to its wall-clock cap: its row stays, so its `running` lease is not swept, and the graph engine reads its verdict once
235
235
  * switched back. Legacy never finishes it as a Task.
236
236
  * @param {Deps} deps @param {Ctx} ctx
237
237
  */
@@ -243,7 +243,7 @@ async function reconcile(deps, ctx) {
243
243
  async (w) => {
244
244
  const caps = config.phases[/** @type {'task'} */ (w.phase)] ?? config.phases.task
245
245
  if (stopOverdue({ procs, store, log: deps.log }, w, caps.wallClockMinutes, ctx.now) !== 'gone') return
246
- if (w.phase === 'dispatch') return
246
+ if (w.phase === 'dispatch' || w.phase === 'rescue') return
247
247
  const snap = ctx.snaps.get(w.repo)
248
248
  if (snap) await finish(deps, ctx, w, snap)
249
249
  },
package/src/upkeep.mjs CHANGED
@@ -12,12 +12,15 @@
12
12
  * so triage looks again under its current rubric: done since (obsolete), now checkable (defer), or now a task.
13
13
  * Not `why:approve` (a merge only a human may make: a re-triage would start a duplicate Task) and not one holding
14
14
  * `running`.
15
+ * - **Rescue** (rescue-engine.mjs, while `phases.rescue.cap > 0`): rescue workers every tick, and every
16
+ * `rescue.everyMinutes` a sweep for stuck items, each diagnosed and unstuck by the rescue graph.
15
17
  */
16
18
  import { drive } from './graph-engine.mjs'
17
19
  import { branchIssue } from './github.mjs'
18
20
  import { iso, key, MIN, setLabels } from './issue-state.mjs'
19
21
  import { move, Q, RUNNING, WHY } from './labels.mjs'
20
22
  import { releaseDue } from './phases/release.mjs'
23
+ import { rescueStep } from './rescue-engine.mjs'
21
24
  import { each } from './tick.mjs'
22
25
 
23
26
  /**
@@ -61,10 +64,14 @@ export function reapVerdict(pr, snap, { now, staleMinutes, orphanDays, busy }) {
61
64
  return { act: 'adopt', issue, link: linked === undefined }
62
65
  }
63
66
 
64
- /** Both clocks, in order: adopt first, so an issue whose PR the reaper takes is not re-triaged. @param {Deps} deps @param {Ctx} ctx */
67
+ /**
68
+ * The clocks, in order: adopt first, so an issue whose PR the reaper takes is not re-triaged; rescue last, so it sees
69
+ * what the others left. @param {Deps} deps @param {Ctx} ctx
70
+ */
65
71
  export async function upkeep(deps, ctx) {
66
72
  await reap(deps, ctx)
67
73
  await retriage(deps, ctx)
74
+ await rescueStep(deps, ctx)
68
75
  }
69
76
 
70
77
  /** @param {Deps} deps @param {Ctx} ctx */