@oakgreencc/cambium 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -10
- package/package.json +15 -2
- package/src/cli.mjs +18 -3
- package/src/config.mjs +4 -0
- package/src/graph/advance.mjs +2 -2
- package/src/graph/langgraph.mjs +200 -0
- package/src/graph-engine.mjs +289 -0
- package/src/graphs/task.mjs +587 -0
- package/src/issue-state.mjs +46 -0
- package/src/tick.mjs +97 -76
|
@@ -0,0 +1,587 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* The Task phase as a graph: the same transitions the legacy stage tick makes (tick.mjs), expressed as typed
|
|
4
|
+
* nodes. Each node declares the queue label it sits in and whether it holds `running`; every label change goes
|
|
5
|
+
* through `move()` from that declaration (see {@link land}).
|
|
6
|
+
*
|
|
7
|
+
* Step nodes write their own transition (marker, then labels) before or after their side effect, exactly where
|
|
8
|
+
* the legacy tick does. A wait node with `enter` is written by the runtime when an edge lands on it.
|
|
9
|
+
*/
|
|
10
|
+
import { z } from 'zod'
|
|
11
|
+
import { defineGraph, defineNode, defineWait, END } from '../graph/index.mjs'
|
|
12
|
+
import { branchName } from '../convention.mjs'
|
|
13
|
+
import { failureSummary, iso, key, markerOf, MIN, setLabels, setMarker } from '../issue-state.mjs'
|
|
14
|
+
import { move, Q, WHY } from '../labels.mjs'
|
|
15
|
+
import { withHistory } from '../marker.mjs'
|
|
16
|
+
import { prFor, taskExit, taskPrompt } from '../phases/task.mjs'
|
|
17
|
+
import { PREPARE_FAILED_LINE } from '../prepare.mjs'
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @typedef {import('../github.mjs').Issue} Issue
|
|
21
|
+
* @typedef {import('../github.mjs').Snapshot} Snapshot
|
|
22
|
+
* @typedef {import('../github.mjs').PullRequest} PullRequest
|
|
23
|
+
* @typedef {import('../marker.mjs').Marker} Marker
|
|
24
|
+
* @typedef {{
|
|
25
|
+
* slug: string, number: number, issue?: Issue, snap?: Snapshot,
|
|
26
|
+
* pr?: PullRequest | null, deleted?: Set<string>, foreground?: boolean, reaped?: Set<number>,
|
|
27
|
+
* started?: import('../workers.mjs').Started & { attempt: number },
|
|
28
|
+
* }} Subject
|
|
29
|
+
* One issue's turn. `pr` is the PR Watch is looking at; `deleted` the branches already deleted this tick;
|
|
30
|
+
* `foreground`/`reaped`/`started` are for `cambium run`.
|
|
31
|
+
* @typedef {import('../tick.mjs').Deps & {
|
|
32
|
+
* tick: import('../tick.mjs').Ctx, subject: Subject, traceparent?: string | null,
|
|
33
|
+
* }} TaskCtx
|
|
34
|
+
* `traceparent`: the running step node's span, when advance is instrumented (tracing.mjs).
|
|
35
|
+
* @typedef {{ node: string, input: unknown }} Position
|
|
36
|
+
* @typedef {{
|
|
37
|
+
* queue: string | null, running: boolean, pollOnEntry?: boolean,
|
|
38
|
+
* enter?: (input: any, ctx: TaskCtx) => Promise<void>,
|
|
39
|
+
* recover?: (ctx: TaskCtx) => Position,
|
|
40
|
+
* }} Declared
|
|
41
|
+
* `queue`/`running`: the labels an issue resting here carries. `pollOnEntry`: a wait polled in the same turn it
|
|
42
|
+
* is entered (otherwise entering a wait ends the turn). `enter`: the write that lands an issue on this wait.
|
|
43
|
+
* `recover`: where a stale `running` lease at this node goes (default: requeue).
|
|
44
|
+
* @typedef {import('../graph/define.mjs').GraphNode & Declared} TaskNode
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @template {import('../graph/define.mjs').GraphNode} N @param {N} node @param {Declared} decl
|
|
49
|
+
* @returns {N & Declared}
|
|
50
|
+
*/
|
|
51
|
+
const declare = (node, decl) => Object.freeze({ ...node, ...decl })
|
|
52
|
+
|
|
53
|
+
/** @param {TaskCtx} ctx */
|
|
54
|
+
function issueOf(ctx) {
|
|
55
|
+
const { issue } = ctx.subject
|
|
56
|
+
if (!issue) throw new Error(`${ctx.subject.slug}#${ctx.subject.number} is not in the snapshot`)
|
|
57
|
+
return issue
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** @param {TaskCtx} ctx */
|
|
61
|
+
function snapOf(ctx) {
|
|
62
|
+
const { snap } = ctx.subject
|
|
63
|
+
if (!snap) throw new Error(`${ctx.subject.slug} has no snapshot this tick`)
|
|
64
|
+
return snap
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** A marker as the graph writes it: no stage-tick `stage`, which would override `node`. @param {Marker} m */
|
|
68
|
+
const graphMarker = (m) => /** @type {Marker} */ (Object.fromEntries(Object.entries(m).filter(([k]) => k !== 'stage')))
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Put the issue at `node`: the marker first, then the labels `node` declares, via `move()`. A half-applied
|
|
72
|
+
* transition is re-done (and over-counted) on the next tick, never under-counted.
|
|
73
|
+
*
|
|
74
|
+
* @param {TaskCtx} ctx @param {string} node @param {Marker} marker
|
|
75
|
+
* @param {{ extra?: string[], status?: Marker['status'] }} [opts]
|
|
76
|
+
*/
|
|
77
|
+
async function land(ctx, node, marker, { extra = [], status = 'waiting' } = {}) {
|
|
78
|
+
const { slug } = ctx.subject
|
|
79
|
+
const issue = issueOf(ctx)
|
|
80
|
+
const decl = nodeOf(node)
|
|
81
|
+
await setMarker(ctx, slug, issue, {
|
|
82
|
+
...graphMarker(marker),
|
|
83
|
+
graph: task.name,
|
|
84
|
+
graphVersion: task.version,
|
|
85
|
+
node,
|
|
86
|
+
status,
|
|
87
|
+
enteredAt: iso(ctx.tick.now),
|
|
88
|
+
})
|
|
89
|
+
await setLabels(ctx, slug, issue, move(issue.labels, decl.queue, { running: decl.running, extra }))
|
|
90
|
+
ctx.tick.touched.add(key(slug, issue.number))
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ---- Schemas
|
|
94
|
+
|
|
95
|
+
const failure = {
|
|
96
|
+
reason: z.string(),
|
|
97
|
+
outcome: z.string(),
|
|
98
|
+
pr: z.number().int().optional(),
|
|
99
|
+
log: z.string().optional(),
|
|
100
|
+
failedSha: z.string().nullable().optional(),
|
|
101
|
+
/** Where the failure happened, when that is not the node the marker rests at. */
|
|
102
|
+
at: z.string().optional(),
|
|
103
|
+
}
|
|
104
|
+
const Failure = z.object(failure)
|
|
105
|
+
const Empty = z.object({})
|
|
106
|
+
const WorkerIn = z.object({ branch: z.string().nullable(), attempt: z.number().int() })
|
|
107
|
+
const PrIn = z.object({ pr: z.number().int().nullable(), branch: z.string().nullable(), log: z.string().optional() })
|
|
108
|
+
const PrOut = z.discriminatedUnion('to', [
|
|
109
|
+
z.object({ to: z.literal('merged'), pr: z.number().int() }),
|
|
110
|
+
z.object({ to: z.literal('queued'), pr: z.number().int(), branch: z.string() }),
|
|
111
|
+
z.object({ to: z.literal('human'), ...failure }),
|
|
112
|
+
z.object({ to: z.literal('requeue'), ...failure }),
|
|
113
|
+
])
|
|
114
|
+
|
|
115
|
+
// ---- Edge predicates (their names label the Mermaid edges)
|
|
116
|
+
|
|
117
|
+
const started = (/** @type {{ to: string }} */ o) => o.to === 'running'
|
|
118
|
+
const budgetSpent = (/** @type {{ to: string }} */ o) => o.to === 'human'
|
|
119
|
+
const issueGone = (/** @type {{ exit: string }} */ o) => o.exit === 'gone'
|
|
120
|
+
const submitted = (/** @type {{ exit: string }} */ o) => o.exit === 'submitted'
|
|
121
|
+
const merged = (/** @type {{ to: string }} */ o) => o.to === 'merged'
|
|
122
|
+
const inMergeQueue = (/** @type {{ to: string }} */ o) => o.to === 'queued'
|
|
123
|
+
const stalled = (/** @type {{ to: string }} */ o) => o.to === 'human'
|
|
124
|
+
const attemptsLeft = (/** @type {{ to: string }} */ o) => o.to === 'start'
|
|
125
|
+
|
|
126
|
+
// ---- Shared checks
|
|
127
|
+
|
|
128
|
+
/** A history entry's `node` from a failure's `at`, when it names one. @param {{ at?: string }} what */
|
|
129
|
+
const nodeAt = (what) => (what.at ? { node: what.at } : {})
|
|
130
|
+
|
|
131
|
+
/** The PR a wait is watching: Watch's pair, else the marker's PR, else the issue's. @param {TaskCtx} ctx @param {{ pr: number | null, branch: string | null }} input */
|
|
132
|
+
function prOf(ctx, input) {
|
|
133
|
+
if (ctx.subject.pr) return ctx.subject.pr
|
|
134
|
+
const snap = snapOf(ctx)
|
|
135
|
+
return (
|
|
136
|
+
(input.pr !== null && snap.prs.find((p) => p.number === input.pr)) ||
|
|
137
|
+
prFor(snap, { issue: ctx.subject.number, branch: input.branch })
|
|
138
|
+
)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Red on a commit CI has not already failed on. @param {PullRequest} pr @param {Marker} m */
|
|
142
|
+
const redCi = (pr, m) =>
|
|
143
|
+
(pr.checks === 'FAILURE' || pr.checks === 'ERROR') && pr.headSha !== m.failedSha
|
|
144
|
+
? /** @type {const} */ ({
|
|
145
|
+
to: 'requeue',
|
|
146
|
+
reason: failureSummary(pr),
|
|
147
|
+
outcome: 'ci-red',
|
|
148
|
+
pr: pr.number,
|
|
149
|
+
failedSha: pr.headSha,
|
|
150
|
+
})
|
|
151
|
+
: null
|
|
152
|
+
|
|
153
|
+
/** Not merged within `mergeQueueMinutes` of submission. @param {TaskCtx} ctx @param {PullRequest} pr @param {Marker} m */
|
|
154
|
+
function stall(ctx, pr, m) {
|
|
155
|
+
const cap = ctx.config.mergeQueueMinutes
|
|
156
|
+
if (!m.submittedAt || ctx.tick.now - Date.parse(m.submittedAt) <= cap * MIN) return null
|
|
157
|
+
return /** @type {const} */ ({
|
|
158
|
+
to: 'human',
|
|
159
|
+
reason: `PR #${pr.number} not merged within ${cap} min of submission`,
|
|
160
|
+
outcome: 'stalled',
|
|
161
|
+
pr: pr.number,
|
|
162
|
+
})
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Delete a merged PR's branch once per tick. @param {TaskCtx} ctx @param {PullRequest} pr */
|
|
166
|
+
export async function deleteMergedBranch(ctx, pr) {
|
|
167
|
+
const deleted = ctx.subject.deleted ?? new Set()
|
|
168
|
+
if (!pr.branchExists || deleted.has(pr.branch)) return
|
|
169
|
+
deleted.add(pr.branch)
|
|
170
|
+
await ctx.github.deleteBranch(ctx.subject.slug, pr.branch)
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// ---- Nodes
|
|
174
|
+
|
|
175
|
+
const startWorker = declare(
|
|
176
|
+
defineNode({
|
|
177
|
+
name: 'start-worker',
|
|
178
|
+
input: Empty,
|
|
179
|
+
output: z.discriminatedUnion('to', [
|
|
180
|
+
z.object({
|
|
181
|
+
to: z.literal('running'),
|
|
182
|
+
pid: z.number().int(),
|
|
183
|
+
branch: z.string(),
|
|
184
|
+
worktree: z.string(),
|
|
185
|
+
attempt: z.number().int(),
|
|
186
|
+
}),
|
|
187
|
+
z.object({ to: z.literal('requeue'), ...failure }),
|
|
188
|
+
z.object({ to: z.literal('human'), ...failure }),
|
|
189
|
+
]),
|
|
190
|
+
/**
|
|
191
|
+
* Claim the issue and start its worker: marker, then `running`, then the process. A crash between the claim
|
|
192
|
+
* and the spawn leaves a stale lease that the sweep recovers.
|
|
193
|
+
* @param {{}} _ @param {TaskCtx} ctx
|
|
194
|
+
*/
|
|
195
|
+
run: async (_, ctx) => {
|
|
196
|
+
const { config, spawner, store } = ctx
|
|
197
|
+
const { slug, snap, foreground = false } = ctx.subject
|
|
198
|
+
const issue = issueOf(ctx)
|
|
199
|
+
const repo = config.repos.find((r) => r.slug === slug)
|
|
200
|
+
if (!repo) throw new Error(`${slug} is not in the config`)
|
|
201
|
+
const caps = config.phases.task
|
|
202
|
+
const m0 = markerOf(issue)
|
|
203
|
+
if (m0.attempt > caps.maxAttempts)
|
|
204
|
+
return { to: 'human', reason: `${caps.maxAttempts} attempts spent`, outcome: 'budget' }
|
|
205
|
+
/** @type {import('../convention.mjs').Convention} */
|
|
206
|
+
let convention
|
|
207
|
+
try {
|
|
208
|
+
convention = spawner.convention(repo)
|
|
209
|
+
} catch (err) {
|
|
210
|
+
return {
|
|
211
|
+
to: 'requeue',
|
|
212
|
+
reason: `could not start the worker: ${/** @type {Error} */ (err)?.message ?? err}`,
|
|
213
|
+
outcome: 'spawn-failed',
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
const known = snap ? prFor(snap, { issue: issue.number, branch: m0.branch }) : null
|
|
217
|
+
const pr = known?.state === 'OPEN' ? known : null
|
|
218
|
+
// Sticky: a retry keeps the branch (and so the worktree topic) its first attempt chose.
|
|
219
|
+
const branch = pr?.branch ?? m0.branch ?? branchName(convention.branch, issue.number, issue.title)
|
|
220
|
+
await land(ctx, 'worker-running', {
|
|
221
|
+
...m0,
|
|
222
|
+
phase: 'task',
|
|
223
|
+
host: config.host,
|
|
224
|
+
start: iso(ctx.tick.now),
|
|
225
|
+
branch,
|
|
226
|
+
pr: pr?.number ?? m0.pr ?? null,
|
|
227
|
+
input: { branch, attempt: m0.attempt },
|
|
228
|
+
})
|
|
229
|
+
const marker = markerOf(issue)
|
|
230
|
+
|
|
231
|
+
/** @type {import('../workers.mjs').Started} */
|
|
232
|
+
let s
|
|
233
|
+
try {
|
|
234
|
+
s = await spawner.start({
|
|
235
|
+
repo,
|
|
236
|
+
convention,
|
|
237
|
+
issue: issue.number,
|
|
238
|
+
attempt: marker.attempt,
|
|
239
|
+
branch,
|
|
240
|
+
phase: 'task',
|
|
241
|
+
maxTurns: caps.maxTurns,
|
|
242
|
+
foreground,
|
|
243
|
+
// TRACING (follow-up): the worker's env gets `TRACEPARENT: ctx.traceparent` here (set by tracing.mjs
|
|
244
|
+
// `advanceHooks` when advance runs with `instrument`), once the Spawner takes an env.
|
|
245
|
+
buildPrompt: (agentsMd) =>
|
|
246
|
+
taskPrompt({ repo: slug, issue, marker, branch, pr, agentsMd, maxAttempts: caps.maxAttempts }),
|
|
247
|
+
})
|
|
248
|
+
} catch (err) {
|
|
249
|
+
// The claim already put the marker at worker-running; the failure is this node's.
|
|
250
|
+
return {
|
|
251
|
+
to: 'requeue',
|
|
252
|
+
reason: `could not start the worker: ${/** @type {Error} */ (err)?.message ?? err}`,
|
|
253
|
+
outcome: 'spawn-failed',
|
|
254
|
+
at: 'start-worker',
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
store.addWorker({
|
|
258
|
+
repo: slug,
|
|
259
|
+
issue: issue.number,
|
|
260
|
+
phase: 'task',
|
|
261
|
+
pid: s.pid,
|
|
262
|
+
worktree: s.worktree,
|
|
263
|
+
branch,
|
|
264
|
+
attempt: marker.attempt,
|
|
265
|
+
startedAt: ctx.tick.now,
|
|
266
|
+
host: config.host,
|
|
267
|
+
stdout: s.stdout,
|
|
268
|
+
stderr: s.stderr,
|
|
269
|
+
})
|
|
270
|
+
const where = { repo: slug, issue: issue.number }
|
|
271
|
+
if (s.reused) ctx.log('warn', `continuing in the previous worktree ${s.worktree}: ${s.reused}`, where)
|
|
272
|
+
ctx.log('info', `started task worker pid ${s.pid} (attempt ${marker.attempt}, ${branch})`, where)
|
|
273
|
+
ctx.subject.started = { ...s, attempt: marker.attempt }
|
|
274
|
+
return { to: 'running', pid: s.pid, branch, worktree: s.worktree, attempt: marker.attempt }
|
|
275
|
+
},
|
|
276
|
+
}),
|
|
277
|
+
{ queue: Q.task, running: false },
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
const workerRunning = declare(
|
|
281
|
+
defineWait({
|
|
282
|
+
name: 'worker-running',
|
|
283
|
+
input: WorkerIn,
|
|
284
|
+
output: z.discriminatedUnion('exit', [
|
|
285
|
+
z.object({ exit: z.literal('submitted'), pr: z.number().int(), branch: z.string(), log: z.string() }),
|
|
286
|
+
z.object({ exit: z.literal('failed'), ...failure }),
|
|
287
|
+
z.object({ exit: z.literal('gone') }),
|
|
288
|
+
]),
|
|
289
|
+
/**
|
|
290
|
+
* Alive and under the wall-clock cap: hold. Over it: kill, then exit-check. Gone: the exit check (a PR that
|
|
291
|
+
* is open or merged and not still on the red commit) or a failure with the stderr tail.
|
|
292
|
+
* @param {unknown} _ @param {TaskCtx} ctx
|
|
293
|
+
*/
|
|
294
|
+
poll: async (_, ctx) => {
|
|
295
|
+
const { config, store, procs, spawner } = ctx
|
|
296
|
+
const { slug, number, snap, issue, reaped } = ctx.subject
|
|
297
|
+
const w = store.worker(slug, number)
|
|
298
|
+
if (!w) {
|
|
299
|
+
const host = issue?.stateComment?.marker.host
|
|
300
|
+
if (host && host !== config.host) return { ready: false }
|
|
301
|
+
return { ready: true, output: { exit: 'failed', ...lostLeaseOf(ctx) } }
|
|
302
|
+
}
|
|
303
|
+
const caps = config.phases[/** @type {'task'} */ (w.phase)] ?? config.phases.task
|
|
304
|
+
if (procs.alive(w.pid) && !reaped?.has(w.pid)) {
|
|
305
|
+
if (!w.killedReason && ctx.tick.now - w.startedAt <= caps.wallClockMinutes * MIN) return { ready: false }
|
|
306
|
+
const reason = w.killedReason ?? `killed: over the ${caps.wallClockMinutes} min wall-clock cap`
|
|
307
|
+
procs.kill(w.pid)
|
|
308
|
+
store.markKilled(w.repo, w.issue, reason)
|
|
309
|
+
w.killedReason = reason
|
|
310
|
+
ctx.log('warn', `${reason} (pid ${w.pid})`, { repo: w.repo, issue: w.issue })
|
|
311
|
+
}
|
|
312
|
+
if (!snap) return { ready: false } // a dead worker waits for a readable snapshot
|
|
313
|
+
if (issue?.state !== 'OPEN') {
|
|
314
|
+
ctx.log('info', 'worker gone; issue is closed or no longer queued', { repo: w.repo, issue: w.issue })
|
|
315
|
+
return { ready: true, output: { exit: 'gone' } }
|
|
316
|
+
}
|
|
317
|
+
// TRACING (follow-up): the worker has ended; emit its spans here with tracing.mjs
|
|
318
|
+
// `workerSpans(<w.stdout log>, { tracer, parent: <start-worker's traceparent> })`.
|
|
319
|
+
const pr = prFor(snap, w)
|
|
320
|
+
const m = markerOf(issue)
|
|
321
|
+
const prepFailed = !w.killedReason && spawner.tail(w.stderr, 15).includes(PREPARE_FAILED_LINE)
|
|
322
|
+
const verdict = w.killedReason
|
|
323
|
+
? { ok: false, reason: w.killedReason }
|
|
324
|
+
: prepFailed
|
|
325
|
+
? { ok: false, reason: 'could not start the worker: prepare failed' }
|
|
326
|
+
: taskExit(pr, m)
|
|
327
|
+
if (verdict.ok && pr)
|
|
328
|
+
return { ready: true, output: { exit: 'submitted', pr: pr.number, branch: pr.branch, log: w.stdout } }
|
|
329
|
+
const tail = spawner.tail(w.stderr, 15)
|
|
330
|
+
return {
|
|
331
|
+
ready: true,
|
|
332
|
+
output: {
|
|
333
|
+
exit: 'failed',
|
|
334
|
+
reason: `${'reason' in verdict ? verdict.reason : 'worker failed'}${tail ? `\n--- stderr (last lines, ${w.stderr})\n${tail}` : ''}`,
|
|
335
|
+
outcome: w.killedReason ? 'killed' : prepFailed ? 'spawn-failed' : 'failed',
|
|
336
|
+
pr: pr?.number,
|
|
337
|
+
log: w.stdout,
|
|
338
|
+
},
|
|
339
|
+
}
|
|
340
|
+
},
|
|
341
|
+
}),
|
|
342
|
+
{
|
|
343
|
+
queue: Q.task,
|
|
344
|
+
running: true,
|
|
345
|
+
recover: (ctx) => ({ node: 'requeue', input: lostLeaseOf(ctx) }),
|
|
346
|
+
},
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
/** @param {TaskCtx} ctx */
|
|
350
|
+
export const lostLeaseOf = (ctx) => ({
|
|
351
|
+
reason: `worker lost: \`running\` on ${ctx.config.host} with no live process`,
|
|
352
|
+
outcome: 'lost',
|
|
353
|
+
})
|
|
354
|
+
|
|
355
|
+
const ciRunning = declare(
|
|
356
|
+
defineWait({
|
|
357
|
+
name: 'ci-running',
|
|
358
|
+
input: PrIn,
|
|
359
|
+
output: PrOut,
|
|
360
|
+
/** @param {z.infer<typeof PrIn>} input @param {TaskCtx} ctx */
|
|
361
|
+
poll: async (input, ctx) => {
|
|
362
|
+
const issue = issueOf(ctx)
|
|
363
|
+
const pr = prOf(ctx, input)
|
|
364
|
+
if (!pr) return { ready: false }
|
|
365
|
+
if (pr.state === 'MERGED') return { ready: true, output: { to: 'merged', pr: pr.number } }
|
|
366
|
+
if (pr.state !== 'OPEN' || issue.state !== 'OPEN') return { ready: false }
|
|
367
|
+
const m = markerOf(issue)
|
|
368
|
+
const red = redCi(pr, m)
|
|
369
|
+
if (red) return { ready: true, output: red }
|
|
370
|
+
if (pr.inMergeQueue) return { ready: true, output: { to: 'queued', pr: pr.number, branch: pr.branch } }
|
|
371
|
+
const late = stall(ctx, pr, m)
|
|
372
|
+
return late ? { ready: true, output: late } : { ready: false }
|
|
373
|
+
},
|
|
374
|
+
}),
|
|
375
|
+
{
|
|
376
|
+
queue: null,
|
|
377
|
+
running: false,
|
|
378
|
+
/** @param {z.infer<typeof PrIn> & { pr: number, branch: string, log?: string }} input @param {TaskCtx} ctx */
|
|
379
|
+
enter: async (input, ctx) => {
|
|
380
|
+
const m = markerOf(issueOf(ctx))
|
|
381
|
+
const at = iso(ctx.tick.now)
|
|
382
|
+
await land(ctx, 'ci-running', {
|
|
383
|
+
...withHistory(m, { at, attempt: m.attempt, outcome: 'submitted', pr: input.pr, log: input.log }),
|
|
384
|
+
pr: input.pr,
|
|
385
|
+
branch: input.branch,
|
|
386
|
+
submittedAt: at,
|
|
387
|
+
queued: false,
|
|
388
|
+
lastFailure: null,
|
|
389
|
+
input: { pr: input.pr, branch: input.branch },
|
|
390
|
+
})
|
|
391
|
+
ctx.log('info', `submitted PR #${input.pr}; watching CI`, { repo: ctx.subject.slug, issue: ctx.subject.number })
|
|
392
|
+
},
|
|
393
|
+
},
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
const mergeQueue = declare(
|
|
397
|
+
defineWait({
|
|
398
|
+
name: 'merge-queue',
|
|
399
|
+
input: PrIn,
|
|
400
|
+
output: PrOut,
|
|
401
|
+
/** @param {z.infer<typeof PrIn>} input @param {TaskCtx} ctx */
|
|
402
|
+
poll: async (input, ctx) => {
|
|
403
|
+
const issue = issueOf(ctx)
|
|
404
|
+
const pr = prOf(ctx, input)
|
|
405
|
+
if (!pr) return { ready: false }
|
|
406
|
+
if (pr.state === 'MERGED') return { ready: true, output: { to: 'merged', pr: pr.number } }
|
|
407
|
+
if (pr.state !== 'OPEN' || issue.state !== 'OPEN') return { ready: false }
|
|
408
|
+
const m = markerOf(issue)
|
|
409
|
+
const red = redCi(pr, m)
|
|
410
|
+
if (red) return { ready: true, output: red }
|
|
411
|
+
const settled = pr.checks !== 'PENDING' && pr.checks !== 'EXPECTED'
|
|
412
|
+
if (!pr.inMergeQueue && !pr.autoMerge && settled)
|
|
413
|
+
return {
|
|
414
|
+
ready: true,
|
|
415
|
+
output: {
|
|
416
|
+
to: 'requeue',
|
|
417
|
+
reason: `PR #${pr.number} left the merge queue without merging`,
|
|
418
|
+
outcome: 'dequeued',
|
|
419
|
+
pr: pr.number,
|
|
420
|
+
},
|
|
421
|
+
}
|
|
422
|
+
const late = stall(ctx, pr, m)
|
|
423
|
+
return late ? { ready: true, output: late } : { ready: false }
|
|
424
|
+
},
|
|
425
|
+
}),
|
|
426
|
+
{
|
|
427
|
+
queue: null,
|
|
428
|
+
running: false,
|
|
429
|
+
pollOnEntry: true,
|
|
430
|
+
/** @param {{ pr: number, branch: string }} input @param {TaskCtx} ctx */
|
|
431
|
+
enter: async (input, ctx) => {
|
|
432
|
+
await land(ctx, 'merge-queue', {
|
|
433
|
+
...markerOf(issueOf(ctx)),
|
|
434
|
+
queued: true,
|
|
435
|
+
input: { pr: input.pr, branch: input.branch },
|
|
436
|
+
})
|
|
437
|
+
},
|
|
438
|
+
},
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
const closeOut = declare(
|
|
442
|
+
defineNode({
|
|
443
|
+
name: 'close-out',
|
|
444
|
+
input: z.object({ pr: z.number().int() }),
|
|
445
|
+
output: Empty,
|
|
446
|
+
/** Merged: done, out of every queue, issue closed, branch deleted. @param {{ pr: number }} input @param {TaskCtx} ctx */
|
|
447
|
+
run: async (input, ctx) => {
|
|
448
|
+
const { slug } = ctx.subject
|
|
449
|
+
const issue = issueOf(ctx)
|
|
450
|
+
const pr = prOf(ctx, { pr: input.pr, branch: null })
|
|
451
|
+
if (issue.state === 'OPEN') {
|
|
452
|
+
const m = markerOf(issue)
|
|
453
|
+
await land(
|
|
454
|
+
ctx,
|
|
455
|
+
'close-out',
|
|
456
|
+
withHistory(m, { at: iso(ctx.tick.now), attempt: m.attempt, outcome: 'merged', pr: input.pr }),
|
|
457
|
+
{ status: 'done' },
|
|
458
|
+
)
|
|
459
|
+
await ctx.github.close(slug, issue.number, `Merged in #${input.pr}. Closed by cambium.`)
|
|
460
|
+
issue.state = 'CLOSED'
|
|
461
|
+
ctx.log('info', `PR #${input.pr} merged; closed`, { repo: slug, issue: issue.number })
|
|
462
|
+
}
|
|
463
|
+
if (pr) await deleteMergedBranch(ctx, pr)
|
|
464
|
+
return {}
|
|
465
|
+
},
|
|
466
|
+
}),
|
|
467
|
+
{ queue: null, running: false },
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
const requeue = declare(
|
|
471
|
+
defineNode({
|
|
472
|
+
name: 'requeue',
|
|
473
|
+
input: Failure,
|
|
474
|
+
output: z.discriminatedUnion('to', [
|
|
475
|
+
z.object({ to: z.literal('start'), attempt: z.number().int() }),
|
|
476
|
+
z.object({ to: z.literal('human'), ...failure }),
|
|
477
|
+
]),
|
|
478
|
+
/** Back to `q:task` with attempt + 1, or to the Human Queue once the attempts are spent. @param {z.infer<typeof Failure>} what @param {TaskCtx} ctx */
|
|
479
|
+
run: async (what, ctx) => {
|
|
480
|
+
const issue = issueOf(ctx)
|
|
481
|
+
const m = markerOf(issue)
|
|
482
|
+
const max = ctx.config.phases.task.maxAttempts
|
|
483
|
+
if (m.attempt >= max) return { ...what, to: 'human', reason: `${max} attempts spent. Last: ${what.reason}` }
|
|
484
|
+
const { reason, outcome, pr, log, failedSha } = what
|
|
485
|
+
await land(ctx, 'start-worker', {
|
|
486
|
+
...withHistory(m, { ...nodeAt(what), at: iso(ctx.tick.now), attempt: m.attempt, outcome, reason, pr, log }),
|
|
487
|
+
attempt: m.attempt + 1,
|
|
488
|
+
lastFailure: reason,
|
|
489
|
+
host: null,
|
|
490
|
+
start: null,
|
|
491
|
+
input: null,
|
|
492
|
+
...(failedSha !== undefined ? { failedSha } : {}),
|
|
493
|
+
})
|
|
494
|
+
ctx.log('info', `requeued (attempt ${m.attempt + 1}/${max}): ${reason.split('\n')[0]}`, {
|
|
495
|
+
repo: ctx.subject.slug,
|
|
496
|
+
issue: issue.number,
|
|
497
|
+
})
|
|
498
|
+
return { to: 'start', attempt: m.attempt + 1 }
|
|
499
|
+
},
|
|
500
|
+
}),
|
|
501
|
+
{ queue: Q.task, running: false },
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
const HumanIn = z.object({ ...failure, why: z.string().optional() })
|
|
505
|
+
const toHuman = declare(
|
|
506
|
+
defineNode({
|
|
507
|
+
name: 'to-human',
|
|
508
|
+
input: HumanIn,
|
|
509
|
+
output: Empty,
|
|
510
|
+
/** Into the Human Queue with a reason label. @param {z.infer<typeof HumanIn>} what @param {TaskCtx} ctx */
|
|
511
|
+
run: async (what, ctx) => {
|
|
512
|
+
const { reason, outcome, pr, log, failedSha, why = WHY.budget } = what
|
|
513
|
+
const issue = issueOf(ctx)
|
|
514
|
+
const m = markerOf(issue)
|
|
515
|
+
await land(
|
|
516
|
+
ctx,
|
|
517
|
+
'to-human',
|
|
518
|
+
{
|
|
519
|
+
...withHistory(m, { ...nodeAt(what), at: iso(ctx.tick.now), attempt: m.attempt, outcome, reason, pr, log }),
|
|
520
|
+
lastFailure: reason,
|
|
521
|
+
host: null,
|
|
522
|
+
start: null,
|
|
523
|
+
input: null,
|
|
524
|
+
...(failedSha !== undefined ? { failedSha } : {}),
|
|
525
|
+
},
|
|
526
|
+
{ extra: [why] },
|
|
527
|
+
)
|
|
528
|
+
ctx.log('warn', `→ q:human (${why}): ${reason.split('\n')[0]}`, { repo: ctx.subject.slug, issue: issue.number })
|
|
529
|
+
return {}
|
|
530
|
+
},
|
|
531
|
+
}),
|
|
532
|
+
{ queue: Q.human, running: false },
|
|
533
|
+
)
|
|
534
|
+
|
|
535
|
+
/** The Task phase. Bump `version` when a node is removed or renamed (see marker.mjs `graphVersionDecision`). */
|
|
536
|
+
export const task = defineGraph({
|
|
537
|
+
name: 'task',
|
|
538
|
+
version: 1,
|
|
539
|
+
entry: { queue: Q.task, node: 'start-worker', input: () => ({}) },
|
|
540
|
+
nodes: [startWorker, workerRunning, ciRunning, mergeQueue, closeOut, requeue, toHuman],
|
|
541
|
+
edges: [
|
|
542
|
+
['start-worker', 'worker-running', started],
|
|
543
|
+
['start-worker', 'to-human', budgetSpent],
|
|
544
|
+
['start-worker', 'requeue'],
|
|
545
|
+
['worker-running', END, issueGone],
|
|
546
|
+
['worker-running', 'ci-running', submitted],
|
|
547
|
+
['worker-running', 'requeue'],
|
|
548
|
+
['ci-running', 'close-out', merged],
|
|
549
|
+
['ci-running', 'merge-queue', inMergeQueue],
|
|
550
|
+
['ci-running', 'to-human', stalled],
|
|
551
|
+
['ci-running', 'requeue'],
|
|
552
|
+
['merge-queue', 'close-out', merged],
|
|
553
|
+
['merge-queue', 'to-human', stalled],
|
|
554
|
+
['merge-queue', 'requeue'],
|
|
555
|
+
['requeue', 'start-worker', attemptsLeft],
|
|
556
|
+
['requeue', 'to-human'],
|
|
557
|
+
['close-out', END],
|
|
558
|
+
['to-human', END],
|
|
559
|
+
],
|
|
560
|
+
})
|
|
561
|
+
|
|
562
|
+
/** @param {string} name @returns {TaskNode} */
|
|
563
|
+
export function nodeOf(name) {
|
|
564
|
+
const node = task.nodes.find((n) => n.name === name)
|
|
565
|
+
if (!node) throw new Error(`task graph has no node "${name}"`)
|
|
566
|
+
return /** @type {TaskNode} */ (node)
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** Nodes that end the graph: an issue resting here has nothing left to run. */
|
|
570
|
+
export const TERMINAL = new Set(task.edges.filter(([, to]) => to === END).map(([from]) => from))
|
|
571
|
+
TERMINAL.delete('worker-running')
|
|
572
|
+
|
|
573
|
+
/** Queue label → owner, for `effectivePosition`: `q:task` belongs to the task graph's nodes that declare it. */
|
|
574
|
+
export const TASK_QUEUES = {
|
|
575
|
+
[Q.task]: {
|
|
576
|
+
graph: task.name,
|
|
577
|
+
entryNode: task.entry.node,
|
|
578
|
+
nodes: task.nodes.filter((n) => /** @type {TaskNode} */ (n).queue === Q.task).map((n) => n.name),
|
|
579
|
+
},
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/** The input a resting node resumes with, rebuilt from the marker. @param {string} node @param {Marker} m */
|
|
583
|
+
export function inputFor(node, m) {
|
|
584
|
+
if (node === 'worker-running') return { branch: m.branch, attempt: m.attempt }
|
|
585
|
+
if (node === 'ci-running' || node === 'merge-queue') return { pr: m.pr, branch: m.branch }
|
|
586
|
+
return {}
|
|
587
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* GitHub writes shared by both engines. Each updates the in-memory issue so
|
|
4
|
+
* later steps in the same tick see it.
|
|
5
|
+
*/
|
|
6
|
+
import { apply } from './labels.mjs'
|
|
7
|
+
import { newMarker, renderMarker } from './marker.mjs'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* @typedef {import('./github.mjs').Issue} Issue
|
|
11
|
+
* @typedef {{ config: import('./config.mjs').Config, github: import('./github.mjs').GitHub }} Writers
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export const MIN = 60_000
|
|
15
|
+
export const iso = (/** @type {number} */ ms) => new Date(ms).toISOString()
|
|
16
|
+
export const key = (/** @type {string} */ slug, /** @type {number} */ n) => `${slug}#${n}`
|
|
17
|
+
|
|
18
|
+
/** @param {Writers} deps @param {string} slug @param {Issue} issue @param {{ add: string[], remove: string[] }} change */
|
|
19
|
+
export async function setLabels(deps, slug, issue, change) {
|
|
20
|
+
if (!change.add.length && !change.remove.length) return
|
|
21
|
+
await deps.github.label(slug, issue.number, change)
|
|
22
|
+
issue.labels = apply(issue.labels, change)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** @param {Writers} deps @param {string} slug @param {Issue} issue @param {import('./marker.mjs').Marker} marker */
|
|
26
|
+
export async function setMarker(deps, slug, issue, marker) {
|
|
27
|
+
const maxAttempts = deps.config.phases[marker.phase]?.maxAttempts
|
|
28
|
+
const id = await deps.github.writeState(
|
|
29
|
+
slug,
|
|
30
|
+
issue.number,
|
|
31
|
+
issue.stateComment?.id ?? null,
|
|
32
|
+
renderMarker(marker, { maxAttempts }),
|
|
33
|
+
)
|
|
34
|
+
issue.stateComment = { id, marker }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** @param {Issue} issue */
|
|
38
|
+
export const markerOf = (issue) => issue.stateComment?.marker ?? newMarker()
|
|
39
|
+
|
|
40
|
+
/** @param {import('./github.mjs').PullRequest} pr */
|
|
41
|
+
export function failureSummary(pr) {
|
|
42
|
+
const lines = pr.failing.map(
|
|
43
|
+
(f) => `${f.name}${f.url ? ` (${f.url})` : ''}${f.summary ? `\n ${f.summary.replace(/\n/g, '\n ')}` : ''}`,
|
|
44
|
+
)
|
|
45
|
+
return `CI red on PR #${pr.number} at ${(pr.headSha ?? '').slice(0, 7)}:\n${lines.join('\n') || '(no failing check reported)'}`
|
|
46
|
+
}
|