@oakgreencc/cambium 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +166 -11
- package/otel/com.oakgreencc.otelcol.plist +33 -0
- package/otel/otelcol-aws.yaml +52 -0
- package/otel/otelcol.yaml +50 -0
- package/package.json +9 -2
- package/src/cli.mjs +18 -3
- package/src/config.mjs +4 -0
- package/src/graph/advance.mjs +19 -8
- package/src/graph-engine.mjs +289 -0
- package/src/graphs/task.mjs +587 -0
- package/src/issue-state.mjs +46 -0
- package/src/tick.mjs +97 -76
- package/src/tracing.mjs +522 -0
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* `engine: "graph"`: the Task-phase steps of the tick, driven through the `task` graph (graphs/task.mjs).
|
|
4
|
+
*
|
|
5
|
+
* The tick keeps its shape (Reconcile → Watch → Sweep → Schedule) so both engines make the same moves in the
|
|
6
|
+
* same order; each step only chooses where an issue's turn starts:
|
|
7
|
+
*
|
|
8
|
+
* - Reconcile: every tracked worker's issue at `worker-running`.
|
|
9
|
+
* - Watch: issues resting at `ci-running` / `merge-queue` (from the marker); a merged PR goes to `close-out`
|
|
10
|
+
* from wherever the issue is.
|
|
11
|
+
* - Sweep: a stale `running` lease runs its node's `recover` (default: `requeue`).
|
|
12
|
+
* - Schedule: the gate on the entry node. Caps and priority pick the issues; each starts at `start-worker`.
|
|
13
|
+
*
|
|
14
|
+
* A turn ({@link drive}) advances one transition at a time and stops when it reaches the entry node again (only
|
|
15
|
+
* Schedule admits entries), when it enters a wait (polled next tick, unless the wait is `pollOnEntry`), at END,
|
|
16
|
+
* or at a wait that is not ready.
|
|
17
|
+
*/
|
|
18
|
+
import { advance, END } from './graph/index.mjs'
|
|
19
|
+
import { deleteMergedBranch, inputFor, lostLeaseOf, nodeOf, task, TASK_QUEUES, TERMINAL } from './graphs/task.mjs'
|
|
20
|
+
import { iso, key, markerOf, setLabels, setMarker } from './issue-state.mjs'
|
|
21
|
+
import { effectivePosition, graphVersionDecision, withHistory } from './marker.mjs'
|
|
22
|
+
import { each, newCtx, staleLeases, taskCandidates, watched, watchPairs } from './tick.mjs'
|
|
23
|
+
import { move, Q } from './labels.mjs'
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @typedef {import('./tick.mjs').Deps} Deps
|
|
27
|
+
* @typedef {import('./tick.mjs').Ctx} Ctx
|
|
28
|
+
* @typedef {import('./graphs/task.mjs').Subject} Subject
|
|
29
|
+
* @typedef {import('./graphs/task.mjs').Position} Position
|
|
30
|
+
* @typedef {import('./graph/advance.mjs').Transition} Transition
|
|
31
|
+
* @typedef {{ transitions: Transition[], reason: 'waiting' | 'gated' | 'end', node: string, input: unknown }} Turn
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/** @param {Deps} deps @param {Ctx} ctx @param {Subject} subject @returns {import('./graphs/task.mjs').TaskCtx} */
|
|
35
|
+
const taskCtx = (deps, ctx, subject) => ({ ...deps, tick: ctx, subject })
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* One issue's turn from `at`. A node error is rethrown as the node's own error (the step logs it, as the legacy
|
|
39
|
+
* tick does); a worker whose `worker-running` was left is torn down only after the turn's writes succeeded.
|
|
40
|
+
*
|
|
41
|
+
* @param {Deps} deps @param {Ctx} ctx @param {Subject} subject @param {Position} at
|
|
42
|
+
* @returns {Promise<Turn>}
|
|
43
|
+
*/
|
|
44
|
+
export async function drive(deps, ctx, subject, at) {
|
|
45
|
+
const c = taskCtx(deps, ctx, subject)
|
|
46
|
+
/** @type {Transition[]} */
|
|
47
|
+
const transitions = []
|
|
48
|
+
let pos = at
|
|
49
|
+
/** @type {Turn['reason']} */
|
|
50
|
+
let reason
|
|
51
|
+
for (;;) {
|
|
52
|
+
const r = await advance(task, pos, c, { maxTransitions: 1 })
|
|
53
|
+
if (r.error) throw r.error.cause instanceof Error ? r.error.cause : r.error
|
|
54
|
+
transitions.push(...r.transitions)
|
|
55
|
+
const t = r.transitions[0]
|
|
56
|
+
if (t && t.to !== END) await nodeOf(t.to).enter?.(t.next, c)
|
|
57
|
+
pos = { node: r.node, input: r.input }
|
|
58
|
+
if (r.reason !== 'limit') {
|
|
59
|
+
reason = r.reason === 'waiting' ? 'waiting' : 'end'
|
|
60
|
+
break
|
|
61
|
+
}
|
|
62
|
+
if (r.node === task.entry.node) {
|
|
63
|
+
reason = 'gated'
|
|
64
|
+
break
|
|
65
|
+
}
|
|
66
|
+
const next = nodeOf(r.node)
|
|
67
|
+
if (next.kind === 'wait' && !next.pollOnEntry) {
|
|
68
|
+
reason = 'waiting'
|
|
69
|
+
break
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (transitions.some((t) => t.from === 'worker-running')) await releaseWorker(deps, subject)
|
|
73
|
+
return { transitions, reason, ...pos }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The worker row goes, then its worktree (kept, with a warning, when it holds local-only work). @param {Deps} deps @param {Subject} s */
|
|
77
|
+
async function releaseWorker(deps, s) {
|
|
78
|
+
const w = deps.store.worker(s.slug, s.number)
|
|
79
|
+
if (!w) return
|
|
80
|
+
deps.store.removeWorker(s.slug, s.number)
|
|
81
|
+
const repo = deps.config.repos.find((r) => r.slug === s.slug)
|
|
82
|
+
if (!repo) return
|
|
83
|
+
const t = await deps.spawner.cleanup({ repo, worktree: w.worktree, branch: w.branch })
|
|
84
|
+
if (!t.removed) deps.log('warn', `kept worktree ${w.worktree}: ${t.reason}`, { repo: s.slug, issue: s.number })
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Where an issue rests in the task graph, from its labels and marker: the label wins (marker.mjs
|
|
89
|
+
* `effectivePosition`), a stage-era `submitted` that the merge queue holds is `merge-queue`, and a node the
|
|
90
|
+
* current graph version no longer has restarts at the entry (with a `graph-changed` history note).
|
|
91
|
+
*
|
|
92
|
+
* @param {import('./github.mjs').Issue} issue @param {number} now
|
|
93
|
+
* @returns {(Position & { note?: import('./marker.mjs').HistoryEntry }) | null} null: not in the task graph
|
|
94
|
+
*/
|
|
95
|
+
export function locate(issue, now) {
|
|
96
|
+
const marker = issue.stateComment?.marker ?? null
|
|
97
|
+
const pos = effectivePosition(issue.labels, marker, TASK_QUEUES)
|
|
98
|
+
if (!pos || pos.graph !== task.name) return null
|
|
99
|
+
if (pos.reset || !marker) return { node: task.entry.node, input: {} }
|
|
100
|
+
const v = graphVersionDecision(marker, {
|
|
101
|
+
version: task.version,
|
|
102
|
+
nodes: task.nodes.map((n) => n.name),
|
|
103
|
+
entryNode: task.entry.node,
|
|
104
|
+
at: iso(now),
|
|
105
|
+
})
|
|
106
|
+
if (v.action === 'restart') return { node: v.node, input: {}, note: v.note }
|
|
107
|
+
const node = pos.node === 'ci-running' && marker.queued ? 'merge-queue' : pos.node
|
|
108
|
+
return { node, input: inputFor(node, marker) }
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* A marker whose node is gone from the graph: back to `q:task` at the entry, with the note in its history.
|
|
113
|
+
* @param {Deps} deps @param {Ctx} ctx @param {string} slug @param {import('./github.mjs').Issue} issue
|
|
114
|
+
* @param {import('./marker.mjs').HistoryEntry} note
|
|
115
|
+
*/
|
|
116
|
+
async function restart(deps, ctx, slug, issue, note) {
|
|
117
|
+
const m = markerOf(issue)
|
|
118
|
+
await setMarker(deps, slug, issue, {
|
|
119
|
+
...withHistory(m, note),
|
|
120
|
+
graph: task.name,
|
|
121
|
+
graphVersion: task.version,
|
|
122
|
+
node: task.entry.node,
|
|
123
|
+
status: 'waiting',
|
|
124
|
+
host: null,
|
|
125
|
+
start: null,
|
|
126
|
+
input: null,
|
|
127
|
+
})
|
|
128
|
+
await setLabels(deps, slug, issue, move(issue.labels, Q.task))
|
|
129
|
+
ctx.touched.add(key(slug, issue.number))
|
|
130
|
+
deps.log('info', note.reason ?? 'graph changed; restarting', { repo: slug, issue: issue.number })
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// ---- The Task-phase steps
|
|
134
|
+
|
|
135
|
+
/** @param {Deps} deps @param {Ctx} ctx */
|
|
136
|
+
async function reconcile(deps, ctx) {
|
|
137
|
+
await each(
|
|
138
|
+
deps,
|
|
139
|
+
deps.store.workers(),
|
|
140
|
+
async (w) => {
|
|
141
|
+
const snap = ctx.snaps.get(w.repo)
|
|
142
|
+
const subject = { slug: w.repo, number: w.issue, snap, issue: snap?.issues.get(w.issue) }
|
|
143
|
+
await drive(deps, ctx, subject, { node: 'worker-running', input: { branch: w.branch, attempt: w.attempt } })
|
|
144
|
+
},
|
|
145
|
+
(w) => ({ repo: w.repo, issue: w.issue }),
|
|
146
|
+
)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** @param {Deps} deps @param {Ctx} ctx */
|
|
150
|
+
async function watch(deps, ctx) {
|
|
151
|
+
for (const [slug, snap] of ctx.snaps) {
|
|
152
|
+
/** @type {Set<string>} */
|
|
153
|
+
const deleted = new Set()
|
|
154
|
+
await each(
|
|
155
|
+
deps,
|
|
156
|
+
watchPairs(snap),
|
|
157
|
+
async ([pr, issue]) => {
|
|
158
|
+
if (ctx.touched.has(key(slug, issue.number))) return
|
|
159
|
+
/** @type {Subject} */
|
|
160
|
+
const subject = { slug, number: issue.number, snap, issue, pr, deleted }
|
|
161
|
+
if (pr.state === 'MERGED') {
|
|
162
|
+
if (issue.state === 'OPEN') await drive(deps, ctx, subject, { node: 'close-out', input: { pr: pr.number } })
|
|
163
|
+
else await deleteMergedBranch(taskCtx(deps, ctx, subject), pr)
|
|
164
|
+
return
|
|
165
|
+
}
|
|
166
|
+
if (pr.state !== 'OPEN' || issue.state !== 'OPEN' || !watched(deps, slug, issue)) return
|
|
167
|
+
const at = locate(issue, ctx.now)
|
|
168
|
+
if (at?.note) return restart(deps, ctx, slug, issue, at.note)
|
|
169
|
+
if (at?.node === 'ci-running' || at?.node === 'merge-queue') await drive(deps, ctx, subject, at)
|
|
170
|
+
},
|
|
171
|
+
([, issue]) => ({ repo: slug, issue: issue.number }),
|
|
172
|
+
)
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** @param {Deps} deps @param {Ctx} ctx */
|
|
177
|
+
async function sweep(deps, ctx) {
|
|
178
|
+
for (const [slug, snap] of ctx.snaps)
|
|
179
|
+
await each(
|
|
180
|
+
deps,
|
|
181
|
+
staleLeases(deps, ctx, slug, snap),
|
|
182
|
+
async (issue) => {
|
|
183
|
+
const subject = { slug, number: issue.number, snap, issue }
|
|
184
|
+
const c = taskCtx(deps, ctx, subject)
|
|
185
|
+
const at = locate(issue, ctx.now)
|
|
186
|
+
const recover = (at && nodeOf(at.node).recover) || defaultRecover
|
|
187
|
+
await drive(deps, ctx, subject, recover(c))
|
|
188
|
+
},
|
|
189
|
+
(issue) => ({ repo: slug, issue: issue.number }),
|
|
190
|
+
)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** @param {Deps} deps @param {Ctx} ctx */
|
|
194
|
+
async function scheduleTasks(deps, ctx) {
|
|
195
|
+
await each(
|
|
196
|
+
deps,
|
|
197
|
+
taskCandidates(deps, ctx),
|
|
198
|
+
async (c) => {
|
|
199
|
+
const subject = { slug: c.slug, number: c.issue.number, snap: c.snap, issue: c.issue }
|
|
200
|
+
await drive(deps, ctx, subject, { node: task.entry.node, input: {} })
|
|
201
|
+
},
|
|
202
|
+
(c) => ({ repo: c.slug, issue: c.issue.number }),
|
|
203
|
+
)
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** A stale lease at a node without its own `recover`: requeue. @param {import('./graphs/task.mjs').TaskCtx} c */
|
|
207
|
+
const defaultRecover = (c) => ({ node: 'requeue', input: lostLeaseOf(c) })
|
|
208
|
+
|
|
209
|
+
/** @type {import('./tick.mjs').Engine} */
|
|
210
|
+
export const GRAPH_ENGINE = { reconcile, watch, sweep, scheduleTasks }
|
|
211
|
+
|
|
212
|
+
// ---- `cambium run <graph> --issue <n> [--follow]`
|
|
213
|
+
|
|
214
|
+
/** Graphs `cambium run` knows, by name. */
|
|
215
|
+
export const GRAPHS = { [task.name]: task }
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* The graph runtime in the foreground for one issue. A `q:task` issue without a lease starts at the entry with a
|
|
219
|
+
* foreground worker, whose exit is awaited; anything else resumes where its marker rests. Stops at the first wait
|
|
220
|
+
* that is not ready, unless `follow`, which re-reads and polls again every `tickSeconds` until the graph ends.
|
|
221
|
+
*
|
|
222
|
+
* @param {Deps} deps
|
|
223
|
+
* @param {{
|
|
224
|
+
* graph: string, slug: string, number: number, follow?: boolean,
|
|
225
|
+
* out: (line: string) => void, sleep?: (ms: number) => Promise<void>, signal?: AbortSignal,
|
|
226
|
+
* }} opts
|
|
227
|
+
* @returns {Promise<{ ok: boolean, error?: string }>}
|
|
228
|
+
*/
|
|
229
|
+
export async function runIssue(deps, { graph, slug, number, follow = false, out, sleep = delay, signal }) {
|
|
230
|
+
if (!Object.hasOwn(GRAPHS, graph))
|
|
231
|
+
return { ok: false, error: `unknown graph "${graph}"; known: ${Object.keys(GRAPHS).join(', ')}` }
|
|
232
|
+
/** @type {Set<number>} */
|
|
233
|
+
const reaped = new Set()
|
|
234
|
+
let first = true
|
|
235
|
+
for (;;) {
|
|
236
|
+
const ctx = newCtx(deps)
|
|
237
|
+
const snap = await deps.github.read(slug)
|
|
238
|
+
ctx.snaps.set(slug, snap)
|
|
239
|
+
const issue = snap.issues.get(number)
|
|
240
|
+
if (issue?.state !== 'OPEN') {
|
|
241
|
+
if (first) return { ok: false, error: `${slug}#${number} is not an open issue` }
|
|
242
|
+
return report(out, slug, number, issue)
|
|
243
|
+
}
|
|
244
|
+
const worker = deps.store.worker(slug, number)
|
|
245
|
+
const fresh = issue.labels.includes(Q.task) && !issue.labels.includes('running') && !worker
|
|
246
|
+
const located = fresh
|
|
247
|
+
? { node: task.entry.node, input: {} }
|
|
248
|
+
: worker
|
|
249
|
+
? { node: 'worker-running', input: { branch: worker.branch, attempt: worker.attempt } }
|
|
250
|
+
: locate(issue, ctx.now)
|
|
251
|
+
if (!located || TERMINAL.has(located.node)) {
|
|
252
|
+
if (first) return { ok: false, error: `${slug}#${number} has nothing to run in the ${graph} graph` }
|
|
253
|
+
return report(out, slug, number, issue)
|
|
254
|
+
}
|
|
255
|
+
if (located.note) await restart(deps, ctx, slug, issue, located.note)
|
|
256
|
+
/** @type {Subject} */
|
|
257
|
+
const subject = { slug, number, snap, issue, foreground: true, reaped }
|
|
258
|
+
const turn = located.note ? null : await drive(deps, ctx, subject, located)
|
|
259
|
+
if (subject.started) {
|
|
260
|
+
const s = subject.started
|
|
261
|
+
out(`worker pid ${s.pid} in ${s.worktree}\n stdout ${s.stdout}\n stderr ${s.stderr}`)
|
|
262
|
+
const code = await s.exited
|
|
263
|
+
out(`worker exited ${code}`)
|
|
264
|
+
reaped.add(s.pid)
|
|
265
|
+
first = false
|
|
266
|
+
continue
|
|
267
|
+
}
|
|
268
|
+
if (first && fresh && !follow)
|
|
269
|
+
return { ok: false, error: `${slug}#${number} was not started; see its state comment` }
|
|
270
|
+
first = false
|
|
271
|
+
if (turn?.reason === 'end' || !follow || signal?.aborted) return report(out, slug, number, issue)
|
|
272
|
+
await sleep(deps.config.tickSeconds * 1000)
|
|
273
|
+
if (signal?.aborted) return report(out, slug, number, issue)
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** @param {number} ms */
|
|
278
|
+
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* @param {(line: string) => void} out @param {string} slug @param {number} n
|
|
282
|
+
* @param {import('./github.mjs').Issue | undefined} issue
|
|
283
|
+
*/
|
|
284
|
+
function report(out, slug, n, issue) {
|
|
285
|
+
const m = issue?.state === 'OPEN' ? issue.stateComment?.marker : null
|
|
286
|
+
const where = m ? `${m.node}${m.status === 'waiting' ? '' : ` (${m.status})`}` : 'closed'
|
|
287
|
+
out(`${slug}#${n}: ${where} · labels ${issue?.labels.join(', ') || '(none)'}`)
|
|
288
|
+
return { ok: true }
|
|
289
|
+
}
|