@edgehero/pi-dispatch 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/.env.example +160 -0
  2. package/deploy/com.pi-dispatch.worker.plist +66 -0
  3. package/deploy/nssm-install.cmd +59 -0
  4. package/deploy/receiver.service +36 -0
  5. package/deploy/worker-env-wrapper.cmd +50 -0
  6. package/deploy/worker-env-wrapper.sh +63 -0
  7. package/deploy/worker.service +55 -0
  8. package/package.json +83 -0
  9. package/src/azure-auth.mjs +61 -0
  10. package/src/azure-host.mjs +236 -0
  11. package/src/azure-identity.mjs +63 -0
  12. package/src/azure-prompt.mjs +118 -0
  13. package/src/branch.mjs +80 -0
  14. package/src/budget.mjs +179 -0
  15. package/src/cli.mjs +208 -0
  16. package/src/config.mjs +329 -0
  17. package/src/connection.mjs +40 -0
  18. package/src/cron.mjs +94 -0
  19. package/src/docker-run.mjs +119 -0
  20. package/src/doctor.mjs +1127 -0
  21. package/src/env-allowlist.mjs +198 -0
  22. package/src/env-file.mjs +153 -0
  23. package/src/exit-code.mjs +32 -0
  24. package/src/flow-gate.mjs +82 -0
  25. package/src/forgejo-auth.mjs +77 -0
  26. package/src/forgejo-host.mjs +172 -0
  27. package/src/forgejo-identity.mjs +74 -0
  28. package/src/forgejo-prompt.mjs +123 -0
  29. package/src/forges.mjs +148 -0
  30. package/src/get-token.mjs +226 -0
  31. package/src/git-dirty.mjs +16 -0
  32. package/src/github-app-setup.mjs +517 -0
  33. package/src/github-host.mjs +159 -0
  34. package/src/github-prompt.mjs +286 -0
  35. package/src/gitlab-auth.mjs +72 -0
  36. package/src/gitlab-host.mjs +200 -0
  37. package/src/gitlab-identity.mjs +61 -0
  38. package/src/gitlab-prompt.mjs +123 -0
  39. package/src/identity.mjs +57 -0
  40. package/src/image-preflight.mjs +180 -0
  41. package/src/import-pi.mjs +451 -0
  42. package/src/index.mjs +177 -0
  43. package/src/init.mjs +77 -0
  44. package/src/job-id.mjs +100 -0
  45. package/src/materialize.mjs +138 -0
  46. package/src/outbox.mjs +179 -0
  47. package/src/packages.mjs +188 -0
  48. package/src/pause-windows.mjs +218 -0
  49. package/src/prepare-github.mjs +260 -0
  50. package/src/prepare-local.mjs +76 -0
  51. package/src/prepare.mjs +199 -0
  52. package/src/pricing.mjs +168 -0
  53. package/src/processor.mjs +360 -0
  54. package/src/queue.mjs +152 -0
  55. package/src/run-container.mjs +133 -0
  56. package/src/run-history.mjs +534 -0
  57. package/src/runtime-settings.mjs +188 -0
  58. package/src/sandbox-cli.mjs +156 -0
  59. package/src/sandbox-store.mjs +269 -0
  60. package/src/sandbox.mjs +171 -0
  61. package/src/scheduler-stall-guard.mjs +67 -0
  62. package/src/schedules.mjs +62 -0
  63. package/src/service.mjs +677 -0
  64. package/src/session-key.mjs +108 -0
  65. package/src/session-store.mjs +249 -0
  66. package/src/start.mjs +502 -0
  67. package/src/subscriptions.mjs +208 -0
  68. package/src/triggers.mjs +491 -0
  69. package/src/up.mjs +315 -0
@@ -0,0 +1,534 @@
1
+ import * as nodeFs from "node:fs";
2
+ import { basename, join } from "node:path";
3
+ import { isForgeKind, targetSeparator } from "./forges.mjs";
4
+
5
+ /**
6
+ * Durable per-run history.
7
+ *
8
+ * The PURE HELPERS are total functions over their arguments -- no filesystem, no clock, no
9
+ * `process.env`, no randomness -- so the record shape and the filename/telemetry parsing are testable
10
+ * without a container, a queue, or a disk: `sanitizeJobId`, `parseExitTurns`, `parseExitTokens`,
11
+ * `parseExitSession`, `parseExitUsage`, `buildRecord`.
12
+ *
13
+ * The I/O factories -- `makeLogSink`, `makeRecordWriter`, `makeFindPreviousRun`, `makeLogReaper` --
14
+ * each inject their own `fs` (and, for the reaper, their own clock via `now`), so they too are testable
15
+ * with a fake and no disk. `makeLogSink` streams a job's raw output to a per-job `.log` and recovers the
16
+ * turn count from a bounded tail; `makeRecordWriter` serialises a finished run to a JSON sidecar;
17
+ * `makeFindPreviousRun` reads a scheduler's most recent prior sidecar back; `makeLogReaper` sweeps aged
18
+ * `.log`/`.json` files at boot.
19
+ */
20
+
21
+ /**
22
+ * Turn an arbitrary job id into a single legal filename segment.
23
+ *
24
+ * BullMQ scheduled ids are `repeat:<schedulerId>:<millis>`; a colon is an illegal NTFS filename
25
+ * character, so writing `<id>.json` verbatim throws on Windows. A strict allowlist (letters, digits,
26
+ * dot, underscore, hyphen) is the safe subset across Windows and POSIX; everything else collapses to
27
+ * `_`. A nullish or empty id yields a fixed sentinel so a downstream writer still produces a file
28
+ * rather than silently dropping the record.
29
+ */
30
+ export function sanitizeJobId(id) {
31
+ if (id === null || id === undefined) return "unknown-job";
32
+ const s = String(id);
33
+ if (s === "") return "unknown-job";
34
+ return s.replace(/[^A-Za-z0-9._-]/g, "_");
35
+ }
36
+
37
+ /**
38
+ * Recover the agent's turn count from buffered container stdout, or `null` if it is not reported.
39
+ *
40
+ * The stream interleaves docker/agent noise and other JSON events (`pi_auto_retry`) with the runner's
41
+ * own lines. Only the success exit line carries `turns` (`image/runner/run-job.mjs:263`); the
42
+ * catch-path exit line (`:277`) omits it. Scan from the end and return the turns of the last `exit`
43
+ * event that reports an integer count.
44
+ *
45
+ * This is read-only telemetry: it MUST NEVER throw and MUST NOT feed exit-code or retry
46
+ * classification -- that is the container exit code's job (INT-RUNNER-EXIT-CODE-PROTOCOL). Every parse
47
+ * is guarded; a truncated or non-JSON line is skipped.
48
+ */
49
+ export function parseExitTurns(text) {
50
+ if (typeof text !== "string") return null;
51
+ const lines = text.split("\n");
52
+ for (let i = lines.length - 1; i >= 0; i--) {
53
+ const line = lines[i].trim();
54
+ if (line === "") continue;
55
+ let parsed;
56
+ try {
57
+ parsed = JSON.parse(line);
58
+ } catch {
59
+ continue; // docker/agent noise or a truncated final line
60
+ }
61
+ if (parsed?.event !== "exit") continue;
62
+ return Number.isInteger(parsed?.turns) ? parsed.turns : null;
63
+ }
64
+ return null;
65
+ }
66
+
67
+ /**
68
+ * Recover the agent's token usage from buffered container stdout, or `null` if it is not reported.
69
+ *
70
+ * Mirrors `parseExitTurns`: scan from the end for the last `exit` event and read its `tokens` object
71
+ * (`{ input, output, total, cost }`). Only the success exit line carries it
72
+ * (`image/runner/run-job.mjs`); the catch-path exit line omits it, and a container that died before the
73
+ * runner's exit line yields none -- all three cases are `null`.
74
+ *
75
+ * Read-only telemetry, exactly like `parseExitTurns`: NEVER throws and MUST NOT feed exit-code or retry
76
+ * classification (INT-RUNNER-EXIT-CODE-PROTOCOL). A malformed or non-object `tokens` (or one missing a
77
+ * numeric `total`) is `null`, never a partial that could poison the daily token counter.
78
+ */
79
+ /**
80
+ * The runner's `session` object off the exit line: `{ resumed: <bool>, reason: "<enum>" }` or null when
81
+ * the container died before emitting one (REQ-RESUMABLE-SESSION).
82
+ *
83
+ * A sibling of parseExitTokens rather than a widening of it, and it reports what pi ACTUALLY did. The
84
+ * host records its own intent separately, and the pair is the point: a host that staged a transcript
85
+ * while the container reports `resumed: false` is a real event -- a corrupt file, a degrade -- and
86
+ * without both numbers it is indistinguishable from an ordinary cold start. A feature that fails open
87
+ * must still say that it did.
88
+ *
89
+ * PII-free by construction: a boolean and a fixed enum. No key, no branch name, no path.
90
+ */
91
+ export function parseExitSession(text) {
92
+ if (typeof text !== "string") return null;
93
+ const lines = text.split("\n");
94
+ for (let i = lines.length - 1; i >= 0; i--) {
95
+ const line = lines[i].trim();
96
+ if (line === "") continue;
97
+ let parsed;
98
+ try {
99
+ parsed = JSON.parse(line);
100
+ } catch {
101
+ continue; // docker/agent noise or a truncated final line
102
+ }
103
+ if (parsed?.event !== "exit") continue;
104
+ const sess = parsed?.session;
105
+ if (sess && typeof sess === "object" && !Array.isArray(sess) && typeof sess.resumed === "boolean") {
106
+ return { resumed: sess.resumed, reason: typeof sess.reason === "string" ? sess.reason : null };
107
+ }
108
+ return null;
109
+ }
110
+ return null;
111
+ }
112
+
113
+ export function parseExitTokens(text) {
114
+ if (typeof text !== "string") return null;
115
+ const lines = text.split("\n");
116
+ for (let i = lines.length - 1; i >= 0; i--) {
117
+ const line = lines[i].trim();
118
+ if (line === "") continue;
119
+ let parsed;
120
+ try {
121
+ parsed = JSON.parse(line);
122
+ } catch {
123
+ continue; // docker/agent noise or a truncated final line
124
+ }
125
+ if (parsed?.event !== "exit") continue;
126
+ const t = parsed?.tokens;
127
+ if (t && typeof t === "object" && !Array.isArray(t) && typeof t.total === "number") return t;
128
+ return null;
129
+ }
130
+ return null;
131
+ }
132
+
133
+ /** The id allowlist for a ledger row's provider/model, applied AFTER lowercasing. The first-char class
134
+ * has no dot, colon or slash, so `.hidden`, `../etc` and `:` shapes fail at character one. */
135
+ const USAGE_ID_PATTERN = /^[a-z0-9][a-z0-9._:/-]{0,63}$/;
136
+
137
+ /** The ten per-row counters, in the row's serialisation order. Absent is an honest zero; anything
138
+ * present must be a finite non-negative number or the whole block is refused. */
139
+ const USAGE_ROW_NUMERIC_KEYS = ["calls", "input", "output", "cacheRead", "cacheWrite", "cacheWrite1h", "reasoning", "total", "cost", "unpriced"];
140
+
141
+ /**
142
+ * The runner's per-(provider,model) usage ledger off the exit line -- `{ v, piAi, truncated, models }`
143
+ * -- REBUILT and validated, or `null` (INT-RUN-HISTORY-FILE-CONTRACT).
144
+ *
145
+ * A sibling of `parseExitSession` rather than a widening of `parseExitTokens`, for the same reason that
146
+ * one was: `tokens` may ride through verbatim only because it holds nothing but numbers, while this
147
+ * block's `provider`/`model` strings are container-emitted and therefore attacker-adjacent. So nothing
148
+ * here is stored as received. Every field is validated and re-written into an explicit literal: ids are
149
+ * lowercased and held to a strict allowlist, the ten per-row counters to finite non-negatives, and ANY
150
+ * violation nulls the WHOLE block, never a partial -- the same malformed->null rule `parseExitTokens`
151
+ * applies to the daily counter, because a half-validated ledger is how sums stop meaning anything.
152
+ *
153
+ * An absent `usage` is the NORMAL case, not an error: the metered:false fallback, the catch-path exit
154
+ * line and a pre-ledger runner image all omit it, and a container may die before any exit line at all.
155
+ *
156
+ * Cross-field sums (the rows against `tokens.total`) are deliberately NOT checked here. The sum is the
157
+ * EMITTER's invariant, asserted where the emitter lives -- in the runner's own meter tests. The host
158
+ * validates fields, not bookkeeping; re-deriving the arithmetic on this side would only manufacture a
159
+ * second source of truth for the first one to drift from.
160
+ *
161
+ * Read-only telemetry, exactly like its siblings: NEVER throws and MUST NOT feed exit-code or retry
162
+ * classification (INT-RUNNER-EXIT-CODE-PROTOCOL).
163
+ */
164
+ export function parseExitUsage(text) {
165
+ if (typeof text !== "string") return null;
166
+ const lines = text.split("\n");
167
+ for (let i = lines.length - 1; i >= 0; i--) {
168
+ const line = lines[i].trim();
169
+ if (line === "") continue;
170
+ let parsed;
171
+ try {
172
+ parsed = JSON.parse(line);
173
+ } catch {
174
+ continue; // docker/agent noise or a truncated final line
175
+ }
176
+ if (parsed?.event !== "exit") continue;
177
+ return rebuildUsage(parsed?.usage);
178
+ }
179
+ return null;
180
+ }
181
+
182
+ /** The validating rebuild behind `parseExitUsage`: explicit literals only, null on ANY violation. */
183
+ function rebuildUsage(u) {
184
+ if (!u || typeof u !== "object" || Array.isArray(u)) return null;
185
+ // v is kept as-is once it passes: readers treat an unknown version as opaque-but-present.
186
+ if (!Number.isInteger(u.v) || u.v < 1) return null;
187
+ let piAi = null;
188
+ if (u.piAi !== undefined && u.piAi !== null) {
189
+ // A rates-provenance stamp is three dot-joined integers or nothing -- any other string is a
190
+ // malformed block, not a value to store.
191
+ if (typeof u.piAi !== "string" || !/^\d+\.\d+\.\d+$/.test(u.piAi)) return null;
192
+ piAi = u.piAi;
193
+ }
194
+ let truncated = 0;
195
+ if (u.truncated !== undefined) {
196
+ if (!Number.isInteger(u.truncated) || u.truncated < 0) return null;
197
+ truncated = u.truncated;
198
+ }
199
+ // 1..9: up to 8 named rows plus at most one {other, other} fold row. Ten rows is not a bigger
200
+ // ledger, it is an emitter that broke its own envelope -- refuse the block whole.
201
+ if (!Array.isArray(u.models) || u.models.length < 1 || u.models.length > 9) return null;
202
+ const models = [];
203
+ for (const row of u.models) {
204
+ if (!row || typeof row !== "object" || Array.isArray(row)) return null;
205
+ if (typeof row.provider !== "string" || typeof row.model !== "string") return null;
206
+ // Lowercase BEFORE the allowlist, so "Anthropic" and "anthropic" are one id and the pattern
207
+ // itself never has to admit uppercase.
208
+ const provider = row.provider.toLowerCase();
209
+ const model = row.model.toLowerCase();
210
+ if (!USAGE_ID_PATTERN.test(provider) || !USAGE_ID_PATTERN.test(model)) return null;
211
+ const nums = {};
212
+ for (const key of USAGE_ROW_NUMERIC_KEYS) {
213
+ const value = row[key] === undefined ? 0 : row[key];
214
+ // null is present-and-wrong, not absent; JSON.parse cannot produce NaN but CAN produce
215
+ // Infinity (1e999), which is why the finite check is load-bearing, not decorative.
216
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return null;
217
+ nums[key] = value;
218
+ }
219
+ // The explicit 12-key literal: an unknown key on the emitted row is dropped HERE, by never
220
+ // being read -- the buildRecord no-spread posture, applied one level down.
221
+ models.push({
222
+ provider,
223
+ model,
224
+ calls: nums.calls,
225
+ input: nums.input,
226
+ output: nums.output,
227
+ cacheRead: nums.cacheRead,
228
+ cacheWrite: nums.cacheWrite,
229
+ cacheWrite1h: nums.cacheWrite1h,
230
+ reasoning: nums.reasoning,
231
+ total: nums.total,
232
+ cost: nums.cost,
233
+ unpriced: nums.unpriced,
234
+ });
235
+ }
236
+ return { v: u.v, piAi, truncated, models };
237
+ }
238
+
239
+ /**
240
+ * Build the durable run record from the full BullMQ job wrapper and the run's outcome.
241
+ *
242
+ * The record is id-only by construction: an EXPLICIT object literal that reads exactly the stable,
243
+ * non-PII fields and never spreads `job.data`, `result`, or `error`. Per `no-pii-in-logs` and
244
+ * `REQ-LOCAL-JOB-VISIBILITY`, a GitHub job's `title`/`body` and a local job's `task` and full `folder`
245
+ * path stay out of the record -- for local jobs only the folder's `basename` is kept, because the full
246
+ * path embeds the operator's OS account name.
247
+ *
248
+ * `reason` is a fixed enum passthrough (worker-abort | over-budget | unprotected-branch |
249
+ * runner-policy | job-image-missing | ...), never free-form or payload text. `exitCode`, `turns`, and `budgetReserved`
250
+ * default to `null` when the outcome does not carry them, so the record shape is stable whether or not
251
+ * the source reports those fields.
252
+ */
253
+ export function buildRecord({ job, result, error, startedAt, endedAt }) {
254
+ const data = job.data ?? {};
255
+ const kind = data.kind ?? job.name;
256
+ const source = result ?? error ?? {};
257
+ return {
258
+ jobId: job.id,
259
+ kind: kind ?? null,
260
+ target: targetFor(kind, data),
261
+ flow: data.flow ?? null,
262
+ startedAt: startedAt ?? null,
263
+ endedAt: endedAt ?? null,
264
+ outcome: result ? (result.outcome ?? null) : "failed",
265
+ reason: source.reason ?? null,
266
+ exitCode: source.exitCode ?? null,
267
+ turns: source.turns ?? null,
268
+ // Token accounting (INT-RUN-HISTORY-FILE-CONTRACT): additive and nullable, an explicit literal, no
269
+ // spread. The runner's per-job usage totals `{ input, output, total, cost }`, or null when the
270
+ // container died before the exit line. PII-free -- integer token counts and numeric cost only.
271
+ tokens: source.tokens ?? null,
272
+ // Usage ledger (INT-RUN-HISTORY-FILE-CONTRACT): all three additive and nullable on the
273
+ // tokens/session precedent, explicit literals, no spread. `usage` is the charset-validated
274
+ // per-model ledger recovered from the exit line exactly as `tokens` is -- but through
275
+ // parseExitUsage's REBUILD, so every provider/model id in it has already passed the lowercased
276
+ // allowlist before it can reach this literal. `provider`/`model` beside it are HOST-effective
277
+ // dispatch facts -- overlay-resolved by the processor, never container strings -- which is what
278
+ // lets a catch-path or pre-exit-line death still attribute the spend it incurred.
279
+ usage: source.usage ?? null,
280
+ provider: source.provider ?? null,
281
+ model: source.model ?? null,
282
+ budgetReserved: source.budgetReserved ?? null,
283
+ attempt: job.attemptsMade ?? 0,
284
+ // Chain telemetry (INT-RUN-HISTORY-FILE-CONTRACT): additive and nullable, explicit literals, no spread.
285
+ // parentJobId/chainDepth come from a chained child's own job.data; chainRefused counts a PARENT's
286
+ // /outbox requests that were refused. A chain refusal is pre-enqueue of the child, so the `reason` enum
287
+ // stays untouched -- chainRefused is a separate int count, never a terminal reason.
288
+ parentJobId: data.parentJobId ?? null,
289
+ chainDepth: data.chainDepth ?? null,
290
+ chainRefused: source.chainRefused ?? null,
291
+ // Replica telemetry (INT-RUN-HISTORY-FILE-CONTRACT, REQ-REPLICA-RUNS): additive and nullable, explicit
292
+ // literals beside the chain fields they mirror, no spread. Both come from this job's own `job.data`
293
+ // and both are INTEGERS -- which is why they can be here at all: the record is PII-free by
294
+ // construction and a replica index carries nothing attacker-chosen. The branch name they imply is
295
+ // deliberately not stored, for the reason `session` states one group below.
296
+ replica: data.replica ?? null,
297
+ replicas: data.replicas ?? null,
298
+ // Session telemetry (INT-RUN-HISTORY-FILE-CONTRACT): additive, nullable, an explicit literal, no
299
+ // spread. `{ resumed, reason, bytes }` -- a boolean, a fixed enum and an integer. THE KEY AND THE
300
+ // BRANCH NAME ARE DELIBERATELY ABSENT: this record's PII-free-by-construction property rests on it
301
+ // holding no attacker-chosen string, and a branch name is exactly that.
302
+ session: source.session ?? null,
303
+ };
304
+ }
305
+
306
+ /**
307
+ * A stable, non-PII target label. Forge jobs read `repo<sep>number`; local jobs read `local:<basename>` --
308
+ * basename only, so the full folder path (which on Windows carries the OS account name) never lands in
309
+ * the record.
310
+ *
311
+ * `#` serves an issue AND a pull request on GitHub because they share one per-repo number sequence, so
312
+ * `repo#7` names exactly one thing. That is a fact about GitHub, not about forges -- a forge with
313
+ * separate sequences needs the target type in the label or `repo#7` is ambiguous.
314
+ *
315
+ * That paragraph was here, correct, and unimplemented: the function enumerated `github` and returned null
316
+ * for everything else, so every GitLab run since #42 recorded `target: null` while
317
+ * INT-RUN-HISTORY-FILE-CONTRACT documented `<project>!<iid>`. Keyed on `isForgeKind` now, with the
318
+ * separator from the table, so the notation a forge uses is the notation its records carry -- and a forge
319
+ * added later inherits a label rather than a null.
320
+ */
321
+ function targetFor(kind, data) {
322
+ if (kind === "local") return `local:${basename(data.folder ?? "")}`;
323
+ if (isForgeKind(kind)) return `${data.repo}${targetSeparator(kind, data.target?.type)}${data.target?.number}`;
324
+ return null;
325
+ }
326
+
327
+ /** Retain only the last ~8KB of container output for turn recovery, so per-job memory stays flat. */
328
+ const TAIL_CAP_BYTES = 8 * 1024;
329
+
330
+ /**
331
+ * The durable log sink: the I/O layer that streams a job's raw container output to a per-job `.log`
332
+ * file and recovers the turn count from a bounded tail of that same output.
333
+ *
334
+ * Fault isolation is the contract, not a nicety. This sits on the money path -- `write` is a stdout
335
+ * `data` listener (run-container.mjs:52) and `close` runs at teardown -- so a throw or an unbounded
336
+ * await here corrupts the run outcome or turns a finished job into a 30-minute timeout
337
+ * (CONST-RETRY-INFRA-ONLY). Therefore `write` and `close` NEVER throw and `close` NEVER hangs: every
338
+ * body is try/catch-swallowed and the flush is a bounded race. This mirrors the "NEVER throws" posture
339
+ * of `makeReaper` and the comment adapter in `start.mjs`.
340
+ *
341
+ * The raw `.log` is written ONLY when `enabled`: raw container output is user-authored data, so it is
342
+ * opt-in per `no-pii-in-logs`. When disabled, no file is opened, but the bounded tail still accumulates
343
+ * so `close` can still report the turn count. The path stays host-side and never reaches the container.
344
+ *
345
+ * Memory is flat regardless of job length: the tail keeps only the last `TAIL_CAP_BYTES`, so a 200-turn
346
+ * job does not accumulate megabytes (REQ-QUEUE-BURST-NO-DROP). The filename runs through `sanitizeJobId`
347
+ * because scheduled ids carry a colon, illegal on NTFS. `fs` is injectable so the sink is testable with
348
+ * a fake writable and no disk.
349
+ */
350
+ export function makeLogSink({ logsDir, enabled, fs = nodeFs, log = () => {} }) {
351
+ try {
352
+ fs.mkdirSync(logsDir, { recursive: true });
353
+ } catch (err) {
354
+ log("logs_dir_error", { reason: err?.message });
355
+ }
356
+
357
+ return function openJobLog(jobId) {
358
+ let tail = "";
359
+ let stream = null;
360
+
361
+ function write(chunk) {
362
+ try {
363
+ tail += typeof chunk === "string" ? chunk : chunk.toString("utf8");
364
+ if (tail.length > TAIL_CAP_BYTES) tail = tail.slice(tail.length - TAIL_CAP_BYTES);
365
+ if (!enabled) return;
366
+ if (stream === null) {
367
+ stream = fs.createWriteStream(join(logsDir, `${sanitizeJobId(jobId)}.log`), { flags: "a" });
368
+ // Attach before the first write so an EPIPE/ENOSPC/EACCES surfaces as a log line rather
369
+ // than an unhandled 'error' that kills the worker.
370
+ stream.on("error", (e) => log("log_sink_error", { jobId, reason: e?.message }));
371
+ }
372
+ stream.write(chunk);
373
+ } catch (err) {
374
+ log("log_sink_error", { jobId, reason: err?.message });
375
+ }
376
+ }
377
+
378
+ async function close({ timeoutMs = 2000 } = {}) {
379
+ // Capture turns/tokens/session/usage from the tail first, so they survive even if the flush errors or times out.
380
+ const turns = parseExitTurns(tail);
381
+ const tokens = parseExitTokens(tail);
382
+ const session = parseExitSession(tail);
383
+ const usage = parseExitUsage(tail);
384
+ try {
385
+ if (stream !== null) {
386
+ const s = stream;
387
+ s.end();
388
+ let timer;
389
+ const timeout = new Promise((resolve) => {
390
+ timer = setTimeout(resolve, timeoutMs);
391
+ });
392
+ try {
393
+ // Bounded race: resolve on flush completion, on stream error, or on the deadline --
394
+ // whichever is first. An unbounded finish-await would turn a completed job into a timeout.
395
+ await Promise.race([
396
+ new Promise((resolve) => s.once("finish", resolve)),
397
+ new Promise((resolve) => s.once("error", resolve)),
398
+ timeout,
399
+ ]);
400
+ } finally {
401
+ clearTimeout(timer);
402
+ }
403
+ }
404
+ } catch (err) {
405
+ log("log_sink_error", { jobId, reason: err?.message });
406
+ }
407
+ return { turns, tokens, session, usage };
408
+ }
409
+
410
+ return { write, close };
411
+ };
412
+ }
413
+
414
+ /**
415
+ * The durable record writer: serialises a finished run's `buildRecord` output to a per-job JSON sidecar.
416
+ *
417
+ * Fault isolation is the contract. The processor wrapper calls `writeRecord` on the money path, so a
418
+ * throw here corrupts a run outcome or turns a finished job into a retry (CONST-RETRY-INFRA-ONLY).
419
+ * Therefore construction and `writeRecord` NEVER throw: the whole write body is try/catch-swallowed,
420
+ * mirroring `makeLogSink`.
421
+ *
422
+ * The write is SYNCHRONOUS by design: it must complete before `process.exit(0)` on shutdown
423
+ * (`index.mjs:83`), and an async write loses that race. `fs.writeFileSync` truncates by default, so a
424
+ * re-run of the same job id overwrites -- last write wins.
425
+ *
426
+ * The failure log carries only `jobId` and `reason`; never the record object or its serialized JSON,
427
+ * which embed target/flow fields (`no-pii-in-logs`). `sanitizeJobId` is applied only at the filename
428
+ * boundary; the record body keeps the raw id. `fs` is injectable so the writer is testable with a fake
429
+ * and no disk.
430
+ */
431
+ export function makeRecordWriter({ logsDir, fs = nodeFs, log = () => {} }) {
432
+ try {
433
+ fs.mkdirSync(logsDir, { recursive: true });
434
+ } catch (err) {
435
+ log("logs_dir_error", { reason: err?.message });
436
+ }
437
+
438
+ return function writeRecord(record) {
439
+ try {
440
+ // Custom: flat JSON sidecar per job over a DB/logging lib -- records are immutable, filename-keyed
441
+ // by jobId, no cross-record queries; DES-RUN-HISTORY-FLAT-FILES-NO-DB; specs/interfaces.md:11
442
+ // (there is deliberately no database).
443
+ const path = join(logsDir, `${sanitizeJobId(record.jobId)}.json`);
444
+ const data = `${JSON.stringify(record)}\n`;
445
+ fs.writeFileSync(path, data);
446
+ } catch (err) {
447
+ log("run_record_failed", { jobId: record?.jobId, reason: err?.message });
448
+ }
449
+ };
450
+ }
451
+
452
+ /**
453
+ * Look up when the previous run of a given scheduler ended, for the cron `/job/event.json`
454
+ * (INT-CONTAINER-JOB-INPUTS).
455
+ *
456
+ * This reads the same per-job files INT-RUN-HISTORY-FILE-CONTRACT defines -- filename-keyed sidecars,
457
+ * no new query surface. A scheduled id is `repeat:<schedulerId>:<millis>` (DES-CRON-VIA-BULLMQ-SCHEDULER),
458
+ * which `makeRecordWriter` stores as `repeat_<schedulerId>_<millis>.json` via `sanitizeJobId`, so the
459
+ * scheduler's prior fires are exactly the files matching that prefix with a pure-digits trailing
460
+ * segment. The digits requirement is what disambiguates a scheduler id that itself contains `_`
461
+ * (schedulers "a" vs "a_1": `repeat_a_1_100.json` has a non-digit tail after "repeat_a_", so it never
462
+ * matches scheduler "a"). The max millis strictly below `beforeMillis` is the previous fire.
463
+ *
464
+ * Returns `record.endedAt ?? record.startedAt ?? null` as an ISO string. `endedAt` first: BullMQ never
465
+ * overlaps two fires of one scheduler, so the prior run's end is the honest high-water mark; `startedAt`
466
+ * covers a crashed run's partial record. ANY failure -- missing dir, no prior run, unreadable file, bad
467
+ * JSON, nullish `beforeMillis` -- yields `null`; the function NEVER throws (the module's fault-isolation
468
+ * posture: this feeds a job input, and a history blip must not fail a prepare). `fs` is injectable so
469
+ * the lookup is testable with a fake and no disk.
470
+ */
471
+ export function makeFindPreviousRun({ logsDir, fs = nodeFs }) {
472
+ // The parameter default keeps the never-throw promise even for an argument-less call: destructuring
473
+ // binds before the try below, so without it `findPreviousRun()` would TypeError past the catch.
474
+ return function findPreviousRun({ schedulerId, beforeMillis } = {}) {
475
+ try {
476
+ if (typeof beforeMillis !== "number" || !Number.isFinite(beforeMillis)) return null;
477
+ const prefix = sanitizeJobId(`repeat:${schedulerId}:`);
478
+ const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
479
+ const pattern = new RegExp(`^${escaped}(\\d+)\\.json$`);
480
+ let best = null;
481
+ for (const name of fs.readdirSync(logsDir)) {
482
+ const m = pattern.exec(name);
483
+ if (m === null) continue;
484
+ const millis = Number(m[1]);
485
+ if (!Number.isFinite(millis) || millis >= beforeMillis) continue;
486
+ if (best === null || millis > best.millis) best = { millis, name };
487
+ }
488
+ if (best === null) return null;
489
+ const record = JSON.parse(fs.readFileSync(join(logsDir, best.name), "utf8"));
490
+ return record.endedAt ?? record.startedAt ?? null;
491
+ } catch {
492
+ return null; // NEVER throws: a history fault must not fail the prepare that asked
493
+ }
494
+ };
495
+ }
496
+
497
+ /**
498
+ * The durable log reaper: a boot-time sweep that deletes `.log` and `.json` history files older than
499
+ * the retention window, keeping the logs directory bounded across restarts.
500
+ *
501
+ * Fault isolation is the contract, mirroring `makeReaper` in `start.mjs`: `reapLogs` NEVER throws under
502
+ * any input. A missing logs directory on first boot (`readdirSync` ENOENT), an unreadable entry, or an
503
+ * unlink failure is caught -- logged as `log_reaper_skipped` -- and boot continues. The per-file
504
+ * try/catch is what keeps one bad entry from aborting the whole sweep.
505
+ *
506
+ * `retentionDays === 0` is the documented keep-forever sentinel: the sweep returns early and touches no
507
+ * file. Age comes from the file's `mtimeMs`, never from any date parsed out of the filename -- the
508
+ * on-disk mtime is the authority. The `isFile()` guard skips a stray `foo.log/` directory so a
509
+ * mis-shaped entry raises no EISDIR/EPERM. `fs` and `now` are injectable so the reaper is testable with
510
+ * a fake and no disk.
511
+ */
512
+ export function makeLogReaper({ logsDir, retentionDays, fs = nodeFs, log = () => {}, now = () => Date.now() }) {
513
+ return function reapLogs() {
514
+ if (retentionDays === 0) return; // keep-forever sentinel: no sweep
515
+ try {
516
+ const names = fs.readdirSync(logsDir);
517
+ const cutoff = now() - retentionDays * 86400000;
518
+ for (const name of names) {
519
+ if (!name.endsWith(".log") && !name.endsWith(".json")) continue;
520
+ try {
521
+ const st = fs.statSync(join(logsDir, name));
522
+ if (st.isFile() && st.mtimeMs < cutoff) {
523
+ fs.unlinkSync(join(logsDir, name));
524
+ log("reaped_log", { file: name });
525
+ }
526
+ } catch (err) {
527
+ log("log_reaper_skipped", { file: name, reason: err?.message });
528
+ }
529
+ }
530
+ } catch (err) {
531
+ log("log_reaper_skipped", { reason: err?.message });
532
+ }
533
+ };
534
+ }