@edgehero/pi-dispatch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +160 -0
- package/deploy/com.pi-dispatch.worker.plist +66 -0
- package/deploy/nssm-install.cmd +59 -0
- package/deploy/receiver.service +36 -0
- package/deploy/worker-env-wrapper.cmd +50 -0
- package/deploy/worker-env-wrapper.sh +63 -0
- package/deploy/worker.service +55 -0
- package/package.json +83 -0
- package/src/azure-auth.mjs +61 -0
- package/src/azure-host.mjs +236 -0
- package/src/azure-identity.mjs +63 -0
- package/src/azure-prompt.mjs +118 -0
- package/src/branch.mjs +80 -0
- package/src/budget.mjs +179 -0
- package/src/cli.mjs +208 -0
- package/src/config.mjs +329 -0
- package/src/connection.mjs +40 -0
- package/src/cron.mjs +94 -0
- package/src/docker-run.mjs +119 -0
- package/src/doctor.mjs +1127 -0
- package/src/env-allowlist.mjs +198 -0
- package/src/env-file.mjs +153 -0
- package/src/exit-code.mjs +32 -0
- package/src/flow-gate.mjs +82 -0
- package/src/forgejo-auth.mjs +77 -0
- package/src/forgejo-host.mjs +172 -0
- package/src/forgejo-identity.mjs +74 -0
- package/src/forgejo-prompt.mjs +123 -0
- package/src/forges.mjs +148 -0
- package/src/get-token.mjs +226 -0
- package/src/git-dirty.mjs +16 -0
- package/src/github-app-setup.mjs +517 -0
- package/src/github-host.mjs +159 -0
- package/src/github-prompt.mjs +286 -0
- package/src/gitlab-auth.mjs +72 -0
- package/src/gitlab-host.mjs +200 -0
- package/src/gitlab-identity.mjs +61 -0
- package/src/gitlab-prompt.mjs +123 -0
- package/src/identity.mjs +57 -0
- package/src/image-preflight.mjs +180 -0
- package/src/import-pi.mjs +451 -0
- package/src/index.mjs +177 -0
- package/src/init.mjs +77 -0
- package/src/job-id.mjs +100 -0
- package/src/materialize.mjs +138 -0
- package/src/outbox.mjs +179 -0
- package/src/packages.mjs +188 -0
- package/src/pause-windows.mjs +218 -0
- package/src/prepare-github.mjs +260 -0
- package/src/prepare-local.mjs +76 -0
- package/src/prepare.mjs +199 -0
- package/src/pricing.mjs +168 -0
- package/src/processor.mjs +360 -0
- package/src/queue.mjs +152 -0
- package/src/run-container.mjs +133 -0
- package/src/run-history.mjs +534 -0
- package/src/runtime-settings.mjs +188 -0
- package/src/sandbox-cli.mjs +156 -0
- package/src/sandbox-store.mjs +269 -0
- package/src/sandbox.mjs +171 -0
- package/src/scheduler-stall-guard.mjs +67 -0
- package/src/schedules.mjs +62 -0
- package/src/service.mjs +677 -0
- package/src/session-key.mjs +108 -0
- package/src/session-store.mjs +249 -0
- package/src/start.mjs +502 -0
- package/src/subscriptions.mjs +208 -0
- package/src/triggers.mjs +491 -0
- package/src/up.mjs +315 -0
|
@@ -0,0 +1,534 @@
|
|
|
1
|
+
import * as nodeFs from "node:fs";
|
|
2
|
+
import { basename, join } from "node:path";
|
|
3
|
+
import { isForgeKind, targetSeparator } from "./forges.mjs";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Durable per-run history.
|
|
7
|
+
*
|
|
8
|
+
* The PURE HELPERS are total functions over their arguments -- no filesystem, no clock, no
|
|
9
|
+
* `process.env`, no randomness -- so the record shape and the filename/telemetry parsing are testable
|
|
10
|
+
* without a container, a queue, or a disk: `sanitizeJobId`, `parseExitTurns`, `parseExitTokens`,
|
|
11
|
+
* `parseExitSession`, `parseExitUsage`, `buildRecord`.
|
|
12
|
+
*
|
|
13
|
+
* The I/O factories -- `makeLogSink`, `makeRecordWriter`, `makeFindPreviousRun`, `makeLogReaper` --
|
|
14
|
+
* each inject their own `fs` (and, for the reaper, their own clock via `now`), so they too are testable
|
|
15
|
+
* with a fake and no disk. `makeLogSink` streams a job's raw output to a per-job `.log` and recovers the
|
|
16
|
+
* turn count from a bounded tail; `makeRecordWriter` serialises a finished run to a JSON sidecar;
|
|
17
|
+
* `makeFindPreviousRun` reads a scheduler's most recent prior sidecar back; `makeLogReaper` sweeps aged
|
|
18
|
+
* `.log`/`.json` files at boot.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Turn an arbitrary job id into a single legal filename segment.
|
|
23
|
+
*
|
|
24
|
+
* BullMQ scheduled ids are `repeat:<schedulerId>:<millis>`; a colon is an illegal NTFS filename
|
|
25
|
+
* character, so writing `<id>.json` verbatim throws on Windows. A strict allowlist (letters, digits,
|
|
26
|
+
* dot, underscore, hyphen) is the safe subset across Windows and POSIX; everything else collapses to
|
|
27
|
+
* `_`. A nullish or empty id yields a fixed sentinel so a downstream writer still produces a file
|
|
28
|
+
* rather than silently dropping the record.
|
|
29
|
+
*/
|
|
30
|
+
export function sanitizeJobId(id) {
|
|
31
|
+
if (id === null || id === undefined) return "unknown-job";
|
|
32
|
+
const s = String(id);
|
|
33
|
+
if (s === "") return "unknown-job";
|
|
34
|
+
return s.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Recover the agent's turn count from buffered container stdout, or `null` if it is not reported.
|
|
39
|
+
*
|
|
40
|
+
* The stream interleaves docker/agent noise and other JSON events (`pi_auto_retry`) with the runner's
|
|
41
|
+
* own lines. Only the success exit line carries `turns` (`image/runner/run-job.mjs:263`); the
|
|
42
|
+
* catch-path exit line (`:277`) omits it. Scan from the end and return the turns of the last `exit`
|
|
43
|
+
* event that reports an integer count.
|
|
44
|
+
*
|
|
45
|
+
* This is read-only telemetry: it MUST NEVER throw and MUST NOT feed exit-code or retry
|
|
46
|
+
* classification -- that is the container exit code's job (INT-RUNNER-EXIT-CODE-PROTOCOL). Every parse
|
|
47
|
+
* is guarded; a truncated or non-JSON line is skipped.
|
|
48
|
+
*/
|
|
49
|
+
export function parseExitTurns(text) {
|
|
50
|
+
if (typeof text !== "string") return null;
|
|
51
|
+
const lines = text.split("\n");
|
|
52
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
53
|
+
const line = lines[i].trim();
|
|
54
|
+
if (line === "") continue;
|
|
55
|
+
let parsed;
|
|
56
|
+
try {
|
|
57
|
+
parsed = JSON.parse(line);
|
|
58
|
+
} catch {
|
|
59
|
+
continue; // docker/agent noise or a truncated final line
|
|
60
|
+
}
|
|
61
|
+
if (parsed?.event !== "exit") continue;
|
|
62
|
+
return Number.isInteger(parsed?.turns) ? parsed.turns : null;
|
|
63
|
+
}
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Recover the agent's token usage from buffered container stdout, or `null` if it is not reported.
|
|
69
|
+
*
|
|
70
|
+
* Mirrors `parseExitTurns`: scan from the end for the last `exit` event and read its `tokens` object
|
|
71
|
+
* (`{ input, output, total, cost }`). Only the success exit line carries it
|
|
72
|
+
* (`image/runner/run-job.mjs`); the catch-path exit line omits it, and a container that died before the
|
|
73
|
+
* runner's exit line yields none -- all three cases are `null`.
|
|
74
|
+
*
|
|
75
|
+
* Read-only telemetry, exactly like `parseExitTurns`: NEVER throws and MUST NOT feed exit-code or retry
|
|
76
|
+
* classification (INT-RUNNER-EXIT-CODE-PROTOCOL). A malformed or non-object `tokens` (or one missing a
|
|
77
|
+
* numeric `total`) is `null`, never a partial that could poison the daily token counter.
|
|
78
|
+
*/
|
|
79
|
+
/**
|
|
80
|
+
* The runner's `session` object off the exit line: `{ resumed: <bool>, reason: "<enum>" }` or null when
|
|
81
|
+
* the container died before emitting one (REQ-RESUMABLE-SESSION).
|
|
82
|
+
*
|
|
83
|
+
* A sibling of parseExitTokens rather than a widening of it, and it reports what pi ACTUALLY did. The
|
|
84
|
+
* host records its own intent separately, and the pair is the point: a host that staged a transcript
|
|
85
|
+
* while the container reports `resumed: false` is a real event -- a corrupt file, a degrade -- and
|
|
86
|
+
* without both numbers it is indistinguishable from an ordinary cold start. A feature that fails open
|
|
87
|
+
* must still say that it did.
|
|
88
|
+
*
|
|
89
|
+
* PII-free by construction: a boolean and a fixed enum. No key, no branch name, no path.
|
|
90
|
+
*/
|
|
91
|
+
export function parseExitSession(text) {
|
|
92
|
+
if (typeof text !== "string") return null;
|
|
93
|
+
const lines = text.split("\n");
|
|
94
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
95
|
+
const line = lines[i].trim();
|
|
96
|
+
if (line === "") continue;
|
|
97
|
+
let parsed;
|
|
98
|
+
try {
|
|
99
|
+
parsed = JSON.parse(line);
|
|
100
|
+
} catch {
|
|
101
|
+
continue; // docker/agent noise or a truncated final line
|
|
102
|
+
}
|
|
103
|
+
if (parsed?.event !== "exit") continue;
|
|
104
|
+
const sess = parsed?.session;
|
|
105
|
+
if (sess && typeof sess === "object" && !Array.isArray(sess) && typeof sess.resumed === "boolean") {
|
|
106
|
+
return { resumed: sess.resumed, reason: typeof sess.reason === "string" ? sess.reason : null };
|
|
107
|
+
}
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function parseExitTokens(text) {
|
|
114
|
+
if (typeof text !== "string") return null;
|
|
115
|
+
const lines = text.split("\n");
|
|
116
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
117
|
+
const line = lines[i].trim();
|
|
118
|
+
if (line === "") continue;
|
|
119
|
+
let parsed;
|
|
120
|
+
try {
|
|
121
|
+
parsed = JSON.parse(line);
|
|
122
|
+
} catch {
|
|
123
|
+
continue; // docker/agent noise or a truncated final line
|
|
124
|
+
}
|
|
125
|
+
if (parsed?.event !== "exit") continue;
|
|
126
|
+
const t = parsed?.tokens;
|
|
127
|
+
if (t && typeof t === "object" && !Array.isArray(t) && typeof t.total === "number") return t;
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** The id allowlist for a ledger row's provider/model, applied AFTER lowercasing. The first-char class
|
|
134
|
+
* has no dot, colon or slash, so `.hidden`, `../etc` and `:` shapes fail at character one. */
|
|
135
|
+
const USAGE_ID_PATTERN = /^[a-z0-9][a-z0-9._:/-]{0,63}$/;
|
|
136
|
+
|
|
137
|
+
/** The ten per-row counters, in the row's serialisation order. Absent is an honest zero; anything
|
|
138
|
+
* present must be a finite non-negative number or the whole block is refused. */
|
|
139
|
+
const USAGE_ROW_NUMERIC_KEYS = ["calls", "input", "output", "cacheRead", "cacheWrite", "cacheWrite1h", "reasoning", "total", "cost", "unpriced"];
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* The runner's per-(provider,model) usage ledger off the exit line -- `{ v, piAi, truncated, models }`
|
|
143
|
+
* -- REBUILT and validated, or `null` (INT-RUN-HISTORY-FILE-CONTRACT).
|
|
144
|
+
*
|
|
145
|
+
* A sibling of `parseExitSession` rather than a widening of `parseExitTokens`, for the same reason that
|
|
146
|
+
* one was: `tokens` may ride through verbatim only because it holds nothing but numbers, while this
|
|
147
|
+
* block's `provider`/`model` strings are container-emitted and therefore attacker-adjacent. So nothing
|
|
148
|
+
* here is stored as received. Every field is validated and re-written into an explicit literal: ids are
|
|
149
|
+
* lowercased and held to a strict allowlist, the ten per-row counters to finite non-negatives, and ANY
|
|
150
|
+
* violation nulls the WHOLE block, never a partial -- the same malformed->null rule `parseExitTokens`
|
|
151
|
+
* applies to the daily counter, because a half-validated ledger is how sums stop meaning anything.
|
|
152
|
+
*
|
|
153
|
+
* An absent `usage` is the NORMAL case, not an error: the metered:false fallback, the catch-path exit
|
|
154
|
+
* line and a pre-ledger runner image all omit it, and a container may die before any exit line at all.
|
|
155
|
+
*
|
|
156
|
+
* Cross-field sums (the rows against `tokens.total`) are deliberately NOT checked here. The sum is the
|
|
157
|
+
* EMITTER's invariant, asserted where the emitter lives -- in the runner's own meter tests. The host
|
|
158
|
+
* validates fields, not bookkeeping; re-deriving the arithmetic on this side would only manufacture a
|
|
159
|
+
* second source of truth for the first one to drift from.
|
|
160
|
+
*
|
|
161
|
+
* Read-only telemetry, exactly like its siblings: NEVER throws and MUST NOT feed exit-code or retry
|
|
162
|
+
* classification (INT-RUNNER-EXIT-CODE-PROTOCOL).
|
|
163
|
+
*/
|
|
164
|
+
export function parseExitUsage(text) {
|
|
165
|
+
if (typeof text !== "string") return null;
|
|
166
|
+
const lines = text.split("\n");
|
|
167
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
168
|
+
const line = lines[i].trim();
|
|
169
|
+
if (line === "") continue;
|
|
170
|
+
let parsed;
|
|
171
|
+
try {
|
|
172
|
+
parsed = JSON.parse(line);
|
|
173
|
+
} catch {
|
|
174
|
+
continue; // docker/agent noise or a truncated final line
|
|
175
|
+
}
|
|
176
|
+
if (parsed?.event !== "exit") continue;
|
|
177
|
+
return rebuildUsage(parsed?.usage);
|
|
178
|
+
}
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** The validating rebuild behind `parseExitUsage`: explicit literals only, null on ANY violation. */
|
|
183
|
+
function rebuildUsage(u) {
|
|
184
|
+
if (!u || typeof u !== "object" || Array.isArray(u)) return null;
|
|
185
|
+
// v is kept as-is once it passes: readers treat an unknown version as opaque-but-present.
|
|
186
|
+
if (!Number.isInteger(u.v) || u.v < 1) return null;
|
|
187
|
+
let piAi = null;
|
|
188
|
+
if (u.piAi !== undefined && u.piAi !== null) {
|
|
189
|
+
// A rates-provenance stamp is three dot-joined integers or nothing -- any other string is a
|
|
190
|
+
// malformed block, not a value to store.
|
|
191
|
+
if (typeof u.piAi !== "string" || !/^\d+\.\d+\.\d+$/.test(u.piAi)) return null;
|
|
192
|
+
piAi = u.piAi;
|
|
193
|
+
}
|
|
194
|
+
let truncated = 0;
|
|
195
|
+
if (u.truncated !== undefined) {
|
|
196
|
+
if (!Number.isInteger(u.truncated) || u.truncated < 0) return null;
|
|
197
|
+
truncated = u.truncated;
|
|
198
|
+
}
|
|
199
|
+
// 1..9: up to 8 named rows plus at most one {other, other} fold row. Ten rows is not a bigger
|
|
200
|
+
// ledger, it is an emitter that broke its own envelope -- refuse the block whole.
|
|
201
|
+
if (!Array.isArray(u.models) || u.models.length < 1 || u.models.length > 9) return null;
|
|
202
|
+
const models = [];
|
|
203
|
+
for (const row of u.models) {
|
|
204
|
+
if (!row || typeof row !== "object" || Array.isArray(row)) return null;
|
|
205
|
+
if (typeof row.provider !== "string" || typeof row.model !== "string") return null;
|
|
206
|
+
// Lowercase BEFORE the allowlist, so "Anthropic" and "anthropic" are one id and the pattern
|
|
207
|
+
// itself never has to admit uppercase.
|
|
208
|
+
const provider = row.provider.toLowerCase();
|
|
209
|
+
const model = row.model.toLowerCase();
|
|
210
|
+
if (!USAGE_ID_PATTERN.test(provider) || !USAGE_ID_PATTERN.test(model)) return null;
|
|
211
|
+
const nums = {};
|
|
212
|
+
for (const key of USAGE_ROW_NUMERIC_KEYS) {
|
|
213
|
+
const value = row[key] === undefined ? 0 : row[key];
|
|
214
|
+
// null is present-and-wrong, not absent; JSON.parse cannot produce NaN but CAN produce
|
|
215
|
+
// Infinity (1e999), which is why the finite check is load-bearing, not decorative.
|
|
216
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) return null;
|
|
217
|
+
nums[key] = value;
|
|
218
|
+
}
|
|
219
|
+
// The explicit 12-key literal: an unknown key on the emitted row is dropped HERE, by never
|
|
220
|
+
// being read -- the buildRecord no-spread posture, applied one level down.
|
|
221
|
+
models.push({
|
|
222
|
+
provider,
|
|
223
|
+
model,
|
|
224
|
+
calls: nums.calls,
|
|
225
|
+
input: nums.input,
|
|
226
|
+
output: nums.output,
|
|
227
|
+
cacheRead: nums.cacheRead,
|
|
228
|
+
cacheWrite: nums.cacheWrite,
|
|
229
|
+
cacheWrite1h: nums.cacheWrite1h,
|
|
230
|
+
reasoning: nums.reasoning,
|
|
231
|
+
total: nums.total,
|
|
232
|
+
cost: nums.cost,
|
|
233
|
+
unpriced: nums.unpriced,
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
return { v: u.v, piAi, truncated, models };
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Build the durable run record from the full BullMQ job wrapper and the run's outcome.
|
|
241
|
+
*
|
|
242
|
+
* The record is id-only by construction: an EXPLICIT object literal that reads exactly the stable,
|
|
243
|
+
* non-PII fields and never spreads `job.data`, `result`, or `error`. Per `no-pii-in-logs` and
|
|
244
|
+
* `REQ-LOCAL-JOB-VISIBILITY`, a GitHub job's `title`/`body` and a local job's `task` and full `folder`
|
|
245
|
+
* path stay out of the record -- for local jobs only the folder's `basename` is kept, because the full
|
|
246
|
+
* path embeds the operator's OS account name.
|
|
247
|
+
*
|
|
248
|
+
* `reason` is a fixed enum passthrough (worker-abort | over-budget | unprotected-branch |
|
|
249
|
+
* runner-policy | job-image-missing | ...), never free-form or payload text. `exitCode`, `turns`, and `budgetReserved`
|
|
250
|
+
* default to `null` when the outcome does not carry them, so the record shape is stable whether or not
|
|
251
|
+
* the source reports those fields.
|
|
252
|
+
*/
|
|
253
|
+
export function buildRecord({ job, result, error, startedAt, endedAt }) {
|
|
254
|
+
const data = job.data ?? {};
|
|
255
|
+
const kind = data.kind ?? job.name;
|
|
256
|
+
const source = result ?? error ?? {};
|
|
257
|
+
return {
|
|
258
|
+
jobId: job.id,
|
|
259
|
+
kind: kind ?? null,
|
|
260
|
+
target: targetFor(kind, data),
|
|
261
|
+
flow: data.flow ?? null,
|
|
262
|
+
startedAt: startedAt ?? null,
|
|
263
|
+
endedAt: endedAt ?? null,
|
|
264
|
+
outcome: result ? (result.outcome ?? null) : "failed",
|
|
265
|
+
reason: source.reason ?? null,
|
|
266
|
+
exitCode: source.exitCode ?? null,
|
|
267
|
+
turns: source.turns ?? null,
|
|
268
|
+
// Token accounting (INT-RUN-HISTORY-FILE-CONTRACT): additive and nullable, an explicit literal, no
|
|
269
|
+
// spread. The runner's per-job usage totals `{ input, output, total, cost }`, or null when the
|
|
270
|
+
// container died before the exit line. PII-free -- integer token counts and numeric cost only.
|
|
271
|
+
tokens: source.tokens ?? null,
|
|
272
|
+
// Usage ledger (INT-RUN-HISTORY-FILE-CONTRACT): all three additive and nullable on the
|
|
273
|
+
// tokens/session precedent, explicit literals, no spread. `usage` is the charset-validated
|
|
274
|
+
// per-model ledger recovered from the exit line exactly as `tokens` is -- but through
|
|
275
|
+
// parseExitUsage's REBUILD, so every provider/model id in it has already passed the lowercased
|
|
276
|
+
// allowlist before it can reach this literal. `provider`/`model` beside it are HOST-effective
|
|
277
|
+
// dispatch facts -- overlay-resolved by the processor, never container strings -- which is what
|
|
278
|
+
// lets a catch-path or pre-exit-line death still attribute the spend it incurred.
|
|
279
|
+
usage: source.usage ?? null,
|
|
280
|
+
provider: source.provider ?? null,
|
|
281
|
+
model: source.model ?? null,
|
|
282
|
+
budgetReserved: source.budgetReserved ?? null,
|
|
283
|
+
attempt: job.attemptsMade ?? 0,
|
|
284
|
+
// Chain telemetry (INT-RUN-HISTORY-FILE-CONTRACT): additive and nullable, explicit literals, no spread.
|
|
285
|
+
// parentJobId/chainDepth come from a chained child's own job.data; chainRefused counts a PARENT's
|
|
286
|
+
// /outbox requests that were refused. A chain refusal is pre-enqueue of the child, so the `reason` enum
|
|
287
|
+
// stays untouched -- chainRefused is a separate int count, never a terminal reason.
|
|
288
|
+
parentJobId: data.parentJobId ?? null,
|
|
289
|
+
chainDepth: data.chainDepth ?? null,
|
|
290
|
+
chainRefused: source.chainRefused ?? null,
|
|
291
|
+
// Replica telemetry (INT-RUN-HISTORY-FILE-CONTRACT, REQ-REPLICA-RUNS): additive and nullable, explicit
|
|
292
|
+
// literals beside the chain fields they mirror, no spread. Both come from this job's own `job.data`
|
|
293
|
+
// and both are INTEGERS -- which is why they can be here at all: the record is PII-free by
|
|
294
|
+
// construction and a replica index carries nothing attacker-chosen. The branch name they imply is
|
|
295
|
+
// deliberately not stored, for the reason `session` states one group below.
|
|
296
|
+
replica: data.replica ?? null,
|
|
297
|
+
replicas: data.replicas ?? null,
|
|
298
|
+
// Session telemetry (INT-RUN-HISTORY-FILE-CONTRACT): additive, nullable, an explicit literal, no
|
|
299
|
+
// spread. `{ resumed, reason, bytes }` -- a boolean, a fixed enum and an integer. THE KEY AND THE
|
|
300
|
+
// BRANCH NAME ARE DELIBERATELY ABSENT: this record's PII-free-by-construction property rests on it
|
|
301
|
+
// holding no attacker-chosen string, and a branch name is exactly that.
|
|
302
|
+
session: source.session ?? null,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* A stable, non-PII target label. Forge jobs read `repo<sep>number`; local jobs read `local:<basename>` --
|
|
308
|
+
* basename only, so the full folder path (which on Windows carries the OS account name) never lands in
|
|
309
|
+
* the record.
|
|
310
|
+
*
|
|
311
|
+
* `#` serves an issue AND a pull request on GitHub because they share one per-repo number sequence, so
|
|
312
|
+
* `repo#7` names exactly one thing. That is a fact about GitHub, not about forges -- a forge with
|
|
313
|
+
* separate sequences needs the target type in the label or `repo#7` is ambiguous.
|
|
314
|
+
*
|
|
315
|
+
* That paragraph was here, correct, and unimplemented: the function enumerated `github` and returned null
|
|
316
|
+
* for everything else, so every GitLab run since #42 recorded `target: null` while
|
|
317
|
+
* INT-RUN-HISTORY-FILE-CONTRACT documented `<project>!<iid>`. Keyed on `isForgeKind` now, with the
|
|
318
|
+
* separator from the table, so the notation a forge uses is the notation its records carry -- and a forge
|
|
319
|
+
* added later inherits a label rather than a null.
|
|
320
|
+
*/
|
|
321
|
+
function targetFor(kind, data) {
|
|
322
|
+
if (kind === "local") return `local:${basename(data.folder ?? "")}`;
|
|
323
|
+
if (isForgeKind(kind)) return `${data.repo}${targetSeparator(kind, data.target?.type)}${data.target?.number}`;
|
|
324
|
+
return null;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** Retain only the last ~8KB of container output for turn recovery, so per-job memory stays flat. */
|
|
328
|
+
const TAIL_CAP_BYTES = 8 * 1024;
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* The durable log sink: the I/O layer that streams a job's raw container output to a per-job `.log`
|
|
332
|
+
* file and recovers the turn count from a bounded tail of that same output.
|
|
333
|
+
*
|
|
334
|
+
* Fault isolation is the contract, not a nicety. This sits on the money path -- `write` is a stdout
|
|
335
|
+
* `data` listener (run-container.mjs:52) and `close` runs at teardown -- so a throw or an unbounded
|
|
336
|
+
* await here corrupts the run outcome or turns a finished job into a 30-minute timeout
|
|
337
|
+
* (CONST-RETRY-INFRA-ONLY). Therefore `write` and `close` NEVER throw and `close` NEVER hangs: every
|
|
338
|
+
* body is try/catch-swallowed and the flush is a bounded race. This mirrors the "NEVER throws" posture
|
|
339
|
+
* of `makeReaper` and the comment adapter in `start.mjs`.
|
|
340
|
+
*
|
|
341
|
+
* The raw `.log` is written ONLY when `enabled`: raw container output is user-authored data, so it is
|
|
342
|
+
* opt-in per `no-pii-in-logs`. When disabled, no file is opened, but the bounded tail still accumulates
|
|
343
|
+
* so `close` can still report the turn count. The path stays host-side and never reaches the container.
|
|
344
|
+
*
|
|
345
|
+
* Memory is flat regardless of job length: the tail keeps only the last `TAIL_CAP_BYTES`, so a 200-turn
|
|
346
|
+
* job does not accumulate megabytes (REQ-QUEUE-BURST-NO-DROP). The filename runs through `sanitizeJobId`
|
|
347
|
+
* because scheduled ids carry a colon, illegal on NTFS. `fs` is injectable so the sink is testable with
|
|
348
|
+
* a fake writable and no disk.
|
|
349
|
+
*/
|
|
350
|
+
export function makeLogSink({ logsDir, enabled, fs = nodeFs, log = () => {} }) {
|
|
351
|
+
try {
|
|
352
|
+
fs.mkdirSync(logsDir, { recursive: true });
|
|
353
|
+
} catch (err) {
|
|
354
|
+
log("logs_dir_error", { reason: err?.message });
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
return function openJobLog(jobId) {
|
|
358
|
+
let tail = "";
|
|
359
|
+
let stream = null;
|
|
360
|
+
|
|
361
|
+
function write(chunk) {
|
|
362
|
+
try {
|
|
363
|
+
tail += typeof chunk === "string" ? chunk : chunk.toString("utf8");
|
|
364
|
+
if (tail.length > TAIL_CAP_BYTES) tail = tail.slice(tail.length - TAIL_CAP_BYTES);
|
|
365
|
+
if (!enabled) return;
|
|
366
|
+
if (stream === null) {
|
|
367
|
+
stream = fs.createWriteStream(join(logsDir, `${sanitizeJobId(jobId)}.log`), { flags: "a" });
|
|
368
|
+
// Attach before the first write so an EPIPE/ENOSPC/EACCES surfaces as a log line rather
|
|
369
|
+
// than an unhandled 'error' that kills the worker.
|
|
370
|
+
stream.on("error", (e) => log("log_sink_error", { jobId, reason: e?.message }));
|
|
371
|
+
}
|
|
372
|
+
stream.write(chunk);
|
|
373
|
+
} catch (err) {
|
|
374
|
+
log("log_sink_error", { jobId, reason: err?.message });
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
async function close({ timeoutMs = 2000 } = {}) {
|
|
379
|
+
// Capture turns/tokens/session/usage from the tail first, so they survive even if the flush errors or times out.
|
|
380
|
+
const turns = parseExitTurns(tail);
|
|
381
|
+
const tokens = parseExitTokens(tail);
|
|
382
|
+
const session = parseExitSession(tail);
|
|
383
|
+
const usage = parseExitUsage(tail);
|
|
384
|
+
try {
|
|
385
|
+
if (stream !== null) {
|
|
386
|
+
const s = stream;
|
|
387
|
+
s.end();
|
|
388
|
+
let timer;
|
|
389
|
+
const timeout = new Promise((resolve) => {
|
|
390
|
+
timer = setTimeout(resolve, timeoutMs);
|
|
391
|
+
});
|
|
392
|
+
try {
|
|
393
|
+
// Bounded race: resolve on flush completion, on stream error, or on the deadline --
|
|
394
|
+
// whichever is first. An unbounded finish-await would turn a completed job into a timeout.
|
|
395
|
+
await Promise.race([
|
|
396
|
+
new Promise((resolve) => s.once("finish", resolve)),
|
|
397
|
+
new Promise((resolve) => s.once("error", resolve)),
|
|
398
|
+
timeout,
|
|
399
|
+
]);
|
|
400
|
+
} finally {
|
|
401
|
+
clearTimeout(timer);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
} catch (err) {
|
|
405
|
+
log("log_sink_error", { jobId, reason: err?.message });
|
|
406
|
+
}
|
|
407
|
+
return { turns, tokens, session, usage };
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
return { write, close };
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* The durable record writer: serialises a finished run's `buildRecord` output to a per-job JSON sidecar.
|
|
416
|
+
*
|
|
417
|
+
* Fault isolation is the contract. The processor wrapper calls `writeRecord` on the money path, so a
|
|
418
|
+
* throw here corrupts a run outcome or turns a finished job into a retry (CONST-RETRY-INFRA-ONLY).
|
|
419
|
+
* Therefore construction and `writeRecord` NEVER throw: the whole write body is try/catch-swallowed,
|
|
420
|
+
* mirroring `makeLogSink`.
|
|
421
|
+
*
|
|
422
|
+
* The write is SYNCHRONOUS by design: it must complete before `process.exit(0)` on shutdown
|
|
423
|
+
* (`index.mjs:83`), and an async write loses that race. `fs.writeFileSync` truncates by default, so a
|
|
424
|
+
* re-run of the same job id overwrites -- last write wins.
|
|
425
|
+
*
|
|
426
|
+
* The failure log carries only `jobId` and `reason`; never the record object or its serialized JSON,
|
|
427
|
+
* which embed target/flow fields (`no-pii-in-logs`). `sanitizeJobId` is applied only at the filename
|
|
428
|
+
* boundary; the record body keeps the raw id. `fs` is injectable so the writer is testable with a fake
|
|
429
|
+
* and no disk.
|
|
430
|
+
*/
|
|
431
|
+
export function makeRecordWriter({ logsDir, fs = nodeFs, log = () => {} }) {
|
|
432
|
+
try {
|
|
433
|
+
fs.mkdirSync(logsDir, { recursive: true });
|
|
434
|
+
} catch (err) {
|
|
435
|
+
log("logs_dir_error", { reason: err?.message });
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
return function writeRecord(record) {
|
|
439
|
+
try {
|
|
440
|
+
// Custom: flat JSON sidecar per job over a DB/logging lib -- records are immutable, filename-keyed
|
|
441
|
+
// by jobId, no cross-record queries; DES-RUN-HISTORY-FLAT-FILES-NO-DB; specs/interfaces.md:11
|
|
442
|
+
// (there is deliberately no database).
|
|
443
|
+
const path = join(logsDir, `${sanitizeJobId(record.jobId)}.json`);
|
|
444
|
+
const data = `${JSON.stringify(record)}\n`;
|
|
445
|
+
fs.writeFileSync(path, data);
|
|
446
|
+
} catch (err) {
|
|
447
|
+
log("run_record_failed", { jobId: record?.jobId, reason: err?.message });
|
|
448
|
+
}
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/**
|
|
453
|
+
* Look up when the previous run of a given scheduler ended, for the cron `/job/event.json`
|
|
454
|
+
* (INT-CONTAINER-JOB-INPUTS).
|
|
455
|
+
*
|
|
456
|
+
* This reads the same per-job files INT-RUN-HISTORY-FILE-CONTRACT defines -- filename-keyed sidecars,
|
|
457
|
+
* no new query surface. A scheduled id is `repeat:<schedulerId>:<millis>` (DES-CRON-VIA-BULLMQ-SCHEDULER),
|
|
458
|
+
* which `makeRecordWriter` stores as `repeat_<schedulerId>_<millis>.json` via `sanitizeJobId`, so the
|
|
459
|
+
* scheduler's prior fires are exactly the files matching that prefix with a pure-digits trailing
|
|
460
|
+
* segment. The digits requirement is what disambiguates a scheduler id that itself contains `_`
|
|
461
|
+
* (schedulers "a" vs "a_1": `repeat_a_1_100.json` has a non-digit tail after "repeat_a_", so it never
|
|
462
|
+
* matches scheduler "a"). The max millis strictly below `beforeMillis` is the previous fire.
|
|
463
|
+
*
|
|
464
|
+
* Returns `record.endedAt ?? record.startedAt ?? null` as an ISO string. `endedAt` first: BullMQ never
|
|
465
|
+
* overlaps two fires of one scheduler, so the prior run's end is the honest high-water mark; `startedAt`
|
|
466
|
+
* covers a crashed run's partial record. ANY failure -- missing dir, no prior run, unreadable file, bad
|
|
467
|
+
* JSON, nullish `beforeMillis` -- yields `null`; the function NEVER throws (the module's fault-isolation
|
|
468
|
+
* posture: this feeds a job input, and a history blip must not fail a prepare). `fs` is injectable so
|
|
469
|
+
* the lookup is testable with a fake and no disk.
|
|
470
|
+
*/
|
|
471
|
+
export function makeFindPreviousRun({ logsDir, fs = nodeFs }) {
|
|
472
|
+
// The parameter default keeps the never-throw promise even for an argument-less call: destructuring
|
|
473
|
+
// binds before the try below, so without it `findPreviousRun()` would TypeError past the catch.
|
|
474
|
+
return function findPreviousRun({ schedulerId, beforeMillis } = {}) {
|
|
475
|
+
try {
|
|
476
|
+
if (typeof beforeMillis !== "number" || !Number.isFinite(beforeMillis)) return null;
|
|
477
|
+
const prefix = sanitizeJobId(`repeat:${schedulerId}:`);
|
|
478
|
+
const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
479
|
+
const pattern = new RegExp(`^${escaped}(\\d+)\\.json$`);
|
|
480
|
+
let best = null;
|
|
481
|
+
for (const name of fs.readdirSync(logsDir)) {
|
|
482
|
+
const m = pattern.exec(name);
|
|
483
|
+
if (m === null) continue;
|
|
484
|
+
const millis = Number(m[1]);
|
|
485
|
+
if (!Number.isFinite(millis) || millis >= beforeMillis) continue;
|
|
486
|
+
if (best === null || millis > best.millis) best = { millis, name };
|
|
487
|
+
}
|
|
488
|
+
if (best === null) return null;
|
|
489
|
+
const record = JSON.parse(fs.readFileSync(join(logsDir, best.name), "utf8"));
|
|
490
|
+
return record.endedAt ?? record.startedAt ?? null;
|
|
491
|
+
} catch {
|
|
492
|
+
return null; // NEVER throws: a history fault must not fail the prepare that asked
|
|
493
|
+
}
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* The durable log reaper: a boot-time sweep that deletes `.log` and `.json` history files older than
|
|
499
|
+
* the retention window, keeping the logs directory bounded across restarts.
|
|
500
|
+
*
|
|
501
|
+
* Fault isolation is the contract, mirroring `makeReaper` in `start.mjs`: `reapLogs` NEVER throws under
|
|
502
|
+
* any input. A missing logs directory on first boot (`readdirSync` ENOENT), an unreadable entry, or an
|
|
503
|
+
* unlink failure is caught -- logged as `log_reaper_skipped` -- and boot continues. The per-file
|
|
504
|
+
* try/catch is what keeps one bad entry from aborting the whole sweep.
|
|
505
|
+
*
|
|
506
|
+
* `retentionDays === 0` is the documented keep-forever sentinel: the sweep returns early and touches no
|
|
507
|
+
* file. Age comes from the file's `mtimeMs`, never from any date parsed out of the filename -- the
|
|
508
|
+
* on-disk mtime is the authority. The `isFile()` guard skips a stray `foo.log/` directory so a
|
|
509
|
+
* mis-shaped entry raises no EISDIR/EPERM. `fs` and `now` are injectable so the reaper is testable with
|
|
510
|
+
* a fake and no disk.
|
|
511
|
+
*/
|
|
512
|
+
export function makeLogReaper({ logsDir, retentionDays, fs = nodeFs, log = () => {}, now = () => Date.now() }) {
|
|
513
|
+
return function reapLogs() {
|
|
514
|
+
if (retentionDays === 0) return; // keep-forever sentinel: no sweep
|
|
515
|
+
try {
|
|
516
|
+
const names = fs.readdirSync(logsDir);
|
|
517
|
+
const cutoff = now() - retentionDays * 86400000;
|
|
518
|
+
for (const name of names) {
|
|
519
|
+
if (!name.endsWith(".log") && !name.endsWith(".json")) continue;
|
|
520
|
+
try {
|
|
521
|
+
const st = fs.statSync(join(logsDir, name));
|
|
522
|
+
if (st.isFile() && st.mtimeMs < cutoff) {
|
|
523
|
+
fs.unlinkSync(join(logsDir, name));
|
|
524
|
+
log("reaped_log", { file: name });
|
|
525
|
+
}
|
|
526
|
+
} catch (err) {
|
|
527
|
+
log("log_reaper_skipped", { file: name, reason: err?.message });
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
} catch (err) {
|
|
531
|
+
log("log_reaper_skipped", { reason: err?.message });
|
|
532
|
+
}
|
|
533
|
+
};
|
|
534
|
+
}
|