@edgehero/pi-dispatch 1.3.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -34,13 +34,57 @@ export function sanitizeJobId(id) {
34
34
  return s.replace(/[^A-Za-z0-9._-]/g, "_");
35
35
  }
36
36
 
37
+ /**
38
+ * Parse one line of the buffered tail as a runner-emitted JSON line, or `null` for noise -- repairing
39
+ * the GLUED case (issue #224, OQ-003).
40
+ *
41
+ * The hazard: the runner's writes used to be newline-terminated but not newline-delimited, so a
42
+ * partial write from anything sharing the container's stdout (a subprocess that never flushed its
43
+ * trailing newline) lands as `<stray bytes><runner line>` in ONE line. A plain JSON.parse skips it,
44
+ * and because every parseExit* scan walks backwards past what does not parse, one un-newlined byte
45
+ * used to lose turns, tokens, usage, session and context at once -- or worse, hand the scan to a
46
+ * forged exit line placed earlier.
47
+ *
48
+ * The repair re-anchors on `{"event":"`, which is collision-free by construction: `event` is the
49
+ * first key both runner writers serialise, and JSON.stringify escapes every quote inside a string
50
+ * value, so these raw bytes cannot occur INSIDE a runner line -- only where one starts. Suffixes are
51
+ * tried left to right, so the leftmost complete object wins: a suffix beginning inside the stray
52
+ * bytes cannot parse to the line's end (nothing can close a JSON container after bytes the runner
53
+ * appended later, and the runner's own quotes terminate any string opened before them), so the first
54
+ * success is the glued runner object itself -- and this left-to-right scan, not just the quote
55
+ * escaping, is what keeps a would-be inner `{"event":"` from being chosen over the outer one.
56
+ * A line whose HEAD the capped tail sliced off is handled correctly either way: if the cut fell in a
57
+ * glued line's stray PREFIX the anchor survives and the genuine object is still repaired, and if it
58
+ * fell in or past the anchor no `{"event":"` survives and the line is skipped. A line truncated at the
59
+ * END (a mid-write death) has no complete object and is skipped too. A fragment is never MISREAD as a
60
+ * value: a broken one does not parse and an anchorless one is not repaired.
61
+ *
62
+ * NEVER throws, like the five scanners that call it.
63
+ */
64
+ function parseTailLine(line) {
65
+ try {
66
+ return JSON.parse(line);
67
+ } catch {
68
+ // docker/agent noise, a truncated line, or a glued one -- try the repair before giving up.
69
+ }
70
+ let from = line.indexOf('{"event":"', 1);
71
+ while (from !== -1) {
72
+ try {
73
+ return JSON.parse(line.slice(from));
74
+ } catch {
75
+ from = line.indexOf('{"event":"', from + 1);
76
+ }
77
+ }
78
+ return null;
79
+ }
80
+
37
81
  /**
38
82
  * Recover the agent's turn count from buffered container stdout, or `null` if it is not reported.
39
83
  *
40
84
  * The stream interleaves docker/agent noise and other JSON events (`pi_auto_retry`) with the runner's
41
85
  * own lines. Only the success exit line carries `turns` (`image/runner/run-job.mjs:263`); the
42
86
  * catch-path exit line (`:277`) omits it. Scan from the end and return the turns of the last `exit`
43
- * event that reports an integer count.
87
+ * event that reports an integer count, repairing a glued line on the way (`parseTailLine`).
44
88
  *
45
89
  * This is read-only telemetry: it MUST NEVER throw and MUST NOT feed exit-code or retry
46
90
  * classification -- that is the container exit code's job (INT-RUNNER-EXIT-CODE-PROTOCOL). Every parse
@@ -52,12 +96,7 @@ export function parseExitTurns(text) {
52
96
  for (let i = lines.length - 1; i >= 0; i--) {
53
97
  const line = lines[i].trim();
54
98
  if (line === "") continue;
55
- let parsed;
56
- try {
57
- parsed = JSON.parse(line);
58
- } catch {
59
- continue; // docker/agent noise or a truncated final line
60
- }
99
+ const parsed = parseTailLine(line);
61
100
  if (parsed?.event !== "exit") continue;
62
101
  return Number.isInteger(parsed?.turns) ? parsed.turns : null;
63
102
  }
@@ -94,12 +133,7 @@ export function parseExitSession(text) {
94
133
  for (let i = lines.length - 1; i >= 0; i--) {
95
134
  const line = lines[i].trim();
96
135
  if (line === "") continue;
97
- let parsed;
98
- try {
99
- parsed = JSON.parse(line);
100
- } catch {
101
- continue; // docker/agent noise or a truncated final line
102
- }
136
+ const parsed = parseTailLine(line);
103
137
  if (parsed?.event !== "exit") continue;
104
138
  const sess = parsed?.session;
105
139
  if (sess && typeof sess === "object" && !Array.isArray(sess) && typeof sess.resumed === "boolean") {
@@ -126,12 +160,7 @@ export function parseExitContext(text) {
126
160
  for (let i = lines.length - 1; i >= 0; i--) {
127
161
  const line = lines[i].trim();
128
162
  if (line === "") continue;
129
- let parsed;
130
- try {
131
- parsed = JSON.parse(line);
132
- } catch {
133
- continue; // docker/agent noise or a truncated final line
134
- }
163
+ const parsed = parseTailLine(line);
135
164
  if (parsed?.event !== "exit") continue;
136
165
  const c = parsed?.context;
137
166
  // A window of 0 is not a denominator, and a negative count is not a measurement. SAFE integers
@@ -153,12 +182,7 @@ export function parseExitTokens(text) {
153
182
  for (let i = lines.length - 1; i >= 0; i--) {
154
183
  const line = lines[i].trim();
155
184
  if (line === "") continue;
156
- let parsed;
157
- try {
158
- parsed = JSON.parse(line);
159
- } catch {
160
- continue; // docker/agent noise or a truncated final line
161
- }
185
+ const parsed = parseTailLine(line);
162
186
  if (parsed?.event !== "exit") continue;
163
187
  const t = parsed?.tokens;
164
188
  if (t && typeof t === "object" && !Array.isArray(t) && typeof t.total === "number") return t;
@@ -204,12 +228,7 @@ export function parseExitUsage(text) {
204
228
  for (let i = lines.length - 1; i >= 0; i--) {
205
229
  const line = lines[i].trim();
206
230
  if (line === "") continue;
207
- let parsed;
208
- try {
209
- parsed = JSON.parse(line);
210
- } catch {
211
- continue; // docker/agent noise or a truncated final line
212
- }
231
+ const parsed = parseTailLine(line);
213
232
  if (parsed?.event !== "exit") continue;
214
233
  return rebuildUsage(parsed?.usage);
215
234
  }
@@ -0,0 +1,277 @@
1
+ /**
2
+ * Scoped limits (issue #242, INT-SCOPED-LIMITS-FILE-CONTRACT): per-scope run caps and per-scope
3
+ * concurrency, where a scope is what `scopeOf` already answers -- the folder for a local job, the repo
4
+ * for a forge one. One `scoped-limits.json` of `{ scope, day?, week?, month?, concurrent? }` entries:
5
+ * the day/week/month caps refuse a job pre-spend (reason `scope-cap`, a policy refusal), `concurrent`
6
+ * defers the excess through the delayed set (never a refusal -- a busy scope is transient state).
7
+ *
8
+ * This module is pure and fs-injectable (mirrors pause-windows.mjs in every respect): `parseScopedLimits`
9
+ * validates the file TEXT fail-loud, `loadScopedLimits` layers the one fs read on top, and the small
10
+ * helpers below are what the processor gate and the budget wiring consume. It also owns the in-process
11
+ * in-flight counter (`makeInFlight`) so the counter is unit-testable without a bullmq import, the same
12
+ * reason job-id.mjs is queue-free. The wiring that consumes all of this (the gate, the budget calls,
13
+ * the admin surfaces) lands in this issue's later slices; the module ships first so the contract has
14
+ * one implementation to bind to -- sentences below describing enforcement describe THOSE slices.
15
+ *
16
+ * The file is a SIBLING of pause-windows.json, not part of the settings overlay, deliberately: the
17
+ * deferral gate runs before the per-job overlay read, so gate-read config must come from a watched
18
+ * mutable ref, and the overlay's KNOWN_KEYS are flat scalars whose only map-shaped precedent
19
+ * (secretProfiles) is deliberately model-unreachable -- the opposite of what these limits need.
20
+ *
21
+ * `version` is REQUIRED and fail-loud-on-newer (subscriptions.mjs's rule, adopted here because this is a
22
+ * MONEY file): unknown fields are silently dropped per the operator-file policy, so a v2 cap field an old
23
+ * worker drops would be a silently WIDENED spend limit. Pause-windows shipping without a version is a
24
+ * sunk decision, not a precedent to extend to enforcement config.
25
+ *
26
+ * Custom: scoped limits validated inline per triggers.mjs/pause-windows.mjs precedent; zod not in deps
27
+ */
28
+
29
+ import { createHash } from "node:crypto";
30
+ import { existsSync as fsExistsSync, readFileSync as fsReadFileSync } from "node:fs";
31
+ import { isAbsolute, resolve } from "node:path";
32
+ import { configError } from "./config.mjs";
33
+ import { scopeOf } from "./pause-windows.mjs";
34
+
35
+ /** The schema version this build reads and writes. A file declaring a higher one is refused loudly. */
36
+ export const SCOPED_LIMITS_VERSION = 1;
37
+
38
+ /** The four limit fields a row may carry, in display order. */
39
+ const LIMIT_FIELDS = ["day", "week", "month", "concurrent"];
40
+
41
+ function isNonEmptyString(value) {
42
+ return typeof value === "string" && value.trim() !== "";
43
+ }
44
+
45
+ /**
46
+ * The canonical scope string for a job: the RESOLVED folder path for a local job, the repo for a forge
47
+ * one. `scopeOf` alone is not enough for enforcement: nothing on the trigger path normalizes
48
+ * `run.folder`, so `/srv/site`, `/srv/site/`, `/srv//site`, `/srv/x/../site` and a padded spelling are
49
+ * five distinct strings naming ONE directory -- an exact-string mutex keyed on the raw value would run
50
+ * them concurrently in one working tree, which is the exact race the mutex exists to close.
51
+ * `path.resolve` (not `normalize`, which keeps trailing slashes and whitespace) collapses them all; a
52
+ * relative folder resolves against the worker's cwd, the same base `prepareWorkspace`'s existence check
53
+ * uses; Unicode is NFC-normalized on both the job and the row side (see below). Two residuals,
54
+ * deliberate: symlinks are NOT resolved (realpath is an fs call on the hot path and can throw), and
55
+ * neither is filesystem case-insensitivity (on a default macOS/APFS volume `/Srv/Site` and `/srv/site`
56
+ * are one directory and two scopes) -- the pause matcher lives with both.
57
+ *
58
+ * The pause matcher itself keeps the RAW `scopeOf` value: resolving there would silently change which
59
+ * jobs an operator's existing trailing-slash window matches. The two features share the folder-vs-repo
60
+ * split (`scopeOf`, defined once) but not the normalization, and this comment is where that difference
61
+ * is recorded.
62
+ *
63
+ * A useful side effect: a resolved local scope is always an absolute path, and a repo string never is,
64
+ * so a folder named `a/b` and a repo named `a/b` can no longer collide in the counters or the mutex.
65
+ */
66
+ export function canonicalScope(job) {
67
+ const scope = scopeOf(job);
68
+ if (!isNonEmptyString(scope)) return null;
69
+ // NFC on both kinds: macOS's filesystem hands paths back NFD while an admin dialog types NFC, so
70
+ // "wéb" can arrive as two byte sequences naming one thing -- without this, an NFD-spelled forge
71
+ // scope silently escapes an NFC-spelled cap (the local side would at least keep the structural
72
+ // mutex). ASCII is fixed under NFC, so no existing key changes.
73
+ return job?.kind === "local" ? resolve(scope.trim().normalize("NFC")) : scope.normalize("NFC");
74
+ }
75
+
76
+ /**
77
+ * Parse, validate, and normalize the scoped-limits file TEXT. Returns the normalized `limits` array
78
+ * (every row rebuilt as an explicit `{ scope, day, week, month, concurrent }` literal, `null` for absent
79
+ * fields, unknown fields dropped -- the operator-file policy). Throws `configError` (fail-loud) on any
80
+ * malformed entry. `path` is for error messages only -- this function touches no filesystem.
81
+ */
82
+ export function parseScopedLimits(text, path) {
83
+ let parsed;
84
+ try {
85
+ parsed = JSON.parse(text);
86
+ } catch (error) {
87
+ throw configError(`scoped-limits file is not valid JSON: ${path} (${error.message})`);
88
+ }
89
+ if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
90
+ throw configError(`scoped-limits file must be an object with "version" and "limits": ${path}`);
91
+ }
92
+ const version = parsed.version;
93
+ if (!Number.isInteger(version) || version < 1) {
94
+ throw configError(`scoped-limits file must have "version": 1 (an integer >= 1): ${path}`);
95
+ }
96
+ if (version > SCOPED_LIMITS_VERSION) {
97
+ throw configError(`scoped-limits file written by a newer pi-dispatch (version ${version}; this build understands ${SCOPED_LIMITS_VERSION}): ${path}`);
98
+ }
99
+ if (!Array.isArray(parsed.limits)) {
100
+ throw configError(`scoped-limits file must have a "limits" array: ${path}`);
101
+ }
102
+ const rows = parsed.limits.map((row, index) => normalizeLimit(row, index, path));
103
+ const seen = new Map();
104
+ rows.forEach((row, index) => {
105
+ if (seen.has(row.scope)) {
106
+ // Two rows for one scope is a precedence question with no right answer; the admin's
107
+ // edit-in-place never produces one, so a duplicate is always a hand-edit mistake.
108
+ throw configError(`scoped limit at index ${index}: duplicate scope ${JSON.stringify(row.scope)} (first at index ${seen.get(row.scope)}): ${path}`);
109
+ }
110
+ seen.set(row.scope, index);
111
+ });
112
+ return rows;
113
+ }
114
+
115
+ function normalizeLimit(row, index, path) {
116
+ const at = `scoped limit at index ${index}`;
117
+ if (row === null || typeof row !== "object" || Array.isArray(row)) {
118
+ throw configError(`${at}: must be an object: ${path}`);
119
+ }
120
+ if (!isNonEmptyString(row.scope)) throw configError(`${at}: scope must be a non-empty string: ${path}`);
121
+ const trimmed = row.scope.trim().normalize("NFC"); // the same NFC canonicalScope applies job-side
122
+ if (trimmed === "*") {
123
+ // "*" as ONE shared counter is redundant with the global caps, so the only useful reading is a
124
+ // per-scope default -- the OPPOSITE of what "*" means one file over (pause-windows: one rule
125
+ // matching all scopes). Refused rather than shipped divergent; a later version may adopt the
126
+ // per-scope-default reading, with an exact row beating "*" (recorded in the contract).
127
+ throw configError(`${at}: "*" is not supported -- add one row per scope (a per-scope default may adopt "*" later): ${path}`);
128
+ }
129
+ if (trimmed.includes("*")) {
130
+ // No globs, enforced rather than described: an exact matcher makes "acme/*" a row that governs
131
+ // nothing, and a silently inert money limit is the failure class this repo refuses outright.
132
+ throw configError(`${at}: scopes match exactly; a scope containing "*" is refused (no globs): ${path}`);
133
+ }
134
+ const norm = {
135
+ // An absolute path is stored resolved so a `/srv/site/` row governs `/srv/site` jobs -- the same
136
+ // collapse canonicalScope applies on the job side. isAbsolute is PLATFORM-NATIVE on purpose, so a
137
+ // foreign-platform row (a windows drive path on a POSIX worker) stays verbatim and is inert here;
138
+ // the doctor's unreferenced-scope advisory names it. Resolving it instead would "work" only by
139
+ // both sides mangling into the same cwd-prefixed string -- a match by accident, not by contract.
140
+ scope: isAbsolute(trimmed) ? resolve(trimmed) : trimmed,
141
+ day: null,
142
+ week: null,
143
+ month: null,
144
+ concurrent: null,
145
+ };
146
+ let any = false;
147
+ for (const field of LIMIT_FIELDS) {
148
+ const value = row[field];
149
+ // Absent-or-null (subscriptions.mjs's rule): null is the normalizer's OWN output for an unset
150
+ // field, so the parser must accept it back or it cannot re-parse what it produced -- the admin's
151
+ // read-modify-write goes through this parser on both edges.
152
+ if (value === undefined || value === null) continue;
153
+ // 0 is refused, not "never run": budget.mjs's caps treat every configured window as >= 1, and
154
+ // "never run this scope" already has two honest spellings (delete the trigger; a pause window).
155
+ // isSafeInteger, not isInteger: 1e21 passes isInteger and reads as a limit while being
156
+ // indistinguishable from unlimited -- a bound that cannot count is not a bound.
157
+ if (!Number.isSafeInteger(value) || value < 1) {
158
+ throw configError(`${at}: ${field} must be an integer >= 1: ${path}`);
159
+ }
160
+ norm[field] = value;
161
+ any = true;
162
+ }
163
+ if (!any) {
164
+ throw configError(`${at}: at least one of day, week, month, concurrent is required (a row that limits nothing is a row an operator sets and then trusts): ${path}`);
165
+ }
166
+ return norm;
167
+ }
168
+
169
+ /**
170
+ * Load and validate the scoped-limits file named by `config.scopedLimitsFile`. Returns `[]` when the file
171
+ * is unset (no scoped caps or concurrency -- a valid deployment; the folder mutex holds regardless, it is
172
+ * code, not configuration). `readFileSync`/`existsSync` are injectable for tests.
173
+ */
174
+ export function loadScopedLimits(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync } = {}) {
175
+ const path = config.scopedLimitsFile;
176
+ if (path === null || path === undefined) return [];
177
+ if (!existsSync(path)) throw configError(`scoped-limits file does not exist: ${path}`);
178
+ return parseScopedLimits(readFileSync(path, "utf8"), path);
179
+ }
180
+
181
+ /**
182
+ * The exact-match row for a canonical scope, or null. Exact string equality only -- the pause matcher's
183
+ * semantics minus its "*" (refused above). With duplicates refused there is no precedence ladder.
184
+ */
185
+ export function limitFor(limits, scope) {
186
+ if (!Array.isArray(limits) || !isNonEmptyString(scope)) return null;
187
+ return limits.find((l) => l.scope === scope) ?? null;
188
+ }
189
+
190
+ /**
191
+ * The scoped budget windows this job reserves against, or null when nothing applies (no row for the
192
+ * scope, or the row is concurrency-only). The returned `scope` is CANONICAL so the redis counters are
193
+ * spelling-stable. Shaped like the global `caps` object so `reserveBudget` consumes it unchanged.
194
+ */
195
+ export function budgetCapsFor(job, limits) {
196
+ const scope = canonicalScope(job);
197
+ const row = limitFor(limits, scope);
198
+ if (!row || (row.day === null && row.week === null && row.month === null)) return null;
199
+ return { scope, caps: { day: row.day, week: row.week, month: row.month } };
200
+ }
201
+
202
+ /**
203
+ * The effective in-flight ceiling for this job's scope: `min(configured concurrent, structural)`, where
204
+ * structural is 1 for a local job -- the folder mutex -- and unbounded otherwise. The mutex is
205
+ * UNCONDITIONAL, in code, with no file configured and no off-switch: two agents in one bind-mounted
206
+ * working tree is the race `run.replicas` is already refused on local jobs for, and a cron trigger
207
+ * reaches it with no operator mistake at all (the scheduler mints the next occurrence at pickup and
208
+ * promotes on time alone, so a slow run overlaps its own successor). A configured `concurrent` above 1
209
+ * on a folder scope silently clamps to 1 rather than refusing at parse: scope strings are not reliably
210
+ * typeable as folder-vs-repo (`"a/b"` is a legal relative folder and a legal repo), so a parse-time
211
+ * classifier would misfire; min() cannot.
212
+ *
213
+ * No scope (a malformed payload) means no gate: Infinity, admit -- the job will fail its own validation
214
+ * downstream, and holding a mutex slot under key `null` helps nobody.
215
+ */
216
+ export function concurrencyFor(job, limits) {
217
+ const scope = canonicalScope(job);
218
+ if (scope === null) return Infinity;
219
+ const structural = job?.kind === "local" ? 1 : Infinity;
220
+ const configured = limitFor(limits, scope)?.concurrent ?? Infinity;
221
+ return Math.min(structural, configured);
222
+ }
223
+
224
+ /**
225
+ * The redis key prefix for a scope's budget windows: `budget:s:<16 hex>`. Handed to
226
+ * `reserveBudget`/`releaseBudget` as `keyPrefix`, so `dayKey`/`weekKey`/`monthKey` compose
227
+ * `budget:s:<h>:YYYY-MM-DD` / `:w:...` / `:m:...` with zero new key-shape logic. A hash (the localJobId
228
+ * idiom: sha256, first 16 hex) rather than an escape: a scope legally contains `:` and `/` (folder
229
+ * paths, gitlab group/subgroup/project), which would collide with budget.mjs's own `w:`/`m:`/`t:`
230
+ * sub-namespaces, and a bijective escape grammar is a new thing to get wrong with unbounded key lengths.
231
+ * The one consumer that must map keys BACK to scopes is the admin's counter display, and it knows the
232
+ * configured scopes -- it recomputes keys through this same export, so unreadability in redis-cli is the
233
+ * accepted cost.
234
+ */
235
+ export function scopeKeyPrefix(scope) {
236
+ const h = createHash("sha256").update(String(scope)).digest("hex").slice(0, 16);
237
+ return `budget:s:${h}`;
238
+ }
239
+
240
+ /**
241
+ * The per-process in-flight counter behind per-scope concurrency and the folder mutex. Process memory is
242
+ * the CORRECT store, not a compromise: one worker per docker daemon is the shape DES-CONCURRENCY-3
243
+ * assumes everywhere and `service install` enforces for installed units (`pi-dispatch start` holds no
244
+ * lock, and two hand-run workers are already unsupported -- the second one's boot reaper kills the
245
+ * first's live containers); the reaper removes every surviving `pi-job-*` container before the worker
246
+ * starts draining, so a fresh, empty map is never wrong about a live container except when the reap
247
+ * itself was skipped (`reaper_skipped`: docker missing/down at boot -- a state where no NEW container
248
+ * can start either); and a Redis-held counter would survive a crash WRONGLY -- a claim for a container
249
+ * the reaper just killed, demanding TTL/heartbeat machinery, a second source of truth about "what is
250
+ * running" (the OQ-008 failure mode).
251
+ *
252
+ * `tryAcquire` is a synchronous check-and-increment: no await between the read and the take, so under
253
+ * Node's single thread no interleaving exists at any concurrency. `release` never throws -- it runs in
254
+ * the processor's finally, where a throw would mask the job's real error -- and clamps at zero.
255
+ */
256
+ export function makeInFlight() {
257
+ const counts = new Map();
258
+ return {
259
+ /** True and counted when under `limit`; false WITHOUT counting when at or over it. */
260
+ tryAcquire(scope, limit) {
261
+ const current = counts.get(scope) ?? 0;
262
+ if (current >= limit) return false;
263
+ counts.set(scope, current + 1);
264
+ return true;
265
+ },
266
+ /** Decrement, deleting at zero; a release without a matching acquire is a no-op, never a throw. */
267
+ release(scope) {
268
+ const current = counts.get(scope) ?? 0;
269
+ if (current <= 1) counts.delete(scope);
270
+ else counts.set(scope, current - 1);
271
+ },
272
+ /** The current in-flight count for a scope (tests and future observability). */
273
+ count(scope) {
274
+ return counts.get(scope) ?? 0;
275
+ },
276
+ };
277
+ }
package/src/start.mjs CHANGED
@@ -1,6 +1,6 @@
1
1
  import { execFile } from "node:child_process";
2
2
  import { watch } from "node:fs";
3
- import { dirname, basename } from "node:path";
3
+ import { dirname, basename, join } from "node:path";
4
4
  import { promisify } from "node:util";
5
5
  import { configError, loadConfig } from "./config.mjs";
6
6
  import { makeRedisClient, parseConnection } from "./connection.mjs";
@@ -22,7 +22,9 @@ import { makeCleanup, makeForgePreparers, makePrepareWorkspace } from "./prepare
22
22
  import { listRunningSandboxes } from "./sandbox.mjs";
23
23
  import { makeSandboxReaper } from "./sandbox-store.mjs";
24
24
  import { makeSessionStore } from "./session-store.mjs";
25
+ import { makeCheckOnceSpent, makeDisarmOnce } from "./triggers-file.mjs";
25
26
  import { loadPauseWindows, pauseUntilMs } from "./pause-windows.mjs";
27
+ import { loadScopedLimits } from "./scoped-limits.mjs";
26
28
  import { makeQueue } from "./queue.mjs";
27
29
  import { makeRunContainer } from "./run-container.mjs";
28
30
  import { makeSecretsResolver } from "./secrets.mjs";
@@ -102,6 +104,42 @@ function watchPauseWindowsFile(config, ref, log) {
102
104
  }
103
105
  }
104
106
 
107
+ /**
108
+ * The scoped-limits reload, EXPORTED apart from its watcher so keep-last-good is unit-testable without
109
+ * fs.watch (its two watcher siblings above bind theirs inline; this one is money config, so the
110
+ * last-good property carries its own test). A bad edit keeps `ref.current` untouched and logs
111
+ * `scoped_limits_reload_invalid` -- the pause-windows posture, INT-SCOPED-LIMITS-FILE-CONTRACT.
112
+ */
113
+ export function reloadScopedLimits(config, ref, log) {
114
+ try {
115
+ ref.current = loadScopedLimits(config);
116
+ log("scoped_limits_reloaded", { count: ref.current.length });
117
+ } catch (err) {
118
+ log("scoped_limits_reload_invalid", { reason: err?.message });
119
+ }
120
+ }
121
+
122
+ /**
123
+ * Watch the scoped-limits file (issue #242) the way the pause-windows watcher above does: the DIRECTORY,
124
+ * for atomic tmp+rename robustness, filtered to the one basename, debounced. Best-effort + unref'd.
125
+ */
126
+ function watchScopedLimitsFile(config, ref, log) {
127
+ const path = config.scopedLimitsFile;
128
+ const dir = dirname(path) || ".";
129
+ const file = basename(path);
130
+ let timer = null;
131
+ try {
132
+ watch(dir, (_event, changed) => {
133
+ if (changed && changed !== file) return;
134
+ clearTimeout(timer);
135
+ timer = setTimeout(() => reloadScopedLimits(config, ref, log), 150);
136
+ }).unref?.();
137
+ log("scoped_limits_watching", { path });
138
+ } catch (err) {
139
+ log("scoped_limits_watch_unavailable", { reason: err?.message });
140
+ }
141
+ }
142
+
105
143
  export function makeReaper({ log }) {
106
144
  return async function reap() {
107
145
  try {
@@ -187,6 +225,10 @@ export async function startWorker(
187
225
  // pauses. Held in a mutable ref so the live-reload watcher can hot-swap it. [] means no scoped pauses.
188
226
  const pauseWindows = { current: loadPauseWindows(config) };
189
227
 
228
+ // Issue #242: same posture for the scoped-limits file -- fail-loud with the operator present, mutable
229
+ // ref for the live-reload watcher, [] when unset (the folder mutex is code and needs no file).
230
+ const scopedLimits = { current: loadScopedLimits(config) };
231
+
190
232
  // The forge a job belongs to is resolved PER JOB from `job.kind`, not bound once for the process.
191
233
  // Each entry is `{ auth, host }`: `auth` is get-token's `{ mintToken, selfId, source }` (null when that
192
234
  // forge is unconfigured or unreachable), `host` is the three methods github-host.mjs returns. The map
@@ -313,8 +355,25 @@ export async function startWorker(
313
355
  } catch (err) {
314
356
  log("session_reaper_skipped", { reason: err?.message });
315
357
  }
316
- const recordRun = ({ job, result, error, startedAt, endedAt }) =>
358
+ // The one-shot file path (issue #231): PI_TRIGGERS_FILE, else ./triggers.json against this process's
359
+ // cwd -- doctor's own fallback, chosen for doctor's own reason ("the two must read the same file"),
360
+ // and deliberately NOT config.triggersFile, whose null means "cron disabled" and must keep meaning
361
+ // that: under that knob the DEFAULT single-host deployment would have a firing receiver and a worker
362
+ // that can neither disarm nor pre-spend-check.
363
+ const onceTriggersFile = env.PI_TRIGGERS_FILE ?? join(process.cwd(), "triggers.json");
364
+ const disarmOnce = makeDisarmOnce({ triggersPath: onceTriggersFile, log });
365
+ const recordRun = ({ job, result, error, startedAt, endedAt }) => {
317
366
  writeRecord(buildRecord({ job, result, error, startedAt, endedAt }));
367
+ // Strictly AFTER the durable record: "fired" means "produced a run record", and the crash
368
+ // direction this ordering buys is the chosen one -- an armed one-shot with a record, never a
369
+ // disarm before writeRecord RETURNED. Returned, not succeeded: the record writer swallows fs
370
+ // errors by contract (run_record_failed), so a full disk still spends the one-shot -- the
371
+ // alternative, skipping the disarm on a failed record write, would re-fire it unbounded. Fire-and-forget: the hook never rejects, and the record path must not
372
+ // wait on a lock retry. An uncontended disarm completes synchronously inside this call; the
373
+ // one loss window is a drain's process.exit landing mid-lock-retry sleep, which loses only
374
+ // the disarm -- the same chosen direction, met at shutdown instead of a crash.
375
+ void disarmOnce({ job, endedAt });
376
+ };
318
377
 
319
378
  // INT-CONFIG-OVERLAY-CONTRACT: the worker reads the runtime-settings overlay at EACH job start, so this
320
379
  // closure -- not a value frozen at boot -- is what the processor calls per job. It resolves the eight
@@ -404,8 +463,16 @@ export async function startWorker(
404
463
  // REQ-SCOPED-PAUSE-WINDOWS: the processor defers a job whose folder/repo is inside an active window.
405
464
  // Reads the live-reloaded ref, so an operator edit takes effect on the next job without a restart.
406
465
  pauseUntil: (job, now) => pauseUntilMs(pauseWindows.current, job, now),
466
+ // Issue #242: the scoped-limits snapshot the pickup gate and the scoped budget read, once per
467
+ // pickup, from the live-reloaded ref -- same next-job grain as pauseUntil above.
468
+ scopedLimits: () => scopedLimits.current,
407
469
  deps: {
408
470
  collectChain,
471
+ // The one-shot pre-spend check (issue #231): reads the same file the disarm writes, refuses
472
+ // only on a FOREIGN positive mark (index.mjs binds the real queue jobId so a retry of the
473
+ // spending delivery is excused). In the compose topology this check is the once-enforcement
474
+ // layer, because the receiver's single-file :ro mount pins a dead inode until restart.
475
+ checkOnceSpent: makeCheckOnceSpent({ triggersPath: onceTriggersFile }),
409
476
  // One deployment default, two consumers, adjacent by construction: the preflight that refuses a missing
410
477
  // image BEFORE the budget slot, and the factory that puts it in the argv. Both resolve a trigger's own
411
478
  // `run.image` through the same resolveJobImage, so the image that was checked is the image that runs.
@@ -572,6 +639,11 @@ export async function startWorker(
572
639
  watchPauseWindowsFile(config, pauseWindows, log);
573
640
  }
574
641
 
642
+ // Issue #242 live edit: hot-swap the scoped limits on file change, keeping last-good on a bad edit.
643
+ if (config.scopedLimitsFile) {
644
+ watchScopedLimitsFile(config, scopedLimits, log);
645
+ }
646
+
575
647
  log("worker_started", {
576
648
  queue: "pi-jobs",
577
649
  concurrency: bootConcurrency, // the slot count the Worker is actually constructed with (overlay may raise/lower it)
@@ -579,6 +651,8 @@ export async function startWorker(
579
651
  weeklyCap: config.weeklyCap, // null when the weekly window is disabled
580
652
  monthlyCap: config.monthlyCap, // null when the monthly window is disabled
581
653
  softHoldPct: config.softHoldPct, // null when the soft-hold band is disabled
654
+ scopedLimitsFile: config.scopedLimitsFile, // null = no scoped caps/concurrency (the folder mutex holds regardless)
655
+ scopedLimits: scopedLimits.current.length, // row count -- money config deserves boot visibility; the watcher logs only changes
582
656
  image: config.jobImage,
583
657
  valkey: config.valkeyUrl,
584
658
  logsDir: config.logsDir,