@edgehero/pi-dispatch 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +2 -0
- package/deploy/docker-compose.yml +5 -1
- package/package.json +3 -1
- package/src/config.mjs +1 -0
- package/src/doctor.mjs +123 -3
- package/src/get-token.mjs +6 -1
- package/src/index.mjs +88 -14
- package/src/init.mjs +6 -0
- package/src/processor.mjs +81 -3
- package/src/queue.mjs +21 -1
- package/src/run-history.mjs +50 -31
- package/src/scoped-limits.mjs +277 -0
- package/src/start.mjs +76 -2
- package/src/triggers-file.mjs +403 -0
- package/src/triggers.mjs +312 -15
package/src/run-history.mjs
CHANGED
|
@@ -34,13 +34,57 @@ export function sanitizeJobId(id) {
|
|
|
34
34
|
return s.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* Parse one line of the buffered tail as a runner-emitted JSON line, or `null` for noise -- repairing
|
|
39
|
+
* the GLUED case (issue #224, OQ-003).
|
|
40
|
+
*
|
|
41
|
+
* The hazard: the runner's writes used to be newline-terminated but not newline-delimited, so a
|
|
42
|
+
* partial write from anything sharing the container's stdout (a subprocess that never flushed its
|
|
43
|
+
* trailing newline) lands as `<stray bytes><runner line>` in ONE line. A plain JSON.parse skips it,
|
|
44
|
+
* and because every parseExit* scan walks backwards past what does not parse, one un-newlined byte
|
|
45
|
+
* used to lose turns, tokens, usage, session and context at once -- or worse, hand the scan to a
|
|
46
|
+
* forged exit line placed earlier.
|
|
47
|
+
*
|
|
48
|
+
* The repair re-anchors on `{"event":"`, which is collision-free by construction: `event` is the
|
|
49
|
+
* first key both runner writers serialise, and JSON.stringify escapes every quote inside a string
|
|
50
|
+
* value, so these raw bytes cannot occur INSIDE a runner line -- only where one starts. Suffixes are
|
|
51
|
+
* tried left to right, so the leftmost complete object wins: a suffix beginning inside the stray
|
|
52
|
+
* bytes cannot parse to the line's end (nothing can close a JSON container after bytes the runner
|
|
53
|
+
* appended later, and the runner's own quotes terminate any string opened before them), so the first
|
|
54
|
+
* success is the glued runner object itself -- and this left-to-right scan, not just the quote
|
|
55
|
+
* escaping, is what keeps a would-be inner `{"event":"` from being chosen over the outer one.
|
|
56
|
+
* A line whose HEAD the capped tail sliced off is handled correctly either way: if the cut fell in a
|
|
57
|
+
* glued line's stray PREFIX the anchor survives and the genuine object is still repaired, and if it
|
|
58
|
+
* fell in or past the anchor no `{"event":"` survives and the line is skipped. A line truncated at the
|
|
59
|
+
* END (a mid-write death) has no complete object and is skipped too. A fragment is never MISREAD as a
|
|
60
|
+
* value: a broken one does not parse and an anchorless one is not repaired.
|
|
61
|
+
*
|
|
62
|
+
* NEVER throws, like the five scanners that call it.
|
|
63
|
+
*/
|
|
64
|
+
function parseTailLine(line) {
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse(line);
|
|
67
|
+
} catch {
|
|
68
|
+
// docker/agent noise, a truncated line, or a glued one -- try the repair before giving up.
|
|
69
|
+
}
|
|
70
|
+
let from = line.indexOf('{"event":"', 1);
|
|
71
|
+
while (from !== -1) {
|
|
72
|
+
try {
|
|
73
|
+
return JSON.parse(line.slice(from));
|
|
74
|
+
} catch {
|
|
75
|
+
from = line.indexOf('{"event":"', from + 1);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
|
|
37
81
|
/**
|
|
38
82
|
* Recover the agent's turn count from buffered container stdout, or `null` if it is not reported.
|
|
39
83
|
*
|
|
40
84
|
* The stream interleaves docker/agent noise and other JSON events (`pi_auto_retry`) with the runner's
|
|
41
85
|
* own lines. Only the success exit line carries `turns` (`image/runner/run-job.mjs:263`); the
|
|
42
86
|
* catch-path exit line (`:277`) omits it. Scan from the end and return the turns of the last `exit`
|
|
43
|
-
* event that reports an integer count.
|
|
87
|
+
* event that reports an integer count, repairing a glued line on the way (`parseTailLine`).
|
|
44
88
|
*
|
|
45
89
|
* This is read-only telemetry: it MUST NEVER throw and MUST NOT feed exit-code or retry
|
|
46
90
|
* classification -- that is the container exit code's job (INT-RUNNER-EXIT-CODE-PROTOCOL). Every parse
|
|
@@ -52,12 +96,7 @@ export function parseExitTurns(text) {
|
|
|
52
96
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
53
97
|
const line = lines[i].trim();
|
|
54
98
|
if (line === "") continue;
|
|
55
|
-
|
|
56
|
-
try {
|
|
57
|
-
parsed = JSON.parse(line);
|
|
58
|
-
} catch {
|
|
59
|
-
continue; // docker/agent noise or a truncated final line
|
|
60
|
-
}
|
|
99
|
+
const parsed = parseTailLine(line);
|
|
61
100
|
if (parsed?.event !== "exit") continue;
|
|
62
101
|
return Number.isInteger(parsed?.turns) ? parsed.turns : null;
|
|
63
102
|
}
|
|
@@ -94,12 +133,7 @@ export function parseExitSession(text) {
|
|
|
94
133
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
95
134
|
const line = lines[i].trim();
|
|
96
135
|
if (line === "") continue;
|
|
97
|
-
|
|
98
|
-
try {
|
|
99
|
-
parsed = JSON.parse(line);
|
|
100
|
-
} catch {
|
|
101
|
-
continue; // docker/agent noise or a truncated final line
|
|
102
|
-
}
|
|
136
|
+
const parsed = parseTailLine(line);
|
|
103
137
|
if (parsed?.event !== "exit") continue;
|
|
104
138
|
const sess = parsed?.session;
|
|
105
139
|
if (sess && typeof sess === "object" && !Array.isArray(sess) && typeof sess.resumed === "boolean") {
|
|
@@ -126,12 +160,7 @@ export function parseExitContext(text) {
|
|
|
126
160
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
127
161
|
const line = lines[i].trim();
|
|
128
162
|
if (line === "") continue;
|
|
129
|
-
|
|
130
|
-
try {
|
|
131
|
-
parsed = JSON.parse(line);
|
|
132
|
-
} catch {
|
|
133
|
-
continue; // docker/agent noise or a truncated final line
|
|
134
|
-
}
|
|
163
|
+
const parsed = parseTailLine(line);
|
|
135
164
|
if (parsed?.event !== "exit") continue;
|
|
136
165
|
const c = parsed?.context;
|
|
137
166
|
// A window of 0 is not a denominator, and a negative count is not a measurement. SAFE integers
|
|
@@ -153,12 +182,7 @@ export function parseExitTokens(text) {
|
|
|
153
182
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
154
183
|
const line = lines[i].trim();
|
|
155
184
|
if (line === "") continue;
|
|
156
|
-
|
|
157
|
-
try {
|
|
158
|
-
parsed = JSON.parse(line);
|
|
159
|
-
} catch {
|
|
160
|
-
continue; // docker/agent noise or a truncated final line
|
|
161
|
-
}
|
|
185
|
+
const parsed = parseTailLine(line);
|
|
162
186
|
if (parsed?.event !== "exit") continue;
|
|
163
187
|
const t = parsed?.tokens;
|
|
164
188
|
if (t && typeof t === "object" && !Array.isArray(t) && typeof t.total === "number") return t;
|
|
@@ -204,12 +228,7 @@ export function parseExitUsage(text) {
|
|
|
204
228
|
for (let i = lines.length - 1; i >= 0; i--) {
|
|
205
229
|
const line = lines[i].trim();
|
|
206
230
|
if (line === "") continue;
|
|
207
|
-
|
|
208
|
-
try {
|
|
209
|
-
parsed = JSON.parse(line);
|
|
210
|
-
} catch {
|
|
211
|
-
continue; // docker/agent noise or a truncated final line
|
|
212
|
-
}
|
|
231
|
+
const parsed = parseTailLine(line);
|
|
213
232
|
if (parsed?.event !== "exit") continue;
|
|
214
233
|
return rebuildUsage(parsed?.usage);
|
|
215
234
|
}
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scoped limits (issue #242, INT-SCOPED-LIMITS-FILE-CONTRACT): per-scope run caps and per-scope
|
|
3
|
+
* concurrency, where a scope is what `scopeOf` already answers -- the folder for a local job, the repo
|
|
4
|
+
* for a forge one. One `scoped-limits.json` of `{ scope, day?, week?, month?, concurrent? }` entries:
|
|
5
|
+
* the day/week/month caps refuse a job pre-spend (reason `scope-cap`, a policy refusal), `concurrent`
|
|
6
|
+
* defers the excess through the delayed set (never a refusal -- a busy scope is transient state).
|
|
7
|
+
*
|
|
8
|
+
* This module is pure and fs-injectable (mirrors pause-windows.mjs in every respect): `parseScopedLimits`
|
|
9
|
+
* validates the file TEXT fail-loud, `loadScopedLimits` layers the one fs read on top, and the small
|
|
10
|
+
* helpers below are what the processor gate and the budget wiring consume. It also owns the in-process
|
|
11
|
+
* in-flight counter (`makeInFlight`) so the counter is unit-testable without a bullmq import, the same
|
|
12
|
+
* reason job-id.mjs is queue-free. The wiring that consumes all of this (the gate, the budget calls,
|
|
13
|
+
* the admin surfaces) lands in this issue's later slices; the module ships first so the contract has
|
|
14
|
+
* one implementation to bind to -- sentences below describing enforcement describe THOSE slices.
|
|
15
|
+
*
|
|
16
|
+
* The file is a SIBLING of pause-windows.json, not part of the settings overlay, deliberately: the
|
|
17
|
+
* deferral gate runs before the per-job overlay read, so gate-read config must come from a watched
|
|
18
|
+
* mutable ref, and the overlay's KNOWN_KEYS are flat scalars whose only map-shaped precedent
|
|
19
|
+
* (secretProfiles) is deliberately model-unreachable -- the opposite of what these limits need.
|
|
20
|
+
*
|
|
21
|
+
* `version` is REQUIRED and fail-loud-on-newer (subscriptions.mjs's rule, adopted here because this is a
|
|
22
|
+
* MONEY file): unknown fields are silently dropped per the operator-file policy, so a v2 cap field an old
|
|
23
|
+
* worker drops would be a silently WIDENED spend limit. Pause-windows shipping without a version is a
|
|
24
|
+
* sunk decision, not a precedent to extend to enforcement config.
|
|
25
|
+
*
|
|
26
|
+
* Custom: scoped limits validated inline per triggers.mjs/pause-windows.mjs precedent; zod not in deps
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { createHash } from "node:crypto";
|
|
30
|
+
import { existsSync as fsExistsSync, readFileSync as fsReadFileSync } from "node:fs";
|
|
31
|
+
import { isAbsolute, resolve } from "node:path";
|
|
32
|
+
import { configError } from "./config.mjs";
|
|
33
|
+
import { scopeOf } from "./pause-windows.mjs";
|
|
34
|
+
|
|
35
|
+
/** The schema version this build reads and writes. A file declaring a higher one is refused loudly. */
|
|
36
|
+
export const SCOPED_LIMITS_VERSION = 1;
|
|
37
|
+
|
|
38
|
+
/** The four limit fields a row may carry, in display order. */
|
|
39
|
+
const LIMIT_FIELDS = ["day", "week", "month", "concurrent"];
|
|
40
|
+
|
|
41
|
+
function isNonEmptyString(value) {
|
|
42
|
+
return typeof value === "string" && value.trim() !== "";
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The canonical scope string for a job: the RESOLVED folder path for a local job, the repo for a forge
|
|
47
|
+
* one. `scopeOf` alone is not enough for enforcement: nothing on the trigger path normalizes
|
|
48
|
+
* `run.folder`, so `/srv/site`, `/srv/site/`, `/srv//site`, `/srv/x/../site` and a padded spelling are
|
|
49
|
+
* five distinct strings naming ONE directory -- an exact-string mutex keyed on the raw value would run
|
|
50
|
+
* them concurrently in one working tree, which is the exact race the mutex exists to close.
|
|
51
|
+
* `path.resolve` (not `normalize`, which keeps trailing slashes and whitespace) collapses them all; a
|
|
52
|
+
* relative folder resolves against the worker's cwd, the same base `prepareWorkspace`'s existence check
|
|
53
|
+
* uses; Unicode is NFC-normalized on both the job and the row side (see below). Two residuals,
|
|
54
|
+
* deliberate: symlinks are NOT resolved (realpath is an fs call on the hot path and can throw), and
|
|
55
|
+
* neither is filesystem case-insensitivity (on a default macOS/APFS volume `/Srv/Site` and `/srv/site`
|
|
56
|
+
* are one directory and two scopes) -- the pause matcher lives with both.
|
|
57
|
+
*
|
|
58
|
+
* The pause matcher itself keeps the RAW `scopeOf` value: resolving there would silently change which
|
|
59
|
+
* jobs an operator's existing trailing-slash window matches. The two features share the folder-vs-repo
|
|
60
|
+
* split (`scopeOf`, defined once) but not the normalization, and this comment is where that difference
|
|
61
|
+
* is recorded.
|
|
62
|
+
*
|
|
63
|
+
* A useful side effect: a resolved local scope is always an absolute path, and a repo string never is,
|
|
64
|
+
* so a folder named `a/b` and a repo named `a/b` can no longer collide in the counters or the mutex.
|
|
65
|
+
*/
|
|
66
|
+
export function canonicalScope(job) {
|
|
67
|
+
const scope = scopeOf(job);
|
|
68
|
+
if (!isNonEmptyString(scope)) return null;
|
|
69
|
+
// NFC on both kinds: macOS's filesystem hands paths back NFD while an admin dialog types NFC, so
|
|
70
|
+
// "wéb" can arrive as two byte sequences naming one thing -- without this, an NFD-spelled forge
|
|
71
|
+
// scope silently escapes an NFC-spelled cap (the local side would at least keep the structural
|
|
72
|
+
// mutex). ASCII is fixed under NFC, so no existing key changes.
|
|
73
|
+
return job?.kind === "local" ? resolve(scope.trim().normalize("NFC")) : scope.normalize("NFC");
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Parse, validate, and normalize the scoped-limits file TEXT. Returns the normalized `limits` array
|
|
78
|
+
* (every row rebuilt as an explicit `{ scope, day, week, month, concurrent }` literal, `null` for absent
|
|
79
|
+
* fields, unknown fields dropped -- the operator-file policy). Throws `configError` (fail-loud) on any
|
|
80
|
+
* malformed entry. `path` is for error messages only -- this function touches no filesystem.
|
|
81
|
+
*/
|
|
82
|
+
export function parseScopedLimits(text, path) {
|
|
83
|
+
let parsed;
|
|
84
|
+
try {
|
|
85
|
+
parsed = JSON.parse(text);
|
|
86
|
+
} catch (error) {
|
|
87
|
+
throw configError(`scoped-limits file is not valid JSON: ${path} (${error.message})`);
|
|
88
|
+
}
|
|
89
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
90
|
+
throw configError(`scoped-limits file must be an object with "version" and "limits": ${path}`);
|
|
91
|
+
}
|
|
92
|
+
const version = parsed.version;
|
|
93
|
+
if (!Number.isInteger(version) || version < 1) {
|
|
94
|
+
throw configError(`scoped-limits file must have "version": 1 (an integer >= 1): ${path}`);
|
|
95
|
+
}
|
|
96
|
+
if (version > SCOPED_LIMITS_VERSION) {
|
|
97
|
+
throw configError(`scoped-limits file written by a newer pi-dispatch (version ${version}; this build understands ${SCOPED_LIMITS_VERSION}): ${path}`);
|
|
98
|
+
}
|
|
99
|
+
if (!Array.isArray(parsed.limits)) {
|
|
100
|
+
throw configError(`scoped-limits file must have a "limits" array: ${path}`);
|
|
101
|
+
}
|
|
102
|
+
const rows = parsed.limits.map((row, index) => normalizeLimit(row, index, path));
|
|
103
|
+
const seen = new Map();
|
|
104
|
+
rows.forEach((row, index) => {
|
|
105
|
+
if (seen.has(row.scope)) {
|
|
106
|
+
// Two rows for one scope is a precedence question with no right answer; the admin's
|
|
107
|
+
// edit-in-place never produces one, so a duplicate is always a hand-edit mistake.
|
|
108
|
+
throw configError(`scoped limit at index ${index}: duplicate scope ${JSON.stringify(row.scope)} (first at index ${seen.get(row.scope)}): ${path}`);
|
|
109
|
+
}
|
|
110
|
+
seen.set(row.scope, index);
|
|
111
|
+
});
|
|
112
|
+
return rows;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function normalizeLimit(row, index, path) {
|
|
116
|
+
const at = `scoped limit at index ${index}`;
|
|
117
|
+
if (row === null || typeof row !== "object" || Array.isArray(row)) {
|
|
118
|
+
throw configError(`${at}: must be an object: ${path}`);
|
|
119
|
+
}
|
|
120
|
+
if (!isNonEmptyString(row.scope)) throw configError(`${at}: scope must be a non-empty string: ${path}`);
|
|
121
|
+
const trimmed = row.scope.trim().normalize("NFC"); // the same NFC canonicalScope applies job-side
|
|
122
|
+
if (trimmed === "*") {
|
|
123
|
+
// "*" as ONE shared counter is redundant with the global caps, so the only useful reading is a
|
|
124
|
+
// per-scope default -- the OPPOSITE of what "*" means one file over (pause-windows: one rule
|
|
125
|
+
// matching all scopes). Refused rather than shipped divergent; a later version may adopt the
|
|
126
|
+
// per-scope-default reading, with an exact row beating "*" (recorded in the contract).
|
|
127
|
+
throw configError(`${at}: "*" is not supported -- add one row per scope (a per-scope default may adopt "*" later): ${path}`);
|
|
128
|
+
}
|
|
129
|
+
if (trimmed.includes("*")) {
|
|
130
|
+
// No globs, enforced rather than described: an exact matcher makes "acme/*" a row that governs
|
|
131
|
+
// nothing, and a silently inert money limit is the failure class this repo refuses outright.
|
|
132
|
+
throw configError(`${at}: scopes match exactly; a scope containing "*" is refused (no globs): ${path}`);
|
|
133
|
+
}
|
|
134
|
+
const norm = {
|
|
135
|
+
// An absolute path is stored resolved so a `/srv/site/` row governs `/srv/site` jobs -- the same
|
|
136
|
+
// collapse canonicalScope applies on the job side. isAbsolute is PLATFORM-NATIVE on purpose, so a
|
|
137
|
+
// foreign-platform row (a windows drive path on a POSIX worker) stays verbatim and is inert here;
|
|
138
|
+
// the doctor's unreferenced-scope advisory names it. Resolving it instead would "work" only by
|
|
139
|
+
// both sides mangling into the same cwd-prefixed string -- a match by accident, not by contract.
|
|
140
|
+
scope: isAbsolute(trimmed) ? resolve(trimmed) : trimmed,
|
|
141
|
+
day: null,
|
|
142
|
+
week: null,
|
|
143
|
+
month: null,
|
|
144
|
+
concurrent: null,
|
|
145
|
+
};
|
|
146
|
+
let any = false;
|
|
147
|
+
for (const field of LIMIT_FIELDS) {
|
|
148
|
+
const value = row[field];
|
|
149
|
+
// Absent-or-null (subscriptions.mjs's rule): null is the normalizer's OWN output for an unset
|
|
150
|
+
// field, so the parser must accept it back or it cannot re-parse what it produced -- the admin's
|
|
151
|
+
// read-modify-write goes through this parser on both edges.
|
|
152
|
+
if (value === undefined || value === null) continue;
|
|
153
|
+
// 0 is refused, not "never run": budget.mjs's caps treat every configured window as >= 1, and
|
|
154
|
+
// "never run this scope" already has two honest spellings (delete the trigger; a pause window).
|
|
155
|
+
// isSafeInteger, not isInteger: 1e21 passes isInteger and reads as a limit while being
|
|
156
|
+
// indistinguishable from unlimited -- a bound that cannot count is not a bound.
|
|
157
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
158
|
+
throw configError(`${at}: ${field} must be an integer >= 1: ${path}`);
|
|
159
|
+
}
|
|
160
|
+
norm[field] = value;
|
|
161
|
+
any = true;
|
|
162
|
+
}
|
|
163
|
+
if (!any) {
|
|
164
|
+
throw configError(`${at}: at least one of day, week, month, concurrent is required (a row that limits nothing is a row an operator sets and then trusts): ${path}`);
|
|
165
|
+
}
|
|
166
|
+
return norm;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Load and validate the scoped-limits file named by `config.scopedLimitsFile`. Returns `[]` when the file
|
|
171
|
+
* is unset (no scoped caps or concurrency -- a valid deployment; the folder mutex holds regardless, it is
|
|
172
|
+
* code, not configuration). `readFileSync`/`existsSync` are injectable for tests.
|
|
173
|
+
*/
|
|
174
|
+
export function loadScopedLimits(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync } = {}) {
|
|
175
|
+
const path = config.scopedLimitsFile;
|
|
176
|
+
if (path === null || path === undefined) return [];
|
|
177
|
+
if (!existsSync(path)) throw configError(`scoped-limits file does not exist: ${path}`);
|
|
178
|
+
return parseScopedLimits(readFileSync(path, "utf8"), path);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* The exact-match row for a canonical scope, or null. Exact string equality only -- the pause matcher's
|
|
183
|
+
* semantics minus its "*" (refused above). With duplicates refused there is no precedence ladder.
|
|
184
|
+
*/
|
|
185
|
+
export function limitFor(limits, scope) {
|
|
186
|
+
if (!Array.isArray(limits) || !isNonEmptyString(scope)) return null;
|
|
187
|
+
return limits.find((l) => l.scope === scope) ?? null;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* The scoped budget windows this job reserves against, or null when nothing applies (no row for the
|
|
192
|
+
* scope, or the row is concurrency-only). The returned `scope` is CANONICAL so the redis counters are
|
|
193
|
+
* spelling-stable. Shaped like the global `caps` object so `reserveBudget` consumes it unchanged.
|
|
194
|
+
*/
|
|
195
|
+
export function budgetCapsFor(job, limits) {
|
|
196
|
+
const scope = canonicalScope(job);
|
|
197
|
+
const row = limitFor(limits, scope);
|
|
198
|
+
if (!row || (row.day === null && row.week === null && row.month === null)) return null;
|
|
199
|
+
return { scope, caps: { day: row.day, week: row.week, month: row.month } };
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* The effective in-flight ceiling for this job's scope: `min(configured concurrent, structural)`, where
|
|
204
|
+
* structural is 1 for a local job -- the folder mutex -- and unbounded otherwise. The mutex is
|
|
205
|
+
* UNCONDITIONAL, in code, with no file configured and no off-switch: two agents in one bind-mounted
|
|
206
|
+
* working tree is the race `run.replicas` is already refused on local jobs for, and a cron trigger
|
|
207
|
+
* reaches it with no operator mistake at all (the scheduler mints the next occurrence at pickup and
|
|
208
|
+
* promotes on time alone, so a slow run overlaps its own successor). A configured `concurrent` above 1
|
|
209
|
+
* on a folder scope silently clamps to 1 rather than refusing at parse: scope strings are not reliably
|
|
210
|
+
* typeable as folder-vs-repo (`"a/b"` is a legal relative folder and a legal repo), so a parse-time
|
|
211
|
+
* classifier would misfire; min() cannot.
|
|
212
|
+
*
|
|
213
|
+
* No scope (a malformed payload) means no gate: Infinity, admit -- the job will fail its own validation
|
|
214
|
+
* downstream, and holding a mutex slot under key `null` helps nobody.
|
|
215
|
+
*/
|
|
216
|
+
export function concurrencyFor(job, limits) {
|
|
217
|
+
const scope = canonicalScope(job);
|
|
218
|
+
if (scope === null) return Infinity;
|
|
219
|
+
const structural = job?.kind === "local" ? 1 : Infinity;
|
|
220
|
+
const configured = limitFor(limits, scope)?.concurrent ?? Infinity;
|
|
221
|
+
return Math.min(structural, configured);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* The redis key prefix for a scope's budget windows: `budget:s:<16 hex>`. Handed to
|
|
226
|
+
* `reserveBudget`/`releaseBudget` as `keyPrefix`, so `dayKey`/`weekKey`/`monthKey` compose
|
|
227
|
+
* `budget:s:<h>:YYYY-MM-DD` / `:w:...` / `:m:...` with zero new key-shape logic. A hash (the localJobId
|
|
228
|
+
* idiom: sha256, first 16 hex) rather than an escape: a scope legally contains `:` and `/` (folder
|
|
229
|
+
* paths, gitlab group/subgroup/project), which would collide with budget.mjs's own `w:`/`m:`/`t:`
|
|
230
|
+
* sub-namespaces, and a bijective escape grammar is a new thing to get wrong with unbounded key lengths.
|
|
231
|
+
* The one consumer that must map keys BACK to scopes is the admin's counter display, and it knows the
|
|
232
|
+
* configured scopes -- it recomputes keys through this same export, so unreadability in redis-cli is the
|
|
233
|
+
* accepted cost.
|
|
234
|
+
*/
|
|
235
|
+
export function scopeKeyPrefix(scope) {
|
|
236
|
+
const h = createHash("sha256").update(String(scope)).digest("hex").slice(0, 16);
|
|
237
|
+
return `budget:s:${h}`;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* The per-process in-flight counter behind per-scope concurrency and the folder mutex. Process memory is
|
|
242
|
+
* the CORRECT store, not a compromise: one worker per docker daemon is the shape DES-CONCURRENCY-3
|
|
243
|
+
* assumes everywhere and `service install` enforces for installed units (`pi-dispatch start` holds no
|
|
244
|
+
* lock, and two hand-run workers are already unsupported -- the second one's boot reaper kills the
|
|
245
|
+
* first's live containers); the reaper removes every surviving `pi-job-*` container before the worker
|
|
246
|
+
* starts draining, so a fresh, empty map is never wrong about a live container except when the reap
|
|
247
|
+
* itself was skipped (`reaper_skipped`: docker missing/down at boot -- a state where no NEW container
|
|
248
|
+
* can start either); and a Redis-held counter would survive a crash WRONGLY -- a claim for a container
|
|
249
|
+
* the reaper just killed, demanding TTL/heartbeat machinery, a second source of truth about "what is
|
|
250
|
+
* running" (the OQ-008 failure mode).
|
|
251
|
+
*
|
|
252
|
+
* `tryAcquire` is a synchronous check-and-increment: no await between the read and the take, so under
|
|
253
|
+
* Node's single thread no interleaving exists at any concurrency. `release` never throws -- it runs in
|
|
254
|
+
* the processor's finally, where a throw would mask the job's real error -- and clamps at zero.
|
|
255
|
+
*/
|
|
256
|
+
export function makeInFlight() {
|
|
257
|
+
const counts = new Map();
|
|
258
|
+
return {
|
|
259
|
+
/** True and counted when under `limit`; false WITHOUT counting when at or over it. */
|
|
260
|
+
tryAcquire(scope, limit) {
|
|
261
|
+
const current = counts.get(scope) ?? 0;
|
|
262
|
+
if (current >= limit) return false;
|
|
263
|
+
counts.set(scope, current + 1);
|
|
264
|
+
return true;
|
|
265
|
+
},
|
|
266
|
+
/** Decrement, deleting at zero; a release without a matching acquire is a no-op, never a throw. */
|
|
267
|
+
release(scope) {
|
|
268
|
+
const current = counts.get(scope) ?? 0;
|
|
269
|
+
if (current <= 1) counts.delete(scope);
|
|
270
|
+
else counts.set(scope, current - 1);
|
|
271
|
+
},
|
|
272
|
+
/** The current in-flight count for a scope (tests and future observability). */
|
|
273
|
+
count(scope) {
|
|
274
|
+
return counts.get(scope) ?? 0;
|
|
275
|
+
},
|
|
276
|
+
};
|
|
277
|
+
}
|
package/src/start.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
2
|
import { watch } from "node:fs";
|
|
3
|
-
import { dirname, basename } from "node:path";
|
|
3
|
+
import { dirname, basename, join } from "node:path";
|
|
4
4
|
import { promisify } from "node:util";
|
|
5
5
|
import { configError, loadConfig } from "./config.mjs";
|
|
6
6
|
import { makeRedisClient, parseConnection } from "./connection.mjs";
|
|
@@ -22,7 +22,9 @@ import { makeCleanup, makeForgePreparers, makePrepareWorkspace } from "./prepare
|
|
|
22
22
|
import { listRunningSandboxes } from "./sandbox.mjs";
|
|
23
23
|
import { makeSandboxReaper } from "./sandbox-store.mjs";
|
|
24
24
|
import { makeSessionStore } from "./session-store.mjs";
|
|
25
|
+
import { makeCheckOnceSpent, makeDisarmOnce } from "./triggers-file.mjs";
|
|
25
26
|
import { loadPauseWindows, pauseUntilMs } from "./pause-windows.mjs";
|
|
27
|
+
import { loadScopedLimits } from "./scoped-limits.mjs";
|
|
26
28
|
import { makeQueue } from "./queue.mjs";
|
|
27
29
|
import { makeRunContainer } from "./run-container.mjs";
|
|
28
30
|
import { makeSecretsResolver } from "./secrets.mjs";
|
|
@@ -102,6 +104,42 @@ function watchPauseWindowsFile(config, ref, log) {
|
|
|
102
104
|
}
|
|
103
105
|
}
|
|
104
106
|
|
|
107
|
+
/**
|
|
108
|
+
* The scoped-limits reload, EXPORTED apart from its watcher so keep-last-good is unit-testable without
|
|
109
|
+
* fs.watch (its two watcher siblings above bind theirs inline; this one is money config, so the
|
|
110
|
+
* last-good property carries its own test). A bad edit keeps `ref.current` untouched and logs
|
|
111
|
+
* `scoped_limits_reload_invalid` -- the pause-windows posture, INT-SCOPED-LIMITS-FILE-CONTRACT.
|
|
112
|
+
*/
|
|
113
|
+
export function reloadScopedLimits(config, ref, log) {
|
|
114
|
+
try {
|
|
115
|
+
ref.current = loadScopedLimits(config);
|
|
116
|
+
log("scoped_limits_reloaded", { count: ref.current.length });
|
|
117
|
+
} catch (err) {
|
|
118
|
+
log("scoped_limits_reload_invalid", { reason: err?.message });
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Watch the scoped-limits file (issue #242) the way the pause-windows watcher above does: the DIRECTORY,
|
|
124
|
+
* for atomic tmp+rename robustness, filtered to the one basename, debounced. Best-effort + unref'd.
|
|
125
|
+
*/
|
|
126
|
+
function watchScopedLimitsFile(config, ref, log) {
|
|
127
|
+
const path = config.scopedLimitsFile;
|
|
128
|
+
const dir = dirname(path) || ".";
|
|
129
|
+
const file = basename(path);
|
|
130
|
+
let timer = null;
|
|
131
|
+
try {
|
|
132
|
+
watch(dir, (_event, changed) => {
|
|
133
|
+
if (changed && changed !== file) return;
|
|
134
|
+
clearTimeout(timer);
|
|
135
|
+
timer = setTimeout(() => reloadScopedLimits(config, ref, log), 150);
|
|
136
|
+
}).unref?.();
|
|
137
|
+
log("scoped_limits_watching", { path });
|
|
138
|
+
} catch (err) {
|
|
139
|
+
log("scoped_limits_watch_unavailable", { reason: err?.message });
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
105
143
|
export function makeReaper({ log }) {
|
|
106
144
|
return async function reap() {
|
|
107
145
|
try {
|
|
@@ -187,6 +225,10 @@ export async function startWorker(
|
|
|
187
225
|
// pauses. Held in a mutable ref so the live-reload watcher can hot-swap it. [] means no scoped pauses.
|
|
188
226
|
const pauseWindows = { current: loadPauseWindows(config) };
|
|
189
227
|
|
|
228
|
+
// Issue #242: same posture for the scoped-limits file -- fail-loud with the operator present, mutable
|
|
229
|
+
// ref for the live-reload watcher, [] when unset (the folder mutex is code and needs no file).
|
|
230
|
+
const scopedLimits = { current: loadScopedLimits(config) };
|
|
231
|
+
|
|
190
232
|
// The forge a job belongs to is resolved PER JOB from `job.kind`, not bound once for the process.
|
|
191
233
|
// Each entry is `{ auth, host }`: `auth` is get-token's `{ mintToken, selfId, source }` (null when that
|
|
192
234
|
// forge is unconfigured or unreachable), `host` is the three methods github-host.mjs returns. The map
|
|
@@ -313,8 +355,25 @@ export async function startWorker(
|
|
|
313
355
|
} catch (err) {
|
|
314
356
|
log("session_reaper_skipped", { reason: err?.message });
|
|
315
357
|
}
|
|
316
|
-
|
|
358
|
+
// The one-shot file path (issue #231): PI_TRIGGERS_FILE, else ./triggers.json against this process's
|
|
359
|
+
// cwd -- doctor's own fallback, chosen for doctor's own reason ("the two must read the same file"),
|
|
360
|
+
// and deliberately NOT config.triggersFile, whose null means "cron disabled" and must keep meaning
|
|
361
|
+
// that: under that knob the DEFAULT single-host deployment would have a firing receiver and a worker
|
|
362
|
+
// that can neither disarm nor pre-spend-check.
|
|
363
|
+
const onceTriggersFile = env.PI_TRIGGERS_FILE ?? join(process.cwd(), "triggers.json");
|
|
364
|
+
const disarmOnce = makeDisarmOnce({ triggersPath: onceTriggersFile, log });
|
|
365
|
+
const recordRun = ({ job, result, error, startedAt, endedAt }) => {
|
|
317
366
|
writeRecord(buildRecord({ job, result, error, startedAt, endedAt }));
|
|
367
|
+
// Strictly AFTER the durable record: "fired" means "produced a run record", and the crash
|
|
368
|
+
// direction this ordering buys is the chosen one -- an armed one-shot with a record, never a
|
|
369
|
+
// disarm before writeRecord RETURNED. Returned, not succeeded: the record writer swallows fs
|
|
370
|
+
// errors by contract (run_record_failed), so a full disk still spends the one-shot -- the
|
|
371
|
+
// alternative, skipping the disarm on a failed record write, would re-fire it unbounded. Fire-and-forget: the hook never rejects, and the record path must not
|
|
372
|
+
// wait on a lock retry. An uncontended disarm completes synchronously inside this call; the
|
|
373
|
+
// one loss window is a drain's process.exit landing mid-lock-retry sleep, which loses only
|
|
374
|
+
// the disarm -- the same chosen direction, met at shutdown instead of a crash.
|
|
375
|
+
void disarmOnce({ job, endedAt });
|
|
376
|
+
};
|
|
318
377
|
|
|
319
378
|
// INT-CONFIG-OVERLAY-CONTRACT: the worker reads the runtime-settings overlay at EACH job start, so this
|
|
320
379
|
// closure -- not a value frozen at boot -- is what the processor calls per job. It resolves the eight
|
|
@@ -404,8 +463,16 @@ export async function startWorker(
|
|
|
404
463
|
// REQ-SCOPED-PAUSE-WINDOWS: the processor defers a job whose folder/repo is inside an active window.
|
|
405
464
|
// Reads the live-reloaded ref, so an operator edit takes effect on the next job without a restart.
|
|
406
465
|
pauseUntil: (job, now) => pauseUntilMs(pauseWindows.current, job, now),
|
|
466
|
+
// Issue #242: the scoped-limits snapshot the pickup gate and the scoped budget read, once per
|
|
467
|
+
// pickup, from the live-reloaded ref -- same next-job grain as pauseUntil above.
|
|
468
|
+
scopedLimits: () => scopedLimits.current,
|
|
407
469
|
deps: {
|
|
408
470
|
collectChain,
|
|
471
|
+
// The one-shot pre-spend check (issue #231): reads the same file the disarm writes, refuses
|
|
472
|
+
// only on a FOREIGN positive mark (index.mjs binds the real queue jobId so a retry of the
|
|
473
|
+
// spending delivery is excused). In the compose topology this check is the once-enforcement
|
|
474
|
+
// layer, because the receiver's single-file :ro mount pins a dead inode until restart.
|
|
475
|
+
checkOnceSpent: makeCheckOnceSpent({ triggersPath: onceTriggersFile }),
|
|
409
476
|
// One deployment default, two consumers, adjacent by construction: the preflight that refuses a missing
|
|
410
477
|
// image BEFORE the budget slot, and the factory that puts it in the argv. Both resolve a trigger's own
|
|
411
478
|
// `run.image` through the same resolveJobImage, so the image that was checked is the image that runs.
|
|
@@ -572,6 +639,11 @@ export async function startWorker(
|
|
|
572
639
|
watchPauseWindowsFile(config, pauseWindows, log);
|
|
573
640
|
}
|
|
574
641
|
|
|
642
|
+
// Issue #242 live edit: hot-swap the scoped limits on file change, keeping last-good on a bad edit.
|
|
643
|
+
if (config.scopedLimitsFile) {
|
|
644
|
+
watchScopedLimitsFile(config, scopedLimits, log);
|
|
645
|
+
}
|
|
646
|
+
|
|
575
647
|
log("worker_started", {
|
|
576
648
|
queue: "pi-jobs",
|
|
577
649
|
concurrency: bootConcurrency, // the slot count the Worker is actually constructed with (overlay may raise/lower it)
|
|
@@ -579,6 +651,8 @@ export async function startWorker(
|
|
|
579
651
|
weeklyCap: config.weeklyCap, // null when the weekly window is disabled
|
|
580
652
|
monthlyCap: config.monthlyCap, // null when the monthly window is disabled
|
|
581
653
|
softHoldPct: config.softHoldPct, // null when the soft-hold band is disabled
|
|
654
|
+
scopedLimitsFile: config.scopedLimitsFile, // null = no scoped caps/concurrency (the folder mutex holds regardless)
|
|
655
|
+
scopedLimits: scopedLimits.current.length, // row count -- money config deserves boot visibility; the watcher logs only changes
|
|
582
656
|
image: config.jobImage,
|
|
583
657
|
valkey: config.valkeyUrl,
|
|
584
658
|
logsDir: config.logsDir,
|