@edgehero/pi-dispatch 1.5.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -0
- package/package.json +3 -1
- package/src/config.mjs +34 -0
- package/src/doctor.mjs +128 -12
- package/src/exit-code.mjs +56 -0
- package/src/index.mjs +333 -5
- package/src/processor.mjs +25 -0
- package/src/queue.mjs +11 -1
- package/src/run-history.mjs +81 -4
- package/src/service.mjs +9 -0
- package/src/start.mjs +44 -1
- package/src/triggers-file.mjs +77 -0
- package/src/triggers.mjs +150 -4
- package/src/wait-check.mjs +172 -0
- package/src/wait-for.mjs +315 -0
- package/src/wait-state.mjs +263 -0
package/src/start.mjs
CHANGED
|
@@ -22,9 +22,11 @@ import { makeCleanup, makeForgePreparers, makePrepareWorkspace } from "./prepare
|
|
|
22
22
|
import { listRunningSandboxes } from "./sandbox.mjs";
|
|
23
23
|
import { makeSandboxReaper } from "./sandbox-store.mjs";
|
|
24
24
|
import { makeSessionStore } from "./session-store.mjs";
|
|
25
|
-
import { makeCheckOnceSpent, makeDisarmOnce } from "./triggers-file.mjs";
|
|
25
|
+
import { makeCheckOnceSpent, makeCheckWaitSkew, makeDisarmOnce } from "./triggers-file.mjs";
|
|
26
26
|
import { loadPauseWindows, pauseUntilMs } from "./pause-windows.mjs";
|
|
27
27
|
import { loadScopedLimits } from "./scoped-limits.mjs";
|
|
28
|
+
import { makeWaitChecker } from "./wait-check.mjs";
|
|
29
|
+
import { makeWaitState } from "./wait-state.mjs";
|
|
28
30
|
import { makeQueue } from "./queue.mjs";
|
|
29
31
|
import { makeRunContainer } from "./run-container.mjs";
|
|
30
32
|
import { makeSecretsResolver } from "./secrets.mjs";
|
|
@@ -466,6 +468,18 @@ export async function startWorker(
|
|
|
466
468
|
// Issue #242: the scoped-limits snapshot the pickup gate and the scoped budget read, once per
|
|
467
469
|
// pickup, from the live-reloaded ref -- same next-job grain as pauseUntil above.
|
|
468
470
|
scopedLimits: () => scopedLimits.current,
|
|
471
|
+
// Issue #230. The `after` ceiling is read per pickup from config rather than frozen into the
|
|
472
|
+
// processor, so it is one value with one home; the wait state shares the budget's redis client
|
|
473
|
+
// because it describes the same delayed jobs that client already reasons about.
|
|
474
|
+
afterMaxMs: () => config.waitAfterMaxMs,
|
|
475
|
+
waitState: makeWaitState({ redis }),
|
|
476
|
+
// The polled tier's bounds, read per pickup from config so they are one value with one home. The
|
|
477
|
+
// slot count is a CEILING the gate clamps against the live concurrency, never the final number.
|
|
478
|
+
checkSlotCount: () => config.waitCheckSlots,
|
|
479
|
+
intervalMs: () => config.waitIntervalMs,
|
|
480
|
+
maxWaitMs: () => config.waitMaxMs,
|
|
481
|
+
maxChecks: () => config.waitMaxChecks,
|
|
482
|
+
maxFaults: () => config.waitMaxFaults,
|
|
469
483
|
deps: {
|
|
470
484
|
collectChain,
|
|
471
485
|
// The one-shot pre-spend check (issue #231): reads the same file the disarm writes, refuses
|
|
@@ -473,6 +487,35 @@ export async function startWorker(
|
|
|
473
487
|
// spending delivery is excused). In the compose topology this check is the once-enforcement
|
|
474
488
|
// layer, because the receiver's single-file :ro mount pins a dead inode until restart.
|
|
475
489
|
checkOnceSpent: makeCheckOnceSpent({ triggersPath: onceTriggersFile }),
|
|
490
|
+
// Issue #230. The same file and the same fail-open posture, but its own mtime-cached read: this one
|
|
491
|
+
// asks whether the AUTHORED entry declares wait conditions the job arrived without, which is how a
|
|
492
|
+
// service below the version floor turns a wait into a paid run nothing can tell from a correct
|
|
493
|
+
// one. In the compose topology the worker's read is the live inode while the receiver's is dead
|
|
494
|
+
// until restart, which is exactly the deployment where the skew happens.
|
|
495
|
+
checkWaitSkew: makeCheckWaitSkew({ triggersPath: onceTriggersFile }),
|
|
496
|
+
// Issue #230. Whether a job the supersede lease names is still in the queue. Without it a holder
|
|
497
|
+
// that vanished by any route except the clean one leaves a key that refuses every later delivery
|
|
498
|
+
// for that target until it expires -- and a refused forge delivery is gone, since no webhook
|
|
499
|
+
// resends it. `getJob` answers from the queue rather than from our own bookkeeping, so the two
|
|
500
|
+
// cannot agree with each other while both being wrong.
|
|
501
|
+
// REQ-WAIT-FOR's polled tier. Built here for the image and egress preflights' reason: one
|
|
502
|
+
// deployment value, one place, so the gate that refuses an undeclared profile and the spawn that
|
|
503
|
+
// runs it cannot disagree about which checks exist. The env-declared table is parsed once at boot
|
|
504
|
+
// (it is env, not overlay -- the gate reads its config above the per-job settings read).
|
|
505
|
+
// The free half of the profile check: whether this deployment declares the name at all. A table
|
|
506
|
+
// lookup, so it belongs with the gate's other free refusals rather than inside the subprocess.
|
|
507
|
+
waitProfileDeclared: (name) => typeof config.waitProfiles[name] === "string",
|
|
508
|
+
checkWait: makeWaitChecker({ profiles: config.waitProfiles, timeoutMs: config.waitCheckTimeoutMs, log }),
|
|
509
|
+
isJobLive: async (id) => {
|
|
510
|
+
const held = await runtimeQueue.getJob(id);
|
|
511
|
+
if (!held) return false;
|
|
512
|
+
// EXISTENCE IS NOT LIVENESS, and the difference decides whether a target stays deafened:
|
|
513
|
+
// `removeOnComplete`/`removeOnFail` keep a finished job's hash for 31 days, so a holder that
|
|
514
|
+
// can never wake again would answer "still waiting" for a month. Only a state it can still be
|
|
515
|
+
// picked up from counts.
|
|
516
|
+
const state = await held.getState();
|
|
517
|
+
return state === "delayed" || state === "waiting" || state === "active" || state === "prioritized" || state === "waiting-children";
|
|
518
|
+
},
|
|
476
519
|
// One deployment default, two consumers, adjacent by construction: the preflight that refuses a missing
|
|
477
520
|
// image BEFORE the budget slot, and the factory that puts it in the argv. Both resolve a trigger's own
|
|
478
521
|
// `run.image` through the same resolveJobImage, so the image that was checked is the image that runs.
|
package/src/triggers-file.mjs
CHANGED
|
@@ -376,6 +376,83 @@ export function makeCheckOnceSpent({ triggersPath, fs = nodeFs }) {
|
|
|
376
376
|
};
|
|
377
377
|
}
|
|
378
378
|
|
|
379
|
+
/**
|
|
380
|
+
* Detect the version skew `run.waitFor` opens (issue #230), pre-spend, from the worker's own file read.
|
|
381
|
+
*
|
|
382
|
+
* The hazard is `DES-TRIGGERS-UNIFIED-FILE`'s widening rule arriving somewhere it has never bitten. Unknown
|
|
383
|
+
* keys DROP, which for every previous field was a harmless no-op: an old parser meeting `run.image` gives
|
|
384
|
+
* you the default image and a job that ran. `waitFor` is the first field whose ABSENCE is destructive -- a
|
|
385
|
+
* receiver below the floor enqueues the job without it and the worker runs it UNHELD, producing a record, a
|
|
386
|
+
* panel row and a log line byte-identical to one that correctly waited. Success is the least detectable
|
|
387
|
+
* failure available, and the whole point of a wait is that running now is the destructive option.
|
|
388
|
+
*
|
|
389
|
+
* `docs/secrets.md`'s answer to the same skew is documentation plus a version floor, and it is not enough
|
|
390
|
+
* here for two reasons: a dropped secret surfaces as a 401 and an agent report that reads wrong, and
|
|
391
|
+
* `doctor` cannot see the receiver's installed version from the worker host, so its warning would fire on
|
|
392
|
+
* every deployment using the feature, forever -- the always-on amber the panel's own design rejects.
|
|
393
|
+
*
|
|
394
|
+
* So the worker checks the authored file itself. It already reads it per job for the one-shot gate, and in
|
|
395
|
+
* the compose topology this read is the authoritative one: the receiver's single-file `:ro` mount pins a
|
|
396
|
+
* dead inode until restart, which is precisely the deployment where the skew bites.
|
|
397
|
+
*
|
|
398
|
+
* FAIL-OPEN throughout, `readDisarmState`'s posture and for its reason: an unreadable or changed file means
|
|
399
|
+
* "run", because a broken read must never wedge every job. Only a positive, identity-confirmed mismatch --
|
|
400
|
+
* this entry authored conditions, this job carries none -- refuses.
|
|
401
|
+
*/
|
|
402
|
+
export function makeCheckWaitSkew({ triggersPath, fs = nodeFs }) {
|
|
403
|
+
// Cached by mtime, unlike `checkOnceSpent` which re-reads every time. The difference is which jobs each
|
|
404
|
+
// one runs for: the one-shot check is gated on `matched.once === true`, so it is rare by construction and
|
|
405
|
+
// its comment can call one read cheap. This one has to look at EVERY forge job, because the whole point
|
|
406
|
+
// is to catch a job that arrived WITHOUT the field, and there is nothing on such a job to narrow by. So a
|
|
407
|
+
// stat replaces a read-and-parse on the hot path. The residual is millisecond mtime granularity: two
|
|
408
|
+
// writes inside one millisecond would serve a stale parse for one job, which fails OPEN like every other
|
|
409
|
+
// uncertainty here.
|
|
410
|
+
let cache = null; // { mtimeMs, size, triggers }
|
|
411
|
+
const read = () => {
|
|
412
|
+
try {
|
|
413
|
+
const st = fs.statSync?.(triggersPath);
|
|
414
|
+
if (cache && st && cache.mtimeMs === st.mtimeMs && cache.size === st.size) return cache.triggers;
|
|
415
|
+
const raw = JSON.parse(fs.readFileSync(triggersPath, "utf8"));
|
|
416
|
+
const triggers = Array.isArray(raw?.triggers) ? raw.triggers : null;
|
|
417
|
+
if (st) cache = { mtimeMs: st.mtimeMs, size: st.size, triggers };
|
|
418
|
+
return triggers;
|
|
419
|
+
} catch {
|
|
420
|
+
cache = null;
|
|
421
|
+
return null;
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
|
|
425
|
+
return async function checkWaitSkew(job) {
|
|
426
|
+
if (typeof triggersPath !== "string" || triggersPath === "") return { ok: true };
|
|
427
|
+
// Cron and CLI jobs carry no matched index, and cannot carry `waitFor` at all.
|
|
428
|
+
const index = job?.trigger?.matched?.index;
|
|
429
|
+
if (!Number.isInteger(index)) return { ok: true };
|
|
430
|
+
// The field arrived. Whether its conditions are SATISFIED is the gate's business, not this check's.
|
|
431
|
+
if (Array.isArray(job?.waitFor) && job.waitFor.length > 0) return { ok: true };
|
|
432
|
+
|
|
433
|
+
const triggers = read();
|
|
434
|
+
if (triggers === null) return { ok: true };
|
|
435
|
+
const entry = triggers[index];
|
|
436
|
+
const authored = entry?.run?.waitFor;
|
|
437
|
+
if (!Array.isArray(authored) || authored.length === 0) return { ok: true };
|
|
438
|
+
|
|
439
|
+
// The identity guard readDisarmState keeps, and WIDER than flow alone. `triggerIndex` is a RAW array
|
|
440
|
+
// position, so an insertion, a reorder, or a stray `triggers.json` at the worker's cwd can all put a
|
|
441
|
+
// different trigger here -- and two rules sharing a flow are indistinguishable on flow alone, which
|
|
442
|
+
// made a false refusal reachable by ordinary editing. Kind and on-type are compared too, on
|
|
443
|
+
// `readDisarmState`'s precedent of checking the item number rather than trusting the index.
|
|
444
|
+
//
|
|
445
|
+
// Every mismatch folds to "run", never to a refusal: the true answer to "is this job missing
|
|
446
|
+
// conditions someone wrote for it?" is then unknown, and unknown must not refuse a paid delivery.
|
|
447
|
+
if (entry?.run?.flow !== job?.flow || entry?.run?.command !== job?.command) return { ok: true };
|
|
448
|
+
if (entry?.run?.kind !== job?.kind) return { ok: true };
|
|
449
|
+
const onType = job?.trigger?.matched?.type;
|
|
450
|
+
if (typeof onType === "string" && entry?.on?.type !== onType) return { ok: true };
|
|
451
|
+
|
|
452
|
+
return { skewed: true, conditions: authored.length };
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
|
|
379
456
|
export function readDisarmState({ triggersPath, index, number, flow, command, fs = nodeFs }) {
|
|
380
457
|
let raw;
|
|
381
458
|
try {
|
package/src/triggers.mjs
CHANGED
|
@@ -30,6 +30,11 @@ import { EGRESS_ENV_VARS, WORKER_ONLY_SECRET_VARS, configError } from "./config.
|
|
|
30
30
|
import { SKILL_NAME_RE } from "./flow-gate.mjs";
|
|
31
31
|
import { FORGE_HOST_VARS, FORGE_KINDS, MINTED_TOKEN_VARS, RUN_KINDS, forgeSpec, isForgeKind } from "./forges.mjs";
|
|
32
32
|
import { CONTAINER_ENV_NAMES } from "./reserved-env.mjs";
|
|
33
|
+
// The wait grammar's two shared halves (issue #230). `afterInstantMs` is imported rather than restated so
|
|
34
|
+
// the loader and the pickup gate cannot disagree about what a legal instant is: a second spelling here is
|
|
35
|
+
// how a file that loads clean starts holding for an instant nobody wrote. wait-for.mjs is pure and
|
|
36
|
+
// fs-free, so importing it keeps parseTriggers pure.
|
|
37
|
+
import { WAIT_CONDITION_KEYS, WAIT_CONDITION_MAX, afterInstantMs } from "./wait-for.mjs";
|
|
33
38
|
|
|
34
39
|
const ON_TYPES = new Set(["cron", "label", "comment", "pull_request", "issue"]);
|
|
35
40
|
|
|
@@ -345,6 +350,9 @@ function normalizeCron(on, run, index, path, state) {
|
|
|
345
350
|
validateDisarmed(on, `cron trigger "${id}"`, path, { onType: "cron" });
|
|
346
351
|
const secrets = validateSecrets(run, `cron trigger "${id}"`, path);
|
|
347
352
|
const secretsProfile = validateSecretsProfile(run, `cron trigger "${id}"`, path);
|
|
353
|
+
// Called for its refusal only: the returned `run` below deliberately grows no `waitFor` key, exactly as
|
|
354
|
+
// it grows no `replicas` one, because a cron entry can never carry either (issue #230).
|
|
355
|
+
validateWaitFor(on, run, `cron trigger "${id}"`, path, { onType: "cron" });
|
|
348
356
|
|
|
349
357
|
// provider/model/maxTurns stay absent when omitted so the value resolves at job start against the
|
|
350
358
|
// settings overlay/env, not a default frozen here (INT-CONFIG-OVERLAY-CONTRACT). github/packages/image stay
|
|
@@ -984,6 +992,140 @@ function validateSecretsProfile(run, at, path) {
|
|
|
984
992
|
return profile;
|
|
985
993
|
}
|
|
986
994
|
|
|
995
|
+
/**
|
|
996
|
+
* `run.waitFor` -- the conditions that must all clear before this trigger's job starts (issue #230).
|
|
997
|
+
*
|
|
998
|
+
* A CONJUNCTION of one-key objects: `{ "after": "<ISO instant>" }` is answered from the clock, and
|
|
999
|
+
* `{ "profile": "<name>" }` is answered by an operator-declared executable. The tiers are evaluated
|
|
1000
|
+
* cheapest-first at the gate rather than in the operator's writing order, which is a deliberate divergence
|
|
1001
|
+
* from the sequential rule `run.secrets` keeps: that rule exists so a broken reference always blames the
|
|
1002
|
+
* same variable, and a conjunction has no such ambiguity to protect.
|
|
1003
|
+
*
|
|
1004
|
+
* Like `run.secretsProfile`, a `profile` names a NAME and never a path, and WHETHER the deployment declares
|
|
1005
|
+
* it cannot be answered here (this validator is pure and fs-free). That refuses pre-spend, per delivery, as
|
|
1006
|
+
* `wait-profile-unknown`.
|
|
1007
|
+
*
|
|
1008
|
+
* UNKNOWN KEYS INSIDE A CONDITION ARE REFUSED, not dropped, which inverts this file's own posture for a
|
|
1009
|
+
* reason `validateDisarmed` already established: everywhere else an unknown key is a field that does
|
|
1010
|
+
* nothing, while here it is a CONDITION that does nothing, and a gate silently missing one of its terms
|
|
1011
|
+
* runs a paid job the operator believed was held.
|
|
1012
|
+
*
|
|
1013
|
+
* Three combination refusals, checked BEFORE any shape check so the message names the combination rather
|
|
1014
|
+
* than the array (`validateReplicas`' ordering lesson, where moving the local gate below the range check
|
|
1015
|
+
* would have answered a cron entry with the wrong complaint):
|
|
1016
|
+
*
|
|
1017
|
+
* - ON CRON. Two mechanisms, either of which is enough. The scheduler advances at PICKUP, so a held
|
|
1018
|
+
* occurrence carries an older `repeat:<id>:<millis>` than the one the scheduler stores; teardown
|
|
1019
|
+
* (`removeJobScheduler`, and therefore both a trigger delete and the stall guard's money backstop)
|
|
1020
|
+
* deletes only the STORED one, so the held job survives its own scheduler and still pays. And a held
|
|
1021
|
+
* occurrence's hash outliving the next `upsertJobScheduler` is a `SchedulerJobIdCollision`, which
|
|
1022
|
+
* `cron.mjs` turns into a config error that FAILS WORKER BOOT. Closing it needs teardown that
|
|
1023
|
+
* enumerates the delayed set, which is a gap to close, not a limit.
|
|
1024
|
+
* - BESIDE `on.once`. The one-shot disarm fires for EVERY run record, outcome-blind ("fired" means
|
|
1025
|
+
* "produced a run record"), so a wait that timed out would permanently spend a one-shot whose
|
|
1026
|
+
* container never started, and `checkOnceSpent` would then refuse every retry until the operator
|
|
1027
|
+
* hand-edited this file. Exempting the wait reasons instead would redefine "fired" for every other
|
|
1028
|
+
* refusal too, which is a gap to close, not a limit.
|
|
1029
|
+
* - BESIDE `run.replicas`. Fanout happens at enqueue, so N replicas are N independent holds: N times the
|
|
1030
|
+
* subprocesses and N times the check contention for ONE external condition. `REPLICAS_MAX` was chosen
|
|
1031
|
+
* against `PI_CONCURRENCY` on the premise that replicas RACE, and a hold inverts it.
|
|
1032
|
+
*
|
|
1033
|
+
* Returns a freshly built array, so nothing unvalidated rides through, and `undefined` when absent so an
|
|
1034
|
+
* unflagged trigger normalizes byte-identically.
|
|
1035
|
+
*/
|
|
1036
|
+
function validateWaitFor(on, run, at, path, { onType }) {
|
|
1037
|
+
// A NEAR-MISS SPELLING IS REFUSED, and this is the one place in this file that looks at a key nobody
|
|
1038
|
+
// wrote. Unknown keys drop here by design, which is harmless for every other field: a misspelled
|
|
1039
|
+
// `run.imgae` gives you the default image and a job that ran. `waitFor` is the first field whose
|
|
1040
|
+
// ABSENCE is destructive -- `waitfor` loads clean, normalizes clean, enqueues clean, and produces a
|
|
1041
|
+
// paid run that is byte-identical in the record, the panel and the log to one that correctly waited.
|
|
1042
|
+
// That is precisely the argument this validator already makes one level down about an unknown key
|
|
1043
|
+
// INSIDE a condition ("a term of the gate that would silently do nothing"), and the gate as a whole is
|
|
1044
|
+
// the bigger casualty. Bounded deliberately to near-misses rather than a general unknown-key sweep:
|
|
1045
|
+
// tolerating unknown `run` keys is this file's documented forward-compatibility posture, and a sweep
|
|
1046
|
+
// would refuse files that load today.
|
|
1047
|
+
// On `run` only a NEAR-MISS is wrong (the exact spelling is the field itself); on `on` every spelling is,
|
|
1048
|
+
// including the correct one, since a wait is a property of the run and `on.waitFor` would be dropped just
|
|
1049
|
+
// as silently as a typo.
|
|
1050
|
+
for (const [label, source, exactIsLegal] of [
|
|
1051
|
+
["run", run, true],
|
|
1052
|
+
["on", on, false],
|
|
1053
|
+
]) {
|
|
1054
|
+
for (const key of Object.keys(source ?? {})) {
|
|
1055
|
+
if (exactIsLegal && key === "waitFor") continue;
|
|
1056
|
+
if (key.replace(/[^a-z0-9]/gi, "").toLowerCase() !== "waitfor") continue;
|
|
1057
|
+
throw configError(`${at}: ${label}.${key} is not a field -- did you mean run.waitFor? A wait condition the loader drops holds nothing while the file reads as though it does, so a near miss is refused rather than dropped: ${path}`);
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
const waitFor = run.waitFor;
|
|
1062
|
+
if (waitFor === undefined) return undefined;
|
|
1063
|
+
|
|
1064
|
+
if (onType === "cron") {
|
|
1065
|
+
throw configError(`${at}: run.waitFor is not yet covered on a cron trigger -- the scheduler advances at pickup, so a held occurrence outlives the teardown that would delete it and still pays, and its surviving job hash can fail worker boot with a scheduler id collision; bounding that needs delayed-set-aware teardown, which is a gap to close, not a limit: ${path}`);
|
|
1066
|
+
}
|
|
1067
|
+
if (on?.once === true) {
|
|
1068
|
+
throw configError(`${at}: run.waitFor and on.once cannot be combined -- the one-shot disarms on every run record, so a wait that never cleared would spend the one-shot with no container ever started and no way to re-arm it but editing this file: ${path}`);
|
|
1069
|
+
}
|
|
1070
|
+
if (run.replicas !== undefined) {
|
|
1071
|
+
throw configError(`${at}: run.waitFor and run.replicas cannot be combined -- replicas fan out at enqueue, so each one would hold and poll the same external condition independently, multiplying the checks by ${run.replicas} for one answer: ${path}`);
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
if (!Array.isArray(waitFor) || waitFor.length === 0) {
|
|
1075
|
+
throw configError(`${at}: run.waitFor must be a non-empty array of conditions when present (got ${JSON.stringify(waitFor)}): ${path}`);
|
|
1076
|
+
}
|
|
1077
|
+
if (waitFor.length > WAIT_CONDITION_MAX) {
|
|
1078
|
+
throw configError(`${at}: run.waitFor names ${waitFor.length} conditions, over the ${WAIT_CONDITION_MAX} cap -- each profile condition is a subprocess run before the container, holding a concurrency slot while it runs: ${path}`);
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
const expected = WAIT_CONDITION_KEYS.join("|");
|
|
1082
|
+
const normalized = [];
|
|
1083
|
+
const profiles = new Set();
|
|
1084
|
+
let seenAfter = false;
|
|
1085
|
+
for (let i = 0; i < waitFor.length; i += 1) {
|
|
1086
|
+
const condition = waitFor[i];
|
|
1087
|
+
const where = `${at}: run.waitFor[${i}]`;
|
|
1088
|
+
if (condition === null || typeof condition !== "object" || Array.isArray(condition)) {
|
|
1089
|
+
throw configError(`${where} must be an object naming exactly one condition (got ${JSON.stringify(condition)}): ${path}`);
|
|
1090
|
+
}
|
|
1091
|
+
const keys = Object.keys(condition);
|
|
1092
|
+
if (keys.length !== 1) {
|
|
1093
|
+
throw configError(`${where} must name exactly one condition (expected ${expected}), got ${keys.length === 0 ? "an empty object" : JSON.stringify(keys)}: ${path}`);
|
|
1094
|
+
}
|
|
1095
|
+
const key = keys[0];
|
|
1096
|
+
if (key === "exclusive") {
|
|
1097
|
+
throw configError(`${where}: exclusive is no longer a condition you write -- the worker already holds at most one local job per folder, always on, with no configuration and no off switch, and scoped-limits.json bounds any scope further: ${path}`);
|
|
1098
|
+
}
|
|
1099
|
+
if (!WAIT_CONDITION_KEYS.includes(key)) {
|
|
1100
|
+
throw configError(`${where} has an unsupported condition ${JSON.stringify(key)} (expected ${expected}) -- an unknown condition here is a term of the gate that would silently do nothing, so it is refused rather than dropped: ${path}`);
|
|
1101
|
+
}
|
|
1102
|
+
if (key === "after") {
|
|
1103
|
+
if (seenAfter) {
|
|
1104
|
+
throw configError(`${where}: run.waitFor names a second "after" -- the conditions are a conjunction, so two instants mean the later one and the file would read as though both mattered: ${path}`);
|
|
1105
|
+
}
|
|
1106
|
+
if (afterInstantMs(condition.after) === null) {
|
|
1107
|
+
throw configError(`${where}: after must be an ISO-8601 instant carrying its own zone, like "2026-09-01T09:00:00Z" or "2026-09-01T11:00:00+02:00" (got ${JSON.stringify(condition.after)}) -- without a zone the same file would hold for a different instant on a worker in a different timezone: ${path}`);
|
|
1108
|
+
}
|
|
1109
|
+
seenAfter = true;
|
|
1110
|
+
normalized.push({ after: condition.after });
|
|
1111
|
+
continue;
|
|
1112
|
+
}
|
|
1113
|
+
const profile = condition.profile;
|
|
1114
|
+
if (!isNonEmptyString(profile)) {
|
|
1115
|
+
throw configError(`${where}: profile must be a non-empty string naming one of the wait profiles this deployment declares (got ${JSON.stringify(profile)}): ${path}`);
|
|
1116
|
+
}
|
|
1117
|
+
if (!ID_CHARSET.test(profile)) {
|
|
1118
|
+
throw configError(`${where}: profile ${JSON.stringify(profile)} may use letters, digits, dot, dash and underscore only -- PI_WAIT_PROFILES is a comma-separated list of name:path pairs, so a name carrying either separator cannot be declared: ${path}`);
|
|
1119
|
+
}
|
|
1120
|
+
if (profiles.has(profile)) {
|
|
1121
|
+
throw configError(`${where}: run.waitFor names the profile ${JSON.stringify(profile)} twice -- the conditions are a conjunction, so the second one can never change the answer while it doubles the subprocesses: ${path}`);
|
|
1122
|
+
}
|
|
1123
|
+
profiles.add(profile);
|
|
1124
|
+
normalized.push({ profile });
|
|
1125
|
+
}
|
|
1126
|
+
return normalized;
|
|
1127
|
+
}
|
|
1128
|
+
|
|
987
1129
|
function normalizeLabel(on, run, index, path) {
|
|
988
1130
|
const at = `trigger at index ${index}`;
|
|
989
1131
|
const predicate = validatePredicate(on, index, path, true);
|
|
@@ -1005,9 +1147,10 @@ function normalizeLabel(on, run, index, path) {
|
|
|
1005
1147
|
const replicas = validateReplicas(run, at, path);
|
|
1006
1148
|
const secrets = validateSecrets(run, at, path);
|
|
1007
1149
|
const secretsProfile = validateSecretsProfile(run, at, path);
|
|
1150
|
+
const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
|
|
1008
1151
|
return {
|
|
1009
1152
|
on: { type: "label", any: predicate.any, all: predicate.all, none: predicate.none },
|
|
1010
|
-
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }) },
|
|
1153
|
+
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
|
|
1011
1154
|
};
|
|
1012
1155
|
}
|
|
1013
1156
|
|
|
@@ -1043,9 +1186,10 @@ function normalizeComment(on, run, index, path, state) {
|
|
|
1043
1186
|
const replicas = validateReplicas(run, at, path);
|
|
1044
1187
|
const secrets = validateSecrets(run, at, path);
|
|
1045
1188
|
const secretsProfile = validateSecretsProfile(run, at, path);
|
|
1189
|
+
const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
|
|
1046
1190
|
return {
|
|
1047
1191
|
on: { type: "comment", phrase: on.phrase },
|
|
1048
|
-
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }) },
|
|
1192
|
+
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
|
|
1049
1193
|
};
|
|
1050
1194
|
}
|
|
1051
1195
|
|
|
@@ -1108,6 +1252,7 @@ function normalizeIssue(on, run, index, path) {
|
|
|
1108
1252
|
const replicas = validateReplicas(run, at, path);
|
|
1109
1253
|
const secrets = validateSecrets(run, at, path);
|
|
1110
1254
|
const secretsProfile = validateSecretsProfile(run, at, path);
|
|
1255
|
+
const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
|
|
1111
1256
|
return {
|
|
1112
1257
|
on: {
|
|
1113
1258
|
type: "issue",
|
|
@@ -1117,7 +1262,7 @@ function normalizeIssue(on, run, index, path) {
|
|
|
1117
1262
|
...(number !== undefined && { number }),
|
|
1118
1263
|
...(once !== undefined && { once }),
|
|
1119
1264
|
},
|
|
1120
|
-
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }) },
|
|
1265
|
+
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
|
|
1121
1266
|
};
|
|
1122
1267
|
}
|
|
1123
1268
|
|
|
@@ -1191,6 +1336,7 @@ function normalizePullRequest(on, run, index, path) {
|
|
|
1191
1336
|
const replicas = validateReplicas(run, at, path);
|
|
1192
1337
|
const secrets = validateSecrets(run, at, path);
|
|
1193
1338
|
const secretsProfile = validateSecretsProfile(run, at, path);
|
|
1339
|
+
const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
|
|
1194
1340
|
return {
|
|
1195
1341
|
on: {
|
|
1196
1342
|
type: "pull_request",
|
|
@@ -1205,7 +1351,7 @@ function normalizePullRequest(on, run, index, path) {
|
|
|
1205
1351
|
...(number !== undefined && { number }),
|
|
1206
1352
|
...(once !== undefined && { once }),
|
|
1207
1353
|
},
|
|
1208
|
-
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }) },
|
|
1354
|
+
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
|
|
1209
1355
|
};
|
|
1210
1356
|
}
|
|
1211
1357
|
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running an operator's wait check (issue #230, `run.waitFor`'s `profile` conditions).
|
|
3
|
+
*
|
|
4
|
+
* `secrets.mjs`'s resolver, reduced. The two seams spawn an operator-written script on the HOST, before
|
|
5
|
+
* anything spends, with the worker's own environment, and read its EXIT CODE. The differences are all
|
|
6
|
+
* subtractions, and each one is a thing the resolver needs that a check does not:
|
|
7
|
+
*
|
|
8
|
+
* - **stdout is a byte counter too, not a value.** A resolver's stdout IS the secret; a check's is
|
|
9
|
+
* incidental output nobody asked for. Reading it would create a channel for a third party's text to
|
|
10
|
+
* reach a panel row or a public comment through an operator's script, which `INT-WAIT-PROFILES-CONTRACT`
|
|
11
|
+
* refuses. Whether a check may return a reason string worth showing is deferred, and deferring it is
|
|
12
|
+
* cheaper than un-shipping it later.
|
|
13
|
+
* - **no size cap, no NUL check, no newline stripping.** Those exist because a resolved value becomes an
|
|
14
|
+
* `-e NAME=VALUE` argv element. Nothing here becomes anything.
|
|
15
|
+
* - **no dedup by reference.** A resolver dedups because two variables may name one item and rotation
|
|
16
|
+
* could straddle the pair. Two conditions naming one profile are refused at load instead.
|
|
17
|
+
* - **four codes rather than three** (`INT-RUNNER-EXIT-CODE-PROTOCOL`'s wait-profile table), because a
|
|
18
|
+
* check has an answer the other participants do not: not yet.
|
|
19
|
+
*
|
|
20
|
+
* What transfers unchanged, because the reasons transfer unchanged: the `name:/abs/path` grammar and its
|
|
21
|
+
* fail-loud parser, the realpath-and-executable probe, `spawn` with an argv ARRAY and `shell: false`,
|
|
22
|
+
* stdin ignored so a script that prompts dies at once instead of blocking to its timeout, stderr counted
|
|
23
|
+
* and never read, SIGTERM then SIGKILL after a grace, a per-invocation timeout, the abort signal, and
|
|
24
|
+
* sequential evaluation in the operator's writing order.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { realpathSync, statSync } from "node:fs";
|
|
28
|
+
import { spawn } from "node:child_process";
|
|
29
|
+
import { EXIT_HOLD, decideWait } from "./exit-code.mjs";
|
|
30
|
+
|
|
31
|
+
/** SIGTERM, then SIGKILL after this. `secrets.mjs`'s grace, for its reason. */
|
|
32
|
+
const KILL_GRACE_MS = 2000;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Build the checker. Returns `async (profile, target, { signal }) => verdict`, where verdict is
|
|
36
|
+
* `{ verdict: "go" | "hold" | "refuse", fault }` or `{ profileUnknown: name }`.
|
|
37
|
+
*
|
|
38
|
+
* NEVER THROWS and never rejects, which is `resolveSecrets`' contract and matters more here: this runs at
|
|
39
|
+
* the pickup gate, ABOVE the processor's `try`, where a rejection would escape into BullMQ's failed-attempt
|
|
40
|
+
* handling and turn a check that could not run into a retried job. The `log` sink is called inside guards
|
|
41
|
+
* for that reason -- an injected sink that throws is the one way this promise could otherwise be broken.
|
|
42
|
+
*/
|
|
43
|
+
export function makeWaitChecker({ profiles = {}, timeoutMs = 10_000, spawnFn = spawn, realExecutablePath = defaultRealExecutablePath, hostEnv = process.env, log = () => {} }) {
|
|
44
|
+
return async function checkWait(profile, target, { signal } = {}) {
|
|
45
|
+
const declared = profiles[profile];
|
|
46
|
+
if (typeof declared !== "string") return { profileUnknown: profile };
|
|
47
|
+
|
|
48
|
+
// Resolved and probed at CHECK time, not at boot: an operator who fixes a path mid-day must not stay
|
|
49
|
+
// refused, and one who deletes a script must not stay admitted. `statSync` rather than `lstat`,
|
|
50
|
+
// deliberately unlike `processor.mjs`'s directory probe, because this must answer the same question
|
|
51
|
+
// `spawn` will: spawn follows symlinks. Mode bits beyond the executable bit are doctor's business, not
|
|
52
|
+
// a job-time refusal -- refusing a group-writable script here would refuse a very ordinary install.
|
|
53
|
+
let path;
|
|
54
|
+
try {
|
|
55
|
+
path = realExecutablePath(declared);
|
|
56
|
+
} catch {
|
|
57
|
+
path = null;
|
|
58
|
+
}
|
|
59
|
+
if (!path) return { profileUnknown: profile };
|
|
60
|
+
|
|
61
|
+
// argv[1] must be a string a script can act on. `targetFor` answers null for any kind that is neither
|
|
62
|
+
// local nor a forge, and `spawn(path, [null])` throws -- which `runCheck` would turn into a silent
|
|
63
|
+
// fault hold, killing the job as `wait-unanswerable` and blaming the operator's check for a shape it
|
|
64
|
+
// never saw. The leading-dash refusal is `run.secrets`' own, for its reason: this is argv[1], where a
|
|
65
|
+
// dash parses as a flag, and we do not pass `--` instead because the option parser is the operator's.
|
|
66
|
+
if (typeof target !== "string" || target === "" || target.startsWith("-")) {
|
|
67
|
+
try {
|
|
68
|
+
log("wait_check_target_unusable", { profile });
|
|
69
|
+
} catch {}
|
|
70
|
+
return { verdict: "hold", fault: true, unusableTarget: true };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const outcome = await runCheck(spawnFn, path, target, { timeoutMs, hostEnv, signal });
|
|
74
|
+
try {
|
|
75
|
+
log("wait_check", { profile, code: outcome.code, detail: outcome.detail, stdoutBytes: outcome.stdoutBytes, stderrBytes: outcome.stderrBytes });
|
|
76
|
+
} catch {}
|
|
77
|
+
return outcome.verdict;
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** The realpath-and-executable probe, `secrets.mjs`'s exactly. */
|
|
82
|
+
function defaultRealExecutablePath(p) {
|
|
83
|
+
const real = realpathSync(p);
|
|
84
|
+
const st = statSync(real);
|
|
85
|
+
return st.isFile() && (st.mode & 0o111) !== 0 ? real : null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Run one check and classify it. Resolves `{ verdict, code, detail, stdoutBytes, stderrBytes }`.
|
|
90
|
+
*
|
|
91
|
+
* A check that could not run AT ALL -- spawn threw, the process died on a signal, the timeout fired, the
|
|
92
|
+
* job was aborted -- is a FAULT hold rather than a refusal, which is `decideWait`'s treatment of exit 1 and
|
|
93
|
+
* for its reason: a check that has not answered has not answered *no*, and dropping a paid delivery over an
|
|
94
|
+
* unreachable dependency is `CONST-RETRY-INFRA-ONLY` in the expensive direction. The fault count is what
|
|
95
|
+
* keeps that from being unbounded.
|
|
96
|
+
*/
|
|
97
|
+
function runCheck(spawnFn, path, target, { timeoutMs, hostEnv, signal }) {
|
|
98
|
+
return new Promise((resolve) => {
|
|
99
|
+
let child;
|
|
100
|
+
try {
|
|
101
|
+
child = spawnFn(path, [target], {
|
|
102
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
103
|
+
env: hostEnv,
|
|
104
|
+
shell: false,
|
|
105
|
+
});
|
|
106
|
+
} catch {
|
|
107
|
+
resolve({ verdict: { verdict: "hold", fault: true }, code: null, detail: "spawn", stdoutBytes: 0, stderrBytes: 0 });
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let stdoutBytes = 0;
|
|
112
|
+
let stderrBytes = 0;
|
|
113
|
+
let done = false;
|
|
114
|
+
let killTimer = null;
|
|
115
|
+
|
|
116
|
+
const finish = (result) => {
|
|
117
|
+
if (done) return;
|
|
118
|
+
done = true;
|
|
119
|
+
clearTimeout(timer);
|
|
120
|
+
signal?.removeEventListener?.("abort", onAbort);
|
|
121
|
+
resolve(result);
|
|
122
|
+
};
|
|
123
|
+
// The kill timer is deliberately NOT cleared by `finish`: the promise settles now, and the escalation
|
|
124
|
+
// still has to land on a child that ignored SIGTERM.
|
|
125
|
+
const stop = (why) => {
|
|
126
|
+
try {
|
|
127
|
+
child.kill();
|
|
128
|
+
} catch {}
|
|
129
|
+
killTimer = setTimeout(() => {
|
|
130
|
+
try {
|
|
131
|
+
child.kill("SIGKILL");
|
|
132
|
+
} catch {}
|
|
133
|
+
}, KILL_GRACE_MS);
|
|
134
|
+
killTimer.unref?.();
|
|
135
|
+
// An ABORT is not a fault. The worker is shutting down or this job was cancelled -- the operator's
|
|
136
|
+
// script did nothing wrong, and counting it would let five restarts that happen to land mid-check
|
|
137
|
+
// terminate a job with a PUBLIC comment blaming their check for the worker's own shutdown. That is
|
|
138
|
+
// the inverse of what the fault count is for, so the caller is told which of the two happened.
|
|
139
|
+
finish({ verdict: { verdict: "hold", fault: why !== "aborted", ...(why === "aborted" && { aborted: true }) }, code: null, detail: why, stdoutBytes, stderrBytes });
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
const timer = setTimeout(() => stop("timeout"), timeoutMs);
|
|
143
|
+
const onAbort = () => stop("aborted");
|
|
144
|
+
signal?.addEventListener?.("abort", onAbort, { once: true });
|
|
145
|
+
|
|
146
|
+
// BOTH streams are counters. stderr for the resolver's stated reason (a script's error text can echo
|
|
147
|
+
// the ticket, the query, or a vendor's own message), and stdout for the same reason one step further:
|
|
148
|
+
// a check's stdout is a third party's words arriving through an operator's script, and the only thing
|
|
149
|
+
// this seam is entitled to read from it is how much there was.
|
|
150
|
+
child.stdout?.on("data", (d) => (stdoutBytes += d.length));
|
|
151
|
+
child.stderr?.on("data", (d) => (stderrBytes += d.length));
|
|
152
|
+
child.on("error", () => finish({ verdict: { verdict: "hold", fault: true }, code: null, detail: "spawn", stdoutBytes, stderrBytes }));
|
|
153
|
+
// `exit`, NOT `close`. `close` waits for the child's stdio pipes to reach EOF, and any process the
|
|
154
|
+
// script leaves in the background inherits those pipes and holds them open -- so a check that answers
|
|
155
|
+
// `exit 0` in a millisecond is reported as a TIMEOUT, charged a fault, and after
|
|
156
|
+
// PI_WAIT_MAX_FAULTS the job dies with a public comment blaming the operator's script for something it
|
|
157
|
+
// did correctly. The resolver next door can afford `close` because it needs the bytes; this seam reads
|
|
158
|
+
// only a COUNT, so waiting for EOF buys nothing and costs the feature its credibility.
|
|
159
|
+
//
|
|
160
|
+
// The counts are then a floor rather than a total, which is what a byte counter is for, and the
|
|
161
|
+
// grandchild that kept the pipe open outlives the kill ladder -- `child.kill` reaches the direct child
|
|
162
|
+
// only. That residual is named in DES-WAIT-FOR-HOLDS-AND-WAIT-PROFILES rather than fixed here, because
|
|
163
|
+
// killing a process GROUP means spawning detached, which changes what the check inherits.
|
|
164
|
+
child.on("exit", (code, sig) => {
|
|
165
|
+
if (done) return;
|
|
166
|
+
// A signalled death reports code null; treat it as unanswered rather than letting `decideWait`
|
|
167
|
+
// read a null as an unrecognised code that means the same thing by accident.
|
|
168
|
+
if (code === null) return finish({ verdict: { verdict: "hold", fault: true }, code: null, detail: sig ? `signal-${sig}` : "no-code", stdoutBytes, stderrBytes });
|
|
169
|
+
finish({ verdict: decideWait(code), code, detail: code === EXIT_HOLD ? "not-yet" : "exit", stdoutBytes, stderrBytes });
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
}
|