@edgehero/pi-dispatch 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +21 -0
- package/package.json +4 -1
- package/src/config.mjs +35 -0
- package/src/doctor.mjs +213 -12
- package/src/exit-code.mjs +56 -0
- package/src/index.mjs +403 -15
- package/src/init.mjs +6 -0
- package/src/processor.mjs +80 -3
- package/src/queue.mjs +11 -1
- package/src/run-history.mjs +6 -1
- package/src/scoped-limits.mjs +277 -0
- package/src/service.mjs +9 -0
- package/src/start.mjs +95 -1
- package/src/triggers-file.mjs +77 -0
- package/src/triggers.mjs +150 -4
- package/src/wait-check.mjs +172 -0
- package/src/wait-for.mjs +315 -0
- package/src/wait-state.mjs +263 -0
package/src/triggers.mjs
CHANGED
|
@@ -30,6 +30,11 @@ import { EGRESS_ENV_VARS, WORKER_ONLY_SECRET_VARS, configError } from "./config.
|
|
|
30
30
|
import { SKILL_NAME_RE } from "./flow-gate.mjs";
|
|
31
31
|
import { FORGE_HOST_VARS, FORGE_KINDS, MINTED_TOKEN_VARS, RUN_KINDS, forgeSpec, isForgeKind } from "./forges.mjs";
|
|
32
32
|
import { CONTAINER_ENV_NAMES } from "./reserved-env.mjs";
|
|
33
|
+
// The wait grammar's two shared halves (issue #230). `afterInstantMs` is imported rather than restated so
|
|
34
|
+
// the loader and the pickup gate cannot disagree about what a legal instant is: a second spelling here is
|
|
35
|
+
// how a file that loads clean starts holding for an instant nobody wrote. wait-for.mjs is pure and
|
|
36
|
+
// fs-free, so importing it keeps parseTriggers pure.
|
|
37
|
+
import { WAIT_CONDITION_KEYS, WAIT_CONDITION_MAX, afterInstantMs } from "./wait-for.mjs";
|
|
33
38
|
|
|
34
39
|
const ON_TYPES = new Set(["cron", "label", "comment", "pull_request", "issue"]);
|
|
35
40
|
|
|
@@ -345,6 +350,9 @@ function normalizeCron(on, run, index, path, state) {
|
|
|
345
350
|
validateDisarmed(on, `cron trigger "${id}"`, path, { onType: "cron" });
|
|
346
351
|
const secrets = validateSecrets(run, `cron trigger "${id}"`, path);
|
|
347
352
|
const secretsProfile = validateSecretsProfile(run, `cron trigger "${id}"`, path);
|
|
353
|
+
// Called for its refusal only: the returned `run` below deliberately grows no `waitFor` key, exactly as
|
|
354
|
+
// it grows no `replicas` one, because a cron entry can never carry either (issue #230).
|
|
355
|
+
validateWaitFor(on, run, `cron trigger "${id}"`, path, { onType: "cron" });
|
|
348
356
|
|
|
349
357
|
// provider/model/maxTurns stay absent when omitted so the value resolves at job start against the
|
|
350
358
|
// settings overlay/env, not a default frozen here (INT-CONFIG-OVERLAY-CONTRACT). github/packages/image stay
|
|
@@ -984,6 +992,140 @@ function validateSecretsProfile(run, at, path) {
|
|
|
984
992
|
return profile;
|
|
985
993
|
}
|
|
986
994
|
|
|
995
|
+
/**
|
|
996
|
+
* `run.waitFor` -- the conditions that must all clear before this trigger's job starts (issue #230).
|
|
997
|
+
*
|
|
998
|
+
* A CONJUNCTION of one-key objects: `{ "after": "<ISO instant>" }` is answered from the clock, and
|
|
999
|
+
* `{ "profile": "<name>" }` is answered by an operator-declared executable. The tiers are evaluated
|
|
1000
|
+
* cheapest-first at the gate rather than in the operator's writing order, which is a deliberate divergence
|
|
1001
|
+
* from the sequential rule `run.secrets` keeps: that rule exists so a broken reference always blames the
|
|
1002
|
+
* same variable, and a conjunction has no such ambiguity to protect.
|
|
1003
|
+
*
|
|
1004
|
+
* Like `run.secretsProfile`, a `profile` names a NAME and never a path, and WHETHER the deployment declares
|
|
1005
|
+
* it cannot be answered here (this validator is pure and fs-free). That refuses pre-spend, per delivery, as
|
|
1006
|
+
* `wait-profile-unknown`.
|
|
1007
|
+
*
|
|
1008
|
+
* UNKNOWN KEYS INSIDE A CONDITION ARE REFUSED, not dropped, which inverts this file's own posture for a
|
|
1009
|
+
* reason `validateDisarmed` already established: everywhere else an unknown key is a field that does
|
|
1010
|
+
* nothing, while here it is a CONDITION that does nothing, and a gate silently missing one of its terms
|
|
1011
|
+
* runs a paid job the operator believed was held.
|
|
1012
|
+
*
|
|
1013
|
+
* Three combination refusals, checked BEFORE any shape check so the message names the combination rather
|
|
1014
|
+
* than the array (`validateReplicas`' ordering lesson, where moving the local gate below the range check
|
|
1015
|
+
* would have answered a cron entry with the wrong complaint):
|
|
1016
|
+
*
|
|
1017
|
+
* - ON CRON. Two mechanisms, either of which is enough. The scheduler advances at PICKUP, so a held
|
|
1018
|
+
* occurrence carries an older `repeat:<id>:<millis>` than the one the scheduler stores; teardown
|
|
1019
|
+
* (`removeJobScheduler`, and therefore both a trigger delete and the stall guard's money backstop)
|
|
1020
|
+
* deletes only the STORED one, so the held job survives its own scheduler and still pays. And a held
|
|
1021
|
+
* occurrence's hash outliving the next `upsertJobScheduler` is a `SchedulerJobIdCollision`, which
|
|
1022
|
+
* `cron.mjs` turns into a config error that FAILS WORKER BOOT. Closing it needs teardown that
|
|
1023
|
+
* enumerates the delayed set, which is a gap to close, not a limit.
|
|
1024
|
+
* - BESIDE `on.once`. The one-shot disarm fires for EVERY run record, outcome-blind ("fired" means
|
|
1025
|
+
* "produced a run record"), so a wait that timed out would permanently spend a one-shot whose
|
|
1026
|
+
* container never started, and `checkOnceSpent` would then refuse every retry until the operator
|
|
1027
|
+
* hand-edited this file. Exempting the wait reasons instead would redefine "fired" for every other
|
|
1028
|
+
* refusal too, which is a gap to close, not a limit.
|
|
1029
|
+
* - BESIDE `run.replicas`. Fanout happens at enqueue, so N replicas are N independent holds: N times the
|
|
1030
|
+
* subprocesses and N times the check contention for ONE external condition. `REPLICAS_MAX` was chosen
|
|
1031
|
+
* against `PI_CONCURRENCY` on the premise that replicas RACE, and a hold inverts it.
|
|
1032
|
+
*
|
|
1033
|
+
* Returns a freshly built array, so nothing unvalidated rides through, and `undefined` when absent so an
|
|
1034
|
+
* unflagged trigger normalizes byte-identically.
|
|
1035
|
+
*/
|
|
1036
|
+
function validateWaitFor(on, run, at, path, { onType }) {
|
|
1037
|
+
// A NEAR-MISS SPELLING IS REFUSED, and this is the one place in this file that looks at a key nobody
|
|
1038
|
+
// wrote. Unknown keys drop here by design, which is harmless for every other field: a misspelled
|
|
1039
|
+
// `run.imgae` gives you the default image and a job that ran. `waitFor` is the first field whose
|
|
1040
|
+
// ABSENCE is destructive -- `waitfor` loads clean, normalizes clean, enqueues clean, and produces a
|
|
1041
|
+
// paid run that is byte-identical in the record, the panel and the log to one that correctly waited.
|
|
1042
|
+
// That is precisely the argument this validator already makes one level down about an unknown key
|
|
1043
|
+
// INSIDE a condition ("a term of the gate that would silently do nothing"), and the gate as a whole is
|
|
1044
|
+
// the bigger casualty. Bounded deliberately to near-misses rather than a general unknown-key sweep:
|
|
1045
|
+
// tolerating unknown `run` keys is this file's documented forward-compatibility posture, and a sweep
|
|
1046
|
+
// would refuse files that load today.
|
|
1047
|
+
// On `run` only a NEAR-MISS is wrong (the exact spelling is the field itself); on `on` every spelling is,
|
|
1048
|
+
// including the correct one, since a wait is a property of the run and `on.waitFor` would be dropped just
|
|
1049
|
+
// as silently as a typo.
|
|
1050
|
+
for (const [label, source, exactIsLegal] of [
|
|
1051
|
+
["run", run, true],
|
|
1052
|
+
["on", on, false],
|
|
1053
|
+
]) {
|
|
1054
|
+
for (const key of Object.keys(source ?? {})) {
|
|
1055
|
+
if (exactIsLegal && key === "waitFor") continue;
|
|
1056
|
+
if (key.replace(/[^a-z0-9]/gi, "").toLowerCase() !== "waitfor") continue;
|
|
1057
|
+
throw configError(`${at}: ${label}.${key} is not a field -- did you mean run.waitFor? A wait condition the loader drops holds nothing while the file reads as though it does, so a near miss is refused rather than dropped: ${path}`);
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
const waitFor = run.waitFor;
|
|
1062
|
+
if (waitFor === undefined) return undefined;
|
|
1063
|
+
|
|
1064
|
+
if (onType === "cron") {
|
|
1065
|
+
throw configError(`${at}: run.waitFor is not yet covered on a cron trigger -- the scheduler advances at pickup, so a held occurrence outlives the teardown that would delete it and still pays, and its surviving job hash can fail worker boot with a scheduler id collision; bounding that needs delayed-set-aware teardown, which is a gap to close, not a limit: ${path}`);
|
|
1066
|
+
}
|
|
1067
|
+
if (on?.once === true) {
|
|
1068
|
+
throw configError(`${at}: run.waitFor and on.once cannot be combined -- the one-shot disarms on every run record, so a wait that never cleared would spend the one-shot with no container ever started and no way to re-arm it but editing this file: ${path}`);
|
|
1069
|
+
}
|
|
1070
|
+
if (run.replicas !== undefined) {
|
|
1071
|
+
throw configError(`${at}: run.waitFor and run.replicas cannot be combined -- replicas fan out at enqueue, so each one would hold and poll the same external condition independently, multiplying the checks by ${run.replicas} for one answer: ${path}`);
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
if (!Array.isArray(waitFor) || waitFor.length === 0) {
|
|
1075
|
+
throw configError(`${at}: run.waitFor must be a non-empty array of conditions when present (got ${JSON.stringify(waitFor)}): ${path}`);
|
|
1076
|
+
}
|
|
1077
|
+
if (waitFor.length > WAIT_CONDITION_MAX) {
|
|
1078
|
+
throw configError(`${at}: run.waitFor names ${waitFor.length} conditions, over the ${WAIT_CONDITION_MAX} cap -- each profile condition is a subprocess run before the container, holding a concurrency slot while it runs: ${path}`);
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
const expected = WAIT_CONDITION_KEYS.join("|");
|
|
1082
|
+
const normalized = [];
|
|
1083
|
+
const profiles = new Set();
|
|
1084
|
+
let seenAfter = false;
|
|
1085
|
+
for (let i = 0; i < waitFor.length; i += 1) {
|
|
1086
|
+
const condition = waitFor[i];
|
|
1087
|
+
const where = `${at}: run.waitFor[${i}]`;
|
|
1088
|
+
if (condition === null || typeof condition !== "object" || Array.isArray(condition)) {
|
|
1089
|
+
throw configError(`${where} must be an object naming exactly one condition (got ${JSON.stringify(condition)}): ${path}`);
|
|
1090
|
+
}
|
|
1091
|
+
const keys = Object.keys(condition);
|
|
1092
|
+
if (keys.length !== 1) {
|
|
1093
|
+
throw configError(`${where} must name exactly one condition (expected ${expected}), got ${keys.length === 0 ? "an empty object" : JSON.stringify(keys)}: ${path}`);
|
|
1094
|
+
}
|
|
1095
|
+
const key = keys[0];
|
|
1096
|
+
if (key === "exclusive") {
|
|
1097
|
+
throw configError(`${where}: exclusive is no longer a condition you write -- the worker already holds at most one local job per folder, always on, with no configuration and no off switch, and scoped-limits.json bounds any scope further: ${path}`);
|
|
1098
|
+
}
|
|
1099
|
+
if (!WAIT_CONDITION_KEYS.includes(key)) {
|
|
1100
|
+
throw configError(`${where} has an unsupported condition ${JSON.stringify(key)} (expected ${expected}) -- an unknown condition here is a term of the gate that would silently do nothing, so it is refused rather than dropped: ${path}`);
|
|
1101
|
+
}
|
|
1102
|
+
if (key === "after") {
|
|
1103
|
+
if (seenAfter) {
|
|
1104
|
+
throw configError(`${where}: run.waitFor names a second "after" -- the conditions are a conjunction, so two instants mean the later one and the file would read as though both mattered: ${path}`);
|
|
1105
|
+
}
|
|
1106
|
+
if (afterInstantMs(condition.after) === null) {
|
|
1107
|
+
throw configError(`${where}: after must be an ISO-8601 instant carrying its own zone, like "2026-09-01T09:00:00Z" or "2026-09-01T11:00:00+02:00" (got ${JSON.stringify(condition.after)}) -- without a zone the same file would hold for a different instant on a worker in a different timezone: ${path}`);
|
|
1108
|
+
}
|
|
1109
|
+
seenAfter = true;
|
|
1110
|
+
normalized.push({ after: condition.after });
|
|
1111
|
+
continue;
|
|
1112
|
+
}
|
|
1113
|
+
const profile = condition.profile;
|
|
1114
|
+
if (!isNonEmptyString(profile)) {
|
|
1115
|
+
throw configError(`${where}: profile must be a non-empty string naming one of the wait profiles this deployment declares (got ${JSON.stringify(profile)}): ${path}`);
|
|
1116
|
+
}
|
|
1117
|
+
if (!ID_CHARSET.test(profile)) {
|
|
1118
|
+
throw configError(`${where}: profile ${JSON.stringify(profile)} may use letters, digits, dot, dash and underscore only -- PI_WAIT_PROFILES is a comma-separated list of name:path pairs, so a name carrying either separator cannot be declared: ${path}`);
|
|
1119
|
+
}
|
|
1120
|
+
if (profiles.has(profile)) {
|
|
1121
|
+
throw configError(`${where}: run.waitFor names the profile ${JSON.stringify(profile)} twice -- the conditions are a conjunction, so the second one can never change the answer while it doubles the subprocesses: ${path}`);
|
|
1122
|
+
}
|
|
1123
|
+
profiles.add(profile);
|
|
1124
|
+
normalized.push({ profile });
|
|
1125
|
+
}
|
|
1126
|
+
return normalized;
|
|
1127
|
+
}
|
|
1128
|
+
|
|
987
1129
|
function normalizeLabel(on, run, index, path) {
|
|
988
1130
|
const at = `trigger at index ${index}`;
|
|
989
1131
|
const predicate = validatePredicate(on, index, path, true);
|
|
@@ -1005,9 +1147,10 @@ function normalizeLabel(on, run, index, path) {
|
|
|
1005
1147
|
const replicas = validateReplicas(run, at, path);
|
|
1006
1148
|
const secrets = validateSecrets(run, at, path);
|
|
1007
1149
|
const secretsProfile = validateSecretsProfile(run, at, path);
|
|
1150
|
+
const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
|
|
1008
1151
|
return {
|
|
1009
1152
|
on: { type: "label", any: predicate.any, all: predicate.all, none: predicate.none },
|
|
1010
|
-
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }) },
|
|
1153
|
+
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
|
|
1011
1154
|
};
|
|
1012
1155
|
}
|
|
1013
1156
|
|
|
@@ -1043,9 +1186,10 @@ function normalizeComment(on, run, index, path, state) {
|
|
|
1043
1186
|
const replicas = validateReplicas(run, at, path);
|
|
1044
1187
|
const secrets = validateSecrets(run, at, path);
|
|
1045
1188
|
const secretsProfile = validateSecretsProfile(run, at, path);
|
|
1189
|
+
const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
|
|
1046
1190
|
return {
|
|
1047
1191
|
on: { type: "comment", phrase: on.phrase },
|
|
1048
|
-
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }) },
|
|
1192
|
+
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
|
|
1049
1193
|
};
|
|
1050
1194
|
}
|
|
1051
1195
|
|
|
@@ -1108,6 +1252,7 @@ function normalizeIssue(on, run, index, path) {
|
|
|
1108
1252
|
const replicas = validateReplicas(run, at, path);
|
|
1109
1253
|
const secrets = validateSecrets(run, at, path);
|
|
1110
1254
|
const secretsProfile = validateSecretsProfile(run, at, path);
|
|
1255
|
+
const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
|
|
1111
1256
|
return {
|
|
1112
1257
|
on: {
|
|
1113
1258
|
type: "issue",
|
|
@@ -1117,7 +1262,7 @@ function normalizeIssue(on, run, index, path) {
|
|
|
1117
1262
|
...(number !== undefined && { number }),
|
|
1118
1263
|
...(once !== undefined && { once }),
|
|
1119
1264
|
},
|
|
1120
|
-
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }) },
|
|
1265
|
+
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
|
|
1121
1266
|
};
|
|
1122
1267
|
}
|
|
1123
1268
|
|
|
@@ -1191,6 +1336,7 @@ function normalizePullRequest(on, run, index, path) {
|
|
|
1191
1336
|
const replicas = validateReplicas(run, at, path);
|
|
1192
1337
|
const secrets = validateSecrets(run, at, path);
|
|
1193
1338
|
const secretsProfile = validateSecretsProfile(run, at, path);
|
|
1339
|
+
const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
|
|
1194
1340
|
return {
|
|
1195
1341
|
on: {
|
|
1196
1342
|
type: "pull_request",
|
|
@@ -1205,7 +1351,7 @@ function normalizePullRequest(on, run, index, path) {
|
|
|
1205
1351
|
...(number !== undefined && { number }),
|
|
1206
1352
|
...(once !== undefined && { once }),
|
|
1207
1353
|
},
|
|
1208
|
-
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }) },
|
|
1354
|
+
run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
|
|
1209
1355
|
};
|
|
1210
1356
|
}
|
|
1211
1357
|
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running an operator's wait check (issue #230, `run.waitFor`'s `profile` conditions).
|
|
3
|
+
*
|
|
4
|
+
* `secrets.mjs`'s resolver, reduced. The two seams spawn an operator-written script on the HOST, before
|
|
5
|
+
* anything spends, with the worker's own environment, and read its EXIT CODE. The differences are all
|
|
6
|
+
* subtractions, and each one is a thing the resolver needs that a check does not:
|
|
7
|
+
*
|
|
8
|
+
* - **stdout is a byte counter too, not a value.** A resolver's stdout IS the secret; a check's is
|
|
9
|
+
* incidental output nobody asked for. Reading it would create a channel for a third party's text to
|
|
10
|
+
* reach a panel row or a public comment through an operator's script, which `INT-WAIT-PROFILES-CONTRACT`
|
|
11
|
+
* refuses. Whether a check may return a reason string worth showing is deferred, and deferring it is
|
|
12
|
+
* cheaper than un-shipping it later.
|
|
13
|
+
* - **no size cap, no NUL check, no newline stripping.** Those exist because a resolved value becomes an
|
|
14
|
+
* `-e NAME=VALUE` argv element. Nothing here becomes anything.
|
|
15
|
+
* - **no dedup by reference.** A resolver dedups because two variables may name one item and rotation
|
|
16
|
+
* could straddle the pair. Two conditions naming one profile are refused at load instead.
|
|
17
|
+
* - **four codes rather than three** (`INT-RUNNER-EXIT-CODE-PROTOCOL`'s wait-profile table), because a
|
|
18
|
+
* check has an answer the other participants do not: not yet.
|
|
19
|
+
*
|
|
20
|
+
* What transfers unchanged, because the reasons transfer unchanged: the `name:/abs/path` grammar and its
|
|
21
|
+
* fail-loud parser, the realpath-and-executable probe, `spawn` with an argv ARRAY and `shell: false`,
|
|
22
|
+
* stdin ignored so a script that prompts dies at once instead of blocking to its timeout, stderr counted
|
|
23
|
+
* and never read, SIGTERM then SIGKILL after a grace, a per-invocation timeout, the abort signal, and
|
|
24
|
+
* sequential evaluation in the operator's writing order.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { realpathSync, statSync } from "node:fs";
|
|
28
|
+
import { spawn } from "node:child_process";
|
|
29
|
+
import { EXIT_HOLD, decideWait } from "./exit-code.mjs";
|
|
30
|
+
|
|
31
|
+
/** SIGTERM, then SIGKILL after this. `secrets.mjs`'s grace, for its reason. */
|
|
32
|
+
const KILL_GRACE_MS = 2000;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Build the checker. Returns `async (profile, target, { signal }) => verdict`, where verdict is
|
|
36
|
+
* `{ verdict: "go" | "hold" | "refuse", fault }` or `{ profileUnknown: name }`.
|
|
37
|
+
*
|
|
38
|
+
* NEVER THROWS and never rejects, which is `resolveSecrets`' contract and matters more here: this runs at
|
|
39
|
+
* the pickup gate, ABOVE the processor's `try`, where a rejection would escape into BullMQ's failed-attempt
|
|
40
|
+
* handling and turn a check that could not run into a retried job. The `log` sink is called inside guards
|
|
41
|
+
* for that reason -- an injected sink that throws is the one way this promise could otherwise be broken.
|
|
42
|
+
*/
|
|
43
|
+
export function makeWaitChecker({ profiles = {}, timeoutMs = 10_000, spawnFn = spawn, realExecutablePath = defaultRealExecutablePath, hostEnv = process.env, log = () => {} }) {
|
|
44
|
+
return async function checkWait(profile, target, { signal } = {}) {
|
|
45
|
+
const declared = profiles[profile];
|
|
46
|
+
if (typeof declared !== "string") return { profileUnknown: profile };
|
|
47
|
+
|
|
48
|
+
// Resolved and probed at CHECK time, not at boot: an operator who fixes a path mid-day must not stay
|
|
49
|
+
// refused, and one who deletes a script must not stay admitted. `statSync` rather than `lstat`,
|
|
50
|
+
// deliberately unlike `processor.mjs`'s directory probe, because this must answer the same question
|
|
51
|
+
// `spawn` will: spawn follows symlinks. Mode bits beyond the executable bit are doctor's business, not
|
|
52
|
+
// a job-time refusal -- refusing a group-writable script here would refuse a very ordinary install.
|
|
53
|
+
let path;
|
|
54
|
+
try {
|
|
55
|
+
path = realExecutablePath(declared);
|
|
56
|
+
} catch {
|
|
57
|
+
path = null;
|
|
58
|
+
}
|
|
59
|
+
if (!path) return { profileUnknown: profile };
|
|
60
|
+
|
|
61
|
+
// argv[1] must be a string a script can act on. `targetFor` answers null for any kind that is neither
|
|
62
|
+
// local nor a forge, and `spawn(path, [null])` throws -- which `runCheck` would turn into a silent
|
|
63
|
+
// fault hold, killing the job as `wait-unanswerable` and blaming the operator's check for a shape it
|
|
64
|
+
// never saw. The leading-dash refusal is `run.secrets`' own, for its reason: this is argv[1], where a
|
|
65
|
+
// dash parses as a flag, and we do not pass `--` instead because the option parser is the operator's.
|
|
66
|
+
if (typeof target !== "string" || target === "" || target.startsWith("-")) {
|
|
67
|
+
try {
|
|
68
|
+
log("wait_check_target_unusable", { profile });
|
|
69
|
+
} catch {}
|
|
70
|
+
return { verdict: "hold", fault: true, unusableTarget: true };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const outcome = await runCheck(spawnFn, path, target, { timeoutMs, hostEnv, signal });
|
|
74
|
+
try {
|
|
75
|
+
log("wait_check", { profile, code: outcome.code, detail: outcome.detail, stdoutBytes: outcome.stdoutBytes, stderrBytes: outcome.stderrBytes });
|
|
76
|
+
} catch {}
|
|
77
|
+
return outcome.verdict;
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** The realpath-and-executable probe, `secrets.mjs`'s exactly. */
|
|
82
|
+
function defaultRealExecutablePath(p) {
|
|
83
|
+
const real = realpathSync(p);
|
|
84
|
+
const st = statSync(real);
|
|
85
|
+
return st.isFile() && (st.mode & 0o111) !== 0 ? real : null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Run one check and classify it. Resolves `{ verdict, code, detail, stdoutBytes, stderrBytes }`.
|
|
90
|
+
*
|
|
91
|
+
* A check that could not run AT ALL -- spawn threw, the process died on a signal, the timeout fired, the
|
|
92
|
+
* job was aborted -- is a FAULT hold rather than a refusal, which is `decideWait`'s treatment of exit 1 and
|
|
93
|
+
* for its reason: a check that has not answered has not answered *no*, and dropping a paid delivery over an
|
|
94
|
+
* unreachable dependency is `CONST-RETRY-INFRA-ONLY` in the expensive direction. The fault count is what
|
|
95
|
+
* keeps that from being unbounded.
|
|
96
|
+
*/
|
|
97
|
+
function runCheck(spawnFn, path, target, { timeoutMs, hostEnv, signal }) {
|
|
98
|
+
return new Promise((resolve) => {
|
|
99
|
+
let child;
|
|
100
|
+
try {
|
|
101
|
+
child = spawnFn(path, [target], {
|
|
102
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
103
|
+
env: hostEnv,
|
|
104
|
+
shell: false,
|
|
105
|
+
});
|
|
106
|
+
} catch {
|
|
107
|
+
resolve({ verdict: { verdict: "hold", fault: true }, code: null, detail: "spawn", stdoutBytes: 0, stderrBytes: 0 });
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let stdoutBytes = 0;
|
|
112
|
+
let stderrBytes = 0;
|
|
113
|
+
let done = false;
|
|
114
|
+
let killTimer = null;
|
|
115
|
+
|
|
116
|
+
const finish = (result) => {
|
|
117
|
+
if (done) return;
|
|
118
|
+
done = true;
|
|
119
|
+
clearTimeout(timer);
|
|
120
|
+
signal?.removeEventListener?.("abort", onAbort);
|
|
121
|
+
resolve(result);
|
|
122
|
+
};
|
|
123
|
+
// The kill timer is deliberately NOT cleared by `finish`: the promise settles now, and the escalation
|
|
124
|
+
// still has to land on a child that ignored SIGTERM.
|
|
125
|
+
const stop = (why) => {
|
|
126
|
+
try {
|
|
127
|
+
child.kill();
|
|
128
|
+
} catch {}
|
|
129
|
+
killTimer = setTimeout(() => {
|
|
130
|
+
try {
|
|
131
|
+
child.kill("SIGKILL");
|
|
132
|
+
} catch {}
|
|
133
|
+
}, KILL_GRACE_MS);
|
|
134
|
+
killTimer.unref?.();
|
|
135
|
+
// An ABORT is not a fault. The worker is shutting down or this job was cancelled -- the operator's
|
|
136
|
+
// script did nothing wrong, and counting it would let five restarts that happen to land mid-check
|
|
137
|
+
// terminate a job with a PUBLIC comment blaming their check for the worker's own shutdown. That is
|
|
138
|
+
// the inverse of what the fault count is for, so the caller is told which of the two happened.
|
|
139
|
+
finish({ verdict: { verdict: "hold", fault: why !== "aborted", ...(why === "aborted" && { aborted: true }) }, code: null, detail: why, stdoutBytes, stderrBytes });
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
const timer = setTimeout(() => stop("timeout"), timeoutMs);
|
|
143
|
+
const onAbort = () => stop("aborted");
|
|
144
|
+
signal?.addEventListener?.("abort", onAbort, { once: true });
|
|
145
|
+
|
|
146
|
+
// BOTH streams are counters. stderr for the resolver's stated reason (a script's error text can echo
|
|
147
|
+
// the ticket, the query, or a vendor's own message), and stdout for the same reason one step further:
|
|
148
|
+
// a check's stdout is a third party's words arriving through an operator's script, and the only thing
|
|
149
|
+
// this seam is entitled to read from it is how much there was.
|
|
150
|
+
child.stdout?.on("data", (d) => (stdoutBytes += d.length));
|
|
151
|
+
child.stderr?.on("data", (d) => (stderrBytes += d.length));
|
|
152
|
+
child.on("error", () => finish({ verdict: { verdict: "hold", fault: true }, code: null, detail: "spawn", stdoutBytes, stderrBytes }));
|
|
153
|
+
// `exit`, NOT `close`. `close` waits for the child's stdio pipes to reach EOF, and any process the
|
|
154
|
+
// script leaves in the background inherits those pipes and holds them open -- so a check that answers
|
|
155
|
+
// `exit 0` in a millisecond is reported as a TIMEOUT, charged a fault, and after
|
|
156
|
+
// PI_WAIT_MAX_FAULTS the job dies with a public comment blaming the operator's script for something it
|
|
157
|
+
// did correctly. The resolver next door can afford `close` because it needs the bytes; this seam reads
|
|
158
|
+
// only a COUNT, so waiting for EOF buys nothing and costs the feature its credibility.
|
|
159
|
+
//
|
|
160
|
+
// The counts are then a floor rather than a total, which is what a byte counter is for, and the
|
|
161
|
+
// grandchild that kept the pipe open outlives the kill ladder -- `child.kill` reaches the direct child
|
|
162
|
+
// only. That residual is named in DES-WAIT-FOR-HOLDS-AND-WAIT-PROFILES rather than fixed here, because
|
|
163
|
+
// killing a process GROUP means spawning detached, which changes what the check inherits.
|
|
164
|
+
child.on("exit", (code, sig) => {
|
|
165
|
+
if (done) return;
|
|
166
|
+
// A signalled death reports code null; treat it as unanswered rather than letting `decideWait`
|
|
167
|
+
// read a null as an unrecognised code that means the same thing by accident.
|
|
168
|
+
if (code === null) return finish({ verdict: { verdict: "hold", fault: true }, code: null, detail: sig ? `signal-${sig}` : "no-code", stdoutBytes, stderrBytes });
|
|
169
|
+
finish({ verdict: decideWait(code), code, detail: code === EXIT_HOLD ? "not-yet" : "exit", stdoutBytes, stderrBytes });
|
|
170
|
+
});
|
|
171
|
+
});
|
|
172
|
+
}
|