@edgehero/pi-dispatch 1.6.1 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +7 -0
- package/package.json +4 -1
- package/src/capabilities.mjs +179 -0
- package/src/cli.mjs +74 -17
- package/src/config.mjs +83 -0
- package/src/cron.mjs +116 -4
- package/src/doctor.mjs +113 -3
- package/src/fingerprint.mjs +78 -0
- package/src/fleet-lease.mjs +179 -0
- package/src/host-registry.mjs +279 -0
- package/src/image-preflight.mjs +8 -3
- package/src/index.mjs +196 -51
- package/src/queue.mjs +130 -2
- package/src/run-history.mjs +23 -1
- package/src/run-mirror.mjs +221 -0
- package/src/schedules.mjs +67 -4
- package/src/start.mjs +240 -28
package/src/queue.mjs
CHANGED
|
@@ -8,10 +8,67 @@ import { PR_CLOSE_ACTIONS } from "./triggers.mjs";
|
|
|
8
8
|
const PR_CLOSE_WORDS = new Set(Object.values(PR_CLOSE_ACTIONS));
|
|
9
9
|
|
|
10
10
|
export const QUEUE = "pi-jobs";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The queue a HOST-AFFINE job goes to (issue #57): work only one machine can do, because the folder, the
|
|
14
|
+
* secret resolver or the wait-check script lives there.
|
|
15
|
+
*
|
|
16
|
+
* `@` is the separator because it is outside the worker-name charset (`[A-Za-z0-9._-]`), so
|
|
17
|
+
* `pi-jobs@<name>` decomposes unambiguously and a name can never contain one. A SUFFIX rather than a
|
|
18
|
+
* prefix so `KEYS bull:pi-jobs*` still shows an operator the whole deployment.
|
|
19
|
+
*
|
|
20
|
+
* Deliberately a separate queue rather than a field the pickup gate filters on. BullMQ has no selective
|
|
21
|
+
* pop, so filtering would mean taking a job and putting it back -- and promotion out of the delayed set is
|
|
22
|
+
* gated on each worker's OWN `Date.now()` in two places, so the host whose clock runs fastest wins every
|
|
23
|
+
* hop deterministically. A job that had to reach a different host might never get there, and jitter cannot
|
|
24
|
+
* fix it: it randomises WHEN the wake is, not WHO wins it.
|
|
25
|
+
*/
|
|
26
|
+
export const hostQueueName = (worker) => `${QUEUE}@${worker}`;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Every queue name this deployment drains: the shared one, plus one per named host (issue #57).
|
|
30
|
+
*
|
|
31
|
+
* Derived from the REGISTRY rather than from configuration, because the reader is usually the admin or
|
|
32
|
+
* the CLI, which know their own host at best and the fleet not at all. A deployment with no named worker
|
|
33
|
+
* that declared NO name yields exactly `[QUEUE]`, so every existing caller is unchanged. Note that this
|
|
34
|
+
* is derived from `routes`, not from a row existing: every worker publishes a row, named or not.
|
|
35
|
+
*
|
|
36
|
+
* This exists because a host queue that no reader knows about is worse than no host queue: the panel
|
|
37
|
+
* would show zero schedulers while cron ran, and `pi-dispatch pause` would stop half a deployment while
|
|
38
|
+
* reporting success -- the silent no-op its own comment already warns about for a mistyped name.
|
|
39
|
+
*/
|
|
40
|
+
export function fleetQueueNames(hosts) {
|
|
41
|
+
const seen = new Set();
|
|
42
|
+
for (const h of hosts ?? []) {
|
|
43
|
+
// A host has a queue only when it DECLARED a name. Every worker publishes a registry row -- that is
|
|
44
|
+
// what lets an unnamed fleet be seen at all -- but an undeclared one drains only the shared queue,
|
|
45
|
+
// so deriving queue names from every row would invent `pi-jobs@<hostname>` for a queue nothing
|
|
46
|
+
// reads: pausing it would create a real key for a phantom, and the counts would be a queue that can
|
|
47
|
+
// never have jobs.
|
|
48
|
+
if (h?.routes !== true && h?.routes !== "true") continue;
|
|
49
|
+
const name = h?.name;
|
|
50
|
+
// VALIDATED, because this is peer-written data crossing a trust boundary. `hostQueueName`'s own
|
|
51
|
+
// contract leans on the charset -- `@` is the separator precisely because a name cannot contain one
|
|
52
|
+
// -- and nothing else re-checks it. A name with a `:` makes `new Queue` throw, which would take the
|
|
53
|
+
// kill switch out entirely; one with an `@` would not decompose.
|
|
54
|
+
if (typeof name !== "string" || !WORKER_NAME_RE.test(name)) continue;
|
|
55
|
+
seen.add(name);
|
|
56
|
+
}
|
|
57
|
+
// Deduped: two rows naming one host would double-count its jobs in a summed status.
|
|
58
|
+
return [QUEUE, ...[...seen].sort().map(hostQueueName)];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The name charset, duplicated from `config.mjs` deliberately: this module imports nothing. */
|
|
62
|
+
const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
63
|
+
|
|
11
64
|
export { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId };
|
|
12
65
|
|
|
13
|
-
|
|
14
|
-
|
|
66
|
+
/**
|
|
67
|
+
* A queue handle. `name` defaults to the shared queue, so every existing caller is unchanged and a
|
|
68
|
+
* single-host deployment never names anything else.
|
|
69
|
+
*/
|
|
70
|
+
export function makeQueue(connection, { name = QUEUE } = {}) {
|
|
71
|
+
return new Queue(name, { connection });
|
|
15
72
|
}
|
|
16
73
|
|
|
17
74
|
/**
|
|
@@ -226,3 +283,74 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
|
|
|
226
283
|
});
|
|
227
284
|
return jobId;
|
|
228
285
|
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Every host queue that EXISTS, read from BullMQ's own keyspace rather than from the host registry.
|
|
289
|
+
*
|
|
290
|
+
* The registry answers "who is alive", and for a kill switch that is the wrong question. A host whose
|
|
291
|
+
* registry writes fail for ninety seconds loses its row while its BullMQ worker -- a separate connection,
|
|
292
|
+
* built with `maxRetriesPerRequest: null` precisely to ride out blips -- keeps draining. Pausing "the
|
|
293
|
+
* fleet" would then miss it and report success. The same gap opens for the ~15s before a booting worker's
|
|
294
|
+
* first beat lands, and on every `service restart`, since a clean shutdown DELs the row.
|
|
295
|
+
*
|
|
296
|
+
* Worse is the direction with no recovery path: pause while a host is live durably pauses its queue, and a
|
|
297
|
+
* later resume while that host is DOWN enumerates nothing for it. The queue stays paused permanently, and
|
|
298
|
+
* no surface can see it, because every surface was reading the registry too.
|
|
299
|
+
*
|
|
300
|
+
* A queue's meta key is durable and outlives its worker, so this asks the only authority that cannot go
|
|
301
|
+
* stale: the queues themselves. It also restores what `host-registry.mjs` claims about itself -- delete the
|
|
302
|
+
* whole `host:*` keyspace and nothing decides differently -- which the registry-derived kill switch had
|
|
303
|
+
* quietly made false.
|
|
304
|
+
*
|
|
305
|
+
* SCAN, not KEYS, and it is why this is NOT on the panel's per-tick path: it is for the rare command where
|
|
306
|
+
* being wrong costs money, not for a reader that runs every second. Fails open to `[]`, so an unreadable
|
|
307
|
+
* keyspace degrades to the registry's answer rather than refusing.
|
|
308
|
+
*/
|
|
309
|
+
export async function discoverHostQueues(redis, { timeoutMs = 2_000, count = 500 } = {}) {
|
|
310
|
+
const prefix = `bull:${QUEUE}@`;
|
|
311
|
+
const names = new Set();
|
|
312
|
+
try {
|
|
313
|
+
const deadline = Date.now() + timeoutMs;
|
|
314
|
+
let cursor = "0";
|
|
315
|
+
do {
|
|
316
|
+
// BOUNDED, because BullMQ's connections carry `maxRetriesPerRequest: null` and a command against
|
|
317
|
+
// an unreachable server therefore QUEUES FOREVER rather than rejecting -- so an unguarded await
|
|
318
|
+
// here would hang the kill switch instead of failing it open. The same trap the registry's
|
|
319
|
+
// `bounded` exists for.
|
|
320
|
+
//
|
|
321
|
+
// CLEARED on the way out, and NOT `unref`'d. Leaving it pending held the event loop open for the
|
|
322
|
+
// rest of the budget after the work was done, so `pi-dispatch pause` sat for two seconds having
|
|
323
|
+
// already paused everything; unref'ing instead would stop it firing when the hang is the last
|
|
324
|
+
// thing on the loop, which is the one case it exists for.
|
|
325
|
+
let timer;
|
|
326
|
+
const [next, keys] = await Promise.race([
|
|
327
|
+
redis.scan(cursor, "MATCH", `${prefix}*:meta`, "COUNT", count),
|
|
328
|
+
new Promise((_, reject) => {
|
|
329
|
+
timer = setTimeout(() => reject(new Error("scan timed out")), Math.max(1, deadline - Date.now()));
|
|
330
|
+
}),
|
|
331
|
+
]).finally(() => clearTimeout(timer));
|
|
332
|
+
cursor = next;
|
|
333
|
+
for (const key of keys ?? []) {
|
|
334
|
+
const name = String(key).slice(prefix.length, -":meta".length);
|
|
335
|
+
// Validated like every other peer-derived name: a key an operator hand-created could hold
|
|
336
|
+
// anything, and `new Queue` throws on a `:`, which would take the kill switch out entirely.
|
|
337
|
+
if (WORKER_NAME_RE.test(name)) names.add(name);
|
|
338
|
+
}
|
|
339
|
+
} while (cursor !== "0" && Date.now() < deadline);
|
|
340
|
+
} catch {
|
|
341
|
+
// Fail open: the registry's answer alone is still better than refusing to pause.
|
|
342
|
+
return [];
|
|
343
|
+
}
|
|
344
|
+
return [...names].sort().map(hostQueueName);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* The union of what is LIVE (the registry) and what EXISTS (the keyspace), which is the set a kill switch
|
|
349
|
+
* must act on: a live host with no queue yet has nothing to pause, and a dead host's queue still holds
|
|
350
|
+
* jobs and still has a paused flag somebody has to be able to clear.
|
|
351
|
+
*/
|
|
352
|
+
export function unionQueueNames(fromRegistry, fromKeyspace) {
|
|
353
|
+
const seen = new Set([...(fromRegistry ?? []), ...(fromKeyspace ?? [])]);
|
|
354
|
+
seen.delete(QUEUE);
|
|
355
|
+
return [QUEUE, ...[...seen].sort()];
|
|
356
|
+
}
|
package/src/run-history.mjs
CHANGED
|
@@ -378,7 +378,7 @@ function rebuildUsage(u) {
|
|
|
378
378
|
* default to `null` when the outcome does not carry them, so the record shape is stable whether or not
|
|
379
379
|
* the source reports those fields.
|
|
380
380
|
*/
|
|
381
|
-
export function buildRecord({ job, result, error, startedAt, endedAt }) {
|
|
381
|
+
export function buildRecord({ job, result, error, startedAt, endedAt, host = null }) {
|
|
382
382
|
const data = job.data ?? {};
|
|
383
383
|
const kind = data.kind ?? job.name;
|
|
384
384
|
const source = result ?? error ?? {};
|
|
@@ -441,6 +441,28 @@ export function buildRecord({ job, result, error, startedAt, endedAt }) {
|
|
|
441
441
|
// BRANCH NAME ARE DELIBERATELY ABSENT: this record's PII-free-by-construction property rests on it
|
|
442
442
|
// holding no attacker-chosen string, and a branch name is exactly that.
|
|
443
443
|
session: source.session ?? null,
|
|
444
|
+
// Which machine ran this (issue #57). Additive, nullable, an explicit literal, TAIL position: every
|
|
445
|
+
// prior addition took the tail, and "field order is the serialisation order" is this record's
|
|
446
|
+
// contract, so the tail is the only placement that leaves twenty-four existing positions untouched.
|
|
447
|
+
// Unconditional rather than a conditional spread, on the same contract sentence and on the
|
|
448
|
+
// `tokens`/`usage`/`session` precedent that null-with-the-key-present is this record's normal case.
|
|
449
|
+
//
|
|
450
|
+
// ADMISSIBILITY, which has to engage this record's own sentences rather than sidestep them. The
|
|
451
|
+
// PII-free-by-construction property rests on holding NO ATTACKER-CHOSEN STRING, and `host`
|
|
452
|
+
// satisfies that absolutely: there is no path from a webhook payload, an issue body, a branch name
|
|
453
|
+
// or a folder to this value. It is fixed once at boot from one environment variable against a
|
|
454
|
+
// charset that excludes `/` and `\`, so it cannot even be path-shaped -- which is the property
|
|
455
|
+
// `targetFor` drops a local folder to its basename to get.
|
|
456
|
+
//
|
|
457
|
+
// What it is NOT is anonymous, and that is worth writing down rather than glossing. The default is
|
|
458
|
+
// `os.hostname()`, and on a personal machine a hostname is often a person's name. The honest word
|
|
459
|
+
// is OPERATOR-DISCLOSED: the operator names their own machines, this record is written to their own
|
|
460
|
+
// disk, that name is already on every packet the machine sends, and `PI_WORKER_NAME` is the
|
|
461
|
+
// documented answer for anyone who wants something else here.
|
|
462
|
+
//
|
|
463
|
+
// Passed in rather than read from a module-level value, so `buildRecord` stays pure and every
|
|
464
|
+
// existing caller keeps getting `null` without knowing this field exists.
|
|
465
|
+
host,
|
|
444
466
|
};
|
|
445
467
|
}
|
|
446
468
|
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A fleet-visible copy of the run history (issue #57, Gap 3).
|
|
3
|
+
*
|
|
4
|
+
* Every worker writes its records to its own `PI_LOGS_DIR`, so on more than one machine each host's panel
|
|
5
|
+
* lists only the runs on its own disk. The operator sees a third of their deployment and has no way to
|
|
6
|
+
* know it.
|
|
7
|
+
*
|
|
8
|
+
* SHARED STORAGE IS THE OTHER ANSWER AND IT IS NOT SECOND-BEST. On a shared `PI_LOGS_DIR` the local read
|
|
9
|
+
* IS the merged read, with no machinery at all, and this module is redundant. It ships because that trade
|
|
10
|
+
* runs both ways and an operator must be allowed to decline it: sharing the directory also shares the
|
|
11
|
+
* PII-bearing raw `.log`, and a mount outage becomes a LOST RECORD where a Valkey outage costs only a
|
|
12
|
+
* fleet view. Both shapes work; `docs/multi-host.md` says which is which.
|
|
13
|
+
*
|
|
14
|
+
* THE FILE IS THE RECORD AND THIS IS A VIEW. Three things follow, and each is load-bearing:
|
|
15
|
+
*
|
|
16
|
+
* - The file is written FIRST, always. A crash between the two leaves a fleet-visible run whose durable
|
|
17
|
+
* source does not exist, which inverts the one claim this design rests on.
|
|
18
|
+
* - The mirror's TTL is never longer than the file retention window, so it can never show a run whose
|
|
19
|
+
* file has already been reaped. A view that outlives its source is a second source of truth, which is
|
|
20
|
+
* exactly what `DES-RUN-HISTORY-FLAT-FILES-NO-DB` refuses.
|
|
21
|
+
* - Nothing derived is stored. The bytes are the sidecar's own bytes, so there is nothing to be stale
|
|
22
|
+
* RELATIVE TO: a retry overwrites the same key exactly as it overwrites the same file, and cost
|
|
23
|
+
* classification is still computed at fold time from `subscriptions.json` rather than frozen here.
|
|
24
|
+
*
|
|
25
|
+
* WHY THE WHOLE RECORD RATHER THAN A PROJECTION. The record is PII-free BY CONSTRUCTION -- it holds no
|
|
26
|
+
* attacker-chosen string, which `INT-RUN-HISTORY-FILE-CONTRACT` states and `buildRecord` enforces field by
|
|
27
|
+
* field. Copying it whole inherits that property; a projection would re-derive it at a second serialiser,
|
|
28
|
+
* where the next person to add a field has to remember this file exists. It is also what the readers need:
|
|
29
|
+
* the cost fold, the graph and the insights view read eleven fields between them.
|
|
30
|
+
*
|
|
31
|
+
* NOT MIRRORED: the raw `.log`. It is the one artifact here that holds issue text, comment text and tool
|
|
32
|
+
* output, and mirroring it would move that off the machine the operator chose to keep it on. A foreign
|
|
33
|
+
* run's record names its host, so the panel can say where the bytes are rather than pretending there are
|
|
34
|
+
* none.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
/** The index: sanitized jobId -> the run's end (or start) in millis. */
|
|
38
|
+
export const RUNS_INDEX = "runs:index";
|
|
39
|
+
|
|
40
|
+
/** One run's own bytes. */
|
|
41
|
+
export const runRecordKey = (sanitizedJobId) => `runs:rec:${sanitizedJobId}`;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The deepest any reader asks. `SCAN_WINDOW_MAX_DAYS` in the admin is 92, so a longer window would hold
|
|
45
|
+
* bytes nothing can request.
|
|
46
|
+
*/
|
|
47
|
+
export const MIRROR_MAX_DAYS = 92;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* A hard ceiling on index members, independent of the time window.
|
|
51
|
+
*
|
|
52
|
+
* The window alone does not bound memory: a deployment running thousands of jobs a day would hold a
|
|
53
|
+
* quarter of a million members for ninety-two days. This caps what the fleet view can cost at roughly the
|
|
54
|
+
* depth a panel can display, and the file on disk remains the complete history either way.
|
|
55
|
+
*/
|
|
56
|
+
export const RUNS_INDEX_MAX = 5_000;
|
|
57
|
+
|
|
58
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
59
|
+
const OP_TIMEOUT_MS = 2_000;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* How long a mirrored record lives.
|
|
63
|
+
*
|
|
64
|
+
* Never longer than the operator's own retention, and never longer than what any reader asks for.
|
|
65
|
+
* `retentionDays: 0` means keep the files forever, which is the one case where the mirror is the shorter
|
|
66
|
+
* of the two, so it clamps to the reader's ceiling rather than to infinity.
|
|
67
|
+
*/
|
|
68
|
+
export function mirrorWindowMs(retentionDays) {
|
|
69
|
+
const days = Number(retentionDays) > 0 ? Math.min(Number(retentionDays), MIRROR_MAX_DAYS) : MIRROR_MAX_DAYS;
|
|
70
|
+
return days * DAY_MS;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* BullMQ's connections carry `maxRetriesPerRequest: null`, so a command against an unreachable server
|
|
75
|
+
* QUEUES FOREVER rather than rejecting. Every await here is bounded for that reason; an unbounded one
|
|
76
|
+
* would not fail the mirror, it would hang the job that was writing to it.
|
|
77
|
+
*/
|
|
78
|
+
function bounded(promise, ms) {
|
|
79
|
+
let timer;
|
|
80
|
+
return Promise.race([
|
|
81
|
+
promise,
|
|
82
|
+
new Promise((_, reject) => {
|
|
83
|
+
timer = setTimeout(() => reject(new Error("mirror timeout")), ms);
|
|
84
|
+
}),
|
|
85
|
+
]).finally(() => clearTimeout(timer));
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* The writer. Returns `{ mirror, close }`; `mirror` never throws and never rejects.
|
|
90
|
+
*
|
|
91
|
+
* A history blip must not fail a job that has already run and already been recorded to disk. Every failure
|
|
92
|
+
* here costs a row in a fleet view and nothing else, which is why the whole body is wrapped and the result
|
|
93
|
+
* is a boolean nobody is obliged to read.
|
|
94
|
+
*/
|
|
95
|
+
export function makeRunMirror({ redis, retentionDays, now = () => Date.now(), log = () => {}, timeoutMs = OP_TIMEOUT_MS, indexMax = RUNS_INDEX_MAX } = {}) {
|
|
96
|
+
const windowMs = mirrorWindowMs(retentionDays);
|
|
97
|
+
let warned = false;
|
|
98
|
+
|
|
99
|
+
return {
|
|
100
|
+
async mirror(record, sanitizedJobId) {
|
|
101
|
+
if (!redis || !record || !sanitizedJobId) return false;
|
|
102
|
+
try {
|
|
103
|
+
const at = Date.parse(record.endedAt ?? record.startedAt ?? "");
|
|
104
|
+
const score = Number.isFinite(at) ? at : now();
|
|
105
|
+
const body = JSON.stringify(record);
|
|
106
|
+
await bounded(redis.set(runRecordKey(sanitizedJobId), body, "PX", windowMs), timeoutMs);
|
|
107
|
+
await bounded(redis.zadd(RUNS_INDEX, score, sanitizedJobId), timeoutMs);
|
|
108
|
+
// Trimmed by the WRITER, twice: by age, and by count. Two `ZREMRANGE`s against a run that
|
|
109
|
+
// took minutes is free, and it means no reader has to pay for a backlog it did not create.
|
|
110
|
+
await bounded(redis.zremrangebyscore(RUNS_INDEX, "-inf", `(${now() - windowMs}`), timeoutMs);
|
|
111
|
+
await bounded(redis.zremrangebyrank(RUNS_INDEX, 0, -indexMax - 1), timeoutMs);
|
|
112
|
+
// ROLLING expiry, deliberately unlike `budget.mjs`'s set-once rule and deliberately like
|
|
113
|
+
// `pi-dispatch:sched-stalls`. A budget window must not be pushed forward by traffic or a busy
|
|
114
|
+
// day never resets; an ACTIVITY index should roll with traffic, because that is what it
|
|
115
|
+
// describes. A fleet that stops running jobs loses its index one window later, which is
|
|
116
|
+
// correct: there is nothing left to show.
|
|
117
|
+
await bounded(redis.pexpire(RUNS_INDEX, windowMs), timeoutMs);
|
|
118
|
+
warned = false;
|
|
119
|
+
return true;
|
|
120
|
+
} catch (err) {
|
|
121
|
+
// Once per transition, not once per job: a Valkey outage during a busy hour must not turn one
|
|
122
|
+
// fault into a thousand log lines (`notePackageKey`'s precedent).
|
|
123
|
+
if (!warned) {
|
|
124
|
+
warned = true;
|
|
125
|
+
log("run_mirror_failed", { jobId: sanitizedJobId, reason: err?.message });
|
|
126
|
+
}
|
|
127
|
+
return false;
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* The reader. Returns `{ runs, degraded }` and never throws.
|
|
135
|
+
*
|
|
136
|
+
* `degraded` is a DISCRIMINATED channel rather than a silence, and two of its values must not collapse:
|
|
137
|
+
* `"off"` means the index is absent, which is what a single-host deployment and a fleet of workers still
|
|
138
|
+
* below the version floor both look like, while `"unreachable"` means we could not tell. A new panel
|
|
139
|
+
* meeting old workers has to read "off", not "error".
|
|
140
|
+
*
|
|
141
|
+
* Two round trips regardless of how many runs come back: one `ZREVRANGEBYSCORE` for the ids, one `MGET`
|
|
142
|
+
* for the bodies. Never one read per run.
|
|
143
|
+
*/
|
|
144
|
+
export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = () => Date.now(), timeoutMs = OP_TIMEOUT_MS } = {}) {
|
|
145
|
+
if (!redis) return { runs: [], degraded: "off" };
|
|
146
|
+
let ids;
|
|
147
|
+
try {
|
|
148
|
+
ids = await bounded(redis.zrevrangebyscore(RUNS_INDEX, "+inf", `(${sinceMs}`, "LIMIT", 0, Math.max(1, limit)), timeoutMs);
|
|
149
|
+
} catch (err) {
|
|
150
|
+
return { runs: [], degraded: `unreachable (${err?.message ?? "?"})` };
|
|
151
|
+
}
|
|
152
|
+
if (!Array.isArray(ids) || ids.length === 0) return { runs: [], degraded: "off" };
|
|
153
|
+
|
|
154
|
+
let bodies;
|
|
155
|
+
try {
|
|
156
|
+
bodies = await bounded(redis.mget(...ids.map(runRecordKey)), timeoutMs);
|
|
157
|
+
} catch (err) {
|
|
158
|
+
return { runs: [], degraded: `unreachable (${err?.message ?? "?"})` };
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const runs = [];
|
|
162
|
+
const stale = [];
|
|
163
|
+
for (let i = 0; i < ids.length; i++) {
|
|
164
|
+
const raw = bodies?.[i];
|
|
165
|
+
if (typeof raw !== "string" || raw === "") {
|
|
166
|
+
// An id whose body has expired: the per-key TTL fired and the index member outlived it. The
|
|
167
|
+
// READER prunes it, which is what `wait:held` does for the same shape and for the same reason --
|
|
168
|
+
// a writer that crashed cannot clean up after itself, and a reader is already here.
|
|
169
|
+
stale.push(ids[i]);
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
const rec = JSON.parse(raw);
|
|
174
|
+
if (rec && typeof rec === "object") runs.push(rec);
|
|
175
|
+
} catch {
|
|
176
|
+
stale.push(ids[i]); // unparseable is indistinguishable from gone, and equally not showable
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
if (stale.length > 0) {
|
|
180
|
+
try {
|
|
181
|
+
await bounded(redis.zrem(RUNS_INDEX, ...stale), timeoutMs);
|
|
182
|
+
} catch {
|
|
183
|
+
// best-effort: a straggler in the index costs one skipped row next time, never a wrong one
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return { runs, degraded: runs.length >= limit ? "truncated" : "ok" };
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* One list from two sources.
|
|
191
|
+
*
|
|
192
|
+
* DEDUP BY LATER `endedAt`, LOCAL BREAKS A TIE. Not decoration: a retry can land on a different host, so
|
|
193
|
+
* host A may hold attempt 0 (failed) while host B mirrored attempt 1 (completed). "Local wins" alone would
|
|
194
|
+
* show the stale one. Local breaking an exact tie keeps a single-host deployment reading its own files.
|
|
195
|
+
*
|
|
196
|
+
* CUT AFTER THE SORT, never before. Slicing first is the defect `held.test.mjs` already exists to prevent:
|
|
197
|
+
* it makes the result depend on which source happened to be longer.
|
|
198
|
+
*/
|
|
199
|
+
export function mergeRuns(local, mirrored, { limit = 50 } = {}) {
|
|
200
|
+
const by = new Map();
|
|
201
|
+
const at = (r) => {
|
|
202
|
+
const t = Date.parse(r?.endedAt ?? r?.startedAt ?? "");
|
|
203
|
+
return Number.isFinite(t) ? t : -Infinity;
|
|
204
|
+
};
|
|
205
|
+
// Mirrored first, so a local record with an equal timestamp overwrites it on the second pass.
|
|
206
|
+
for (const r of Array.isArray(mirrored) ? mirrored : []) if (r?.jobId) by.set(r.jobId, r);
|
|
207
|
+
for (const r of Array.isArray(local) ? local : []) {
|
|
208
|
+
if (!r?.jobId) continue;
|
|
209
|
+
const seen = by.get(r.jobId);
|
|
210
|
+
if (!seen || at(r) >= at(seen)) by.set(r.jobId, r);
|
|
211
|
+
}
|
|
212
|
+
const out = [...by.values()].sort((a, b) => at(b) - at(a));
|
|
213
|
+
return out.slice(0, Math.max(0, limit));
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** The distinct hosts a merged list came from, computed from the RECORDS rather than from the mirror. */
|
|
217
|
+
export function hostsIn(runs) {
|
|
218
|
+
const names = new Set();
|
|
219
|
+
for (const r of runs ?? []) if (typeof r?.host === "string" && r.host !== "") names.add(r.host);
|
|
220
|
+
return [...names].sort();
|
|
221
|
+
}
|
package/src/schedules.mjs
CHANGED
|
@@ -23,7 +23,7 @@ import { parseTriggers } from "./triggers.mjs";
|
|
|
23
23
|
* valid deployment. `readFileSync`/`existsSync` are injectable so tests exercise the full path with no
|
|
24
24
|
* real filesystem.
|
|
25
25
|
*/
|
|
26
|
-
export function loadSchedules(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync } = {}) {
|
|
26
|
+
export function loadSchedules(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync, fleet = false } = {}) {
|
|
27
27
|
const path = config.triggersFile;
|
|
28
28
|
if (path === null || path === undefined) return []; // cron disabled
|
|
29
29
|
|
|
@@ -33,14 +33,77 @@ export function loadSchedules(config, { readFileSync = fsReadFileSync, existsSyn
|
|
|
33
33
|
|
|
34
34
|
const triggers = parseTriggers(readFileSync(path, "utf8"), path);
|
|
35
35
|
|
|
36
|
-
return triggers.filter((t) => t.on.type === "cron").map((t) => normalizeCronSchedule(t, path, existsSync));
|
|
36
|
+
return triggers.filter((t) => t.on.type === "cron").map((t) => normalizeCronSchedule(t, path, existsSync, fleet));
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
/**
|
|
40
|
+
* The cron set AS AUTHORED, before any placement decision (issue #57).
|
|
41
|
+
*
|
|
42
|
+
* This is the object two hosts have to agree about, and it is deliberately not `loadSchedules`'s output.
|
|
43
|
+
* That function resolves PLACEMENT -- it replaces every trigger whose folder is on another machine with
|
|
44
|
+
* a stub -- so its result differs per host BY CONSTRUCTION. Fingerprinting it would make every correctly
|
|
45
|
+
* configured fleet refuse itself forever: mini1 owns `/a`, mini2 owns `/b`, their sets never match, and
|
|
46
|
+
* neither ever reconciles again. What they share is the FILE, so the file is what gets hashed.
|
|
47
|
+
*
|
|
48
|
+
* Pure and fs-free apart from the read: no `existsSync`, because existence is exactly the question that
|
|
49
|
+
* makes two honest hosts differ.
|
|
50
|
+
*/
|
|
51
|
+
export function authoredCron(config, { readFileSync = fsReadFileSync, existsSync = fsExistsSync } = {}) {
|
|
52
|
+
const path = config.triggersFile;
|
|
53
|
+
if (path === null || path === undefined) return null; // cron disabled: no opinion at all (see cronFingerprint)
|
|
54
|
+
if (!existsSync(path)) return null;
|
|
55
|
+
try {
|
|
56
|
+
return parseTriggers(readFileSync(path, "utf8"), path)
|
|
57
|
+
.filter((t) => t.on.type === "cron")
|
|
58
|
+
.map((t) => ({ schedulerId: t.on.id, pattern: t.on.pattern, run: t.run }));
|
|
59
|
+
} catch {
|
|
60
|
+
// A file this host cannot parse is not an opinion about what should be scheduled. It refuses boot
|
|
61
|
+
// elsewhere and keeps last-good on reload; here it must not become a fingerprint that disagrees
|
|
62
|
+
// with every peer.
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Split a schedule set into the triggers THIS host serves and the ones it does not (issue #57).
|
|
69
|
+
*
|
|
70
|
+
* `loadSchedules` already refused everything a pure validator could refuse and everything the filesystem
|
|
71
|
+
* could answer for a trigger this host owns. What is left is the third question, and it is the one Gap 2
|
|
72
|
+
* is about: a folder that is not here is not necessarily a mistake, it may simply be another machine's.
|
|
73
|
+
*
|
|
74
|
+
* Exported so the split is testable without an fs, and so a caller can report what it will not be running.
|
|
75
|
+
*/
|
|
76
|
+
export function servedSchedules(schedules) {
|
|
77
|
+
const served = [];
|
|
78
|
+
const unserved = [];
|
|
79
|
+
for (const s of schedules) (s.unserved ? unserved : served).push(s);
|
|
80
|
+
return { served, unserved };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function normalizeCronSchedule({ on, run }, path, existsSync, fleet) {
|
|
40
84
|
// The pure validator already guaranteed a non-empty, `:`-free, charset-valid, unique id and a
|
|
41
85
|
// well-formed pattern; folder existence is the one fs-dependent check it deferred to here.
|
|
86
|
+
//
|
|
87
|
+
// ON A FLEET THAT IS THE WRONG QUESTION. `INT-TRIGGERS-FILE-CONTRACT` splits this as "type here,
|
|
88
|
+
// reality where it can be known", and issue #57 adds a third level: PLACEMENT, where the fleet is
|
|
89
|
+
// known. A folder that is absent on THIS machine may simply belong to another one, and refusing the
|
|
90
|
+
// worker's boot for it takes every unrelated trigger -- every forge job, every other folder -- offline
|
|
91
|
+
// with it. That is the sentence #57's own acceptance forbids.
|
|
92
|
+
//
|
|
93
|
+
// `fleet` is `PI_WORKER_NAME` being DECLARED, deliberately, and not a registry read. Two reasons, and
|
|
94
|
+
// both are failures I would otherwise have shipped. A registry read here would make a fleet-wide
|
|
95
|
+
// restart into a fleet-wide boot refusal, because every host would come up seeing no peers yet. And it
|
|
96
|
+
// would have to happen after the Valkey client exists, which is BELOW the four destructive boot sweeps
|
|
97
|
+
// -- so a single-host deployment with one typo'd folder would reap containers, prune history and delete
|
|
98
|
+
// sandboxes on every restart before refusing. Declaring a name is the operator saying "this is a
|
|
99
|
+
// fleet", it is known before anything runs, and it keeps a single-host deployment byte-identical.
|
|
42
100
|
if (!existsSync(run.folder)) {
|
|
43
|
-
|
|
101
|
+
if (!fleet) {
|
|
102
|
+
throw configError(`cron trigger "${on.id}": run.folder does not exist: ${run.folder} (${path})`);
|
|
103
|
+
}
|
|
104
|
+
// Not mine. Its skillsDir is not my business either: `isAbsolute` is OS-dependent, and judging
|
|
105
|
+
// another host's path on my platform is the exact mistake the shared validator refuses to make.
|
|
106
|
+
return { schedulerId: on.id, unserved: "folder-absent" };
|
|
44
107
|
}
|
|
45
108
|
|
|
46
109
|
// `run.skillsDir` gets the same treatment, and for the same reason (REQ-PER-TRIGGER-SKILLS): the pure
|