@edgehero/pi-dispatch 1.10.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +300 -148
- package/README.md +50 -0
- package/deploy/com.pi-dispatch.worker.plist +9 -3
- package/deploy/docker-compose.yml +49 -16
- package/deploy/egress-proxy.conf +32 -2
- package/deploy/nssm-install.cmd +12 -6
- package/deploy/pi-dispatch-egress-out.network +10 -0
- package/deploy/pi-dispatch-egress-proxy.container +50 -0
- package/deploy/pi-dispatch-netns-keeper.container +80 -0
- package/deploy/pi-dispatch-netns-keeper.network +18 -0
- package/deploy/pi-dispatch-valkey.container +51 -0
- package/deploy/pi-dispatch-valkey.network +16 -0
- package/deploy/receiver.service +6 -0
- package/deploy/worker-env-wrapper.cmd +11 -0
- package/deploy/worker-env-wrapper.sh +60 -34
- package/deploy/worker.service +18 -8
- package/package.json +14 -4
- package/src/azure-host.mjs +19 -0
- package/src/azure-identity.mjs +18 -2
- package/src/backend-conformance.mjs +71 -18
- package/src/backend-local.mjs +637 -21
- package/src/backend-podman.mjs +1168 -0
- package/src/backend-registry.mjs +86 -3
- package/src/backends.mjs +489 -37
- package/src/branch.mjs +7 -2
- package/src/cancel-cli.mjs +174 -0
- package/src/cancel-state.mjs +125 -0
- package/src/cli.mjs +188 -90
- package/src/config.mjs +503 -43
- package/src/connection.mjs +374 -8
- package/src/container-spec.mjs +102 -7
- package/src/daemon-facts.mjs +167 -0
- package/src/deployment-venue.mjs +158 -0
- package/src/docker-run.mjs +146 -15
- package/src/doctor.mjs +4701 -414
- package/src/egress-conf-copy.mjs +166 -0
- package/src/egress-proxy-state.mjs +151 -0
- package/src/egress.mjs +455 -25
- package/src/entry.mjs +27 -0
- package/src/env-allowlist.mjs +222 -40
- package/src/env-file.mjs +1869 -33
- package/src/exit-code.mjs +15 -0
- package/src/flow-gate.mjs +5 -3
- package/src/forgejo-host.mjs +19 -0
- package/src/forgejo-identity.mjs +21 -2
- package/src/get-token.mjs +67 -18
- package/src/git-dirty.mjs +9 -1
- package/src/git-hardening.mjs +33 -0
- package/src/github-app-setup.mjs +29 -12
- package/src/github-prompt.mjs +4 -1
- package/src/gitlab-host.mjs +19 -0
- package/src/gitlab-identity.mjs +19 -2
- package/src/host-registry.mjs +29 -2
- package/src/identity.mjs +29 -4
- package/src/image-preflight.mjs +46 -11
- package/src/image-ref.mjs +21 -0
- package/src/index.mjs +363 -13
- package/src/init.mjs +197 -38
- package/src/job-user.mjs +252 -0
- package/src/json-duplicates.mjs +204 -0
- package/src/live-probes.mjs +1020 -0
- package/src/materialize.mjs +4 -11
- package/src/netns-keeper.mjs +264 -0
- package/src/on-failure.mjs +119 -0
- package/src/outbox.mjs +7 -0
- package/src/podman-stack.mjs +1304 -0
- package/src/prepare-github.mjs +6 -6
- package/src/prepare-local.mjs +51 -17
- package/src/prepare.mjs +27 -6
- package/src/processor.mjs +505 -26
- package/src/provider-key.mjs +41 -0
- package/src/provider-steering.mjs +144 -0
- package/src/queue.mjs +35 -8
- package/src/redact.mjs +84 -0
- package/src/reserved-env.mjs +7 -3
- package/src/retention-sweep.mjs +178 -0
- package/src/run-container.mjs +181 -14
- package/src/run-history.mjs +105 -16
- package/src/runtime-observations.mjs +1152 -0
- package/src/runtime-settings.mjs +13 -8
- package/src/sandbox-cli.mjs +100 -95
- package/src/sandbox-store.mjs +612 -45
- package/src/sandbox.mjs +1459 -37
- package/src/schedules.mjs +16 -3
- package/src/secret-profiles.mjs +2 -1
- package/src/secrets.mjs +23 -6
- package/src/service-env.mjs +247 -0
- package/src/service.mjs +618 -28
- package/src/session-store.mjs +678 -53
- package/src/start.mjs +1348 -326
- package/src/transient.mjs +240 -0
- package/src/triggers-file.mjs +71 -15
- package/src/triggers.mjs +176 -19
- package/src/up.mjs +1399 -85
- package/src/valkey-auth.mjs +529 -0
- package/src/valkey-endpoint.mjs +367 -0
- package/src/watch-closer.mjs +158 -0
package/src/cli.mjs
CHANGED
|
@@ -3,8 +3,10 @@ import { existsSync } from "node:fs";
|
|
|
3
3
|
import { resolve } from "node:path";
|
|
4
4
|
import { parseArgs } from "node:util";
|
|
5
5
|
import { loadConfig } from "./config.mjs";
|
|
6
|
-
import { EXIT_POLICY } from "./exit-code.mjs";
|
|
6
|
+
import { EXIT_POLICY, installRejectionPrinter } from "./exit-code.mjs";
|
|
7
|
+
import { isEntryModule } from "./entry.mjs";
|
|
7
8
|
import { gitDirty } from "./git-dirty.mjs";
|
|
9
|
+
import { imageRefProblem } from "./image-ref.mjs";
|
|
8
10
|
|
|
9
11
|
/** How long the kill switch waits on the host registry before acting on the shared queue alone. */
|
|
10
12
|
const FLEET_READ_TIMEOUT_MS = 2_000;
|
|
@@ -12,7 +14,9 @@ const FLEET_READ_TIMEOUT_MS = 2_000;
|
|
|
12
14
|
const USAGE = `pi-dispatch — run pi coding-agent flows on your own folders
|
|
13
15
|
|
|
14
16
|
pi-dispatch init scaffold .env + triggers.json + pause-windows.json + pi-packages.json + subscriptions.json here
|
|
15
|
-
pi-dispatch doctor [--fix]
|
|
17
|
+
pi-dispatch doctor [--fix] [--live]
|
|
18
|
+
preflight Docker, Valkey, the job image, and your provider key; --fix offers to run each fix (y/N per action);
|
|
19
|
+
--live reads the backend declarations back off short-lived real containers (shown in docker ps while they run)
|
|
16
20
|
pi-dispatch up [--yes] one consented pass: pull+tag the job image, start Valkey, init, doctor
|
|
17
21
|
pi-dispatch setup github mint GitHub App credentials in one browser click (App Manifest flow);
|
|
18
22
|
every write shown first and individually consented — no --yes here
|
|
@@ -30,7 +34,7 @@ const USAGE = `pi-dispatch — run pi coding-agent flows on your own folders
|
|
|
30
34
|
what is still re-openable, for how long, and what is running now
|
|
31
35
|
|
|
32
36
|
pi-dispatch worker drain the queue (run this in another terminal, or as a service)
|
|
33
|
-
pi-dispatch-receiver webhook receiver for forge triggers
|
|
37
|
+
pi-dispatch-receiver webhook receiver for forge triggers, its own bin (see docs/github.md)
|
|
34
38
|
pi-dispatch service <render|install|uninstall|status|start|stop|restart> [--receiver] [--user|--system] [--force]
|
|
35
39
|
run the worker (or --receiver) as an OS service — the deploy/ templates
|
|
36
40
|
rendered with this host's real paths, installed user-level;
|
|
@@ -38,6 +42,9 @@ const USAGE = `pi-dispatch — run pi coding-agent flows on your own folders
|
|
|
38
42
|
pi-dispatch pause stop taking new jobs (durable; survives worker restart)
|
|
39
43
|
pi-dispatch resume resume taking jobs
|
|
40
44
|
pi-dispatch status show paused state + job counts
|
|
45
|
+
pi-dispatch cancel <jobId> stop one job: a queued or held job is removed (the line says whether it had
|
|
46
|
+
made attempts; cancel records nothing), a running one is aborted on whichever
|
|
47
|
+
host owns it (its record says operator-cancel)
|
|
41
48
|
|
|
42
49
|
Config comes from the environment (see .env.example); flags override it per run.
|
|
43
50
|
Prefer being walked through all of this? The operator panel's /dispatch setup does every step
|
|
@@ -47,18 +54,20 @@ with a consent per action: pi install npm:@edgehero/pi-dispatch-admin`;
|
|
|
47
54
|
// injects a collector instead of reassigning `process.stdout.write`. That matters because `node --test`
|
|
48
55
|
// runs each file in a child process that serialises its own results over that same stdout, so a test
|
|
49
56
|
// holding a replacement across an `await` swallows the runner's result frames (issue #266).
|
|
50
|
-
export async function main(argv = process.argv.slice(2), env = process.env, { write = (chunk) => process.stdout.write(chunk) } = {}) {
|
|
57
|
+
export async function main(argv = process.argv.slice(2), env = process.env, { write = (chunk) => process.stdout.write(chunk), valkeyRefusal = valkeyRefusalAtStart } = {}) {
|
|
51
58
|
const cmd = argv[0];
|
|
52
59
|
|
|
53
60
|
if (cmd === "init") {
|
|
54
61
|
const { runInit } = await import("./init.mjs");
|
|
55
|
-
|
|
62
|
+
// The environment rides along so the next steps match the venue (issue #453): PI_BACKENDS=podman alone gets
|
|
63
|
+
// the podman ladder.
|
|
64
|
+
return runInit(process.cwd(), { env });
|
|
56
65
|
}
|
|
57
66
|
|
|
58
67
|
if (cmd === "doctor") {
|
|
59
68
|
const { runDoctor } = await import("./doctor.mjs");
|
|
60
|
-
// `fix`
|
|
61
|
-
return runDoctor(env, { fix: argv.slice(1).includes("--fix") });
|
|
69
|
+
// `fix` and `live` ride in the deps position (runDoctor(env, depsOrOpts)) — one options bag, no third arg.
|
|
70
|
+
return runDoctor(env, { fix: argv.slice(1).includes("--fix"), live: argv.slice(1).includes("--live") });
|
|
62
71
|
}
|
|
63
72
|
|
|
64
73
|
if (cmd === "up") {
|
|
@@ -115,6 +124,12 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
|
|
|
115
124
|
const folder = positionals[0] && resolve(positionals[0]);
|
|
116
125
|
if (!folder || !existsSync(folder)) return fail(`folder not found: ${positionals[0] ?? "(none given)"}`);
|
|
117
126
|
if (!values.task) return fail("a --task is required");
|
|
127
|
+
// The one image rule (`image-ref.mjs`, issue #471 gate round 1), before anything is queued: a dash-leading `--image`
|
|
128
|
+
// was enqueued as it was and reached the runtime's argv as a flag at job start. Empty stays "the default", as below.
|
|
129
|
+
if (values.image) {
|
|
130
|
+
const problem = imageRefProblem(values.image);
|
|
131
|
+
if (problem) return fail(`--image ${problem.reason} (got ${JSON.stringify(values.image)})`);
|
|
132
|
+
}
|
|
118
133
|
|
|
119
134
|
// A local job edits the folder IN PLACE with no undo (SECURITY.md). Refuse a dirty working
|
|
120
135
|
// tree unless --force, so a bad run cannot mix with uncommitted work the operator can't
|
|
@@ -126,14 +141,21 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
|
|
|
126
141
|
}
|
|
127
142
|
|
|
128
143
|
const config = loadConfig(env);
|
|
129
|
-
const { parseConnection } = await import("./connection.mjs");
|
|
144
|
+
const { cliValkeyUrl, parseConnection } = await import("./connection.mjs");
|
|
145
|
+
// PR #475's review: VALKEY_URL as the password is read, this shell's else the deployment .env's (a disagreement
|
|
146
|
+
// named), not the shell's alone: from the folder of a Valkey on another port, `run` dialled 6379.
|
|
147
|
+
const valkeyUrl = cliValkeyUrl(env);
|
|
130
148
|
const { makeQueue, enqueueLocalJob, hostQueueName } = await import("./queue.mjs");
|
|
131
149
|
// failFast: a one-shot enqueue must not hang forever if Valkey is down -- error clearly.
|
|
132
150
|
// Onto THIS host's queue when the deployment declares a name (issue #57). The folder was checked
|
|
133
151
|
// against this machine's filesystem a few lines up, so this machine is the only one that can run it;
|
|
134
152
|
// enqueueing it where every host drains would be handing a job to a peer that has no such folder.
|
|
135
153
|
const hq = config.workerNameDeclared ? hostQueueName(config.workerName) : null;
|
|
136
|
-
|
|
154
|
+
// Issue #464 (gate round 3): judged before anything is sent, so a refused Valkey is said as the refusal it is, from
|
|
155
|
+
// any directory. Only a refusal stops here; nothing answering is the "could not reach" below.
|
|
156
|
+
const refused = await valkeyRefusal(valkeyUrl, env);
|
|
157
|
+
if (refused) return fail(refused);
|
|
158
|
+
const queue = makeQueue(parseConnection(valkeyUrl, { failFast: true }), { ...(hq ? { name: hq } : {}) });
|
|
137
159
|
try {
|
|
138
160
|
// Absent flags stay absent (undefined) so the value resolves at job start against the
|
|
139
161
|
// settings overlay/env, not a default frozen here (INT-CONFIG-OVERLAY-CONTRACT).
|
|
@@ -150,7 +172,7 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
|
|
|
150
172
|
});
|
|
151
173
|
write(`queued ${jobId} — folder ${folder}\nrun \`pi-dispatch worker\` to process it.\n`);
|
|
152
174
|
} catch (error) {
|
|
153
|
-
return fail(`could not reach Valkey at ${
|
|
175
|
+
return fail(error?.valkeyRefused ? error.message : `could not reach Valkey at ${(await import("./connection.mjs")).urlShown(valkeyUrl)}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(valkeyUrl)}\n ${error.message}`);
|
|
154
176
|
} finally {
|
|
155
177
|
await queue.close().catch(() => {});
|
|
156
178
|
}
|
|
@@ -159,89 +181,147 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
|
|
|
159
181
|
|
|
160
182
|
if (cmd === "pause" || cmd === "resume" || cmd === "status") {
|
|
161
183
|
// The kill switch reads ONLY VALKEY_URL, not the full loadConfig -- it must work even when
|
|
162
|
-
// GitHub auth is misconfigured, so an operator can always stop the queue.
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
//
|
|
168
|
-
//
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
// `readLiveHosts` RETURNS `{unreachable}` rather than rejecting, so `blind` is a branch on its
|
|
177
|
-
// value and the `.catch` below is only for a client that throws before it can answer.
|
|
178
|
-
const probe = makeRedisClient(url);
|
|
179
|
-
// Without this, a down Valkey dumps nine `[ioredis] Unhandled error event` traces before the one clean
|
|
180
|
-
// line -- the exact noise `defaultProbeValkey` exists to suppress.
|
|
181
|
-
probe.on?.("error", () => {});
|
|
182
|
-
// Both reads, concurrently, sharing one budget. The registry answers WHO IS LIVE; BullMQ's own meta
|
|
183
|
-
// keys answer WHICH QUEUES EXIST, and for a kill switch the second is the question that matters. A
|
|
184
|
-
// host whose registry writes fail for ninety seconds loses its row while its worker keeps draining,
|
|
185
|
-
// and a resume that misses a queue leaves it paused forever with no surface able to name it. A meta
|
|
186
|
-
// key outlives its worker; a registry row does not.
|
|
187
|
-
const [fleet, existing] = await Promise.all([
|
|
188
|
-
readLiveHosts(probe, { timeoutMs: FLEET_READ_TIMEOUT_MS }).catch((error) => ({ unreachable: error?.message ?? String(error) })),
|
|
189
|
-
discoverHostQueues(probe, { timeoutMs: FLEET_READ_TIMEOUT_MS }),
|
|
190
|
-
]);
|
|
191
|
-
probe.disconnect?.();
|
|
192
|
-
const blind = fleet?.unreachable ?? null;
|
|
193
|
-
const names = unionQueueNames(fleetQueueNames(fleet?.hosts), existing);
|
|
194
|
-
// The registry being unreadable no longer means we saw one queue: the keyspace scan may well have
|
|
195
|
-
// found them. Report the count we ACTED on, and name the degraded read separately.
|
|
196
|
-
const span = `${names.length > 1 ? ` [${names.length} queues]` : ""}${blind ? ` [registry unreadable: ${blind}]` : ""}`;
|
|
197
|
-
const queues = [];
|
|
198
|
-
try {
|
|
199
|
-
// Constructed INSIDE the try: `makeQueue` can throw on a malformed peer-written name, and a throw
|
|
200
|
-
// at index k > 0 would otherwise leak the k connections already opened.
|
|
201
|
-
for (const name of names) queues.push(makeQueue(parseConnection(url, { failFast: true }), { name }));
|
|
202
|
-
if (cmd === "pause" || cmd === "resume") {
|
|
203
|
-
const done = [];
|
|
204
|
-
try {
|
|
205
|
-
for (const q of queues) {
|
|
206
|
-
await (cmd === "pause" ? q.pause() : q.resume());
|
|
207
|
-
done.push(q.name);
|
|
208
|
-
}
|
|
209
|
-
} catch (error) {
|
|
210
|
-
// A mid-loop failure leaves the deployment HALF switched. Naming what did change is the whole
|
|
211
|
-
// difference between an operator who knows to finish the job and one who reads "unreachable"
|
|
212
|
-
// as "nothing happened" and walks away from a fleet with one host still spending.
|
|
213
|
-
return fail(`could not ${cmd} the whole deployment at ${url}\n ${done.length > 0 ? `${cmd}d: ${done.join(", ")}` : "nothing changed"}\n failed at: ${names[done.length]}\n ${error.message}`);
|
|
214
|
-
}
|
|
215
|
-
write(cmd === "pause" ? `paused — worker will stop taking new jobs (jobs still enqueue)${span}\n` : `resumed${span}\n`);
|
|
216
|
-
} else {
|
|
217
|
-
// "paused" is included in the counts because jobs enqueued while paused land in the
|
|
218
|
-
// `paused` list, not `wait` -- omitting it would report backlog 0 in the exact state
|
|
219
|
-
// the pause switch creates. `pausedState` (the boolean) is named apart from the
|
|
220
|
-
// `paused` count `getJobCounts` returns, so the two do not collide in the output.
|
|
221
|
-
const states = await Promise.all(queues.map((q) => q.isPaused()));
|
|
222
|
-
const per = await Promise.all(queues.map((q) => q.getJobCounts("waiting", "active", "paused", "delayed", "failed")));
|
|
223
|
-
const counts = per.reduce((acc, c) => {
|
|
224
|
-
for (const [k, v] of Object.entries(c ?? {})) acc[k] = (acc[k] ?? 0) + (Number(v) || 0);
|
|
225
|
-
return acc;
|
|
226
|
-
}, {});
|
|
227
|
-
// Summed counts with a boolean from ONE queue would report a half-paused deployment as fully
|
|
228
|
-
// one or fully the other. `pausedPartial` is the third state, and the dangerous direction is
|
|
229
|
-
// the one it makes visible: pause ran while a host was invisible, so that host still spends.
|
|
230
|
-
const pausedState = states.every(Boolean);
|
|
231
|
-
const pausedPartial = !pausedState && states.some(Boolean);
|
|
232
|
-
const out = { pausedState, ...(pausedPartial ? { pausedPartial, pausedQueues: names.filter((_, i) => states[i]) } : {}), ...counts, ...(blind ? { fleet: blind } : {}) };
|
|
233
|
-
write(`${JSON.stringify(out)}\n`);
|
|
234
|
-
}
|
|
235
|
-
} catch (error) {
|
|
236
|
-
return fail(`could not reach Valkey at ${url} — is it running? (docker compose up)\n ${error.message}`);
|
|
237
|
-
} finally {
|
|
238
|
-
for (const q of queues) await q.close().catch(() => {});
|
|
184
|
+
// GitHub auth is misconfigured, so an operator can always stop the queue. Which VALKEY_URL (PR #475's review,
|
|
185
|
+
// rounds 1 and 2): `--valkey-url <url>` when the operator names one; else this shell's and the deployment .env's
|
|
186
|
+
// (the one resolver, `valkeyUrlFor`). When those two DISAGREE, a stale export must not make "paused" true of the
|
|
187
|
+
// wrong Valkey (measured: `pause` paused the shell's while the service's kept taking jobs), so the kill switch
|
|
188
|
+
// keeps its promise the safe way: `pause` pauses BOTH and says so, `status` shows both, and `resume`, which would
|
|
189
|
+
// START spending, refuses until the operator names which. Every URL is printed through `urlShown`: a password in
|
|
190
|
+
// one never reaches the terminal.
|
|
191
|
+
const { urlShown } = await import("./connection.mjs");
|
|
192
|
+
const picked = await killSwitchUrls(argv.slice(1), env);
|
|
193
|
+
if (picked.error) return fail(picked.error);
|
|
194
|
+
if (picked.positionals.length > 0) return fail(`pi-dispatch ${cmd} takes no argument but --valkey-url <url> (got ${picked.positionals.map((p) => JSON.stringify(p)).join(" ")})`);
|
|
195
|
+
if (picked.urls.length > 1) {
|
|
196
|
+
if (cmd === "resume") return fail(`${picked.disagreement}: resume would start jobs on one of them, so it names neither. Say which: pi-dispatch resume --valkey-url <url>`);
|
|
197
|
+
process.stderr.write(`warning: ${picked.disagreement}: ${cmd === "pause" ? "pausing both" : "showing both"}\n`);
|
|
239
198
|
}
|
|
240
|
-
|
|
199
|
+
let code = 0;
|
|
200
|
+
for (const url of picked.urls) {
|
|
201
|
+
const label = picked.urls.length > 1 ? `[${urlShown(url)}] ` : "";
|
|
202
|
+
code = Math.max(code, await killSwitch(cmd, url, { env, write, label, urlShown, valkeyRefusal }));
|
|
203
|
+
}
|
|
204
|
+
return code;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
if (cmd === "cancel") {
|
|
208
|
+
// The kill switch's doctrine (issue #287): VALKEY_URL only, never loadConfig, so one misbehaving
|
|
209
|
+
// job can be stopped even when everything else about the deployment is misconfigured. On a shell/.env
|
|
210
|
+
// disagreement it refuses until the operator names which (PR #475's review): a job id belongs to one Valkey.
|
|
211
|
+
const picked = await killSwitchUrls(argv.slice(1), env);
|
|
212
|
+
if (picked.error) return fail(picked.error);
|
|
213
|
+
if (picked.positionals.length > 1) return fail(`pi-dispatch cancel takes one job id (got ${picked.positionals.map((p) => JSON.stringify(p)).join(" ")})`);
|
|
214
|
+
const jobId = picked.positionals[0];
|
|
215
|
+
if (picked.urls.length > 1) return fail(`${picked.disagreement}: a job lives in one of them. Say which: pi-dispatch cancel ${jobId ?? "<jobId>"} --valkey-url <url>`);
|
|
216
|
+
const { runCancel } = await import("./cancel-cli.mjs");
|
|
217
|
+
return runCancel(jobId, picked.urls[0], { write });
|
|
241
218
|
}
|
|
242
219
|
|
|
243
220
|
write(`${USAGE}\n`);
|
|
244
|
-
return cmd ? 1 : 0;
|
|
221
|
+
return cmd && cmd !== "--help" && cmd !== "-h" ? 1 : 0; // asked-for help is success; a typo is not (the receiver's rule)
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* The Valkey(s) a kill-switch verb acts on: the one rule of `killSwitchValkeyUrls` (valkey-endpoint.mjs), shared with the
|
|
226
|
+
* panel. The verb's own flags are parsed with parseArgs over what follows the verb (PR #475's review, round 3: a hand
|
|
227
|
+
* scan took `--valkey-url` for the job id in `cancel --valkey-url URL j1`). `{ urls, disagreement, positionals }` or
|
|
228
|
+
* `{ error }`; a note (a URL that matches neither side) is written to stderr.
|
|
229
|
+
*/
|
|
230
|
+
async function killSwitchUrls(args, env) {
|
|
231
|
+
let parsed;
|
|
232
|
+
try {
|
|
233
|
+
parsed = parseArgs({ args, allowPositionals: true, options: { "valkey-url": { type: "string" } } });
|
|
234
|
+
} catch (error) {
|
|
235
|
+
return { error: error.message };
|
|
236
|
+
}
|
|
237
|
+
const { killSwitchValkeyUrls } = await import("./connection.mjs");
|
|
238
|
+
const picked = killSwitchValkeyUrls({ env, flagUrl: parsed.values["valkey-url"] ?? null });
|
|
239
|
+
if (picked.error) return picked;
|
|
240
|
+
if (picked.note) process.stderr.write(`warning: ${picked.note}\n`);
|
|
241
|
+
return { ...picked, positionals: parsed.positionals };
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** One Valkey's pause, resume or status: every queue the deployment drains there (issue #57). Returns the exit code. */
|
|
245
|
+
async function killSwitch(cmd, url, { env, write, label, urlShown, valkeyRefusal }) {
|
|
246
|
+
const { parseConnection, makeRedisClient } = await import("./connection.mjs");
|
|
247
|
+
const { fleetQueueNames, discoverHostQueues, unionQueueNames, makeQueue } = await import("./queue.mjs");
|
|
248
|
+
const { readLiveHosts } = await import("./host-registry.mjs");
|
|
249
|
+
// EVERY queue this deployment drains (issue #57), not just the shared one. This is the kill switch:
|
|
250
|
+
// pausing `pi-jobs` alone would stop forge deliveries while a named host's cron, chained children
|
|
251
|
+
// and manual runs kept spending -- and would print "paused" for having done it. That is the silent
|
|
252
|
+
// no-op the comment here already warned about for a mistyped name, arriving through a new door.
|
|
253
|
+
//
|
|
254
|
+
// Both reads fail OPEN -- between them an unreadable registry and an unreadable keyspace yield the
|
|
255
|
+
// shared queue alone, which is exactly what this command did before, so a Valkey blip can never make
|
|
256
|
+
// the kill switch refuse. But it fails open LOUDLY: a degraded read is NAMED in the output rather
|
|
257
|
+
// than left indistinguishable from a single-host success while a named host keeps spending.
|
|
258
|
+
// `readLiveHosts` RETURNS `{unreachable}` rather than rejecting, so `blind` is a branch on its
|
|
259
|
+
// value and the `.catch` below is only for a client that throws before it can answer.
|
|
260
|
+
const refused = await valkeyRefusal(url, env);
|
|
261
|
+
if (refused) return fail(refused);
|
|
262
|
+
const probe = makeRedisClient(url);
|
|
263
|
+
// Without this, a down Valkey dumps nine `[ioredis] Unhandled error event` traces before the one clean
|
|
264
|
+
// line -- the exact noise `defaultProbeValkey` exists to suppress.
|
|
265
|
+
probe.on?.("error", () => {});
|
|
266
|
+
// Both reads, concurrently, sharing one budget. The registry answers WHO IS LIVE; BullMQ's own meta
|
|
267
|
+
// keys answer WHICH QUEUES EXIST, and for a kill switch the second is the question that matters. A
|
|
268
|
+
// host whose registry writes fail for ninety seconds loses its row while its worker keeps draining,
|
|
269
|
+
// and a resume that misses a queue leaves it paused forever with no surface able to name it. A meta
|
|
270
|
+
// key outlives its worker; a registry row does not.
|
|
271
|
+
const [fleet, existing] = await Promise.all([
|
|
272
|
+
readLiveHosts(probe, { timeoutMs: FLEET_READ_TIMEOUT_MS }).catch((error) => ({ unreachable: error?.message ?? String(error) })),
|
|
273
|
+
discoverHostQueues(probe, { timeoutMs: FLEET_READ_TIMEOUT_MS }),
|
|
274
|
+
]);
|
|
275
|
+
probe.disconnect?.();
|
|
276
|
+
const blind = fleet?.unreachable ?? null;
|
|
277
|
+
const names = unionQueueNames(fleetQueueNames(fleet?.hosts), existing);
|
|
278
|
+
// The registry being unreadable no longer means we saw one queue: the keyspace scan may well have
|
|
279
|
+
// found them. Report the count we ACTED on, and name the degraded read separately.
|
|
280
|
+
const span = `${names.length > 1 ? ` [${names.length} queues]` : ""}${blind ? ` [registry unreadable: ${blind}]` : ""}`;
|
|
281
|
+
const queues = [];
|
|
282
|
+
try {
|
|
283
|
+
// Constructed INSIDE the try: `makeQueue` can throw on a malformed peer-written name, and a throw
|
|
284
|
+
// at index k > 0 would otherwise leak the k connections already opened.
|
|
285
|
+
for (const name of names) queues.push(makeQueue(parseConnection(url, { failFast: true }), { name }));
|
|
286
|
+
if (cmd === "pause" || cmd === "resume") {
|
|
287
|
+
const done = [];
|
|
288
|
+
try {
|
|
289
|
+
for (const q of queues) {
|
|
290
|
+
await (cmd === "pause" ? q.pause() : q.resume());
|
|
291
|
+
done.push(q.name);
|
|
292
|
+
}
|
|
293
|
+
} catch (error) {
|
|
294
|
+
// A mid-loop failure leaves the deployment HALF switched. Naming what did change is the whole
|
|
295
|
+
// difference between an operator who knows to finish the job and one who reads "unreachable"
|
|
296
|
+
// as "nothing happened" and walks away from a fleet with one host still spending.
|
|
297
|
+
return fail(`could not ${cmd} the whole deployment at ${urlShown(url)}\n ${done.length > 0 ? `${cmd}d: ${done.join(", ")}` : "nothing changed"}\n failed at: ${names[done.length]}\n ${error.message}`);
|
|
298
|
+
}
|
|
299
|
+
write(`${label}${cmd === "pause" ? `paused: worker will stop taking new jobs (jobs still enqueue)${span}` : `resumed${span}`}\n`);
|
|
300
|
+
} else {
|
|
301
|
+
// "paused" is included in the counts because jobs enqueued while paused land in the
|
|
302
|
+
// `paused` list, not `wait` -- omitting it would report backlog 0 in the exact state
|
|
303
|
+
// the pause switch creates. `pausedState` (the boolean) is named apart from the
|
|
304
|
+
// `paused` count `getJobCounts` returns, so the two do not collide in the output.
|
|
305
|
+
const states = await Promise.all(queues.map((q) => q.isPaused()));
|
|
306
|
+
const per = await Promise.all(queues.map((q) => q.getJobCounts("waiting", "active", "paused", "delayed", "failed")));
|
|
307
|
+
const counts = per.reduce((acc, c) => {
|
|
308
|
+
for (const [k, v] of Object.entries(c ?? {})) acc[k] = (acc[k] ?? 0) + (Number(v) || 0);
|
|
309
|
+
return acc;
|
|
310
|
+
}, {});
|
|
311
|
+
// Summed counts with a boolean from ONE queue would report a half-paused deployment as fully
|
|
312
|
+
// one or fully the other. `pausedPartial` is the third state, and the dangerous direction is
|
|
313
|
+
// the one it makes visible: pause ran while a host was invisible, so that host still spends.
|
|
314
|
+
const pausedState = states.every(Boolean);
|
|
315
|
+
const pausedPartial = !pausedState && states.some(Boolean);
|
|
316
|
+
const out = { ...(label ? { valkey: urlShown(url) } : {}), pausedState, ...(pausedPartial ? { pausedPartial, pausedQueues: names.filter((_, i) => states[i]) } : {}), ...counts, ...(blind ? { fleet: blind } : {}) };
|
|
317
|
+
write(`${JSON.stringify(out)}\n`);
|
|
318
|
+
}
|
|
319
|
+
} catch (error) {
|
|
320
|
+
return fail(error?.valkeyRefused ? error.message : `could not reach Valkey at ${urlShown(url)}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(url)}\n ${error.message}`);
|
|
321
|
+
} finally {
|
|
322
|
+
for (const q of queues) await q.close().catch(() => {});
|
|
323
|
+
}
|
|
324
|
+
return 0;
|
|
245
325
|
}
|
|
246
326
|
|
|
247
327
|
function fail(message) {
|
|
@@ -259,7 +339,10 @@ export function entryExitCode(err) {
|
|
|
259
339
|
}
|
|
260
340
|
|
|
261
341
|
// Entry point when run as a bin. Kept out of the exported main so tests can call main() directly.
|
|
262
|
-
if (import.meta.url
|
|
342
|
+
if (isEntryModule(import.meta.url)) {
|
|
343
|
+
// A promise nobody handled is printed as its message alone (PR #475's review): Node's own print shows the whole
|
|
344
|
+
// reason, which for a Valkey client's error could carry what it sent.
|
|
345
|
+
installRejectionPrinter();
|
|
263
346
|
main()
|
|
264
347
|
.then((code) => {
|
|
265
348
|
if (code) process.exitCode = code;
|
|
@@ -269,3 +352,18 @@ if (import.meta.url === `file://${process.argv[1]}` || process.argv[1]?.endsWith
|
|
|
269
352
|
process.exitCode = entryExitCode(err);
|
|
270
353
|
});
|
|
271
354
|
}
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* The refusal of the Valkey at `url`, judged once before a CLI command connects (issue #464, gate round 3), or null. A
|
|
358
|
+
* judgement that may succeed later (nothing answers, a name that does not resolve) is not a refusal: the command's own
|
|
359
|
+
* connect then says it could not reach Valkey. A seam of `main` (`valkeyRefusal`), so a test can stand in for the host.
|
|
360
|
+
*/
|
|
361
|
+
async function valkeyRefusalAtStart(url, env) {
|
|
362
|
+
const { judgeValkeyAtStart, valkeyClientContext } = await import("./connection.mjs");
|
|
363
|
+
try {
|
|
364
|
+
await judgeValkeyAtStart(url, valkeyClientContext({ env }), { waitMs: 0 });
|
|
365
|
+
return null;
|
|
366
|
+
} catch (error) {
|
|
367
|
+
return error?.valkeyRefused ? error.message : null;
|
|
368
|
+
}
|
|
369
|
+
}
|