@edgehero/pi-dispatch 1.10.2 → 2.0.0

package/src/branch.mjs

@@ -29,7 +29,11 @@
 export function normalizeNumber(number) {
 	const n = Number(number);
 	if (!Number.isInteger(n) || n <= 0) {
-		const error = new Error(`invalid target number (must be a positive integer): ${String(number)}`);
+		// The TYPE, never the value: `number` is a forge-payload field, and this message's history is
+		// exactly the leak class -- since #310 the piDispatchConfig classifier contains it, but a message
+		// that would carry attacker text if the classifier ever missed is a property held by luck, not
+		// locally (issue #289 made it local; failedReason and the job_failed line are where it would land).
+		const error = new Error(`invalid target number (must be a positive integer; got ${typeof number})`);
 		error.piDispatchConfig = true;
 		throw error;
 	}
@@ -47,7 +51,8 @@ export function normalizeNumber(number) {
  */
 function positiveReplica(replica) {
 	if (!Number.isInteger(replica) || replica <= 0) {
-		const error = new Error(`invalid replica index (must be a positive integer): ${String(replica)}`);
+		// Type-only for normalizeNumber's reason -- a caller bug's value adds nothing a type does not.
+		const error = new Error(`invalid replica index (must be a positive integer; got ${typeof replica})`);
 		error.piDispatchConfig = true;
 		throw error;
 	}

package/src/cancel-cli.mjs

@@ -0,0 +1,174 @@
+/**
+ * `pi-dispatch cancel <jobId>` (issue #287): the operator's stop for ONE job, whatever state it is in.
+ *
+ * VALKEY_URL-only like `pause`, and for the same reason: stopping a job must work even when forge auth is
+ * misconfigured. The verb finds the job across every queue this deployment drains (the kill switch's own
+ * fleet discovery), then dispatches on what the job IS:
+ *
+ *   held on run.waitFor  -> the shared removeHeldJob sequence (hold keys first, then the job); no record
+ *                           (the dispatch_wait_cancel rule).
+ *   delayed / queued     -> job.remove(); no record, same rule.
+ *   active               -> a `cancel:req` key + a brief ack poll (cancel-state.mjs), because the abort can
+ *                           only be raised by the process that holds the job. No ack within the window is a
+ *                           NAMED failure, never a silent one: the job may be on a host that is down, or on
+ *                           a worker predating this verb.
+ *
+ * Held is checked BEFORE the plain state dispatch: a held job IS delayed, and removing it through the plain
+ * path would strand its wait:* keys as a panel row for a job that no longer exists.
+ *
+ * What the line says about the job's PAST is read off the job, not assumed from its state (issue #477): a delayed job
+ * can be one waiting to retry after a failed attempt, which usually wrote a run record and may have left a retained
+ * sandbox, and "it never ran" was printed about it. `ranBefore` (cancel-state.mjs, shared with the panel) words it from the
+ * job's attempt count.
+ */
+
+import { cancelReqKey, ranBefore, removeHeldJob, requestCancel } from "./cancel-state.mjs";
+import { jobKey } from "./wait-state.mjs";
+
+/** Same budget the kill switch gives the host registry before acting on what the keyspace alone says. */
+const FLEET_READ_TIMEOUT_MS = 2_000;
+
+/**
+ * Run the cancel. Returns the process exit code. Every collaborator is a seam with the production default,
+ * so tests drive states and races without a queue; `write` is the stdout seam cli.mjs already injects.
+ */
+export async function runCancel(jobId, url, { write = (chunk) => process.stdout.write(chunk), errWrite = (chunk) => process.stderr.write(chunk), redisFn, queueFn, parseConnectionFn, readLiveHostsFn, discoverHostQueuesFn, fleetQueueNamesFn, unionQueueNamesFn, ackTimeoutMs = 10_000, pollMs = 250, sleep, refusalFn } = {}) {
+	const fail = (message) => {
+		errWrite(`error: ${message}\n`);
+		return 1;
+	};
+	if (typeof jobId !== "string" || jobId === "") {
+		return fail("a job id is required: pi-dispatch cancel <jobId>  (ids are in `pi-dispatch status`, the panel, and the run log)");
+	}
+	// Lazy imports, the pause verb's shape: a mistyped id must not load bullmq before it can be refused.
+	const { parseConnection, makeRedisClient } = await import("./connection.mjs");
+	const { fleetQueueNames, discoverHostQueues, unionQueueNames, makeQueue } = await import("./queue.mjs");
+	const { readLiveHosts } = await import("./host-registry.mjs");
+	const parseConn = parseConnectionFn ?? parseConnection;
+	const mkRedis = redisFn ?? makeRedisClient;
+	const mkQueue = queueFn ?? makeQueue;
+	const liveHosts = readLiveHostsFn ?? readLiveHosts;
+	const discover = discoverHostQueuesFn ?? discoverHostQueues;
+	const fleetNames = fleetQueueNamesFn ?? fleetQueueNames;
+	const unionNames = unionQueueNamesFn ?? unionQueueNames;
+
+	// Issue #464 (gate round 3): judged before anything is sent, so a refused Valkey is said as the refusal it is. Only
+	// where this command builds its own clients (a test's fakes stand in for Valkey itself), or a test's `refusalFn`.
+	const refusalOf =
+		refusalFn ??
+		(redisFn
+			? async () => null
+			: async (u) => {
+					const { judgeValkeyAtStart, defaultValkeyContext } = await import("./connection.mjs");
+					try {
+						await judgeValkeyAtStart(u, defaultValkeyContext(), { waitMs: 0 });
+						return null;
+					} catch (error) {
+						return error?.valkeyRefused ? error.message : null;
+					}
+				});
+	const refused = await refusalOf(url);
+	if (refused) return fail(refused);
+	const probe = mkRedis(url);
+	probe.on?.("error", () => {});
+	const queues = [];
+	let requested = false;
+	try {
+		// The job could sit on the shared queue or on any host's own (issue #57): find it the way pause
+		// spans them, and fail OPEN but LOUDLY on a degraded registry read, exactly as the kill switch does.
+		const [fleet, existing] = await Promise.all([
+			liveHosts(probe, { timeoutMs: FLEET_READ_TIMEOUT_MS }).catch((error) => ({ unreachable: error?.message ?? String(error) })),
+			discover(probe, { timeoutMs: FLEET_READ_TIMEOUT_MS }),
+		]);
+		const blind = fleet?.unreachable ?? null;
+		const names = unionNames(fleetNames(fleet?.hosts), existing);
+
+		// Constructed INSIDE the try (the pause loop's leak posture); first queue that knows the id wins.
+		// Job ids are delivery GUIDs, `local-<hex>` or `repeat:<id>:<millis>`, so a cross-queue duplicate
+		// is not a state this project can produce.
+		let job = null;
+		let owner = null;
+		for (const name of names) {
+			const q = mkQueue(parseConn(url, { failFast: true }), { name });
+			queues.push(q);
+			job = await q.getJob(jobId);
+			if (job) {
+				owner = q;
+				break;
+			}
+		}
+		if (!job) {
+			return fail(`no job ${jobId} in ${names.length} queue(s)${blind ? ` [registry unreadable: ${blind} — a host that is not registered may still hold it]` : ""}`);
+		}
+
+		// Held FIRST (see the header). The hash tells held apart from plain-delayed; `since` is the clock
+		// that makes it a real hold rather than a counter key.
+		const hash = await probe.hgetall(jobKey(jobId)).catch(() => null);
+		if (hash?.since) {
+			const res = await removeHeldJob({ redis: probe, queue: owner, jobId });
+			if (res.ok) {
+				// A hold consumes no attempt (`skipAttempt`), so a held job that ran is a retry held again by its conditions.
+				write(`cancelled ${jobId}: it was held on its wait condition, and ${ranBefore(res)}\n`);
+				return 0;
+			}
+			return fail(`could not cancel held job ${jobId}: ${res.invalid}`);
+		}
+
+		let state = await job.getState().catch(() => "unknown");
+		if (state === "delayed" || state === "waiting" || state === "prioritized" || state === "paused") {
+			try {
+				await job.remove();
+				write(`removed ${jobId} (${state}): ${ranBefore(job)}\n`);
+				return 0;
+			} catch (error) {
+				// The one race worth handling by name: picked up between getState and remove. remove() throws
+				// on a locked job, so re-read and fall through to the active path rather than failing an
+				// operator whose job is now exactly the case the active path exists for.
+				state = await job.getState().catch(() => "unknown");
+				// The re-read state and the throw's own words together, labelled apart: the error came from
+				// the remove attempt, the state from after it, and gluing them unlabelled read as one
+				// mismatched diagnosis (review finding).
+				if (state !== "active") return fail(`could not remove ${jobId} — it is now ${state}; the remove failed with: ${error.message}`);
+			}
+		}
+
+		if (state === "active") {
+			requested = true;
+			const res = await requestCancel({ redis: probe, jobId, ackTimeoutMs, pollMs, ...(sleep ? { sleep } : {}) });
+			if (res.ack !== undefined) {
+				write(`cancel accepted by ${res.ack === "" ? "the worker" : res.ack} — stopping the container (allow ~30s); the run record will say operator-cancel\n`);
+				return 0;
+			}
+			// Gate round 2 of PR #479: the worker stops its cancel poll before it holds or retries a job, so a request that
+			// lands in that moment is never acknowledged and the job is back in the queue. Re-read, so the words are true.
+			// Gate round 3: by the state, since a job that finished or left the queue cannot be removed by a second cancel.
+			const after = await job.getState().catch(() => "unknown");
+			const waited = `no worker acknowledged within ${Math.round(ackTimeoutMs / 1000)}s`;
+			if (after === "waiting" || after === "delayed" || after === "prioritized" || after === "paused") return fail(`${waited}: the job went back to the queue before its worker read the cancel (it is now ${after}), and nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again to remove it`);
+			if (after === "completed" || after === "failed") return fail(`${waited}: the job finished (${after}) before its worker read the cancel, and nothing was changed`);
+			if (after !== "active") return fail(`${waited}: the job is no longer in the queue (its state reads ${after}), and nothing was changed`);
+			return fail(`no worker acknowledged within ${Math.round(ackTimeoutMs / 1000)}s — the job is active but no reachable worker owns it (host down, or a worker predating cancel); nothing was changed`);
+		}
+
+		return fail(`job ${jobId} is ${state} — nothing to cancel`);
+	} catch (error) {
+		// Best-effort, and ONLY when a request was actually placed: an abandoned request must not fire
+		// after the operator read an error and walked away -- but against a Valkey that is DOWN, a bare
+		// del would sit in ioredis's retry queue and hold this process open long past the error message.
+		// The bound timer is CLEARED when the del wins the race (discoverHostQueues' own rule): cli.mjs
+		// sets exitCode rather than exiting, so a stray pending timer is a second of pure hang.
+		if (requested) {
+			let bound;
+			await Promise.race([
+				Promise.resolve(probe.del?.(cancelReqKey(jobId))).catch(() => {}),
+				new Promise((r) => (bound = setTimeout(r, 1000))),
+			]);
+			clearTimeout(bound);
+		}
+		return fail(error?.valkeyRefused ? error.message : `could not reach Valkey at ${(await import("./connection.mjs")).urlShown(url)}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(url)}\n  ${error.message}`);
+	} finally {
+		probe.disconnect?.();
+		for (const q of queues) await q.close().catch(() => {});
+	}
+}
+

package/src/cancel-state.mjs

@@ -0,0 +1,125 @@
+/**
+ * The `cancel:` keyspace: how an operator's cancel reaches the worker that owns a running job (issue #287).
+ *
+ * BullMQ's `cancelJob` aborts an entry in ONE process's tracked-job map -- it is unreachable from another
+ * process, returns `false` silently for a job it does not hold, and nothing anywhere maps a jobId to the
+ * host draining it. So the CLI and the panel cannot call it; they can only ask. This keyspace is the ask:
+ *
+ *   cancel:req:<jobId>        STRING "operator-cancel", TTL 60s -- the request. Written by the CLI or the
+ *                             panel; consumed (DEL) by the worker that holds the job; deleted by the
+ *                             requester itself when it gives up, so a request the operator was told failed
+ *                             cannot fire a minute later. The TTL is the backstop for a requester that died.
+ *   cancel:ack:<jobId>        STRING <workerName> -- the answer, written by the worker AFTER it raised the
+ *                             abort. A name, possibly "", never a path (INT-HOST-REGISTRY-CONTRACT's content
+ *                             rule). TTL 5m: long enough for a requester that polls slowly, gone before the
+ *                             id could plausibly be reused.
+ *
+ * WHY A DURABLE KEY AND NOT PUB/SUB. A subscriber is a dedicated connection type this repo has zero of, and
+ * fire-and-forget loses a cancel across a worker restart or a blip with no error anywhere -- the silent
+ * no-op this project refuses. The key survives until someone answers or the TTL says nobody will, it is
+ * operator-inspectable (`KEYS cancel:*`, the wait-state doctrine), and the ack the acceptance requires
+ * ("a cancel of a job this host does not own says so") needs a readable key regardless.
+ *
+ * WHY THIS IS NOT THE REDIS STATE OQ-008 REFUSED. That refusal is about durable CONFIG whose deletion
+ * silently loses an operator's edit. This is a transient, attended, TTL-bounded one-shot whose loss is
+ * REPORTED: the requester is watching the ack poll, and a request that reaches nobody comes back as a
+ * named timeout, never as silence.
+ */
+
+import { HELD_SET, jobKey, leaseKey } from "./wait-state.mjs";
+
+/** How long an unanswered request may sit before redis reaps it (the requester deletes it sooner). */
+export const CANCEL_REQ_TTL_MS = 60_000;
+
+/** How long the worker's answer stays readable. Generous: an ack outliving its poller costs nothing. */
+export const CANCEL_ACK_TTL_MS = 5 * 60_000;
+
+export const cancelReqKey = (jobId) => `cancel:req:${jobId}`;
+export const cancelAckKey = (jobId) => `cancel:ack:${jobId}`;
+
+/**
+ * Ask whichever worker holds `jobId` to cancel it, and wait briefly for the answer.
+ *
+ * Resolves `{ ack: host }` when a worker acknowledged (host may be ""), or `{ timeout: true }` when nobody
+ * did within `ackTimeoutMs` -- in which case the request key is deleted first, so the operator's "nothing
+ * was changed" stays true after they walk away. Redis errors propagate: the callers (CLI, panel deps) each
+ * already own a could-not-reach-Valkey message, and swallowing here would turn an unreachable Valkey into
+ * a lying "no worker acknowledged".
+ *
+ * `pollMs`/`sleep` are injectable so tests never wait on a wall clock.
+ */
+export async function requestCancel({ redis, jobId, ackTimeoutMs = 10_000, pollMs = 250, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+	await redis.set(cancelReqKey(jobId), "operator-cancel", "PX", CANCEL_REQ_TTL_MS);
+	for (let waited = 0; ; waited += pollMs) {
+		const host = await redis.get(cancelAckKey(jobId));
+		if (host !== null && host !== undefined) return { ack: host };
+		if (waited + pollMs > ackTimeoutMs) break;
+		await sleep(pollMs);
+	}
+	// Best-effort: losing this DEL to a blip leaves the 60s TTL as the backstop, and the worker-side GET
+	// racing it by one tick is a named residual (the operator is told to re-check `status`).
+	await redis.del(cancelReqKey(jobId)).catch(() => {});
+	return { timeout: true };
+}
+
+/**
+ * What a removed job had done before a cancel removed it, in the CLI's and the panel's words (issue #477), from BullMQ's
+ * own counter. `attemptsMade` counts the attempts that FINISHED (moveToFinished/moveToFailed, bullmq 5.80.4). Most of
+ * them wrote a run record, but not all: a gate above the processor's `try` (a `moveToDelayed` or a wait-state read that
+ * rejects) fails the attempt with nothing recorded, by design (index.mjs). So the sentence says an attempt was made and
+ * that whatever it recorded stays, never that a record exists. NOT `attemptsStarted`: a hold on run.waitFor, a
+ * pause-gate move and a rootful-Podman deferral each take the job and hand it back to the delayed set with
+ * `skipAttempt`, which counts a start and no attempt (moveToDelayed), so that counter is non-zero on jobs that never
+ * ran. Cancel itself writes no record in any case.
+ */
+export function ranBefore(job) {
+	const made = Number.isInteger(job?.attemptsMade) && job.attemptsMade > 0 ? job.attemptsMade : 0;
+	if (made === 0) return "it never ran; no record written";
+	return made === 1
+		? "it made 1 attempt before and was waiting to retry; whatever that attempt recorded stays, and cancel writes none"
+		: `it made ${made} attempts before and was waiting to retry; whatever those attempts recorded stays, and cancel writes none`;
+}
+
+/**
+ * Remove a job that is held on a `run.waitFor` condition: hold keys first, then the job (issue #230's
+ * sequence, issue #287's shared home). This is the inner body of the admin's `cancelHeldJob`, moved here
+ * so the CLI's held path and the panel's are ONE sequence rather than two copies that can drift -- the
+ * read-model's own rule is that key derivations are imported from the worker, never re-implemented.
+ *
+ * Returns `{ ok: true, jobId, attemptsMade }` or `{ invalid: <reason> }`; never throws. Writes no run record of its
+ * own (INT-RUN-HISTORY-FILE-CONTRACT records terminal states of runs, and a removed hold is none); `attemptsMade` is
+ * what the job had done before, for the caller's sentence (`ranBefore`, issue #477): a job held again on a retry has
+ * made attempts, and whatever they recorded stays.
+ */
+export async function removeHeldJob({ redis, queue, jobId }) {
+	try {
+		const hash = await redis.hgetall(jobKey(jobId));
+		// A hold is a hash with a CLOCK, not merely a non-empty hash: the worker's own counters create this
+		// key before anything is held, so "non-empty" would let this tool reach a job that is not waiting.
+		if (!hash || !hash.since) return { invalid: `job ${jobId} is not waiting on a condition` };
+		const job = await queue.getJob(jobId);
+		if (!job) return { invalid: `job ${jobId} is no longer in the queue` };
+		// STATE, not existence. `release` is fail-open by design, so a redis blip can leave the hash behind
+		// while the job wakes, runs and completes -- and bullmq will happily `remove()` a completed job. That
+		// would answer `applied: true` to an operator who approved a dialog reading "It will never run", for
+		// a job whose run record is already on disk.
+		const state = await job.getState().catch(() => null);
+		if (state !== "delayed" && state !== "waiting" && state !== "prioritized") {
+			return { invalid: `job ${jobId} is ${state ?? "in an unknown state"}, not waiting -- it has already left the hold` };
+		}
+		// The hold goes FIRST. If `remove` throws (an active job is locked) or a caller's timeout fires
+		// mid-sequence, an orphaned hash would keep a row on the panel for a job that no longer exists; an
+		// orphaned JOB is merely a job that still runs, which is the state the operator was already in.
+		await redis.del(jobKey(jobId));
+		await redis.srem(HELD_SET, jobId).catch(() => {});
+		if (hash.dedupId) {
+			const holder = await redis.get(leaseKey(hash.dedupId));
+			if (holder === jobId) await redis.del(leaseKey(hash.dedupId));
+		}
+		await job.remove();
+		// Issue #477: what the job had done, for the caller's sentence (`ranBefore`); read off the job just removed.
+		return { ok: true, jobId, attemptsMade: Number.isInteger(job.attemptsMade) && job.attemptsMade > 0 ? job.attemptsMade : 0 };
+	} catch (err) {
+		return { invalid: err?.message ?? String(err) };
+	}
+}