@edgehero/pi-dispatch 5.1.0 → 5.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch",
-  "version": "5.1.0",
+  "version": "5.2.1",
   "type": "module",
   "description": "The pi-dispatch worker and CLI: go from one AI coding agent to many in the background, running the pi coding agent as a self-hosted service, one locked down Docker or Podman container per job, with spend caps checked before anything is spent, plus init, up, doctor and service install.",
   "keywords": [

package/src/auth-cli.mjs

@@ -2,7 +2,9 @@ import { createInterface } from "node:readline/promises";
 import { loadConfig } from "./config.mjs";
 import { configuredSubscriptions } from "./subscription-config.mjs";
 import { subscriptionIdentity } from "./subscription-runtime.mjs";
-import { makeRedisClient } from "./connection.mjs";
+import { makeRedisClient, urlShown, VALKEY_COMMAND_TIMEOUT_MS } from "./connection.mjs";
+import { valkeyDownHint } from "./valkey-auth.mjs";
+import { boundedText } from "./cancel-cli.mjs";
 import { openBrowser } from "./open-browser.mjs";
 import { authError } from "./auth-store.mjs";
 import { OPENAI_LOGIN_INSTRUCTIONS } from "./oauth-openai.mjs";
@@ -22,6 +24,26 @@ export function authSelectAnswer(question, answer) {
 	return input;
 }
 
+/**
+ * The line for a Valkey failure (issue #867), or null when `error` is no Valkey failure: Valkey's own refusal (a reply),
+ * a configuration refusal this project raised (`valkeyRefused`), or a command that got no answer (a bound that fired, or
+ * one that reached a Valkey and lost its answer). `seen` is the client's last `error` event, which says why a client
+ * that never connected got none ("nothing answers VALKEY_URL ...") where the timeout says only that it got none. After a
+ * login or a logout the failure is the status read's, since neither sends anything to Valkey. `casLost` says the
+ * trial's compare-and-set itself was sent and its answer lost, so the grant may have landed and is not said to have
+ * failed; a read lost before it means the compare-and-set was never sent, and nothing is said of a grant.
+ */
+export function authValkeyLine(error, { command, url, seen = null, casLost = false }) {
+	if (error?.valkeyRefused === true) return `Subscription authentication: ${boundedText(error.message)}`;
+	const refused = error?.name === "ReplyError";
+	// ioredis' own timeout by its words, not by `ETIMEDOUT` alone, which a provider's request can carry too.
+	if (!refused && error?.message !== "Command timed out" && error?.valkeyMayStillRun !== true) return null;
+	const lead = `Subscription authentication: ${command === "login" || command === "logout" ? `the ${command} finished, but ` : ""}Valkey at ${urlShown(url)}`;
+	if (refused) return `${lead} refused ${boundedText(typeof error?.command?.name === "string" ? error.command.name.toUpperCase() : "a command")} (${boundedText(error.message)}).`;
+	const trial = command === "trial" && casLost === true ? " The trial grant may or may not have been made." : "";
+	return `${lead} did not answer (${boundedText(seen ?? error?.message)}).${trial} Run \`pi-dispatch auth status\` once it answers: ${valkeyDownHint(url)}`;
+}
+
 export async function runAuth(args, { env = process.env, write = (value) => process.stdout.write(value), writeErr = (value) => process.stderr.write(value), configure = configuredSubscriptions, configFn = loadConfig, redisFn = makeRedisClient, interaction = null, now = Date.now, terminalInput = process.stdin, terminalOutput = process.stdout, createPrompt = createInterface, openBrowserFn = openBrowser } = {}) {
 	const [command = "status", id, ...rest] = args;
 	if (!["status", "login", "logout", "usage", "trial"].includes(command) || rest.length > 0 || (command === "status" ? id !== undefined : !/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(id ?? ""))) {
@@ -30,10 +52,40 @@ export async function runAuth(args, { env = process.env, write = (value) => proc
 	}
 	let redis;
 	let prompt;
+	let config;
+	// The client's last error event since it was last ready, for the line a Valkey failure ends with: cleared when it goes
+	// ready, so a line after a long login does not quote an outage that ended during it.
+	let seen = null;
+	let valkeyFailed = false;
+	// Whether the one write `auth` sends, a quota compare-and-set (EVAL), was sent and lost its answer, so the line says a
+	// trial grant may or may not have been made only then, never of a read lost before the write was sent.
+	let casLost = false;
 	try {
-		const config = configFn(env);
+		config = configFn(env);
 		if (!config.authConnectionsFile) { writeErr("Set PI_AUTH_CONNECTIONS_FILE to an operator-reviewed subscription connection policy.\n"); return 2; }
-		redis = redisFn(config.valkeyUrl);
+		// BOUNDED (issue #867): every command this client sends is a short read or one compare-and-set of a quota row, so one
+		// that waits past VALKEY_COMMAND_TIMEOUT_MS is a Valkey that is gone or stopped answering. Without the bound `auth
+		// status` and `auth usage` waited for as long as it stayed so (measured: `auth usage` on a closed port printed 18 stack
+		// traces in 8 s, one per reconnect attempt, while its read sat in ioredis' offline queue for as long as Valkey stayed
+		// gone). The bound is on each command, never on the client's life, so a login
+		// that waits minutes on a browser is not cut short: the login itself sends nothing to Valkey, only the status read
+		// after it. `disconnectTimeoutMs: 0`, as `cancel`'s probe, so the socket of a Valkey that stopped answering does not
+		// hold the process past its last line.
+		redis = redisFn(config.valkeyUrl, { commandTimeoutMs: VALKEY_COMMAND_TIMEOUT_MS, disconnectTimeoutMs: 0 });
+		// Without a listener ioredis prints a stack trace per reconnect attempt, for as long as an outage outlasts this call.
+		redis?.on?.("error", (err) => {
+			seen = err?.message ?? String(err);
+		});
+		redis?.on?.("ready", () => {
+			seen = null;
+		});
+		if (typeof redis?.eval === "function") {
+			const evaluate = redis.eval.bind(redis);
+			redis.eval = (...args) => Promise.resolve(evaluate(...args)).catch((err) => {
+				if (err?.valkeyMayStillRun === true) casLost = true;
+				throw err;
+			});
+		}
 		const service = configure({ config, redis, now });
 		if (!interaction) interaction = {
 			async notify(event) {
@@ -70,17 +122,36 @@ export async function runAuth(args, { env = process.env, write = (value) => proc
 			const logout = { id, state: "disconnected", remoteRevocationConfirmed: result?.remoteRevocationConfirmed === true };
 			let status;
 			try { status = await service.runtime.status(); }
-			catch { status = { state: "unavailable", partial: true, connections: [] }; }
+			catch (error) {
+				status = { state: "unavailable", partial: true, connections: [] };
+				// The logout is done (it sends nothing to Valkey) and its evidence printed below; a Valkey failure of the status
+				// read is still said as the one line, and the client is dropped rather than asked to QUIT (issue #867).
+				const line = authValkeyLine(error, { command, url: config.valkeyUrl, seen });
+				if (line !== null) {
+					valkeyFailed = true;
+					writeErr(`${line}\n`);
+				}
+			}
 			result = { ...status, logout };
 		}
 		write(`${JSON.stringify(result, (key, value) => ["accountGroup", "generation", "accountDigest", "credential", "registration"].includes(key) ? undefined : value, 2)}\n`);
 		return result?.ok === false || result?.kind === "unavailable" ? 2 : 0;
 	} catch (error) {
+		// A Valkey failure is one plain line naming Valkey, exit 1 as the transient failure it is: "auth-unavailable" and
+		// "Run auth status" said nothing of Valkey, and `auth status` needs the same Valkey.
+		const line = config?.valkeyUrl ? authValkeyLine(error, { command, url: config.valkeyUrl, seen, casLost }) : null;
+		if (line !== null) {
+			valkeyFailed = true;
+			writeErr(`${line}\n`);
+			return 1;
+		}
 		const code = typeof error?.code === "string" && /^(?:auth|subscription)-[a-z-]+$/.test(error.code) ? error.code : "auth-unavailable";
 		writeErr(`Subscription authentication: ${code}. Run auth status on the connection owner host.\n`);
 		return error?.transient === true ? 1 : 2;
 	} finally {
 		prompt?.close();
-		try { await redis?.quit(); } catch { redis?.disconnect?.(); }
+		// After a Valkey failure QUIT would wait out one more bound on a Valkey that stopped answering, for nothing.
+		if (valkeyFailed) redis?.disconnect?.();
+		else try { await redis?.quit(); } catch { redis?.disconnect?.(); }
 	}
 }

package/src/cancel-cli.mjs

@@ -22,9 +22,15 @@
  * can be one waiting to retry after a failed attempt, which usually wrote a run record and may have left a retained
  * sandbox, and "it never ran" was printed about it. `ranBefore` (cancel-state.mjs, shared with the panel) words it from the
  * job's attempt count.
+ *
+ * Every command it sends has a bound (issue #846 for its own client, issue #867 for its queues), so a Valkey that is
+ * connected and stopped answering ends the verb with a line rather than holding it for as long as it stays so. The line
+ * says what the point it stopped at allows: the lookup changes nothing about the job (each queue writes its own meta row
+ * on connect, as every BullMQ queue does), so nothing was changed; a removal or a hold's delete that was sent and not
+ * answered may or may not have been made, and `cancel` again settles it once Valkey answers.
  */
 
-import { cancelReqKey, ranBefore, removeQueuedJob, requestCancel } from "./cancel-state.mjs";
+import { CANCEL_REQ_TTL_MS, cancelAckKey, cancelReqKey, ranBefore, removeQueuedJob, requestCancel } from "./cancel-state.mjs";
 
 /** Same budget the kill switch gives the host registry before acting on what the keyspace alone says. */
 const FLEET_READ_TIMEOUT_MS = 2_000;
@@ -42,11 +48,69 @@ export function boundedText(text) {
 	return String(text ?? "no reason given").replace(/[\u0000-\u001f\u007f-\u009f]/g, " ").slice(0, 200);
 }
 
+/** Whether `error` is Valkey's own refusal (a reply), as against no answer at all: the two are different sentences. */
+function refusedByValkey(error) {
+	return error?.name === "ReplyError";
+}
+
+/** What went wrong, as the start of a line: Valkey refused a command (it answered), or it did not answer. */
+function valkeyFault(error, shown) {
+	const command = typeof error?.command?.name === "string" ? error.command.name.toUpperCase() : "a command";
+	return refusedByValkey(error) ? `Valkey at ${shown} refused ${boundedText(command)}: ${boundedText(error?.message)}` : `Valkey at ${shown} stopped answering (${boundedText(error?.message)})`;
+}
+
+/** A bound in words: whole seconds, or milliseconds below one (a test's bound read "0s"). */
+function spoken(ms) {
+	return ms >= 1000 ? `${Math.round(ms / 1000)}s` : `${ms} ms`;
+}
+
+/** Whether `error` is a command that got no answer: a bound that fired, or one sent whose answer was lost. */
+function unanswered(error) {
+	return error?.code === "ETIMEDOUT" || error?.valkeyMayStillRun === true;
+}
+
+/**
+ * The job's state, read under the same bound as every other command here: `{ state }`, or `{ unread }` with why it could
+ * not be read. Issue #846: a read that failed was `"unknown"`, which is also what BullMQ says of a job that left its
+ * queue, so a job still running was reported as gone "and nothing was changed".
+ */
+async function boundedState(job, timeoutMs) {
+	let timer;
+	try {
+		return await Promise.race([
+			Promise.resolve(job.getState()).then((state) => ({ state })),
+			new Promise((resolve) => {
+				timer = setTimeout(() => resolve({ unread: `no answer within ${spoken(timeoutMs)}` }), timeoutMs);
+			}),
+		]);
+	} catch (error) {
+		return { unread: error?.message ?? String(error) };
+	} finally {
+		clearTimeout(timer);
+	}
+}
+
+/**
+ * `promise`'s answer, or a rejection once `ms` passed without one, marked as no answer (`ETIMEDOUT`, `lookupBound`). Each
+ * queue's lookup has this bound of its own, as one step (issue #867): BullMQ connects inside the queue, and on a Valkey
+ * that takes the connection and never answers, ioredis' handshake (CLIENT SETINFO, then the ready check's INFO) waits
+ * one command bound each, on each of the three connections `failFast` tries, so the per-command bound alone ended a
+ * lookup after about six of them (measured: 6.4 s at a 1 s bound). A lookup changes nothing about the job, so what it
+ * leaves running changes nothing either, and its queue is dropped on the way out (`closeWithin`).
+ */
+function answeredWithin(promise, ms) {
+	let timer;
+	return new Promise((resolve, reject) => {
+		timer = setTimeout(() => reject(Object.assign(new Error(`no answer within ${spoken(ms)}`), { code: "ETIMEDOUT", lookupBound: true })), ms);
+		Promise.resolve(promise).then(resolve, reject);
+	}).finally(() => clearTimeout(timer));
+}
+
 /**
  * Run the cancel. Returns the process exit code. Every collaborator is a seam with the production default,
  * so tests drive states and races without a queue; `write` is the stdout seam cli.mjs already injects.
  */
-export async function runCancel(jobId, url, { write = (chunk) => process.stdout.write(chunk), errWrite = (chunk) => process.stderr.write(chunk), redisFn, queueFn, parseConnectionFn, readLiveHostsFn, discoverHostQueuesFn, fleetQueueNamesFn, unionQueueNamesFn, ackTimeoutMs = 10_000, pollMs = 250, sleep, refusalFn, touchStore, touchOutbox = null, now = Date.now } = {}) {
+export async function runCancel(jobId, url, { write = (chunk) => process.stdout.write(chunk), errWrite = (chunk) => process.stderr.write(chunk), redisFn, queueFn, parseConnectionFn, readLiveHostsFn, discoverHostQueuesFn, fleetQueueNamesFn, unionQueueNamesFn, ackTimeoutMs = 10_000, pollMs = 250, sleep, refusalFn, touchStore, touchOutbox = null, now = Date.now, commandTimeoutMs = null, closeTimeoutMs = 1000 } = {}) {
 	const fail = (message) => {
 		errWrite(`error: ${message}\n`);
 		return 1;
@@ -55,7 +119,9 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 		return fail("a job id is required: pi-dispatch cancel <jobId>  (ids are in `pi-dispatch status`, the panel, and the run log)");
 	}
 	// Lazy imports, the pause verb's shape: a mistyped id must not load bullmq before it can be refused.
-	const { parseConnection, makeRedisClient } = await import("./connection.mjs");
+	const { parseConnection, makeRedisClient, VALKEY_COMMAND_TIMEOUT_MS, urlShown } = await import("./connection.mjs");
+	// The bound on each command and on the state reads: the console's, unless a test asks for another.
+	const bound = Number.isSafeInteger(commandTimeoutMs) && commandTimeoutMs > 0 ? commandTimeoutMs : VALKEY_COMMAND_TIMEOUT_MS;
 	const { fleetQueueNames, discoverHostQueues, unionQueueNames, makeQueue } = await import("./queue.mjs");
 	const { readLiveHosts } = await import("./host-registry.mjs");
 	const parseConn = parseConnectionFn ?? parseConnection;
@@ -83,10 +149,23 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 				});
 	const refused = await refusalOf(url);
 	if (refused) return fail(refused);
-	const probe = mkRedis(url);
+	// BOUNDED (issue #846): each command waits at most VALKEY_COMMAND_TIMEOUT_MS for its answer, and one that timed out is
+	// never sent again. The unbounded client this was waited for as long as Valkey was gone: killed during the ack poll it
+	// printed nothing until the operator gave up, and on a partition its DEL never reached the catch below.
+	const probe = mkRedis(url, { commandTimeoutMs: bound, disconnectTimeoutMs: 0 });
 	probe.on?.("error", () => {});
+	// Whether a connection to Valkey was ever made (issue #867, the review): a lookup that got no answer from a Valkey that
+	// took the connection says it stopped answering; one where nothing ever took it (an address that drops the connect, a
+	// host that is off) says it could not be reached, with the hint, as before #867. The probe connects first, so it tells.
+	let connected = false;
+	probe.on?.("connect", () => {
+		connected = true;
+	});
 	const queues = [];
+	// Whether the job was found: until then nothing about the job changed (issue #867), so a failure says nothing was changed.
+	let found = false;
 	let requested = false;
+	let requestedAt = null;
 	try {
 		// The job could sit on the shared queue or on any host's own (issue #57): find it the way pause
 		// spans them, and fail OPEN but LOUDLY on a degraded registry read, exactly as the kill switch does.
@@ -100,14 +179,19 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 		// Constructed INSIDE the try (the pause loop's leak posture); first queue that knows the id wins.
 		// Job ids are delivery GUIDs, `local-<hex>` or `repeat:<id>:<millis>`, so a cross-queue duplicate
 		// is not a state this project can produce.
+		// BOUNDED (issue #867), each command at the probe's bound (`parseConnection`'s `commandTimeoutMs`) and each queue's
+		// lookup as one step (`answeredWithin`): `failFast` alone refuses while nothing is connected, and a command a Valkey
+		// took and never answered (`kill -STOP`, a partition) held the verb for as long as it stayed so, in the lookup and in
+		// the state read and the remove of `removeQueuedJob`, which go through this queue too.
 		let job = null;
 		let owner = null;
 		for (const name of names) {
-			const q = mkQueue(parseConn(url, { failFast: true }), { name });
+			const q = mkQueue(parseConn(url, { failFast: true, commandTimeoutMs: bound }), { name });
 			queues.push(q);
-			job = await q.getJob(jobId);
+			job = await answeredWithin(q.getJob(jobId), bound);
 			if (job) {
 				owner = q;
+				found = true;
 				break;
 			}
 		}
@@ -126,6 +210,26 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			write(`cancelled ${jobId}: it was held on its wait condition, and ${ranBefore(queued)}\n`);
 			return 0;
 		}
+		// Issue #867: a write of the held sequence that was sent and not answered may still run when Valkey wakes, so the line
+		// names which write that was and says it may or may not have been made. The cleared line below would say the job
+		// "could not be removed" of a removal that may yet land, and its hold "was cleared" of a delete that may not.
+		const shown = urlShown(url);
+		// Issue #867 (the review): a hold that could not be read refuses, so a held job is never removed by the plain path,
+		// which strands its hold as a panel row. Nothing was sent. The outcome comes before the failure's words, which are
+		// the only part bounded, so a long id never cuts it off.
+		if (typeof queued.holdUnread === "string") {
+			if (queued.noAnswer !== undefined) return fail(`Valkey at ${shown} stopped answering (${boundedText(queued.noAnswer)}) while cancel read whether ${jobId} is held, so nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again once Valkey answers`);
+			return fail(`could not read whether ${jobId} is held on its wait condition, so nothing was changed: ${boundedText(queued.holdUnread)}`);
+		}
+		if (queued.invalid !== undefined && queued.cleared !== true && queued.noAnswer !== undefined) {
+			return fail(`could not cancel held job ${jobId}: Valkey at ${shown} did not answer (${boundedText(queued.noAnswer)}), so nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again once Valkey answers`);
+		}
+		if (queued.invalid !== undefined && queued.cleared === true && queued.unanswered === "hold") {
+			return fail(`Valkey at ${shown} stopped answering (${boundedText(queued.invalid)}) after the delete of held job ${jobId}'s hold was sent, so the hold may or may not be gone, and the job itself was not removed. Once Valkey answers, run \`pi-dispatch cancel ${jobId}\` again to remove it`);
+		}
+		if (queued.invalid !== undefined && queued.cleared === true && queued.unanswered === "job") {
+			return fail(`Valkey at ${shown} stopped answering (${boundedText(queued.invalid)}) after the removal of held job ${jobId} was sent: its hold was cleared, and the job may or may not have been removed. ${settleLater(jobId)}`);
+		}
 		// A removal that failed after the hold's keys were deleted changed something: the job lost its hold and was not removed,
 		// so the line says so and how to finish, never only the error (which reads as "nothing happened"). Every error a
 		// line carries is bounded (`boundedText`): it is another module's words, of any length and with any bytes.
@@ -138,6 +242,18 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			return 0;
 		}
 		const state = queued.state;
+		// Issue #867: a remove that was sent and not answered, with the job not read back in any of BullMQ's lists (it was
+		// read as "unknown", or not read at all), may have run: never "nothing was changed", which the re-read below would
+		// say of a state it cannot read. A job read back in a list was not removed (its remove is one script), and goes on.
+		if (queued.raced && queued.error?.valkeyMayStillRun === true && state === "unknown") {
+			return fail(`${valkeyFault(queued.error, shown)} after the removal of ${jobId} was sent, so it may or may not have been removed. ${settleLater(jobId)}`);
+		}
+		// Issue #846: `removeQueuedJob` reads a state it could not read as "unknown" (the console's sequence shares it), which
+		// BullMQ also says of a job that left its queue. Asked again under the bound, so a read that failed says so.
+		if (state === "unknown") {
+			const again = await boundedState(job, bound);
+			if (again.unread !== undefined) return fail(`could not read the state of ${jobId} (${boundedText(again.unread)}), so whether it is running is not known; nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again once Valkey answers`);
+		}
 		// The one race worth handling by name: picked up between getState and remove. remove() throws on a locked job, so
 		// the state is re-read and an active job falls through to the active path rather than failing an operator whose
 		// job is now exactly the case the active path exists for. Otherwise the re-read state and the throw's own words
@@ -147,7 +263,8 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 
 		if (state === "active") {
 			requested = true;
-			const res = await requestCancel({ redis: probe, jobId, ackTimeoutMs, pollMs, ...(sleep ? { sleep } : {}) });
+			requestedAt = now();
+			const res = await requestCancel({ redis: probe, jobId, ackTimeoutMs, pollMs, now, onWritten: () => (requestedAt = now()), ...(sleep ? { sleep } : {}) });
 			if (res.ack !== undefined) {
 				write(`cancel accepted by ${res.ack === "" ? "the worker" : boundedText(res.ack)} — stopping the container (allow ~30s); the run record will say operator-cancel\n`);
 				return 0;
@@ -155,8 +272,11 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			// Gate round 2 of PR #479: the worker stops its cancel poll before it holds or retries a job, so a request that
 			// lands in that moment is never acknowledged and the job is back in the queue. Re-read, so the words are true.
 			// Gate round 3: by the state, since a job that finished or left the queue cannot be removed by a second cancel.
-			const after = await job.getState().catch(() => "unknown");
+			const read = await boundedState(job, bound);
 			const waited = `no worker acknowledged within ${Math.round(ackTimeoutMs / 1000)}s`;
+			// Issue #846: a state that could not be read is said as that, never as BullMQ's "unknown" (a job that left).
+			if (read.unread !== undefined) return fail(`${waited}, and the job's state could not be read (${boundedText(read.unread)}), so whether it still runs is not known. The cancel request was withdrawn, so nothing was changed: run \`pi-dispatch status\`, and \`pi-dispatch cancel ${jobId}\` again if it still runs`);
+			const after = read.state;
 			if (after === "waiting" || after === "delayed" || after === "prioritized" || after === "paused") return fail(`${waited}: the job went back to the queue before its worker read the cancel (it is now ${after}), and nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again to remove it`);
 			if (after === "completed" || after === "failed") return fail(`${waited}: the job finished (${after}) before its worker read the cancel, and nothing was changed`);
 			if (after !== "active") return fail(`${waited}: the job is no longer in the queue (its state reads ${after}), and nothing was changed`);
@@ -170,18 +290,75 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 		// del would sit in ioredis's retry queue and hold this process open long past the error message.
 		// The bound timer is CLEARED when the del wins the race (discoverHostQueues' own rule): cli.mjs
 		// sets exitCode rather than exiting, so a stray pending timer is a second of pure hang.
+		// What the withdrawing DEL ANSWERED decides the sentence (issue #846, the second adversary round): 1, it was taken back;
+		// 0, it was already gone, because its worker read it (and may have acknowledged) or it expired; no answer, not known.
+		// Reading "answered" as "withdrawn" said "nothing was changed" of a cancel its worker had taken and acted on, when a
+		// poll was lost to a Valkey that died and came back within the bound.
+		const within = (promise, ms) => {
+			let timer;
+			return Promise.race([Promise.resolve(promise).then((value) => ({ value }), (error) => ({ lost: true, error })), new Promise((r) => (timer = setTimeout(() => r({ lost: true }), ms)))]).finally(() => clearTimeout(timer));
+		};
+		let withdrawal = null;
+		let acked = null;
 		if (requested) {
-			let bound;
-			await Promise.race([
-				Promise.resolve(probe.del?.(cancelReqKey(jobId))).catch(() => {}),
-				new Promise((r) => (bound = setTimeout(r, 1000))),
-			]);
-			clearTimeout(bound);
+			withdrawal = await within(probe.del?.(cancelReqKey(jobId)), 1000);
+			if (withdrawal.value !== undefined && Number(withdrawal.value) === 0) {
+				const ack = await within(probe.get?.(cancelAckKey(jobId)), 1000);
+				if (typeof ack.value === "string") acked = ack.value;
+			}
 		}
-		return fail(error?.valkeyRefused ? error.message : `could not reach Valkey at ${(await import("./connection.mjs")).urlShown(url)}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(url)}\n  ${boundedText(error?.message)}`);
+		if (acked !== null) {
+			write(`cancel accepted by ${acked === "" ? "the worker" : boundedText(acked)} — stopping the container (allow ~30s); the run record will say operator-cancel\n`);
+			return 0;
+		}
+		if (error?.valkeyRefused) return fail(error.message);
+		const shown = urlShown(url);
+		// Issue #846: after the request went out, what is said is what became of it. The old line said only "could not reach
+		// Valkey ... is it running?", also when Valkey answered with a refusal, and never that a request may still stand.
+		if (requested && error?.valkeyMayStillRun) {
+			const removed = withdrawal?.value !== undefined ? Number(withdrawal.value) : null;
+			if (removed !== null && removed > 0) return fail(`${valkeyFault(error, shown)} after the cancel request for ${jobId} was written. The request was withdrawn again, so nothing was changed, unless its worker read it in that moment: run \`pi-dispatch status\` to see whether ${jobId} still runs`);
+			if (removed === 0) return fail(`${valkeyFault(error, shown)} after the cancel request for ${jobId} was written, and the request was gone when the CLI went to withdraw it: its worker may have read it, or it expired, and no acknowledgment was found. Run \`pi-dispatch status\` to see whether ${jobId} still runs`);
+			const until = new Date((requestedAt ?? now()) + CANCEL_REQ_TTL_MS).toISOString();
+			const then = refusedByValkey(error) || refusedByValkey(withdrawal?.error) ? "run `pi-dispatch status` to see whether it did" : "run `pi-dispatch status` once Valkey answers to see whether it did";
+			return fail(`${valkeyFault(error, shown)} after the cancel request for ${jobId} was written, and its withdrawal was not answered, so it may stand until ${until}, ${Math.round(CANCEL_REQ_TTL_MS / 1000)}s after it was written: a worker that reads it before then stops the job. ${then.charAt(0).toUpperCase()}${then.slice(1)}`);
+		}
+		if (refusedByValkey(error)) return fail(`${valkeyFault(error, shown)}; nothing was changed`);
+		// Issue #867: the lookup changes nothing about the job, so a failure there changed nothing, and the line says so: a
+		// Valkey that took a connection and stopped answering as that, with when to ask again; one never reached (nothing took
+		// a connection, so the lookup's own bound fired, or the queue gave up) as before #867, with the hint. A lost answer
+		// (`valkeyMayStillRun`) was on a socket, so it was reached.
+		const reached = connected || error?.valkeyMayStillRun === true;
+		if (!found && unanswered(error) && reached) return fail(`${valkeyFault(error, shown)} while cancel looked for ${jobId}, so nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again once Valkey answers`);
+		const hint = (await import("./valkey-auth.mjs")).valkeyDownHint(url);
+		const words = error?.lookupBound === true && !reached ? `no connection within ${spoken(bound)}` : error?.message;
+		if (!found) return fail(`could not reach Valkey at ${shown} while cancel looked for ${jobId}, so nothing was changed: ${hint}\n  ${boundedText(words)}`);
+		return fail(`could not reach Valkey at ${shown}: ${hint}\n  ${boundedText(error?.message)}`);
 	} finally {
 		probe.disconnect?.();
-		for (const q of queues) await q.close().catch(() => {});
+		// BOUNDED (issue #846): `close()` waits for the answer to QUIT, so on a Valkey that stopped answering the verb could hold
+		// its process after its last line. Past the bound the queue is dropped (`disconnect`, no QUIT; ioredis may still take
+		// its own disconnect timeout to let the socket go), which nothing here needs an answer to.
+		await Promise.all(queues.map((q) => closeWithin(q, closeTimeoutMs)));
 	}
 }
 
+/**
+ * What to do about a removal that was sent and not answered (issue #867): run the verb again once Valkey answers, which
+ * finds the job and removes it, or says no queue holds it. Not `pi-dispatch status`, whose counts name no job.
+ */
+function settleLater(jobId) {
+	return `Once Valkey answers, run \`pi-dispatch cancel ${jobId}\` again: it removes the job if it is still queued, and says so if it is gone`;
+}
+
+/** Close `queue`, or drop its connection when the close is not answered within `ms`. Never throws. */
+async function closeWithin(queue, ms) {
+	let timer;
+	const closed = await Promise.race([
+		Promise.resolve(queue.close()).then(() => true, () => true),
+		new Promise((resolve) => (timer = setTimeout(() => resolve(false), ms))),
+	]);
+	clearTimeout(timer);
+	if (!closed) await Promise.resolve(queue.disconnect?.()).catch(() => {});
+}
+