@edgehero/pi-dispatch 5.2.0 → 5.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch",
-  "version": "5.2.0",
+  "version": "5.2.1",
   "type": "module",
   "description": "The pi-dispatch worker and CLI: go from one AI coding agent to many in the background, running the pi coding agent as a self-hosted service, one locked down Docker or Podman container per job, with spend caps checked before anything is spent, plus init, up, doctor and service install.",
   "keywords": [

package/src/auth-cli.mjs

@@ -2,7 +2,9 @@ import { createInterface } from "node:readline/promises";
 import { loadConfig } from "./config.mjs";
 import { configuredSubscriptions } from "./subscription-config.mjs";
 import { subscriptionIdentity } from "./subscription-runtime.mjs";
-import { makeRedisClient } from "./connection.mjs";
+import { makeRedisClient, urlShown, VALKEY_COMMAND_TIMEOUT_MS } from "./connection.mjs";
+import { valkeyDownHint } from "./valkey-auth.mjs";
+import { boundedText } from "./cancel-cli.mjs";
 import { openBrowser } from "./open-browser.mjs";
 import { authError } from "./auth-store.mjs";
 import { OPENAI_LOGIN_INSTRUCTIONS } from "./oauth-openai.mjs";
@@ -22,6 +24,26 @@ export function authSelectAnswer(question, answer) {
 	return input;
 }
 
+/**
+ * The line for a Valkey failure (issue #867), or null when `error` is no Valkey failure: Valkey's own refusal (a reply),
+ * a configuration refusal this project raised (`valkeyRefused`), or a command that got no answer (a bound that fired, or
+ * one that reached a Valkey and lost its answer). `seen` is the client's last `error` event, which says why a client
+ * that never connected got none ("nothing answers VALKEY_URL ...") where the timeout says only that it got none. After a
+ * login or a logout the failure is the status read's, since neither sends anything to Valkey. `casLost` says the
+ * trial's compare-and-set itself was sent and its answer lost, so the grant may have landed and is not said to have
+ * failed; a read lost before it means the compare-and-set was never sent, and nothing is said of a grant.
+ */
+export function authValkeyLine(error, { command, url, seen = null, casLost = false }) {
+	if (error?.valkeyRefused === true) return `Subscription authentication: ${boundedText(error.message)}`;
+	const refused = error?.name === "ReplyError";
+	// ioredis' own timeout by its words, not by `ETIMEDOUT` alone, which a provider's request can carry too.
+	if (!refused && error?.message !== "Command timed out" && error?.valkeyMayStillRun !== true) return null;
+	const lead = `Subscription authentication: ${command === "login" || command === "logout" ? `the ${command} finished, but ` : ""}Valkey at ${urlShown(url)}`;
+	if (refused) return `${lead} refused ${boundedText(typeof error?.command?.name === "string" ? error.command.name.toUpperCase() : "a command")} (${boundedText(error.message)}).`;
+	const trial = command === "trial" && casLost === true ? " The trial grant may or may not have been made." : "";
+	return `${lead} did not answer (${boundedText(seen ?? error?.message)}).${trial} Run \`pi-dispatch auth status\` once it answers: ${valkeyDownHint(url)}`;
+}
+
 export async function runAuth(args, { env = process.env, write = (value) => process.stdout.write(value), writeErr = (value) => process.stderr.write(value), configure = configuredSubscriptions, configFn = loadConfig, redisFn = makeRedisClient, interaction = null, now = Date.now, terminalInput = process.stdin, terminalOutput = process.stdout, createPrompt = createInterface, openBrowserFn = openBrowser } = {}) {
 	const [command = "status", id, ...rest] = args;
 	if (!["status", "login", "logout", "usage", "trial"].includes(command) || rest.length > 0 || (command === "status" ? id !== undefined : !/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(id ?? ""))) {
@@ -30,10 +52,40 @@ export async function runAuth(args, { env = process.env, write = (value) => proc
 	}
 	let redis;
 	let prompt;
+	let config;
+	// The client's last error event since it was last ready, for the line a Valkey failure ends with: cleared when it goes
+	// ready, so a line after a long login does not quote an outage that ended during it.
+	let seen = null;
+	let valkeyFailed = false;
+	// Whether the one write `auth` sends, a quota compare-and-set (EVAL), was sent and lost its answer, so the line says a
+	// trial grant may or may not have been made only then, never of a read lost before the write was sent.
+	let casLost = false;
 	try {
-		const config = configFn(env);
+		config = configFn(env);
 		if (!config.authConnectionsFile) { writeErr("Set PI_AUTH_CONNECTIONS_FILE to an operator-reviewed subscription connection policy.\n"); return 2; }
-		redis = redisFn(config.valkeyUrl);
+		// BOUNDED (issue #867): every command this client sends is a short read or one compare-and-set of a quota row, so one
+		// that waits past VALKEY_COMMAND_TIMEOUT_MS is a Valkey that is gone or stopped answering. Without the bound `auth
+		// status` and `auth usage` waited for as long as it stayed so (measured: `auth usage` on a closed port printed 18 stack
+		// traces in 8 s, one per reconnect attempt, while its read sat in ioredis' offline queue for as long as Valkey stayed
+		// gone). The bound is on each command, never on the client's life, so a login
+		// that waits minutes on a browser is not cut short: the login itself sends nothing to Valkey, only the status read
+		// after it. `disconnectTimeoutMs: 0`, as `cancel`'s probe, so the socket of a Valkey that stopped answering does not
+		// hold the process past its last line.
+		redis = redisFn(config.valkeyUrl, { commandTimeoutMs: VALKEY_COMMAND_TIMEOUT_MS, disconnectTimeoutMs: 0 });
+		// Without a listener ioredis prints a stack trace per reconnect attempt, for as long as an outage outlasts this call.
+		redis?.on?.("error", (err) => {
+			seen = err?.message ?? String(err);
+		});
+		redis?.on?.("ready", () => {
+			seen = null;
+		});
+		if (typeof redis?.eval === "function") {
+			const evaluate = redis.eval.bind(redis);
+			redis.eval = (...args) => Promise.resolve(evaluate(...args)).catch((err) => {
+				if (err?.valkeyMayStillRun === true) casLost = true;
+				throw err;
+			});
+		}
 		const service = configure({ config, redis, now });
 		if (!interaction) interaction = {
 			async notify(event) {
@@ -70,17 +122,36 @@ export async function runAuth(args, { env = process.env, write = (value) => proc
 			const logout = { id, state: "disconnected", remoteRevocationConfirmed: result?.remoteRevocationConfirmed === true };
 			let status;
 			try { status = await service.runtime.status(); }
-			catch { status = { state: "unavailable", partial: true, connections: [] }; }
+			catch (error) {
+				status = { state: "unavailable", partial: true, connections: [] };
+				// The logout is done (it sends nothing to Valkey) and its evidence printed below; a Valkey failure of the status
+				// read is still said as the one line, and the client is dropped rather than asked to QUIT (issue #867).
+				const line = authValkeyLine(error, { command, url: config.valkeyUrl, seen });
+				if (line !== null) {
+					valkeyFailed = true;
+					writeErr(`${line}\n`);
+				}
+			}
 			result = { ...status, logout };
 		}
 		write(`${JSON.stringify(result, (key, value) => ["accountGroup", "generation", "accountDigest", "credential", "registration"].includes(key) ? undefined : value, 2)}\n`);
 		return result?.ok === false || result?.kind === "unavailable" ? 2 : 0;
 	} catch (error) {
+		// A Valkey failure is one plain line naming Valkey, exit 1 as the transient failure it is: "auth-unavailable" and
+		// "Run auth status" said nothing of Valkey, and `auth status` needs the same Valkey.
+		const line = config?.valkeyUrl ? authValkeyLine(error, { command, url: config.valkeyUrl, seen, casLost }) : null;
+		if (line !== null) {
+			valkeyFailed = true;
+			writeErr(`${line}\n`);
+			return 1;
+		}
 		const code = typeof error?.code === "string" && /^(?:auth|subscription)-[a-z-]+$/.test(error.code) ? error.code : "auth-unavailable";
 		writeErr(`Subscription authentication: ${code}. Run auth status on the connection owner host.\n`);
 		return error?.transient === true ? 1 : 2;
 	} finally {
 		prompt?.close();
-		try { await redis?.quit(); } catch { redis?.disconnect?.(); }
+		// After a Valkey failure QUIT would wait out one more bound on a Valkey that stopped answering, for nothing.
+		if (valkeyFailed) redis?.disconnect?.();
+		else try { await redis?.quit(); } catch { redis?.disconnect?.(); }
 	}
 }

package/src/cancel-cli.mjs

@@ -22,6 +22,12 @@
  * can be one waiting to retry after a failed attempt, which usually wrote a run record and may have left a retained
  * sandbox, and "it never ran" was printed about it. `ranBefore` (cancel-state.mjs, shared with the panel) words it from the
  * job's attempt count.
+ *
+ * Every command it sends has a bound (issue #846 for its own client, issue #867 for its queues), so a Valkey that is
+ * connected and stopped answering ends the verb with a line rather than holding it for as long as it stays so. The line
+ * says what the point it stopped at allows: the lookup changes nothing about the job (each queue writes its own meta row
+ * on connect, as every BullMQ queue does), so nothing was changed; a removal or a hold's delete that was sent and not
+ * answered may or may not have been made, and `cancel` again settles it once Valkey answers.
  */
 
 import { CANCEL_REQ_TTL_MS, cancelAckKey, cancelReqKey, ranBefore, removeQueuedJob, requestCancel } from "./cancel-state.mjs";
@@ -53,6 +59,16 @@ function valkeyFault(error, shown) {
 	return refusedByValkey(error) ? `Valkey at ${shown} refused ${boundedText(command)}: ${boundedText(error?.message)}` : `Valkey at ${shown} stopped answering (${boundedText(error?.message)})`;
 }
 
+/** A bound in words: whole seconds, or milliseconds below one (a test's bound read "0s"). */
+function spoken(ms) {
+	return ms >= 1000 ? `${Math.round(ms / 1000)}s` : `${ms} ms`;
+}
+
+/** Whether `error` is a command that got no answer: a bound that fired, or one sent whose answer was lost. */
+function unanswered(error) {
+	return error?.code === "ETIMEDOUT" || error?.valkeyMayStillRun === true;
+}
+
 /**
  * The job's state, read under the same bound as every other command here: `{ state }`, or `{ unread }` with why it could
  * not be read. Issue #846: a read that failed was `"unknown"`, which is also what BullMQ says of a job that left its
@@ -64,7 +80,7 @@ async function boundedState(job, timeoutMs) {
 		return await Promise.race([
 			Promise.resolve(job.getState()).then((state) => ({ state })),
 			new Promise((resolve) => {
-				timer = setTimeout(() => resolve({ unread: `no answer within ${Math.round(timeoutMs / 1000)}s` }), timeoutMs);
+				timer = setTimeout(() => resolve({ unread: `no answer within ${spoken(timeoutMs)}` }), timeoutMs);
 			}),
 		]);
 	} catch (error) {
@@ -74,6 +90,22 @@ async function boundedState(job, timeoutMs) {
 	}
 }
 
+/**
+ * `promise`'s answer, or a rejection once `ms` passed without one, marked as no answer (`ETIMEDOUT`, `lookupBound`). Each
+ * queue's lookup has this bound of its own, as one step (issue #867): BullMQ connects inside the queue, and on a Valkey
+ * that takes the connection and never answers, ioredis' handshake (CLIENT SETINFO, then the ready check's INFO) waits
+ * one command bound each, on each of the three connections `failFast` tries, so the per-command bound alone ended a
+ * lookup after about six of them (measured: 6.4 s at a 1 s bound). A lookup changes nothing about the job, so what it
+ * leaves running changes nothing either, and its queue is dropped on the way out (`closeWithin`).
+ */
+function answeredWithin(promise, ms) {
+	let timer;
+	return new Promise((resolve, reject) => {
+		timer = setTimeout(() => reject(Object.assign(new Error(`no answer within ${spoken(ms)}`), { code: "ETIMEDOUT", lookupBound: true })), ms);
+		Promise.resolve(promise).then(resolve, reject);
+	}).finally(() => clearTimeout(timer));
+}
+
 /**
  * Run the cancel. Returns the process exit code. Every collaborator is a seam with the production default,
  * so tests drive states and races without a queue; `write` is the stdout seam cli.mjs already injects.
@@ -87,7 +119,7 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 		return fail("a job id is required: pi-dispatch cancel <jobId>  (ids are in `pi-dispatch status`, the panel, and the run log)");
 	}
 	// Lazy imports, the pause verb's shape: a mistyped id must not load bullmq before it can be refused.
-	const { parseConnection, makeRedisClient, VALKEY_COMMAND_TIMEOUT_MS } = await import("./connection.mjs");
+	const { parseConnection, makeRedisClient, VALKEY_COMMAND_TIMEOUT_MS, urlShown } = await import("./connection.mjs");
 	// The bound on each command and on the state reads: the console's, unless a test asks for another.
 	const bound = Number.isSafeInteger(commandTimeoutMs) && commandTimeoutMs > 0 ? commandTimeoutMs : VALKEY_COMMAND_TIMEOUT_MS;
 	const { fleetQueueNames, discoverHostQueues, unionQueueNames, makeQueue } = await import("./queue.mjs");
@@ -122,7 +154,16 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 	// printed nothing until the operator gave up, and on a partition its DEL never reached the catch below.
 	const probe = mkRedis(url, { commandTimeoutMs: bound, disconnectTimeoutMs: 0 });
 	probe.on?.("error", () => {});
+	// Whether a connection to Valkey was ever made (issue #867, the review): a lookup that got no answer from a Valkey that
+	// took the connection says it stopped answering; one where nothing ever took it (an address that drops the connect, a
+	// host that is off) says it could not be reached, with the hint, as before #867. The probe connects first, so it tells.
+	let connected = false;
+	probe.on?.("connect", () => {
+		connected = true;
+	});
 	const queues = [];
+	// Whether the job was found: until then nothing about the job changed (issue #867), so a failure says nothing was changed.
+	let found = false;
 	let requested = false;
 	let requestedAt = null;
 	try {
@@ -138,14 +179,19 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 		// Constructed INSIDE the try (the pause loop's leak posture); first queue that knows the id wins.
 		// Job ids are delivery GUIDs, `local-<hex>` or `repeat:<id>:<millis>`, so a cross-queue duplicate
 		// is not a state this project can produce.
+		// BOUNDED (issue #867), each command at the probe's bound (`parseConnection`'s `commandTimeoutMs`) and each queue's
+		// lookup as one step (`answeredWithin`): `failFast` alone refuses while nothing is connected, and a command a Valkey
+		// took and never answered (`kill -STOP`, a partition) held the verb for as long as it stayed so, in the lookup and in
+		// the state read and the remove of `removeQueuedJob`, which go through this queue too.
 		let job = null;
 		let owner = null;
 		for (const name of names) {
-			const q = mkQueue(parseConn(url, { failFast: true }), { name });
+			const q = mkQueue(parseConn(url, { failFast: true, commandTimeoutMs: bound }), { name });
 			queues.push(q);
-			job = await q.getJob(jobId);
+			job = await answeredWithin(q.getJob(jobId), bound);
 			if (job) {
 				owner = q;
+				found = true;
 				break;
 			}
 		}
@@ -164,6 +210,26 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			write(`cancelled ${jobId}: it was held on its wait condition, and ${ranBefore(queued)}\n`);
 			return 0;
 		}
+		// Issue #867: a write of the held sequence that was sent and not answered may still run when Valkey wakes, so the line
+		// names which write that was and says it may or may not have been made. The cleared line below would say the job
+		// "could not be removed" of a removal that may yet land, and its hold "was cleared" of a delete that may not.
+		const shown = urlShown(url);
+		// Issue #867 (the review): a hold that could not be read refuses, so a held job is never removed by the plain path,
+		// which strands its hold as a panel row. Nothing was sent. The outcome comes before the failure's words, which are
+		// the only part bounded, so a long id never cuts it off.
+		if (typeof queued.holdUnread === "string") {
+			if (queued.noAnswer !== undefined) return fail(`Valkey at ${shown} stopped answering (${boundedText(queued.noAnswer)}) while cancel read whether ${jobId} is held, so nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again once Valkey answers`);
+			return fail(`could not read whether ${jobId} is held on its wait condition, so nothing was changed: ${boundedText(queued.holdUnread)}`);
+		}
+		if (queued.invalid !== undefined && queued.cleared !== true && queued.noAnswer !== undefined) {
+			return fail(`could not cancel held job ${jobId}: Valkey at ${shown} did not answer (${boundedText(queued.noAnswer)}), so nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again once Valkey answers`);
+		}
+		if (queued.invalid !== undefined && queued.cleared === true && queued.unanswered === "hold") {
+			return fail(`Valkey at ${shown} stopped answering (${boundedText(queued.invalid)}) after the delete of held job ${jobId}'s hold was sent, so the hold may or may not be gone, and the job itself was not removed. Once Valkey answers, run \`pi-dispatch cancel ${jobId}\` again to remove it`);
+		}
+		if (queued.invalid !== undefined && queued.cleared === true && queued.unanswered === "job") {
+			return fail(`Valkey at ${shown} stopped answering (${boundedText(queued.invalid)}) after the removal of held job ${jobId} was sent: its hold was cleared, and the job may or may not have been removed. ${settleLater(jobId)}`);
+		}
 		// A removal that failed after the hold's keys were deleted changed something: the job lost its hold and was not removed,
 		// so the line says so and how to finish, never only the error (which reads as "nothing happened"). Every error a
 		// line carries is bounded (`boundedText`): it is another module's words, of any length and with any bytes.
@@ -176,6 +242,12 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			return 0;
 		}
 		const state = queued.state;
+		// Issue #867: a remove that was sent and not answered, with the job not read back in any of BullMQ's lists (it was
+		// read as "unknown", or not read at all), may have run: never "nothing was changed", which the re-read below would
+		// say of a state it cannot read. A job read back in a list was not removed (its remove is one script), and goes on.
+		if (queued.raced && queued.error?.valkeyMayStillRun === true && state === "unknown") {
+			return fail(`${valkeyFault(queued.error, shown)} after the removal of ${jobId} was sent, so it may or may not have been removed. ${settleLater(jobId)}`);
+		}
 		// Issue #846: `removeQueuedJob` reads a state it could not read as "unknown" (the console's sequence shares it), which
 		// BullMQ also says of a job that left its queue. Asked again under the bound, so a read that failed says so.
 		if (state === "unknown") {
@@ -240,7 +312,7 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			return 0;
 		}
 		if (error?.valkeyRefused) return fail(error.message);
-		const shown = (await import("./connection.mjs")).urlShown(url);
+		const shown = urlShown(url);
 		// Issue #846: after the request went out, what is said is what became of it. The old line said only "could not reach
 		// Valkey ... is it running?", also when Valkey answered with a refusal, and never that a request may still stand.
 		if (requested && error?.valkeyMayStillRun) {
@@ -252,7 +324,16 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			return fail(`${valkeyFault(error, shown)} after the cancel request for ${jobId} was written, and its withdrawal was not answered, so it may stand until ${until}, ${Math.round(CANCEL_REQ_TTL_MS / 1000)}s after it was written: a worker that reads it before then stops the job. ${then.charAt(0).toUpperCase()}${then.slice(1)}`);
 		}
 		if (refusedByValkey(error)) return fail(`${valkeyFault(error, shown)}; nothing was changed`);
-		return fail(`could not reach Valkey at ${shown}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(url)}\n  ${boundedText(error?.message)}`);
+		// Issue #867: the lookup changes nothing about the job, so a failure there changed nothing, and the line says so: a
+		// Valkey that took a connection and stopped answering as that, with when to ask again; one never reached (nothing took
+		// a connection, so the lookup's own bound fired, or the queue gave up) as before #867, with the hint. A lost answer
+		// (`valkeyMayStillRun`) was on a socket, so it was reached.
+		const reached = connected || error?.valkeyMayStillRun === true;
+		if (!found && unanswered(error) && reached) return fail(`${valkeyFault(error, shown)} while cancel looked for ${jobId}, so nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again once Valkey answers`);
+		const hint = (await import("./valkey-auth.mjs")).valkeyDownHint(url);
+		const words = error?.lookupBound === true && !reached ? `no connection within ${spoken(bound)}` : error?.message;
+		if (!found) return fail(`could not reach Valkey at ${shown} while cancel looked for ${jobId}, so nothing was changed: ${hint}\n  ${boundedText(words)}`);
+		return fail(`could not reach Valkey at ${shown}: ${hint}\n  ${boundedText(error?.message)}`);
 	} finally {
 		probe.disconnect?.();
 		// BOUNDED (issue #846): `close()` waits for the answer to QUIT, so on a Valkey that stopped answering the verb could hold
@@ -262,6 +343,14 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 	}
 }
 
+/**
+ * What to do about a removal that was sent and not answered (issue #867): run the verb again once Valkey answers, which
+ * finds the job and removes it, or says no queue holds it. Not `pi-dispatch status`, whose counts name no job.
+ */
+function settleLater(jobId) {
+	return `Once Valkey answers, run \`pi-dispatch cancel ${jobId}\` again: it removes the job if it is still queued, and says so if it is gone`;
+}
+
 /** Close `queue`, or drop its connection when the close is not answered within `ms`. Never throws. */
 async function closeWithin(queue, ms) {
 	let timer;

package/src/cancel-state.mjs

@@ -143,17 +143,38 @@ export function ranBefore(job) {
 
 /**
  * Whether the hold is still there, read back after its delete failed: its CLOCK (`since`), the field that makes the hash
- * a hold, since the worker's own counters can re-create the key without it. False when it cannot be read.
+ * a hold, since the worker's own counters can re-create the key without it. True or false as read, null when it cannot
+ * be read (issue #867: a hold read back gone is known gone, one not read back is not known either way).
  */
 async function holdInPlace(redis, jobId) {
-	if (typeof redis?.hexists !== "function") return false;
+	if (typeof redis?.hexists !== "function") return null;
 	try {
 		return (await redis.hexists(jobKey(jobId), "since")) === 1;
 	} catch {
-		return false;
+		return null;
 	}
 }
 
+/**
+ * Whether a command's failure means it was never sent: a bounded client's timeout that came while the command still
+ * waited for a connection (`makeRedisClient`'s `BoundedRedis` takes such a command out of its queue, so it is never sent
+ * later either), told apart from one that reached a socket by the missing `valkeyMayStillRun`.
+ */
+function neverSent(err) {
+	return err?.code === "ETIMEDOUT" && err?.message === "Command timed out" && err?.valkeyMayStillRun !== true;
+}
+
+/**
+ * Whether a failure is Valkey giving no answer (issue #867): a bounded client's timeout, or a command it sent whose answer
+ * was lost (`valkeyMayStillRun`). An answer that changed nothing carries the failure's words as `noAnswer`, so a caller can
+ * say Valkey did not answer, and when to ask again, rather than quote it as a refusal; a caller that does not read it
+ * says what it said before.
+ */
+function noAnswer(err) {
+	return err?.code === "ETIMEDOUT" || err?.valkeyMayStillRun === true;
+}
+const noAnswerOf = (err) => (noAnswer(err) ? { noAnswer: err?.message ?? String(err) } : {});
+
 /**
  * Remove a job that is held on a `run.waitFor` condition: hold keys first, then the job (issue #230's
  * sequence, issue #287's shared home). This is the inner body of the admin's `cancelHeldJob`, moved here
@@ -165,8 +186,14 @@ async function holdInPlace(redis, jobId) {
  * been sent for deletion (most often `remove` refusing a job a worker locked in between): the hold is gone, or partly
  * gone, while the job's own removal failed, which a caller must never word as "nothing was changed". A first delete
  * that failed answers `{ invalid }` only when the hold is read back in place afterwards (Valkey refused the command:
- * NOPERM, READONLY, MISCONF); a hold not read back, or a read that fails, keeps the mark, since a refusal can answer a
- * resend of a delete that already ran. Never throws.
+ * NOPERM, READONLY, MISCONF), or when it was never sent (`neverSent`); a hold not read back, or a read that fails, keeps
+ * the mark, since a refusal can answer a resend of a delete that already ran. Never throws.
+ * Issue #867: a cleared answer whose failure was a lost answer (`valkeyMayStillRun`: the command reached a Valkey that
+ * then did not answer, so it may still run) says which write that was, `unanswered`: "hold" for the hold's first delete,
+ * when the hold could not be read back either (the hold may or may not be gone, and the job was not asked to go), "job"
+ * for the job's own removal (the hold is gone, and the job may or may not be). A job whose state cannot be read is
+ * refused as that, never as one that left the hold. An answer that changed nothing because Valkey gave none (a read past
+ * its bound, a delete never sent) carries the failure's words as `noAnswer`.
  * Writes no run record of its own (INT-RUN-HISTORY-FILE-CONTRACT records terminal states of runs, and a removed hold
  * is none); `attemptsMade` is what the job had done before, for the caller's sentence (`ranBefore`, issue #477): a job
  * held again on a retry has made attempts, and whatever they recorded stays. Once the job is removed, its model attempt
@@ -177,6 +204,9 @@ async function holdInPlace(redis, jobId) {
 export async function removeHeldJob({ redis, queue, jobId, timeoutMs = 2_000, dropDefer = false, touchStore = undefined, touchOutbox = null, now = Date.now, interventionOrigin = "human" }) {
 	// Set as the first hold key's delete is sent, before it is answered: a delete lost in transit may still have landed.
 	let cleared = false;
+	// The write in flight, for `unanswered`: the hold's first delete, then the job's removal; null between them, where a
+	// failure leaves the hold known gone and the job not asked.
+	let step = null;
 	try {
 		const hash = await redis.hgetall(jobKey(jobId));
 		// A hold is a hash with a CLOCK, not merely a non-empty hash: the worker's own counters create this
@@ -188,7 +218,12 @@ export async function removeHeldJob({ redis, queue, jobId, timeoutMs = 2_000, dr
 		// while the job wakes, runs and completes -- and bullmq will happily `remove()` a completed job. That
 		// would answer `applied: true` to an operator who approved a dialog reading "It will never run", for
 		// a job whose run record is already on disk.
-		const state = await job.getState().catch(() => null);
+		let unread = null;
+		const state = await job.getState().catch((err) => ((unread = err), null));
+		// Issue #867: a read that failed (on a bounded queue, one Valkey did not answer) says nothing of where the job is,
+		// so it is no job that "has already left the hold": nothing was sent, and that is what is said, before the
+		// failure's own words, which a caller may cut.
+		if (unread !== null) return { invalid: `the state of job ${jobId} could not be read, so nothing was changed: ${unread?.message ?? String(unread)}`, ...noAnswerOf(unread) };
 		if (state !== "delayed" && state !== "waiting" && state !== "prioritized") {
 			return { invalid: `job ${jobId} is ${state ?? "in an unknown state"}, not waiting -- it has already left the hold` };
 		}
@@ -196,28 +231,40 @@ export async function removeHeldJob({ redis, queue, jobId, timeoutMs = 2_000, dr
 		// mid-sequence, an orphaned hash would keep a row on the panel for a job that no longer exists; an
 		// orphaned JOB is merely a job that still runs, which is the state the operator was already in.
 		cleared = true;
+		step = "hold";
 		try {
 			await redis.del(jobKey(jobId));
 		} catch (err) {
 			// A failed DEL may still have run: ioredis resends an unanswered command after a reconnect, and the refusal
 			// (an ACL without DEL, a read-only replica, a failed save) can be the answer to that resend. So the hold is
 			// read back, and only a hold still in place makes "nothing was changed" true; a read that fails keeps the mark.
-			if (await holdInPlace(redis, jobId)) cleared = false;
+			// A DEL never sent changed nothing either, whatever the read says. One read back gone ran.
+			const inPlace = await holdInPlace(redis, jobId);
+			if (inPlace === true || neverSent(err)) cleared = false;
+			else if (inPlace === false) step = null;
 			throw err;
 		}
+		step = null;
 		await redis.srem(HELD_SET, jobId).catch(() => {});
 		if (hash.dedupId) {
 			const holder = await redis.get(leaseKey(hash.dedupId));
 			if (holder === jobId) await redis.del(leaseKey(hash.dedupId));
 		}
 		const store = touchStore === undefined ? (typeof redis?.eval === "function" ? makeTouchStore({ redis, now }) : null) : touchStore;
-		const removal = interventionOrigin === "agent" ? (await job.remove(), { ok: true }) : await trackTouchAction({ store, outbox: touchOutbox, event: stopTouch(job, queue, jobId, now), act: async () => { await job.remove(); return { ok: true }; }, succeeded: () => !(job.attemptsMade > 0) && isWorkload(job) });
+		const remove = async () => {
+			step = "job";
+			await job.remove();
+			return { ok: true };
+		};
+		const removal = interventionOrigin === "agent" ? await remove() : await trackTouchAction({ store, outbox: touchOutbox, event: stopTouch(job, queue, jobId, now), act: remove, succeeded: () => !(job.attemptsMade > 0) && isWorkload(job) });
+		step = null;
 		await Promise.all([dropDefer ? dropDeferRecord(redis, jobId, timeoutMs) : null, dropAttemptOf(redis, queue, jobId, timeoutMs)]);
 		// Issue #477: what the job had done, for the caller's sentence (`ranBefore`); read off the job just removed.
 		return { ok: true, jobId, attemptsMade: Number.isInteger(job.attemptsMade) && job.attemptsMade > 0 ? job.attemptsMade : 0, ...(removal.touchWarning ? { touchWarning: removal.touchWarning } : {}) };
 	} catch (err) {
 		const invalid = err?.message ?? String(err);
-		return cleared ? { invalid, cleared: true } : { invalid };
+		if (!cleared) return { invalid, ...noAnswerOf(err) };
+		return { invalid, cleared: true, ...(err?.valkeyMayStillRun === true && step !== null ? { unanswered: step } : {}) };
 	}
 }
 
@@ -238,8 +285,10 @@ export const QUEUED_STATES = Object.freeze(["delayed", "waiting", "prioritized",
  *   any other state                                      -> nothing is changed.
  *
  * A HOLD IS JUDGED FIRST: a held job IS delayed, and the plain path would strand its `wait:*` keys as a panel row for a
- * job that no longer exists. A hash that cannot be read is not a hold, as the CLI has always read it. `state` is the
- * state the caller already read, or null to read it here.
+ * job that no longer exists. So a hash that cannot be read refuses, nothing changed (issue #867): it was read as no hold,
+ * and a read that timed out on a Valkey that then woke removed a held job by the plain path, leaving its hash and its
+ * `wait:held` member as a panel row for a job that was gone. A client with no `hgetall` (a bare test wiring) has no hold
+ * to read. `state` is the state the caller already read, or null to read it here.
  *
  * `guard` (optional; the CLI passes none) is a caller's last check, asked with `job` right before the removal is sent:
  * before `removeHeldJob` for a held job, and after the state check, before `remove()`, otherwise. It answers null to go
@@ -253,10 +302,13 @@ export const QUEUED_STATES = Object.freeze(["delayed", "waiting", "prioritized",
  * (issue #812), beside the record, so the two deletes share one bound: here for a waiting job, inside `removeHeldJob`
  * (`dropDefer`) for a held one. Returns, and never throws:
  *   `{ ok: true, held: true, jobId, attemptsMade }`         a held job, removed;
- *   `{ invalid }`                                            `removeHeldJob`'s own refusal of a held job, nothing changed;
- *   `{ invalid, cleared: true }`                             `removeHeldJob` failed after its hold's keys were deleted: the
+ *   `{ invalid, holdUnread, noAnswer? }`                     the hold's hash could not be read (`holdUnread`, the failure's
+ *                                                            words; `noAnswer` when Valkey gave none): nothing was sent;
+ *   `{ invalid, noAnswer? }`                                 `removeHeldJob`'s own refusal of a held job, nothing changed;
+ *   `{ invalid, cleared: true, unanswered? }`                `removeHeldJob` failed after its hold's keys were deleted: the
  *                                                            hold is gone and the job's own removal failed (never "nothing
- *                                                            was changed");
+ *                                                            was changed"), or a write of it was not answered
+ *                                                            (`unanswered`, removeHeldJob's);
  *   `{ refused, error? }`                                    the guard's answer (`refused`), or its throw (`refused` null,
  *                                                            `error` the throw); nothing was sent;
  *   `{ notQueued: true, state }`                             not waiting to start (`active`, `completed`, `unknown`...);
@@ -264,17 +316,24 @@ export const QUEUED_STATES = Object.freeze(["delayed", "waiting", "prioritized",
  *   `{ raced: true, state, error }`                          `remove()` threw (most often: a worker took the job between
  *                                                            the read and the remove, or it is a job scheduler's next
  *                                                            run, which BullMQ refuses to remove); `state` is read again.
+ *                                                            An `error` with `valkeyMayStillRun` was sent and not answered,
+ *                                                            so the remove may yet run (issue #867).
  * `attemptsMade` is the job's own count for the caller's sentence (`ranBefore`), 0 when it is not a positive whole number.
  */
 export async function removeQueuedJob({ redis, queue, job, jobId, state = null, timeoutMs = 2_000, guard = null, touchStore = undefined, touchOutbox = null, now = Date.now, interventionOrigin = "human" }) {
-	const hash = await Promise.resolve()
+	let holdUnread = null;
+	const hash = typeof redis?.hgetall !== "function" ? null : await Promise.resolve()
 		.then(() => redis.hgetall(jobKey(jobId)))
-		.catch(() => null);
+		.catch((err) => ((holdUnread = err), null));
+	if (holdUnread !== null) {
+		const words = holdUnread?.message ?? String(holdUnread);
+		return { invalid: `could not read whether job ${jobId} is held on a wait condition, so nothing was changed: ${words}`, holdUnread: words, ...noAnswerOf(holdUnread) };
+	}
 	if (hash?.since) {
 		const stop = await guarded(guard, job);
 		if (stop !== null) return stop;
 		const res = await removeHeldJob({ redis, queue, jobId, timeoutMs, dropDefer: true, touchStore, touchOutbox, now, interventionOrigin });
-		if (!res.ok) return res.cleared === true ? { invalid: res.invalid, cleared: true } : { invalid: res.invalid };
+		if (!res.ok) return res.cleared === true ? { invalid: res.invalid, cleared: true, ...(res.unanswered ? { unanswered: res.unanswered } : {}) } : { invalid: res.invalid, ...(res.noAnswer !== undefined ? { noAnswer: res.noAnswer } : {}) };
 		return { ok: true, held: true, jobId: res.jobId, attemptsMade: res.attemptsMade, ...(res.touchWarning ? { touchWarning: res.touchWarning } : {}) };
 	}
 	if (!job) return { notQueued: true, state: "unknown" };