@edgehero/pi-dispatch 5.1.0 → 5.2.1

package/src/cancel-state.mjs

@@ -86,12 +86,18 @@ export function withdrawCancel({ redis, jobId, interventionOrigin = "human" }) {
  * already own a could-not-reach-Valkey message, and swallowing here would turn an unreachable Valkey into
  * a lying "no worker acknowledged".
  *
- * `pollMs`/`sleep` are injectable so tests never wait on a wall clock.
+ * `pollMs`/`sleep` are injectable so tests never wait on a wall clock. The wait ends at `ackTimeoutMs` by whichever runs
+ * out first: the sleeps it took, or the time `now` says has passed (issue #846). Counting sleeps alone, a GET that waited
+ * out a Valkey restart in the client's queue was not counted at all, so a 10 s wait could last as long as the outage did.
  */
-export async function requestCancel({ redis, jobId, interventionOrigin = "human", ackTimeoutMs = 10_000, pollMs = 250, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+export async function requestCancel({ redis, jobId, interventionOrigin = "human", ackTimeoutMs = 10_000, pollMs = 250, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), now = Date.now, onWritten = null }) {
 	const requestKey = cancelRequestKeyOf(jobId, interventionOrigin);
 	const ackKey = interventionOrigin === "agent" ? agentCancelAckKey(jobId) : cancelAckKey(jobId);
 	await redis.set(requestKey, interventionOrigin === "agent" ? "agent-cancel" : "operator-cancel", "PX", CANCEL_REQ_TTL_MS);
+	// When the request's TTL started, for a caller that names until when it may stand (the CLI, issue #846), and the start
+	// of the wait for its acknowledgment, so a slow SET is not counted against it.
+	onWritten?.();
+	const deadline = now() + ackTimeoutMs;
 	for (let waited = 0; ; waited += pollMs) {
 		let host;
 		try {
@@ -102,7 +108,7 @@ export async function requestCancel({ redis, jobId, interventionOrigin = "human"
 			throw err;
 		}
 		if (host !== null && host !== undefined) return { ack: host };
-		if (waited + pollMs > ackTimeoutMs) break;
+		if (waited + pollMs > ackTimeoutMs || now() >= deadline) break;
 		await sleep(pollMs);
 	}
 	// Losing this DEL leaves the request standing for its 60s TTL, where a worker can still read it, so that is no "nothing was
@@ -137,17 +143,38 @@ export function ranBefore(job) {
 
 /**
  * Whether the hold is still there, read back after its delete failed: its CLOCK (`since`), the field that makes the hash
- * a hold, since the worker's own counters can re-create the key without it. False when it cannot be read.
+ * a hold, since the worker's own counters can re-create the key without it. True or false as read, null when it cannot
+ * be read (issue #867: a hold read back gone is known gone, one not read back is not known either way).
  */
 async function holdInPlace(redis, jobId) {
-	if (typeof redis?.hexists !== "function") return false;
+	if (typeof redis?.hexists !== "function") return null;
 	try {
 		return (await redis.hexists(jobKey(jobId), "since")) === 1;
 	} catch {
-		return false;
+		return null;
 	}
 }
 
+/**
+ * Whether a command's failure means it was never sent: a bounded client's timeout that came while the command still
+ * waited for a connection (`makeRedisClient`'s `BoundedRedis` takes such a command out of its queue, so it is never sent
+ * later either), told apart from one that reached a socket by the missing `valkeyMayStillRun`.
+ */
+function neverSent(err) {
+	return err?.code === "ETIMEDOUT" && err?.message === "Command timed out" && err?.valkeyMayStillRun !== true;
+}
+
+/**
+ * Whether a failure is Valkey giving no answer (issue #867): a bounded client's timeout, or a command it sent whose answer
+ * was lost (`valkeyMayStillRun`). An answer that changed nothing carries the failure's words as `noAnswer`, so a caller can
+ * say Valkey did not answer, and when to ask again, rather than quote it as a refusal; a caller that does not read it
+ * says what it said before.
+ */
+function noAnswer(err) {
+	return err?.code === "ETIMEDOUT" || err?.valkeyMayStillRun === true;
+}
+const noAnswerOf = (err) => (noAnswer(err) ? { noAnswer: err?.message ?? String(err) } : {});
+
 /**
  * Remove a job that is held on a `run.waitFor` condition: hold keys first, then the job (issue #230's
  * sequence, issue #287's shared home). This is the inner body of the admin's `cancelHeldJob`, moved here
@@ -159,8 +186,14 @@ async function holdInPlace(redis, jobId) {
  * been sent for deletion (most often `remove` refusing a job a worker locked in between): the hold is gone, or partly
  * gone, while the job's own removal failed, which a caller must never word as "nothing was changed". A first delete
  * that failed answers `{ invalid }` only when the hold is read back in place afterwards (Valkey refused the command:
- * NOPERM, READONLY, MISCONF); a hold not read back, or a read that fails, keeps the mark, since a refusal can answer a
- * resend of a delete that already ran. Never throws.
+ * NOPERM, READONLY, MISCONF), or when it was never sent (`neverSent`); a hold not read back, or a read that fails, keeps
+ * the mark, since a refusal can answer a resend of a delete that already ran. Never throws.
+ * Issue #867: a cleared answer whose failure was a lost answer (`valkeyMayStillRun`: the command reached a Valkey that
+ * then did not answer, so it may still run) says which write that was, `unanswered`: "hold" for the hold's first delete,
+ * when the hold could not be read back either (the hold may or may not be gone, and the job was not asked to go), "job"
+ * for the job's own removal (the hold is gone, and the job may or may not be). A job whose state cannot be read is
+ * refused as that, never as one that left the hold. An answer that changed nothing because Valkey gave none (a read past
+ * its bound, a delete never sent) carries the failure's words as `noAnswer`.
  * Writes no run record of its own (INT-RUN-HISTORY-FILE-CONTRACT records terminal states of runs, and a removed hold
  * is none); `attemptsMade` is what the job had done before, for the caller's sentence (`ranBefore`, issue #477): a job
  * held again on a retry has made attempts, and whatever they recorded stays. Once the job is removed, its model attempt
@@ -171,6 +204,9 @@ async function holdInPlace(redis, jobId) {
 export async function removeHeldJob({ redis, queue, jobId, timeoutMs = 2_000, dropDefer = false, touchStore = undefined, touchOutbox = null, now = Date.now, interventionOrigin = "human" }) {
 	// Set as the first hold key's delete is sent, before it is answered: a delete lost in transit may still have landed.
 	let cleared = false;
+	// The write in flight, for `unanswered`: the hold's first delete, then the job's removal; null between them, where a
+	// failure leaves the hold known gone and the job not asked.
+	let step = null;
 	try {
 		const hash = await redis.hgetall(jobKey(jobId));
 		// A hold is a hash with a CLOCK, not merely a non-empty hash: the worker's own counters create this
@@ -182,7 +218,12 @@ export async function removeHeldJob({ redis, queue, jobId, timeoutMs = 2_000, dr
 		// while the job wakes, runs and completes -- and bullmq will happily `remove()` a completed job. That
 		// would answer `applied: true` to an operator who approved a dialog reading "It will never run", for
 		// a job whose run record is already on disk.
-		const state = await job.getState().catch(() => null);
+		let unread = null;
+		const state = await job.getState().catch((err) => ((unread = err), null));
+		// Issue #867: a read that failed (on a bounded queue, one Valkey did not answer) says nothing of where the job is,
+		// so it is no job that "has already left the hold": nothing was sent, and that is what is said, before the
+		// failure's own words, which a caller may cut.
+		if (unread !== null) return { invalid: `the state of job ${jobId} could not be read, so nothing was changed: ${unread?.message ?? String(unread)}`, ...noAnswerOf(unread) };
 		if (state !== "delayed" && state !== "waiting" && state !== "prioritized") {
 			return { invalid: `job ${jobId} is ${state ?? "in an unknown state"}, not waiting -- it has already left the hold` };
 		}
@@ -190,28 +231,40 @@ export async function removeHeldJob({ redis, queue, jobId, timeoutMs = 2_000, dr
 		// mid-sequence, an orphaned hash would keep a row on the panel for a job that no longer exists; an
 		// orphaned JOB is merely a job that still runs, which is the state the operator was already in.
 		cleared = true;
+		step = "hold";
 		try {
 			await redis.del(jobKey(jobId));
 		} catch (err) {
 			// A failed DEL may still have run: ioredis resends an unanswered command after a reconnect, and the refusal
 			// (an ACL without DEL, a read-only replica, a failed save) can be the answer to that resend. So the hold is
 			// read back, and only a hold still in place makes "nothing was changed" true; a read that fails keeps the mark.
-			if (await holdInPlace(redis, jobId)) cleared = false;
+			// A DEL never sent changed nothing either, whatever the read says. One read back gone ran.
+			const inPlace = await holdInPlace(redis, jobId);
+			if (inPlace === true || neverSent(err)) cleared = false;
+			else if (inPlace === false) step = null;
 			throw err;
 		}
+		step = null;
 		await redis.srem(HELD_SET, jobId).catch(() => {});
 		if (hash.dedupId) {
 			const holder = await redis.get(leaseKey(hash.dedupId));
 			if (holder === jobId) await redis.del(leaseKey(hash.dedupId));
 		}
 		const store = touchStore === undefined ? (typeof redis?.eval === "function" ? makeTouchStore({ redis, now }) : null) : touchStore;
-		const removal = interventionOrigin === "agent" ? (await job.remove(), { ok: true }) : await trackTouchAction({ store, outbox: touchOutbox, event: stopTouch(job, queue, jobId, now), act: async () => { await job.remove(); return { ok: true }; }, succeeded: () => !(job.attemptsMade > 0) && isWorkload(job) });
+		const remove = async () => {
+			step = "job";
+			await job.remove();
+			return { ok: true };
+		};
+		const removal = interventionOrigin === "agent" ? await remove() : await trackTouchAction({ store, outbox: touchOutbox, event: stopTouch(job, queue, jobId, now), act: remove, succeeded: () => !(job.attemptsMade > 0) && isWorkload(job) });
+		step = null;
 		await Promise.all([dropDefer ? dropDeferRecord(redis, jobId, timeoutMs) : null, dropAttemptOf(redis, queue, jobId, timeoutMs)]);
 		// Issue #477: what the job had done, for the caller's sentence (`ranBefore`); read off the job just removed.
 		return { ok: true, jobId, attemptsMade: Number.isInteger(job.attemptsMade) && job.attemptsMade > 0 ? job.attemptsMade : 0, ...(removal.touchWarning ? { touchWarning: removal.touchWarning } : {}) };
 	} catch (err) {
 		const invalid = err?.message ?? String(err);
-		return cleared ? { invalid, cleared: true } : { invalid };
+		if (!cleared) return { invalid, ...noAnswerOf(err) };
+		return { invalid, cleared: true, ...(err?.valkeyMayStillRun === true && step !== null ? { unanswered: step } : {}) };
 	}
 }
 
@@ -232,8 +285,10 @@ export const QUEUED_STATES = Object.freeze(["delayed", "waiting", "prioritized",
  *   any other state                                      -> nothing is changed.
  *
  * A HOLD IS JUDGED FIRST: a held job IS delayed, and the plain path would strand its `wait:*` keys as a panel row for a
- * job that no longer exists. A hash that cannot be read is not a hold, as the CLI has always read it. `state` is the
- * state the caller already read, or null to read it here.
+ * job that no longer exists. So a hash that cannot be read refuses, nothing changed (issue #867): it was read as no hold,
+ * and a read that timed out on a Valkey that then woke removed a held job by the plain path, leaving its hash and its
+ * `wait:held` member as a panel row for a job that was gone. A client with no `hgetall` (a bare test wiring) has no hold
+ * to read. `state` is the state the caller already read, or null to read it here.
  *
  * `guard` (optional; the CLI passes none) is a caller's last check, asked with `job` right before the removal is sent:
  * before `removeHeldJob` for a held job, and after the state check, before `remove()`, otherwise. It answers null to go
@@ -247,10 +302,13 @@ export const QUEUED_STATES = Object.freeze(["delayed", "waiting", "prioritized",
  * (issue #812), beside the record, so the two deletes share one bound: here for a waiting job, inside `removeHeldJob`
  * (`dropDefer`) for a held one. Returns, and never throws:
  *   `{ ok: true, held: true, jobId, attemptsMade }`         a held job, removed;
- *   `{ invalid }`                                            `removeHeldJob`'s own refusal of a held job, nothing changed;
- *   `{ invalid, cleared: true }`                             `removeHeldJob` failed after its hold's keys were deleted: the
+ *   `{ invalid, holdUnread, noAnswer? }`                     the hold's hash could not be read (`holdUnread`, the failure's
+ *                                                            words; `noAnswer` when Valkey gave none): nothing was sent;
+ *   `{ invalid, noAnswer? }`                                 `removeHeldJob`'s own refusal of a held job, nothing changed;
+ *   `{ invalid, cleared: true, unanswered? }`                `removeHeldJob` failed after its hold's keys were deleted: the
  *                                                            hold is gone and the job's own removal failed (never "nothing
- *                                                            was changed");
+ *                                                            was changed"), or a write of it was not answered
+ *                                                            (`unanswered`, removeHeldJob's);
  *   `{ refused, error? }`                                    the guard's answer (`refused`), or its throw (`refused` null,
  *                                                            `error` the throw); nothing was sent;
  *   `{ notQueued: true, state }`                             not waiting to start (`active`, `completed`, `unknown`...);
@@ -258,17 +316,24 @@ export const QUEUED_STATES = Object.freeze(["delayed", "waiting", "prioritized",
  *   `{ raced: true, state, error }`                          `remove()` threw (most often: a worker took the job between
  *                                                            the read and the remove, or it is a job scheduler's next
  *                                                            run, which BullMQ refuses to remove); `state` is read again.
+ *                                                            An `error` with `valkeyMayStillRun` was sent and not answered,
+ *                                                            so the remove may yet run (issue #867).
  * `attemptsMade` is the job's own count for the caller's sentence (`ranBefore`), 0 when it is not a positive whole number.
  */
 export async function removeQueuedJob({ redis, queue, job, jobId, state = null, timeoutMs = 2_000, guard = null, touchStore = undefined, touchOutbox = null, now = Date.now, interventionOrigin = "human" }) {
-	const hash = await Promise.resolve()
+	let holdUnread = null;
+	const hash = typeof redis?.hgetall !== "function" ? null : await Promise.resolve()
 		.then(() => redis.hgetall(jobKey(jobId)))
-		.catch(() => null);
+		.catch((err) => ((holdUnread = err), null));
+	if (holdUnread !== null) {
+		const words = holdUnread?.message ?? String(holdUnread);
+		return { invalid: `could not read whether job ${jobId} is held on a wait condition, so nothing was changed: ${words}`, holdUnread: words, ...noAnswerOf(holdUnread) };
+	}
 	if (hash?.since) {
 		const stop = await guarded(guard, job);
 		if (stop !== null) return stop;
 		const res = await removeHeldJob({ redis, queue, jobId, timeoutMs, dropDefer: true, touchStore, touchOutbox, now, interventionOrigin });
-		if (!res.ok) return res.cleared === true ? { invalid: res.invalid, cleared: true } : { invalid: res.invalid };
+		if (!res.ok) return res.cleared === true ? { invalid: res.invalid, cleared: true, ...(res.unanswered ? { unanswered: res.unanswered } : {}) } : { invalid: res.invalid, ...(res.noAnswer !== undefined ? { noAnswer: res.noAnswer } : {}) };
 		return { ok: true, held: true, jobId: res.jobId, attemptsMade: res.attemptsMade, ...(res.touchWarning ? { touchWarning: res.touchWarning } : {}) };
 	}
 	if (!job) return { notQueued: true, state: "unknown" };

package/src/connection.mjs

@@ -2,8 +2,8 @@ import { AbstractConnector, Command, Redis } from "ioredis";
 import { Socket, connect as netConnect } from "node:net";
 import { connect as tlsConnect } from "node:tls";
 import { valkeyAuthRefusal } from "./valkey-auth.mjs";
-import { valkeySchemeOf } from "./podman-stack.mjs";
-import { decodedUserinfo, judgeValkeyAtStart as judgeEndpointAtStart, judgedEndpoint, killSwitchValkeyUrls, defaultValkeyContext, urlShown, useValkeyContext, valkeyClientContext, valkeyContextFromKeys, VALKEY_CONTEXT_KEYS, valkeyPasswordFor, valkeyDbRangeSentence, valkeyRefusal, valkeyUrlFor, valkeyUrlProblem } from "./valkey-endpoint.mjs";
+import { valkeySchemeOf, valkeyTarget } from "./podman-stack.mjs";
+import { DEFAULT_VALKEY_URL, decodedUserinfo, judgeValkeyAtStart as judgeEndpointAtStart, judgedEndpoint, killSwitchValkeyUrls, defaultValkeyContext, urlShown, useValkeyContext, valkeyClientContext, valkeyContextFromKeys, VALKEY_CONTEXT_KEYS, valkeyPasswordFor, valkeyDbRangeSentence, valkeyRefusal, valkeyUrlFor, valkeyUrlProblem } from "./valkey-endpoint.mjs";
 
 // Re-exported for the receiver and the admin, which import this module from the package (issues #464 and #468).
 export { killSwitchValkeyUrls, defaultValkeyContext, urlShown, useValkeyContext, valkeyClientContext, valkeyContextFromKeys, VALKEY_CONTEXT_KEYS, valkeyPasswordFor, valkeyUrlFor, valkeyUrlProblem };
@@ -89,6 +89,18 @@ if (!Redis.prototype[SCRUBS]) {
  */
 export const DB_REFUSED = Symbol.for("pi-dispatch.valkey-db-refused");
 const SELECT_HOOK = Symbol.for("pi-dispatch.valkey-select-refusal");
+
+/**
+ * The database a client holds a refusal of (`DB_REFUSED`), as `{ db }` (null where the client no longer says which), or
+ * null when it holds none. Issue #845: the refusal is held for the client's life, so only a restart of a long-lived
+ * process tries the database again, and the console says so on every page from this.
+ */
+export function refusedDatabaseOf(client) {
+	if (!client?.[DB_REFUSED]) return null;
+	const db = client.condition?.select ?? client.options?.db;
+	return { db: Number.isSafeInteger(db) ? db : null };
+}
+
 /** Whether `err` is a server's refusal of a SELECT for a database it does not have. */
 function isSelectRangeError(err) {
 	return Boolean(err) && typeof err === "object" && String(err.command?.name ?? "").toLowerCase() === "select" && /DB index is out of range|invalid DB index/i.test(String(err.message ?? ""));
@@ -172,16 +184,30 @@ export function givesUp(connection) {
  *
  * `freshContext` is for a persistent client of a long-running surface that names no context: each connect judges by the
  * context as it is then, not as it was at construction, so an operator's fix of the deployment's `.env` reaches a client
- * that has been refused since it started. Off by default, so a worker's judgement stays the one it booted on. The password
- * is still the one read at construction, since ioredis has settled it before the connector runs.
+ * that has been refused since it started. Off by default, so a worker's judgement stays the one it booted on. Its password
+ * is read again before each connect too (issue #845, `freshPassword` below), so a password rotated in that `.env`, or one
+ * a bad line kept from being read at start, is what the next connect sends. Not its URL: a changed `VALKEY_URL` is
+ * another Valkey, which a running client is never moved to (the console says it needs a restart for that instead).
+ *
+ * `commandTimeoutMs` (issue #867) is a bound on each command of a `failFast` client, for a one-shot BullMQ queue that
+ * must not wait for as long as a Valkey that is connected and stopped answering (`kill -STOP`, a partition) stays so:
+ * `failFast` refuses while nothing is connected, and a command already sent had no bound. A command past it rejects with
+ * ioredis' "Command timed out"; one that was sent may still run when Valkey wakes, so its error says so
+ * (`valkeyMayStillRun`, the hook below), and it is never sent again on a later connection. It needs `failFast`: that
+ * client has no offline queue, and on one that has, ioredis sends a command that timed out there once Valkey returns.
+ * `makeRedisClient` takes such a command out of its own queue (`BoundedRedis`), but BullMQ builds this client itself, so
+ * nothing could. Never for a Worker: its blocking reads wait longer than any bound by design, and its connection is a
+ * persistent one, which this refuses.
  */
-export function parseConnection(url, { failFast = false, servername = null, context = null, judge = null, withoutPassword = false, freshContext = false } = {}) {
+export function parseConnection(url, { failFast = false, servername = null, context = null, judge = null, withoutPassword = false, freshContext = false, commandTimeoutMs = null } = {}) {
 	const u = new URL(url);
 	// PR #478's gate: a path that is not a database number was `db: NaN`, and the SELECT ioredis then sent failed outside
 	// every caller's await (an unhandled rejection). Refused here, where every client of the project is made, as the
 	// configuration refusal it is; `judgeValkeyAtStart` says it first at each start.
 	const dbProblem = valkeyUrlProblem(url);
 	if (dbProblem) throw valkeyRefusal(dbProblem);
+	const bounded = Number.isSafeInteger(commandTimeoutMs) && commandTimeoutMs > 0;
+	if (bounded && !failFast) throw new TypeError("parseConnection: a command bound (commandTimeoutMs) is for a failFast client only, which has no offline queue to send a command that timed out from later");
 	const where = context ?? defaultValkeyContext();
 	// Issue #468: the password this client sends, by `valkeyPasswordFor`'s rule (the URL's own, else VALKEY_PASSWORD from
 	// the environment, else from the deployment `.env` for a loopback host). Here and nowhere else, so every client of the
@@ -194,7 +220,9 @@ export function parseConnection(url, { failFast = false, servername = null, cont
 		// reports. `context` is where the client stands (`valkeyClientContext`), the process's own when not given.
 		Connector: JudgedConnector,
 		// `judge` is a test seam only: the function asked for the address (judgedEndpoint when not given).
-		[JUDGE]: { url, context: where, ...(judge ? { judge } : {}), ...(freshContext && !context ? { freshContext: true } : {}), ...(failFast ? { failFast: true } : {}) },
+		// `freshPassword` (issue #845): a client that judges by the context as it is at each connect sends the password that
+		// context gives then, unless it was built to send none (`withoutPassword`, a probe of what another account sees).
+		[JUDGE]: { url, context: where, ...(judge ? { judge } : {}), ...(freshContext && !context ? { freshContext: true, ...(withoutPassword ? {} : { freshPassword: true }) } : {}), ...(failFast ? { failFast: true } : {}), ...(bounded ? { commandBound: true } : {}) },
 		// WHATWG URL keeps an IPv6 literal's brackets in `hostname` (`[::1]`), and net.connect then looks up the name
 		// "[::1]", which never resolves, so a `redis://[::1]:6379` VALKEY_URL never connected (issue #464, measured on
 		// Fedora 44: ETIMEDOUT). The address is the part inside them.
@@ -216,9 +244,44 @@ export function parseConnection(url, { failFast = false, servername = null, cont
 					retryStrategy: (attempts) => (attempts > 2 ? null : 200), // give up after ~2 tries
 				}
 			: {}),
+		// ioredis' resend of what was on a connection that died is off with the bound: the caller was told such a command
+		// failed, and a resend on the next connection would make it run after that.
+		...(bounded ? { commandTimeout: commandTimeoutMs, autoResendUnfulfilledCommands: false } : {}),
 	};
 }
 
+/**
+ * What `BoundedRedis` does for its own clients, done for a client `parseConnection` built with a command bound (issue
+ * #867), which BullMQ constructs itself as a plain ioredis client, so it cannot be a subclass. On the prototype, like the
+ * scrub above, and inert for every client without the bound (a worker's, a receiver's, every `makeRedisClient` client).
+ *
+ * Two things. The command's timer is unref'd, since ioredis' is ref'd and a one-shot that has said its last line must not
+ * wait out a bound on a command nobody awaits. And a command that was WRITTEN (it is the newest entry of the client's
+ * command queue right after the send; without an offline queue, one that was not is refused at once and never sent) and
+ * then rejected with anything but Valkey's own reply carries `valkeyMayStillRun`: it reached a socket, so Valkey may run
+ * it when it wakes, or ran it before its answer was lost. Measured against the pinned ioredis: a SET past the bound on a
+ * `kill -STOP`ed Valkey rejects with "Command timed out" and is applied once the process is continued. Besides the
+ * timeout that is "Connection is closed.", for a command on a connection whose retries ran out. A refusal this project
+ * raised itself (`valkeyRefused`, a database the server does not have) is no lost answer.
+ */
+const COMMAND_BOUND_HOOK = Symbol.for("pi-dispatch.valkey-command-bound");
+if (!Redis.prototype[COMMAND_BOUND_HOOK]) {
+	const sendCommand = Redis.prototype.sendCommand;
+	Redis.prototype.sendCommand = function sendCommandWithBound(command, stream) {
+		const answer = sendCommand.call(this, command, stream);
+		if (this.options?.[JUDGE]?.commandBound !== true) return answer;
+		command._commandTimeoutTimer?.unref?.();
+		if (this.commandQueue?.peekBack?.()?.command !== command) return answer;
+		answer?.catch?.((err) => {
+			if (!err || typeof err !== "object" || err.name === "ReplyError" || err.valkeyRefused) return;
+			if (err.message === "Command timed out") err.code ??= "ETIMEDOUT";
+			err.valkeyMayStillRun = true;
+		});
+		return answer;
+	};
+	Redis.prototype[COMMAND_BOUND_HOOK] = true;
+}
+
 /**
  * A client whose commands have a bound. Its caller has been told a command that timed out failed, so that command must
  * not go out later, and ioredis would send it: the timer rejects it and leaves it in the offline queue for when Valkey
@@ -250,6 +313,13 @@ class BoundedRedis extends Redis {
 	}
 }
 
+/**
+ * How long one command of a short-lived client may wait for its answer (`commandTimeoutMs`), in the offline queue or on a
+ * Valkey that stopped answering: the console's own clients wait as long, and `pi-dispatch cancel` (issue #846) takes it
+ * from here, so a cancel against a Valkey that went away ends with a sentence rather than waiting as long as it is gone.
+ */
+export const VALKEY_COMMAND_TIMEOUT_MS = 5_000;
+
 /**
  * A raw ioredis client (the budget's INCR/EXPIRE, the registry and fleet reads, doctor's PING). Built from
  * `parseConnection`'s options, so it connects through `JudgedConnector` like every other client. `lazyConnect` and
@@ -375,6 +445,60 @@ export function authRefusalFor(state, url, context) {
  */
 const JUDGE = "piDispatchValkeyJudge";
 
+/** The connector's method that reads a `freshPassword` client's password again (`JudgedConnector`, the hook below it). */
+const FRESH_PASSWORD = Symbol.for("pi-dispatch.valkey-fresh-password");
+
+/** Set on a connector whose last refresh declined a `.env` password other than the one its client holds (see there). */
+const PASSWORD_DECLINED = Symbol.for("pi-dispatch.valkey-password-declined");
+
+/**
+ * Whether two Valkey URLs name one Valkey server, by `valkeyTarget`'s host and port (the scheme alias, a database path
+ * and a trailing `/` aside): a password belongs to the server. False for one that does not parse. `localhost` and
+ * `127.0.0.1` are two names here, as for `valkeyEndpointTag`: which address a name reaches is the judge's to say.
+ */
+function sameValkeyServer(a, b) {
+	const x = valkeyTarget(a);
+	const y = valkeyTarget(b);
+	return !x.error && !y.error && x.host === y.host && x.port === y.port;
+}
+
+/**
+ * What a `freshPassword` client's refresh does with `context` as it is now (`JudgedConnector`'s method below has the
+ * rules): `{ take: true, password }` to send `password` (null: none) from the next connect on; `{ take: false, declined }`
+ * to keep the one it holds (`held`), where `declined` is true when the `.env` gives another password than `held` for a
+ * Valkey the client does not dial as it dials it, false when nothing is being held back, and null when the context
+ * cannot tell (it could not be read, finds no file, or is another deployment's). Pure: one rule for the refresh before
+ * each connect and for `passwordDeclinedOf`, so the two never disagree about what is held back.
+ */
+function freshPasswordVerdict(judge, context, held) {
+	if (!context || context.error || context.absent === true) return { take: false, declined: null };
+	if ((context.envPath ?? null) !== (judge?.context?.envPath ?? null)) return { take: false, declined: null };
+	const { password, from } = valkeyPasswordFor(judge.url, context);
+	const fromFile = from !== "VALKEY_URL" && from !== "the environment";
+	if (fromFile && !sameValkeyServer(context.url?.file ?? DEFAULT_VALKEY_URL, judge.url)) return { take: false, declined: (password ?? null) !== (held ?? null) };
+	return { take: true, password: password ?? null };
+}
+
+/**
+ * Issue #845, review of PR #878: whether a `freshPassword` client holds back a password the deployment's `.env` gives,
+ * other than the one it holds, because the file does not name the Valkey the client dials as it dials it. True only
+ * while a reconnect has declined one (recorded by the refresh, which runs on connect alone) AND the `.env` as it is now
+ * still gives such a password (`freshPasswordVerdict` over the context read now): a file put back to the held password,
+ * or changed to name the client's Valkey, clears it at once, with no reconnect, and so does a file that cannot be read
+ * now (the console says that instead). Reads the client's context once; touches no Valkey.
+ */
+export function passwordDeclinedOf(client) {
+	const connector = client?.connector;
+	if (connector?.[PASSWORD_DECLINED] !== true) return false;
+	const judge = connector.options?.[JUDGE];
+	if (judge?.freshPassword !== true) return false;
+	try {
+		return freshPasswordVerdict(judge, defaultValkeyContext(), connector.options.password).declined === true;
+	} catch {
+		return false;
+	}
+}
+
 /**
  * ioredis' connector for every client of this project (issue #464, gate round 2 follow-up): on each connect it asks
  * `judgedEndpoint` for the address, then dials that literal (with a `rediss:` URL's name kept as the TLS servername),
@@ -401,6 +525,57 @@ export class JudgedConnector extends AbstractConnector {
 		return !this[DB_REFUSED];
 	}
 
+	/**
+	 * Issue #845: a `freshPassword` client's password, as `valkeyPasswordFor` reads it from the context as it is now, put
+	 * where ioredis reads it for the connect about to start (the hook below). Here and not in `connect()`: by the time
+	 * ioredis asks the connector for a stream it has already built that connect's AUTH from `options.password`. The options
+	 * are the client's own object (ioredis hands its `options` to the Connector it constructs), so this is what the client
+	 * sends. A password from VALKEY_URL's userinfo comes back as it is, so it stays fixed. The last password is kept by a
+	 * context that:
+	 *   - throws, or names an `error` (a `.env` that cannot be read, or a line in it the service's loader reads
+	 *     differently): it cannot say which password the file holds, and dropping the one that worked would refuse a client
+	 *     that is fine;
+	 *   - finds no file (`absent`): a `.env` gone, for good or for the moment an editor replaces it, is not a file that
+	 *     sets no password, so it does not drop one either (the console says to put the file back before a restart);
+	 *   - would answer from the `.env` while that file does not name the Valkey this client dials: a password the file
+	 *     gives (or the lack of one) belongs to the Valkey the file's VALKEY_URL names, the default one when it names
+	 *     none, and a held client never moves to another Valkey. Review of PR #878, measured with two Valkeys: one
+	 *     deployment moving to another Valkey changes both lines together, a symlinked deployment folder repointed reads
+	 *     another deployment's file through the same path, and a shell that exports VALKEY_URL keeps the client on its URL
+	 *     whatever the file says; in each the new Valkey's password went to the old one, which refused it, and the pages
+	 *     lost the old Valkey too. The check is against the URL the client dials, not the context's own choice of URL
+	 *     (`valkeyUrlFor`, which prefers this shell's and so never moved), and by the server each names (`sameValkeyServer`:
+	 *     `valkeyTarget`'s host and port, so a scheme alias, a database path or a trailing `/` is the same Valkey, while
+	 *     `localhost` and `127.0.0.1` stay two names, as `valkeyEndpointTag` keeps them). A rotation keeps the file's URL,
+	 *     so it is taken, and so is a password a bad line hid at start once the line is fixed, when the file names the
+	 *     client's Valkey. A password declined here that is not the one the client holds is recorded (`PASSWORD_DECLINED`,
+	 *     `passwordDeclinedOf`), so the console names VALKEY_PASSWORD as needing a restart rather than going on refused in
+	 *     silence (a shell that exports `redis://localhost:6440` over a file's `redis://127.0.0.1:6440`, say);
+	 *   - is read from another `.env` path than the one the client was built with: the pi extension's context follows the
+	 *     deployment pointer, and that alone says the file is another deployment's.
+	 * A password from this process's environment, or from VALKEY_URL's own userinfo, is not the file's, and is taken as it
+	 * comes. A context that passes all of these and gives no password drops it, as a restart would.
+	 */
+	[FRESH_PASSWORD]() {
+		const judge = this.options?.[JUDGE];
+		if (judge?.freshPassword !== true) return;
+		let context;
+		try {
+			context = defaultValkeyContext();
+		} catch {
+			return;
+		}
+		const verdict = freshPasswordVerdict(judge, context, this.options.password);
+		if (!verdict.take) {
+			// Recorded only where the context could tell; `passwordDeclinedOf` reads it against the file as it is then.
+			if (verdict.declined !== null) this[PASSWORD_DECLINED] = verdict.declined;
+			return;
+		}
+		this[PASSWORD_DECLINED] = false;
+		if (verdict.password) this.options.password = verdict.password;
+		else delete this.options.password;
+	}
+
 	connect() {
 		const { options } = this;
 		// A database the server refused stays refused: no reconnect goes ready on another one.
@@ -438,6 +613,39 @@ export class JudgedConnector extends AbstractConnector {
 /** How `JudgedConnector` dials, net and tls: a seam, so a test sees the options a pinned connect is made with. */
 JudgedConnector.dial = { net: netConnect, tls: tlsConnect };
 
+/**
+ * Where a `freshPassword` client's password is read again (issue #845): before EVERY connect, its first and each
+ * reconnect, on the prototype like the hooks above, so a client ioredis or BullMQ duplicates keeps it too. Measured
+ * against the pinned ioredis 5.11.1 (`worker/test/valkey-clients.test.mjs` holds each line): `connect()` runs
+ * `_connect()`, which builds the connect's AUTH (`this.condition.auth`) from `this.options.password` before it asks the
+ * connector for a stream; a refused AUTH (WRONGPASS) and a refused ready check (NOAUTH) go to `recoverFromFatalError`,
+ * which disconnects to reconnect, and the reconnect timer calls `self.connect()`. So a rotated password reaches a client
+ * that has been refused since the rotation at its next try, by its own retry strategy, with no rebuild.
+ *
+ * Delegated to the client's connector, so the context read is the one of the copy of this module that built the client
+ * (a bundle carries its own copy beside the shared ioredis, and the first copy loaded installs this hook). Inert for every
+ * other client. A throw here would escape ioredis' reconnect timer, which does not catch one, so it is caught: the last
+ * password is kept.
+ *
+ * REJECTED: rebuilding the client when its password changes, which every holder of one (the console's audit copy, the
+ * fleet reader, a page's reads) would have to learn to swap; wrapping `connect` on each client `makeRedisClient` makes,
+ * which misses a duplicate and a client BullMQ builds from `parseConnection`'s options; and reading the password in
+ * `JudgedConnector.connect`, which is too late for that connect's AUTH (above).
+ */
+const FRESH_PASSWORD_HOOK = Symbol.for("pi-dispatch.valkey-fresh-password-hook");
+if (!Redis.prototype[FRESH_PASSWORD_HOOK]) {
+	const connect = Redis.prototype.connect;
+	Redis.prototype.connect = function connectWithFreshPassword(...args) {
+		try {
+			this.connector?.[FRESH_PASSWORD]?.();
+		} catch {
+			// The last password stays.
+		}
+		return connect.apply(this, args);
+	};
+	Redis.prototype[FRESH_PASSWORD_HOOK] = true;
+}
+
 /**
  * Whether `connection` (options for BullMQ, or an ioredis client) connects through `JudgedConnector`. `makeQueue` and the
  * worker refuse any other, so no Queue or Worker of this project can be built on an unjudged connection.

package/src/doctor-bundles.mjs

@@ -49,7 +49,7 @@ export function parseSecretBundleProfilesSafe(raw) {
  *   - `realExecutablePath`: the resolver's own probe (secrets.mjs `realExecutable`), or null when pi did not load;
  *   - `resolveBundle`: `(binding) => resolve result`, only with `--secrets`.
  */
-export async function secretBundleChecks({ bundles = {}, triggersPresent = true, projects = [], projectsError = null, envRaw, overlayBundles = {}, roots = [], realExecutablePath = null, resolveBundle = null } = {}) {
+export async function secretBundleChecks({ bundles = {}, triggersPresent = true, projects = [], projectsError = null, projectsUnread = false, envRaw, overlayBundles = {}, roots = [], realExecutablePath = null, resolveBundle = null } = {}) {
 	const checks = [];
 	const { bundles: envBundles, error } = parseSecretBundleProfilesSafe(envRaw);
 	if (error) {
@@ -91,7 +91,11 @@ export async function secretBundleChecks({ bundles = {}, triggersPresent = true,
 		} else if (!fromEnv && !Object.hasOwn(usableOverlay, b.profile)) {
 			checks.push({ ok: false, label: `secret bundle ${id} names profile ${b.profile}, declared in the settings overlay but not usable`, fix: roots.length === 0 ? "PI_SECRET_RESOLVER_ROOTS is unset, so the worker uses no overlay-declared resolver: set it, or declare the profile in PI_SECRET_BUNDLES" : "the resolver must be an executable file whose real path sits inside PI_SECRET_RESOLVER_ROOTS" });
 		}
-		if (projectsError !== null) {
+		if (projectsUnread) {
+			// Issue #859: the `.env` line naming the projects file could not be read (its own line says why), so whether the
+			// service's file holds the project is not known: a warning, never the ✗ a file with no such project earns.
+			checks.push({ ok: false, warn: true, label: `secret bundle ${id} names project ${b.project}, and whether the projects file the service reads holds it is not known: doctor could not read what the deployment .env sets for PI_PROJECTS_FILE`, fix: "fix what the boot-file line above names, then re-run doctor" });
+		} else if (projectsError !== null) {
 			checks.push({ ok: false, label: `secret bundle ${id} names project ${b.project}, and projects.json does not load`, fix: "fix projects.json (its own line above says why) -- a bundle applies to a project's jobs only once the worker knows the project" });
 		} else if (!projectIds.has(b.project)) {
 			checks.push({ ok: false, label: `secret bundle ${id} names project ${b.project}, which projects.json does not hold`, fix: "add the project to PI_PROJECTS_FILE, or correct the bundle's project -- until then the bundle reaches no job, while the file reads as granting it" });

package/src/doctor-model-proposals.mjs

@@ -77,7 +77,7 @@ export function cheaperRouteChecks(roles, routes, models, endpoints, builtin = g
 	return checks;
 }
 
-export async function doctorModelProposalChecks(env, { cwd, credentials = {}, now = Date.now, read = safeRead, fetch = fetchLifecycle, endpoints: suppliedEndpoints } = {}) {
+export async function doctorModelProposalChecks(env, { cwd, credentials = {}, now = Date.now, read = safeRead, fetch = fetchLifecycle, endpoints: suppliedEndpoints, declaredEndpoints = null } = {}) {
 	let config;
 	try { config = parseProposalConfig(read(join(cwd, "model-proposals.json"))); }
 	catch (error) {
@@ -102,7 +102,9 @@ export async function doctorModelProposalChecks(env, { cwd, credentials = {}, no
 	if (config.routes?.length) try {
 		const parsed = parseModelsJson(read(join(env.PI_GLOBAL_PI_DIR, "models.json")));
 		if (parsed.error) throw new Error("overlay unavailable");
-		const endpoints = suppliedEndpoints ?? loadModelEndpoints({ modelEndpointsFile: env.PI_MODEL_ENDPOINTS_FILE }, { cwd });
+		// Issue #859: the endpoints as doctor's other endpoint checks read them (`declaredEndpointsIn`: this shell's file, else
+		// the deployment `.env`'s), handed in and asked only here; this shell's alone when a caller hands none.
+		const endpoints = suppliedEndpoints ?? (typeof declaredEndpoints === "function" ? declaredEndpoints() : loadModelEndpoints({ modelEndpointsFile: env.PI_MODEL_ENDPOINTS_FILE }, { cwd }));
 		checks.push(...cheaperRouteChecks(roles, config.routes, parsed.value, endpoints));
 	} catch { checks.push({ proposal: { kind: "unavailable" }, ok: false, warn: true, label: "Model proposals: declared route prices or endpoints unavailable", fix: "check the overlay models.json and declared endpoints; no route was changed" }); }
 	return checks;