@edgehero/pi-dispatch 5.0.0 → 5.2.0

package/README.md

@@ -43,6 +43,7 @@ with at least one commit, and `run` refuses any other folder before it queues an
 |---|---|
 | `init` | write `.env` and the config files into this folder (never overwrites) |
 | `up` | one pass that asks before each container action: image, Valkey, egress proxy, `init`, `doctor` |
+| `console` | get into the web console: starts its service if it stopped, then opens your browser signed in |
 | `doctor [--fix] [--live] [--openshell] [--secrets] [--no-publish]` | check the whole setup; `--live` reads the isolation guarantees back from real containers; each run publishes its checks (paths, URLs and secret values cut out) for the console and the panel, unless `--no-publish` |
 | `worker` | run jobs from the queue |
 | `run <folder> --task "..."` | queue one job against a local folder |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch",
-  "version": "5.0.0",
+  "version": "5.2.0",
   "type": "module",
   "description": "The pi-dispatch worker and CLI: go from one AI coding agent to many in the background, running the pi coding agent as a self-hosted service, one locked down Docker or Podman container per job, with spend caps checked before anything is spent, plus init, up, doctor and service install.",
   "keywords": [
@@ -54,6 +54,7 @@
     "./git-hardening": "./src/git-hardening.mjs",
     "./http-body": "./src/http-body.mjs",
     "./console-tokens": "./src/console-tokens.mjs",
+    "./console-devices": "./src/console-devices.mjs",
     "./console-probe": "./src/console-probe.mjs",
     "./console-audit": "./src/console-audit.mjs",
     "./console-answer": "./src/console-answer.mjs",
@@ -207,8 +208,8 @@
     "start": "node src/cli.mjs worker"
   },
   "dependencies": {
-    "@earendil-works/pi-ai": "1.5.0",
-    "@earendil-works/pi-tui": "1.5.0",
+    "@earendil-works/pi-ai": "1.5.2",
+    "@earendil-works/pi-tui": "1.5.2",
     "@octokit/auth-app": "8.2.0",
     "@octokit/rest": "22.0.1",
     "bullmq": "5.80.4",

package/src/cancel-cli.mjs

@@ -24,7 +24,7 @@
  * job's attempt count.
  */
 
-import { cancelReqKey, ranBefore, removeQueuedJob, requestCancel } from "./cancel-state.mjs";
+import { CANCEL_REQ_TTL_MS, cancelAckKey, cancelReqKey, ranBefore, removeQueuedJob, requestCancel } from "./cancel-state.mjs";
 
 /** Same budget the kill switch gives the host registry before acting on what the keyspace alone says. */
 const FLEET_READ_TIMEOUT_MS = 2_000;
@@ -42,11 +42,43 @@ export function boundedText(text) {
 	return String(text ?? "no reason given").replace(/[\u0000-\u001f\u007f-\u009f]/g, " ").slice(0, 200);
 }
 
+/** Whether `error` is Valkey's own refusal (a reply), as against no answer at all: the two are different sentences. */
+function refusedByValkey(error) {
+	return error?.name === "ReplyError";
+}
+
+/** What went wrong, as the start of a line: Valkey refused a command (it answered), or it did not answer. */
+function valkeyFault(error, shown) {
+	const command = typeof error?.command?.name === "string" ? error.command.name.toUpperCase() : "a command";
+	return refusedByValkey(error) ? `Valkey at ${shown} refused ${boundedText(command)}: ${boundedText(error?.message)}` : `Valkey at ${shown} stopped answering (${boundedText(error?.message)})`;
+}
+
+/**
+ * The job's state, read under the same bound as every other command here: `{ state }`, or `{ unread }` with why it could
+ * not be read. Issue #846: a read that failed was `"unknown"`, which is also what BullMQ says of a job that left its
+ * queue, so a job still running was reported as gone "and nothing was changed".
+ */
+async function boundedState(job, timeoutMs) {
+	let timer;
+	try {
+		return await Promise.race([
+			Promise.resolve(job.getState()).then((state) => ({ state })),
+			new Promise((resolve) => {
+				timer = setTimeout(() => resolve({ unread: `no answer within ${Math.round(timeoutMs / 1000)}s` }), timeoutMs);
+			}),
+		]);
+	} catch (error) {
+		return { unread: error?.message ?? String(error) };
+	} finally {
+		clearTimeout(timer);
+	}
+}
+
 /**
  * Run the cancel. Returns the process exit code. Every collaborator is a seam with the production default,
  * so tests drive states and races without a queue; `write` is the stdout seam cli.mjs already injects.
  */
-export async function runCancel(jobId, url, { write = (chunk) => process.stdout.write(chunk), errWrite = (chunk) => process.stderr.write(chunk), redisFn, queueFn, parseConnectionFn, readLiveHostsFn, discoverHostQueuesFn, fleetQueueNamesFn, unionQueueNamesFn, ackTimeoutMs = 10_000, pollMs = 250, sleep, refusalFn, touchStore, touchOutbox = null, now = Date.now } = {}) {
+export async function runCancel(jobId, url, { write = (chunk) => process.stdout.write(chunk), errWrite = (chunk) => process.stderr.write(chunk), redisFn, queueFn, parseConnectionFn, readLiveHostsFn, discoverHostQueuesFn, fleetQueueNamesFn, unionQueueNamesFn, ackTimeoutMs = 10_000, pollMs = 250, sleep, refusalFn, touchStore, touchOutbox = null, now = Date.now, commandTimeoutMs = null, closeTimeoutMs = 1000 } = {}) {
 	const fail = (message) => {
 		errWrite(`error: ${message}\n`);
 		return 1;
@@ -55,7 +87,9 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 		return fail("a job id is required: pi-dispatch cancel <jobId>  (ids are in `pi-dispatch status`, the panel, and the run log)");
 	}
 	// Lazy imports, the pause verb's shape: a mistyped id must not load bullmq before it can be refused.
-	const { parseConnection, makeRedisClient } = await import("./connection.mjs");
+	const { parseConnection, makeRedisClient, VALKEY_COMMAND_TIMEOUT_MS } = await import("./connection.mjs");
+	// The bound on each command and on the state reads: the console's, unless a test asks for another.
+	const bound = Number.isSafeInteger(commandTimeoutMs) && commandTimeoutMs > 0 ? commandTimeoutMs : VALKEY_COMMAND_TIMEOUT_MS;
 	const { fleetQueueNames, discoverHostQueues, unionQueueNames, makeQueue } = await import("./queue.mjs");
 	const { readLiveHosts } = await import("./host-registry.mjs");
 	const parseConn = parseConnectionFn ?? parseConnection;
@@ -83,10 +117,14 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 				});
 	const refused = await refusalOf(url);
 	if (refused) return fail(refused);
-	const probe = mkRedis(url);
+	// BOUNDED (issue #846): each command waits at most VALKEY_COMMAND_TIMEOUT_MS for its answer, and one that timed out is
+	// never sent again. The unbounded client this was waited for as long as Valkey was gone: killed during the ack poll it
+	// printed nothing until the operator gave up, and on a partition its DEL never reached the catch below.
+	const probe = mkRedis(url, { commandTimeoutMs: bound, disconnectTimeoutMs: 0 });
 	probe.on?.("error", () => {});
 	const queues = [];
 	let requested = false;
+	let requestedAt = null;
 	try {
 		// The job could sit on the shared queue or on any host's own (issue #57): find it the way pause
 		// spans them, and fail OPEN but LOUDLY on a degraded registry read, exactly as the kill switch does.
@@ -138,6 +176,12 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			return 0;
 		}
 		const state = queued.state;
+		// Issue #846: `removeQueuedJob` reads a state it could not read as "unknown" (the console's sequence shares it), which
+		// BullMQ also says of a job that left its queue. Asked again under the bound, so a read that failed says so.
+		if (state === "unknown") {
+			const again = await boundedState(job, bound);
+			if (again.unread !== undefined) return fail(`could not read the state of ${jobId} (${boundedText(again.unread)}), so whether it is running is not known; nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again once Valkey answers`);
+		}
 		// The one race worth handling by name: picked up between getState and remove. remove() throws on a locked job, so
 		// the state is re-read and an active job falls through to the active path rather than failing an operator whose
 		// job is now exactly the case the active path exists for. Otherwise the re-read state and the throw's own words
@@ -147,7 +191,8 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 
 		if (state === "active") {
 			requested = true;
-			const res = await requestCancel({ redis: probe, jobId, ackTimeoutMs, pollMs, ...(sleep ? { sleep } : {}) });
+			requestedAt = now();
+			const res = await requestCancel({ redis: probe, jobId, ackTimeoutMs, pollMs, now, onWritten: () => (requestedAt = now()), ...(sleep ? { sleep } : {}) });
 			if (res.ack !== undefined) {
 				write(`cancel accepted by ${res.ack === "" ? "the worker" : boundedText(res.ack)} — stopping the container (allow ~30s); the run record will say operator-cancel\n`);
 				return 0;
@@ -155,8 +200,11 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 			// Gate round 2 of PR #479: the worker stops its cancel poll before it holds or retries a job, so a request that
 			// lands in that moment is never acknowledged and the job is back in the queue. Re-read, so the words are true.
 			// Gate round 3: by the state, since a job that finished or left the queue cannot be removed by a second cancel.
-			const after = await job.getState().catch(() => "unknown");
+			const read = await boundedState(job, bound);
 			const waited = `no worker acknowledged within ${Math.round(ackTimeoutMs / 1000)}s`;
+			// Issue #846: a state that could not be read is said as that, never as BullMQ's "unknown" (a job that left).
+			if (read.unread !== undefined) return fail(`${waited}, and the job's state could not be read (${boundedText(read.unread)}), so whether it still runs is not known. The cancel request was withdrawn, so nothing was changed: run \`pi-dispatch status\`, and \`pi-dispatch cancel ${jobId}\` again if it still runs`);
+			const after = read.state;
 			if (after === "waiting" || after === "delayed" || after === "prioritized" || after === "paused") return fail(`${waited}: the job went back to the queue before its worker read the cancel (it is now ${after}), and nothing was changed. Run \`pi-dispatch cancel ${jobId}\` again to remove it`);
 			if (after === "completed" || after === "failed") return fail(`${waited}: the job finished (${after}) before its worker read the cancel, and nothing was changed`);
 			if (after !== "active") return fail(`${waited}: the job is no longer in the queue (its state reads ${after}), and nothing was changed`);
@@ -170,18 +218,58 @@ export async function runCancel(jobId, url, { write = (chunk) => process.stdout.
 		// del would sit in ioredis's retry queue and hold this process open long past the error message.
 		// The bound timer is CLEARED when the del wins the race (discoverHostQueues' own rule): cli.mjs
 		// sets exitCode rather than exiting, so a stray pending timer is a second of pure hang.
+		// What the withdrawing DEL ANSWERED decides the sentence (issue #846, the second adversary round): 1, it was taken back;
+		// 0, it was already gone, because its worker read it (and may have acknowledged) or it expired; no answer, not known.
+		// Reading "answered" as "withdrawn" said "nothing was changed" of a cancel its worker had taken and acted on, when a
+		// poll was lost to a Valkey that died and came back within the bound.
+		const within = (promise, ms) => {
+			let timer;
+			return Promise.race([Promise.resolve(promise).then((value) => ({ value }), (error) => ({ lost: true, error })), new Promise((r) => (timer = setTimeout(() => r({ lost: true }), ms)))]).finally(() => clearTimeout(timer));
+		};
+		let withdrawal = null;
+		let acked = null;
 		if (requested) {
-			let bound;
-			await Promise.race([
-				Promise.resolve(probe.del?.(cancelReqKey(jobId))).catch(() => {}),
-				new Promise((r) => (bound = setTimeout(r, 1000))),
-			]);
-			clearTimeout(bound);
+			withdrawal = await within(probe.del?.(cancelReqKey(jobId)), 1000);
+			if (withdrawal.value !== undefined && Number(withdrawal.value) === 0) {
+				const ack = await within(probe.get?.(cancelAckKey(jobId)), 1000);
+				if (typeof ack.value === "string") acked = ack.value;
+			}
 		}
-		return fail(error?.valkeyRefused ? error.message : `could not reach Valkey at ${(await import("./connection.mjs")).urlShown(url)}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(url)}\n  ${boundedText(error?.message)}`);
+		if (acked !== null) {
+			write(`cancel accepted by ${acked === "" ? "the worker" : boundedText(acked)} — stopping the container (allow ~30s); the run record will say operator-cancel\n`);
+			return 0;
+		}
+		if (error?.valkeyRefused) return fail(error.message);
+		const shown = (await import("./connection.mjs")).urlShown(url);
+		// Issue #846: after the request went out, what is said is what became of it. The old line said only "could not reach
+		// Valkey ... is it running?", also when Valkey answered with a refusal, and never that a request may still stand.
+		if (requested && error?.valkeyMayStillRun) {
+			const removed = withdrawal?.value !== undefined ? Number(withdrawal.value) : null;
+			if (removed !== null && removed > 0) return fail(`${valkeyFault(error, shown)} after the cancel request for ${jobId} was written. The request was withdrawn again, so nothing was changed, unless its worker read it in that moment: run \`pi-dispatch status\` to see whether ${jobId} still runs`);
+			if (removed === 0) return fail(`${valkeyFault(error, shown)} after the cancel request for ${jobId} was written, and the request was gone when the CLI went to withdraw it: its worker may have read it, or it expired, and no acknowledgment was found. Run \`pi-dispatch status\` to see whether ${jobId} still runs`);
+			const until = new Date((requestedAt ?? now()) + CANCEL_REQ_TTL_MS).toISOString();
+			const then = refusedByValkey(error) || refusedByValkey(withdrawal?.error) ? "run `pi-dispatch status` to see whether it did" : "run `pi-dispatch status` once Valkey answers to see whether it did";
+			return fail(`${valkeyFault(error, shown)} after the cancel request for ${jobId} was written, and its withdrawal was not answered, so it may stand until ${until}, ${Math.round(CANCEL_REQ_TTL_MS / 1000)}s after it was written: a worker that reads it before then stops the job. ${then.charAt(0).toUpperCase()}${then.slice(1)}`);
+		}
+		if (refusedByValkey(error)) return fail(`${valkeyFault(error, shown)}; nothing was changed`);
+		return fail(`could not reach Valkey at ${shown}: ${(await import("./valkey-auth.mjs")).valkeyDownHint(url)}\n  ${boundedText(error?.message)}`);
 	} finally {
 		probe.disconnect?.();
-		for (const q of queues) await q.close().catch(() => {});
+		// BOUNDED (issue #846): `close()` waits for the answer to QUIT, so on a Valkey that stopped answering the verb could hold
+		// its process after its last line. Past the bound the queue is dropped (`disconnect`, no QUIT; ioredis may still take
+		// its own disconnect timeout to let the socket go), which nothing here needs an answer to.
+		await Promise.all(queues.map((q) => closeWithin(q, closeTimeoutMs)));
 	}
 }
 
+/** Close `queue`, or drop its connection when the close is not answered within `ms`. Never throws. */
+async function closeWithin(queue, ms) {
+	let timer;
+	const closed = await Promise.race([
+		Promise.resolve(queue.close()).then(() => true, () => true),
+		new Promise((resolve) => (timer = setTimeout(() => resolve(false), ms))),
+	]);
+	clearTimeout(timer);
+	if (!closed) await Promise.resolve(queue.disconnect?.()).catch(() => {});
+}
+

package/src/cancel-state.mjs

@@ -86,12 +86,18 @@ export function withdrawCancel({ redis, jobId, interventionOrigin = "human" }) {
  * already own a could-not-reach-Valkey message, and swallowing here would turn an unreachable Valkey into
  * a lying "no worker acknowledged".
  *
- * `pollMs`/`sleep` are injectable so tests never wait on a wall clock.
+ * `pollMs`/`sleep` are injectable so tests never wait on a wall clock. The wait ends at `ackTimeoutMs` by whichever runs
+ * out first: the sleeps it took, or the time `now` says has passed (issue #846). Counting sleeps alone, a GET that waited
+ * out a Valkey restart in the client's queue was not counted at all, so a 10 s wait could last as long as the outage did.
  */
-export async function requestCancel({ redis, jobId, interventionOrigin = "human", ackTimeoutMs = 10_000, pollMs = 250, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) }) {
+export async function requestCancel({ redis, jobId, interventionOrigin = "human", ackTimeoutMs = 10_000, pollMs = 250, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), now = Date.now, onWritten = null }) {
 	const requestKey = cancelRequestKeyOf(jobId, interventionOrigin);
 	const ackKey = interventionOrigin === "agent" ? agentCancelAckKey(jobId) : cancelAckKey(jobId);
 	await redis.set(requestKey, interventionOrigin === "agent" ? "agent-cancel" : "operator-cancel", "PX", CANCEL_REQ_TTL_MS);
+	// When the request's TTL started, for a caller that names until when it may stand (the CLI, issue #846), and the start
+	// of the wait for its acknowledgment, so a slow SET is not counted against it.
+	onWritten?.();
+	const deadline = now() + ackTimeoutMs;
 	for (let waited = 0; ; waited += pollMs) {
 		let host;
 		try {
@@ -102,7 +108,7 @@ export async function requestCancel({ redis, jobId, interventionOrigin = "human"
 			throw err;
 		}
 		if (host !== null && host !== undefined) return { ack: host };
-		if (waited + pollMs > ackTimeoutMs) break;
+		if (waited + pollMs > ackTimeoutMs || now() >= deadline) break;
 		await sleep(pollMs);
 	}
 	// Losing this DEL leaves the request standing for its 60s TTL, where a worker can still read it, so that is no "nothing was

package/src/cli.mjs

@@ -27,7 +27,10 @@ const USAGE = `pi-dispatch — run pi coding-agent flows on your own folders
                            --secrets resolves each project secret bundle here and prints its variable names, never values;
                            every run publishes its checks to Valkey for the console and the panel (labels with paths
                            and secret values redacted, kept 7 days), and --no-publish skips that
+  pi-dispatch console [--name NAME] [--next /setup] [--port N] [--deployment DIR]
+      get into the web console: starts its service if it stopped, then opens your browser signed in
   pi-dispatch up [--console] [--yes] [--adopt-volume] [--restart-proxy] [--refresh-proxy-rules]
+      with --console also: [--ask-each] [--no-open] [--name NAME]
                            one consented pass: pull+tag the job image, start Valkey, init, doctor
   pi-dispatch setup github mint GitHub App credentials in one browser click (App Manifest flow);
                            every write shown first and individually consented — no --yes here
@@ -161,6 +164,11 @@ export async function main(argv = process.argv.slice(2), env = process.env, { wr
 		return runUp(argv.slice(1), { env });
 	}
 
+	if (cmd === "console") {
+		const { runConsoleOpen } = await import("./console-open.mjs");
+		return runConsoleOpen(argv.slice(1), { env });
+	}
+
 	if (cmd === "setup") {
 		// Subcommand shape (`setup <forge>`) so future forges can land beside github without a new
 		// top-level verb; an unknown or missing target prints guidance rather than guessing.

package/src/connection.mjs

@@ -250,6 +250,13 @@ class BoundedRedis extends Redis {
 	}
 }
 
+/**
+ * How long one command of a short-lived client may wait for its answer (`commandTimeoutMs`), in the offline queue or on a
+ * Valkey that stopped answering: the console's own clients wait as long, and `pi-dispatch cancel` (issue #846) takes it
+ * from here, so a cancel against a Valkey that went away ends with a sentence rather than waiting as long as it is gone.
+ */
+export const VALKEY_COMMAND_TIMEOUT_MS = 5_000;
+
 /**
  * A raw ioredis client (the budget's INCR/EXPIRE, the registry and fleet reads, doctor's PING). Built from
  * `parseConnection`'s options, so it connects through `JudgedConnector` like every other client. `lazyConnect` and

package/src/console-audit.mjs

@@ -39,7 +39,7 @@ import { escapeControls } from "./projects.mjs";
 
 export const CONSOLE_AUDIT_DIR = "console-audit";
 export const CONSOLE_AUDIT_KEY = "console:audit";
-export const CONSOLE_AUDIT_ACTIONS = Object.freeze(["login", "login-failed", "login-code", "logout", "refused", "write", "write-applied", "token-added", "token-revoked", "token-rotated", "sessions-ended", "serve-started"]);
+export const CONSOLE_AUDIT_ACTIONS = Object.freeze(["login", "login-failed", "login-code", "logout", "refused", "write", "write-applied", "token-added", "token-revoked", "token-rotated", "sessions-ended", "serve-started", "login-device", "device-remembered", "device-ended"]);
 export const CONSOLE_AUDIT_OUTCOMES = Object.freeze(["ok", "refused", "failed"]);
 // A string field's cap, in code points, AFTER its controls are written out: a row stays small enough for the bounded tail
 // read below whatever a caller hands in (a Host header, a refusal reason), and no field can bury the others.