@edgehero/pi-dispatch 4.3.1 → 5.1.0

package/src/connection.mjs

@@ -119,12 +119,40 @@ if (!Redis.prototype[SELECT_HOOK]) {
  * The `error` listener every BullMQ Queue and Worker of the project carries (issue #468): without one, BullMQ prints
  * the whole error object with `console.error`. This writes one line, the message only (scrubbed of any command), and
  * `what` naming whose error it is.
+ *
+ * `quietly`: the same line for the same `what` is said once per `VALKEY_ERROR_QUIET_MS`, across every emitter of the
+ * process, for the reader's raw client and for a queue that gives up (`makeQueue`): a reader that replaces its queues
+ * (`createFleetReader`) repeats the line on a new emitter per read, and a one-shot queue of a CLI verb or a worker's boot says
+ * it once instead of a few times. A message not said in that minute is said at once, so a new fault is not held back; two
+ * faults taking turns are each said once a minute. The raw client going ready ends the quiet for every line (a BullMQ queue
+ * does not pass that event on), since a fault after that is a new outage in the same words. Off for a client that rides a
+ * restart out: a running worker's and a receiver's retry lines are how a second outage shows in their logs.
  */
-export function onValkeyError(emitter, what, write = (line) => process.stderr.write(line)) {
-	emitter.on?.("error", (err) => write(`[pi-dispatch] Valkey error (${what}): ${String(scrubValkeyError(err)?.message ?? err)}\n`));
+export const VALKEY_ERROR_QUIET_MS = 60_000;
+const VALKEY_ERRORS_HELD = 256;
+const saidAt = new Map();
+export function onValkeyError(emitter, what, write = (line) => process.stderr.write(line), { quietly = false, now = () => Date.now() } = {}) {
+	if (quietly) emitter.on?.("ready", () => saidAt.clear());
+	emitter.on?.("error", (err) => {
+		const message = String(scrubValkeyError(err)?.message ?? err);
+		if (quietly) {
+			const key = `${what}\u0000${message}`;
+			const at = now();
+			const last = saidAt.get(key);
+			if (last !== undefined && at >= last && at - last < VALKEY_ERROR_QUIET_MS) return;
+			if (saidAt.size >= VALKEY_ERRORS_HELD) saidAt.clear();
+			saidAt.set(key, at);
+		}
+		write(`[pi-dispatch] Valkey error (${what}): ${message}\n`);
+	});
 	return emitter;
 }
 
+/** Whether `parseConnection` built this connection to give up after a couple of tries (`failFast`). */
+export function givesUp(connection) {
+	return connection?.[JUDGE]?.failFast === true;
+}
+
 /**
  * Connection helpers for BullMQ and the budget's raw Redis client, both from one VALKEY_URL.
  *
@@ -139,8 +167,15 @@ export function onValkeyError(emitter, what, write = (line) => process.stderr.wr
  * `failFast` is for the CLI producer (a one-shot enqueue): if Valkey is unreachable it should
  * error in a couple of seconds with a clear message, not hang forever. The long-running WORKER
  * uses the default (persistent) options -- it should ride out a Valkey restart, not give up.
+ * A failFast client whose retries ran out has ended for good, so anything that holds one for longer than a question must
+ * replace it (`createFleetReader`), or hold a persistent one instead (`noOfflineQueue` on `makeRedisClient`).
+ *
+ * `freshContext` is for a persistent client of a long-running surface that names no context: each connect judges by the
+ * context as it is then, not as it was at construction, so an operator's fix of the deployment's `.env` reaches a client
+ * that has been refused since it started. Off by default, so a worker's judgement stays the one it booted on. The password
+ * is still the one read at construction, since ioredis has settled it before the connector runs.
  */
-export function parseConnection(url, { failFast = false, servername = null, context = null, judge = null, withoutPassword = false } = {}) {
+export function parseConnection(url, { failFast = false, servername = null, context = null, judge = null, withoutPassword = false, freshContext = false } = {}) {
 	const u = new URL(url);
 	// PR #478's gate: a path that is not a database number was `db: NaN`, and the SELECT ioredis then sent failed outside
 	// every caller's await (an unhandled rejection). Refused here, where every client of the project is made, as the
@@ -159,7 +194,7 @@ export function parseConnection(url, { failFast = false, servername = null, cont
 		// reports. `context` is where the client stands (`valkeyClientContext`), the process's own when not given.
 		Connector: JudgedConnector,
 		// `judge` is a test seam only: the function asked for the address (judgedEndpoint when not given).
-		[JUDGE]: { url, context: where, ...(judge ? { judge } : {}) },
+		[JUDGE]: { url, context: where, ...(judge ? { judge } : {}), ...(freshContext && !context ? { freshContext: true } : {}), ...(failFast ? { failFast: true } : {}) },
 		// WHATWG URL keeps an IPv6 literal's brackets in `hostname` (`[::1]`), and net.connect then looks up the name
 		// "[::1]", which never resolves, so a `redis://[::1]:6379` VALKEY_URL never connected (issue #464, measured on
 		// Fedora 44: ETIMEDOUT). The address is the part inside them.
@@ -184,15 +219,54 @@ export function parseConnection(url, { failFast = false, servername = null, cont
 	};
 }
 
+/**
+ * A client whose commands have a bound. Its caller has been told a command that timed out failed, so that command must
+ * not go out later, and ioredis would send it: the timer rejects it and leaves it in the offline queue for when Valkey
+ * returns. A cancel the page said was not made then stopped the run hours later. One already on the socket of a Valkey
+ * that stopped answering cannot be taken back, so its error says so (`valkeyMayStillRun`), and the caller can too.
+ *
+ * ioredis' timer is also ref'd, so a client told to stop with commands still waiting held its process for up to one more
+ * bound. Reaches into two internals of the pinned ioredis, both held by
+ * `worker/test/connection.test.mjs`. REJECTED: `enableOfflineQueue: false`, which refuses a client's first command too,
+ * before it has had a chance to connect.
+ */
+class BoundedRedis extends Redis {
+	sendCommand(command, stream) {
+		const answer = super.sendCommand(command, stream);
+		command._commandTimeoutTimer?.unref?.();
+		answer?.catch?.((err) => {
+			if (err?.message !== "Command timed out") return;
+			err.code ??= "ETIMEDOUT";
+			const queue = this.offlineQueue;
+			for (let i = 0; i < queue.length; i += 1) {
+				if (queue.peekAt(i)?.command === command) {
+					queue.removeOne(i);
+					return;
+				}
+			}
+			err.valkeyMayStillRun = true;
+		});
+		return answer;
+	}
+}
+
 /**
  * A raw ioredis client (the budget's INCR/EXPIRE, the registry and fleet reads, doctor's PING). Built from
  * `parseConnection`'s options, so it connects through `JudgedConnector` like every other client. `lazyConnect` and
  * `failFast` as ioredis and `parseConnection` read them.
  */
-export function makeRedisClient(url, { servername = null, context = null, failFast = false, lazyConnect = false, judge = null, withoutPassword = false, disconnectTimeoutMs = null } = {}) {
+export function makeRedisClient(url, { servername = null, context = null, failFast = false, lazyConnect = false, judge = null, withoutPassword = false, disconnectTimeoutMs = null, noOfflineQueue = false, commandTimeoutMs = null, freshContext = false } = {}) {
 	// `disconnectTimeoutMs` (issue #599): how long ioredis's `disconnect()` waits for a socket to close before it destroys
 	// it (its default is 2 s, on a ref'd timer). A one-shot read that must not hold its process passes 0.
-	return new Redis({ ...parseConnection(url, { failFast, servername, context, judge, withoutPassword }), ...(lazyConnect ? { lazyConnect: true } : {}), ...(Number.isSafeInteger(disconnectTimeoutMs) && disconnectTimeoutMs >= 0 ? { disconnectTimeout: disconnectTimeoutMs } : {}) });
+	// A shell monitor must reconnect across an outage without accumulating stale control commands (#771).
+	// Keep the persistent retry strategy, but reject each command while disconnected and bound in-flight retries.
+	// `commandTimeoutMs`: how long one command may wait for its answer, in the offline queue or on a Valkey that stopped
+	// answering. Without it a persistent client's command waits for as long as the outage lasts. With it, what timed out is
+	// never sent again: `BoundedRedis` takes it out of the offline queue, and no resend covers what was on a connection that
+	// died.
+	const bounded = Number.isSafeInteger(commandTimeoutMs) && commandTimeoutMs > 0;
+	const options = { ...parseConnection(url, { failFast, servername, context, judge, withoutPassword, freshContext }), ...(noOfflineQueue ? { enableOfflineQueue: false, maxRetriesPerRequest: 1 } : {}), ...(lazyConnect ? { lazyConnect: true } : {}), ...(Number.isSafeInteger(disconnectTimeoutMs) && disconnectTimeoutMs >= 0 ? { disconnectTimeout: disconnectTimeoutMs } : {}), ...(bounded ? { commandTimeout: commandTimeoutMs, autoResendUnfulfilledCommands: false } : {}) };
+	return bounded ? new BoundedRedis(options) : new Redis(options);
 }
 
 /**
@@ -334,7 +408,9 @@ export class JudgedConnector extends AbstractConnector {
 		this.connecting = true;
 		const judge = options[JUDGE];
 		if (!judge) return Promise.reject(new Error("a Valkey client was built without parseConnection's judged options"));
-		return (judge.judge ?? judgedEndpoint)(judge.url, judge.context).then(
+		// A promise chain, so a context read that throws takes the same path as a judgement that does: a client's connect
+		// must not reject (below), or ioredis ends it for good.
+		return new Promise((resolve) => resolve((judge.judge ?? judgedEndpoint)(judge.url, judge.freshContext ? defaultValkeyContext() : judge.context))).then(
 			(endpoint) => {
 				if (!this.connecting) throw new Error("Connection is closed.");
 				const target = { host: endpoint.host, port: options.port, ...(options.family != null ? { family: options.family } : {}) };

package/src/console-answer.mjs

@@ -0,0 +1,113 @@
+/**
+ * An answer to a parked run typed into the operator console (issue #642, REQ-AWAIT-INPUT's console source).
+ *
+ * Shared by the two sides that must agree on it: the admin, which bounds and scrubs the text before it builds the
+ * reply job (`planParkedAnswer`), and the worker, which posts the one trail comment that says on the forge who
+ * answered (processor.mjs, after the `not-parked` gate and before any spend). One module, so the text the job carries
+ * and the text the trail quotes are scrubbed by one rule.
+ *
+ * WHY A TRAIL AT ALL. A forge reply leaves its own comment on the issue: anyone reading the thread sees who answered
+ * and what. A console answer reaches the job without one, so without the trail the issue would show a question, then
+ * a run, with the answer that steered it visible nowhere a reviewer looks. The trail is posted as the deployment's own
+ * identity (the `comment` adapter), which the receiver's self-guard drops as `self`, so it never starts a job.
+ *
+ * Import-light (console-tokens.mjs for the name rule, which imports only node built-ins), so the receiver-free admin
+ * bundle can take it.
+ */
+
+import { TOKEN_NAME_RE } from "./console-tokens.mjs";
+
+/**
+ * The `by` an answer from the panel's `/dispatch chat` carries (issue #647, part 2): it has no console token, so it is
+ * signed with this reserved word, and its trail says it came from the panel rather than naming a token. RESERVED: the
+ * console refuses to answer as a token of this name (`planParkedAnswer`'s `from`), so no token can sign as the panel.
+ */
+export const PANEL_ANSWER_BY = "panel";
+
+/** The most an answer may carry, in UTF-8 bytes AFTER scrubbing. A person's reply to a question, not a document. */
+export const ANSWER_TEXT_MAX_BYTES = 8 * 1024;
+
+/**
+ * Who answered, as `job.console.by` carries it: a console token's name (`TOKEN_NAME_RE`: lowercase letters, digits,
+ * dot, dash and underscore, at most 32), else null. The charset is what makes it safe to print on a forge: no `@`, no
+ * markdown, no control byte, no space.
+ */
+export function consoleAnswerBy(meta) {
+	const by = meta?.by;
+	return typeof by === "string" && TOKEN_NAME_RE.test(by) ? by : null;
+}
+
+/** The provenance is all-or-none. A model's free-text source label never reaches the trail or touch index. */
+export function consoleDraftOf(meta) {
+	const fields = ["draftRunId", "draftDigest", "draftSourceRunId"];
+	if (!fields.some(k => meta && Object.hasOwn(meta, k))) return null;
+	if (!/^[A-Za-z0-9._:-]{1,200}$/.test(meta?.draftRunId ?? "") || !/^[a-f0-9]{64}$/.test(meta?.draftDigest ?? "") || !/^[A-Za-z0-9._:-]{1,200}$/.test(meta?.draftSourceRunId ?? "")) return { invalid: true };
+	return { draftRunId: meta.draftRunId, draftDigest: meta.draftDigest, draftSourceRunId: meta.draftSourceRunId };
+}
+
+/**
+ * The answer text as a job may carry it. Never throws; a non-string is "". Newlines are kept: a reply can be several
+ * lines.
+ *   - LINE ENDINGS: `\r\n` and a lone `\r` become `\n`, and so do the Unicode line and paragraph separators (U+2028,
+ *     U+2029). Made newlines rather than removed: a person who typed a separator meant a line break, and removing it
+ *     would join two words the person kept apart. On a forge they would otherwise render as a break INSIDE one quoted
+ *     line, or not at all, so they never reach the trail as themselves.
+ *   - A TAB becomes a space; every other C0 control, DEL and every C1 control is removed.
+ *   - EVERY BIDIRECTIONAL FORMAT CHARACTER is removed: the embeddings and overrides (U+202A-U+202E), the isolates
+ *     (U+2066-U+2069) and the marks (U+200E, U+200F and the Arabic letter mark U+061C). A forge renders them, so a quoted
+ *     line could otherwise display in an order that is not the order it was typed (the "Trojan Source" class), in a
+ *     comment that claims to quote the operator verbatim. The marks are not overrides, but a mark beside a neutral
+ *     character moves it across a run boundary, which is the same lie at a smaller scale; nothing an operator types to
+ *     answer a question needs one.
+ */
+export function scrubAnswerText(text) {
+	if (typeof text !== "string") return "";
+	return text
+		.replace(/\r\n?/g, "\n")
+		.replace(/[\u2028\u2029]/g, "\n")
+		.replace(/\t/g, " ")
+		.replace(/[\u0000-\u0009\u000b-\u001f\u007f-\u009f\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]/g, "");
+}
+
+/**
+ * Bound a scrubbed answer: `{ text }` when it has something to say and fits `ANSWER_TEXT_MAX_BYTES`, else `{ refusal }`.
+ * Refused rather than cut: a truncated answer is an answer the person did not give.
+ */
+export function boundAnswerText(raw) {
+	const text = scrubAnswerText(raw);
+	if (text.trim() === "") return { refusal: "the answer is empty (after control characters are removed); nothing was queued" };
+	const bytes = Buffer.byteLength(text, "utf8");
+	if (bytes > ANSWER_TEXT_MAX_BYTES) return { refusal: `the answer is ${bytes} bytes, above the ${ANSWER_TEXT_MAX_BYTES}-byte limit; shorten it. Nothing was queued` };
+	return { text };
+}
+
+/**
+ * The trail comment for an answer: the name, then the text as a markdown blockquote, one `> ` per line (an empty line
+ * too, `> ` alone, so the quote never breaks and nothing below it reads as the deployment's own words).
+ *
+ * HTML IS ESCAPED FIRST: `&`, `<` and `>` become `&amp;`, `&lt;` and `&gt;`. Every forge renders HTML inside a comment's
+ * markdown, so without it an answer holding `<!--` hid every quoted line after it (the forge showed a truncated quote
+ * under the operator's name), a tag could restyle or hide the rest, and an entity such as `&#64;` decoded to an `@`
+ * AFTER the mention neutraliser below had looked for one. With `&` escaped no entity survives, with `<` escaped no
+ * comment or tag opens, and with `>` escaped a line cannot nest a deeper quote. The escape runs before the
+ * neutraliser, so the zero-width space it adds is never inside an entity.
+ *
+ * MENTIONS ARE NEUTRALISED by a zero-width space (U+200B) after every `@`: `@alice` renders as the same text but is no
+ * longer a mention, so an operator's answer cannot ping people or teams on the forge under the deployment's identity.
+ * Chosen over wrapping in backticks, which a backtick in the text could close. U+200B is a format character, not a
+ * control one, and the scrub above has already removed every control the text had. The name needs no neutralising:
+ * its charset holds no `@`. The panel's reserved name (`PANEL_ANSWER_BY`) gets its own head line.
+ */
+export function answerTrailComment(by, text, draftRunId = null) {
+	const quoted = scrubAnswerText(text)
+		.replace(/&/g, "&amp;")
+		.replace(/</g, "&lt;")
+		.replace(/>/g, "&gt;")
+		.replace(/@/g, "@\u200b")
+		.split("\n")
+		.map((line) => `> ${line}`)
+		.join("\n");
+	const head = by === PANEL_ANSWER_BY ? "Answered from an operator's /dispatch panel" : `Answered from the operator console by ${by}`;
+	const draft = typeof draftRunId === "string" && /^[A-Za-z0-9._:-]{1,200}$/.test(draftRunId) ? `\n\nDrafted in run ${draftRunId}.` : "";
+	return `${head}:\n\n${quoted}${draft}`;
+}