@edgehero/pi-dispatch 4.3.0 → 5.0.0

package/src/queue.mjs

@@ -1,9 +1,15 @@
+import { makeApprovalStore } from "./approval-state.mjs";
+import { validateFork } from "./fork-job.mjs";
 import { randomUUID } from "node:crypto";
 import { Queue } from "bullmq";
-import { assertJudgedConnection, onValkeyError } from "./connection.mjs";
-import { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId, manualTriggerJobId } from "./job-id.mjs";
+import { metaKeyPattern, queueNameOfMetaKey } from "./bullmq-internals.mjs";
+import { assertJudgedConnection, givesUp, onValkeyError } from "./connection.mjs";
+import { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId, manualCanaryJobId, manualTriggerJobId } from "./job-id.mjs";
 import { targetSeparator } from "./forges.mjs";
 import { PR_CLOSE_ACTIONS } from "./triggers.mjs";
+import { consoleAnswerBy, consoleDraftOf } from "./console-answer.mjs";
+import { JOB_DATA_VERSION_FIELD, withJobDataVersion } from "./job-data-version.mjs";
+import { FORGE_JOB_RETENTION_S, LOCAL_JOB_KEPT_COMPLETED_S, LOCAL_JOB_KEPT_FAILED_S } from "./retention.mjs";
 
 // The close words in every forge's spelling, derived from the one table (never re-typed here): a
 // matched PR action in this set marks a close job for the semantic-key discriminant below.
@@ -11,6 +17,48 @@ const PR_CLOSE_WORDS = new Set(Object.values(PR_CLOSE_ACTIONS));
 
 export const QUEUE = "pi-jobs";
 
+/**
+ * A builder's job data stamped with the version it needs (issue #761, `DES-STORE-VERSION-WHEN-NEEDED`). `dataVersion`
+ * is the version the builder's INPUT arrived with: a job a newer release stored that this build re-enqueues (a held
+ * reply, a late message's fallback, a rerun), so the stamp never falls below it. Throws when that version is one this
+ * build cannot read, above its own or not a whole number of 1 or more (`withJobDataVersion`'s RangeError): such input
+ * cannot be rebuilt here without dropping what this build does not know, and the error carries `code: "newer-release"`,
+ * so a caller that falls back to a reply logs why. Nothing is stamped while the data needs 1, so every job of today's
+ * fields is stored byte for byte as before. `dataVersions` (`{ table, valueTable, supported }`) is the version rule,
+ * injectable for tests.
+ */
+function stampJobData(data, dataVersion, dataVersions = {}) {
+	try {
+		return withJobDataVersion(dataVersion === undefined ? data : { ...data, [JOB_DATA_VERSION_FIELD]: dataVersion }, dataVersions);
+	} catch (error) {
+		if (error instanceof RangeError && error.code === undefined) error.code = "newer-release";
+		throw error;
+	}
+}
+
+/** Deterministic source/run binding survives a lost enqueue acknowledgement. No question or draft is queued. */
+export async function enqueueAnswerDraftJob(queue, { runId, ruleId, draftRunId }) {
+	if (!/^[A-Za-z0-9._:-]{1,200}$/.test(runId ?? "") || !/^[a-z][a-z0-9_-]{0,63}$/.test(ruleId ?? "") || !/^answer-draft-[a-f0-9]{40}$/.test(draftRunId ?? "")) throw new Error("invalid answer draft reference");
+	await queue.add("answer-draft", { kind: "answer-draft", purpose: "answer-draft", answerDraft: { runId, ruleId } }, { jobId: draftRunId, attempts: 2, backoff: { type: "exponential", delay: 5000 }, removeOnComplete: { age: FORGE_JOB_RETENTION_S, count: 5000 }, removeOnFail: { age: FORGE_JOB_RETENTION_S, count: 5000 } });
+	return draftRunId;
+}
+
+/** A private evidence reference, never collector-supplied execution authority. */
+export async function enqueueWatchJob(queue, fields, { now = Date.now, phase = "dispatch" } = {}) {
+	if (!/^[a-f0-9]{64}$/.test(fields.id) || !/^[A-Za-z0-9_-]{1,128}$/.test(fields.ruleId) || !["dispatch", "action"].includes(phase)) throw new Error("invalid watch queue reference");
+	const id = `watch-${phase}-${fields.id}`;
+	await queue.add("watch", { kind: "watch", watch: { findingId: fields.id, ruleId: fields.ruleId, phase } }, { jobId: id, delay: Math.max(0, (fields.dueAt ?? now()) - now()), attempts: 2, backoff: { type: "exponential", delay: 5000 }, removeOnComplete: { age: 7 * 86400, count: 10000 }, removeOnFail: { age: 7 * 86400, count: 10000 } });
+	return id;
+}
+
+/** A confirmed grouping request has no private text and one paid attempt. Collection failures never buy another. */
+export async function enqueueQuestionGroupJob(queue, { runId, digest, configDigest }, { nonce = randomUUID } = {}) {
+	if (!/^[A-Za-z0-9._:-]{1,200}$/.test(runId ?? "") || !/^[a-f0-9]{64}$/.test(digest ?? "") || !/^[a-f0-9]{64}$/.test(configDigest ?? "")) throw new Error("invalid question grouping reference");
+	const jobId = `question-group-${nonce()}`;
+	await queue.add("question-grouping", { kind: "question-grouping", questionGrouping: { runId, digest, configDigest } }, { jobId, attempts: 1, removeOnComplete: { age: FORGE_JOB_RETENTION_S, count: 5000 }, removeOnFail: { age: FORGE_JOB_RETENTION_S, count: 5000 } });
+	return jobId;
+}
+
 /**
  * The queue a HOST-AFFINE job goes to (issue #57): work only one machine can do, because the folder, the
  * secret resolver or the wait-check script lives there.
@@ -63,7 +111,40 @@ export function fleetQueueNames(hosts) {
 /** The name charset, duplicated from `config.mjs` deliberately: this module imports nothing. */
 const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
 
-export { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId, manualTriggerJobId };
+export { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeliveryJobId, manualCanaryJobId, manualTriggerJobId };
+
+/**
+ * BullMQ's `jobScheduler` and `repeat` getters are `new Promise(async (resolve) => { ... await this.client ... })` (read
+ * at 5.80.4). A connection that gave up rejects `this.client` inside that async executor, where no caller can reach it:
+ * the outer promise never settles and the rejection is unhandled, which ends the process. One `getJobSchedulers` on a
+ * refused or absent Valkey took the whole console down, from behind every `.catch` and timeout its readers carry.
+ *
+ * So the connection is awaited FIRST, out here, where a rejection is the caller's. Past that `this.client` is settled and
+ * resolved, so the getter's own await cannot reject. Guarding each call site instead was the other way, and one of them
+ * already did (`await q.client` in trigger-schedule-zones): the next caller would have to know, and `makeQueue` is the one
+ * place every queue of this project is built.
+ *
+ * The constructor is the same class of fault at another address. `RedisConnection.close` removes every listener, and a
+ * connect still in flight then fails into `emit("error")`, which with no listener throws out of a promise chain nothing can
+ * reach, so a read that closed its queue at its own bound ended the process a moment after it answered. A queue that was
+ * closed has nobody left to tell, so that one emit says nothing.
+ */
+class SettledQueue extends Queue {
+	constructor(name, options) {
+		super(name, options);
+		const { connection } = this;
+		const emit = connection.emit.bind(connection);
+		connection.emit = (event, ...args) => (event === "error" && connection.listenerCount("error") === 0 ? false : emit(event, ...args));
+	}
+
+	get jobScheduler() {
+		return this.client.then(() => super.jobScheduler);
+	}
+
+	get repeat() {
+		return this.client.then(() => super.repeat);
+	}
+}
 
 /**
  * A queue handle. `name` defaults to the shared queue, so every existing caller is unchanged and a
@@ -72,9 +153,10 @@ export { chainedJobId, localJobId, deliveryJobId, gitlabDeliveryJobId, forgeDeli
 export function makeQueue(connection, { name = QUEUE } = {}) {
 	// Issue #464: only a connection `parseConnection` built, which judges and pins the Valkey it dials.
 	assertJudgedConnection(connection);
-	const queue = new Queue(name, { connection });
+	const queue = new SettledQueue(name, { connection });
 	// Issue #468: an error of this queue is one line, its message, never BullMQ's console.error of the whole object.
-	onValkeyError(queue, `queue ${name}`);
+	// A queue that gives up is the kind a long-lived reader replaces, so it is the kind that repeats the line per read.
+	onValkeyError(queue, `queue ${name}`, undefined, { quietly: givesUp(connection) });
 	return queue;
 }
 
@@ -85,8 +167,8 @@ export function makeQueue(connection, { name = QUEUE } = {}) {
  * removeOnComplete keeps the dedup window ~= the retention. Unlike webhooks, local jobs are not
  * redelivered, so a modest window is enough.
  */
-export async function enqueueLocalJob(queue, fields) {
-	return (await addLocalJob(queue, fields)).id;
+export async function enqueueLocalJob(queue, fields, { dataVersions } = {}) {
+	return (await addLocalJob(queue, fields, { dataVersions })).id;
 }
 
 /**
@@ -119,8 +201,8 @@ export async function enqueueLocalJob(queue, fields) {
  * done rather than inventing one. A read that fails outright returns `existing: undefined`: whether this
  * call queued anything is then unknown, and the caller says so instead of guessing either way.
  */
-export async function enqueueLocalJobReporting(queue, fields) {
-	return addReporting(queue, (enqueueNonce) => addLocalJob(queue, { ...fields, enqueueNonce }));
+export async function enqueueLocalJobReporting(queue, fields, { dataVersions } = {}) {
+	return addReporting(queue, (enqueueNonce) => addLocalJob(queue, { ...fields, enqueueNonce }, { dataVersions }));
 }
 
 /**
@@ -135,8 +217,7 @@ export async function enqueueLocalJobReporting(queue, fields) {
  * job is not retried, as a scheduled tick is not (a scheduler job carries no `attempts`). The id is
  * `manualTriggerJobId`'s, so `localEventContext` reports `source: "cron"` with `scheduledFor: null`.
  */
-export async function enqueueTriggerRunReporting(queue, schedule, { now = new Date() } = {}) {
-	const jobId = manualTriggerJobId({ triggerId: schedule.schedulerId, now });
+export async function enqueueTriggerRunReporting(queue, schedule, { now = new Date(), jobId = manualTriggerJobId({ triggerId: schedule.schedulerId, now }) } = {}) {
 	return addReporting(queue, async (enqueueNonce) => {
 		await queue.add(schedule.name, { ...schedule.data, enqueueNonce }, { ...schedule.opts, jobId });
 		return { id: jobId };
@@ -176,7 +257,7 @@ export function swallowedRunSentence(jobId, existing) {
 	return `an identical run was queued${hhmm} as ${jobId} (${existing?.state ?? "already queued or done"}); nothing new was queued.`;
 }
 
-async function addLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, models, maxCostUsd, image, backend, excludeTools, mcp, skillsDir, secrets, secretsProfile, chainDepth, parentJobId, jobId, enqueueNonce, now = new Date() }) {
+async function addLocalJob(queue, { folder, flow, task, command, provider, model, maxTurns, models, fallbackModels, maxCostUsd, packages, image, backend, excludeTools, excludeExtensions, excludeExtensionTools, mcp, skillsDir, secrets, secretsProfile, recover, steer, chainDepth, parentJobId, jobId, enqueueNonce, dataVersion, now = new Date() }, { dataVersions } = {}) {
 	const minute = now.toISOString().slice(0, 16); // YYYY-MM-DDTHH:MM -- the dedup window
 	// A caller-supplied jobId (the outbox collector's retry-idempotent chainedJobId) wins; otherwise the
 	// minute-windowed localJobId is the dedup key. A command job (issue #189) fills the flow slot with
@@ -185,11 +266,13 @@ async function addLocalJob(queue, { folder, flow, task, command, provider, model
 	// vanish silently -- and the `cmd:` prefix keeps a command named X from colliding with a flow named X
 	// (`:` is outside the skill-name charset, so no real flow can spell the prefixed form). A flow job's
 	// key is byte-identical to before the feature.
-	const id = jobId ?? localJobId({ folder, flow: command !== undefined ? `cmd:${command}` : flow, task, minute, provider, model, models });
+	const id = jobId ?? localJobId({ folder, flow: command !== undefined ? `cmd:${command}` : flow, task, minute, provider, model, models, fallbackModels });
 	// image/chainDepth/parentJobId land on `data` only when present, so a plain non-chained job's data is
 	// byte-identical. `image` is the container image this job runs in (INT-TRIGGERS-FILE-CONTRACT); absent
 	// resolves the deployment default at job start, never a value frozen here.
-	const data = {
+	// Stamped with the version its fields need (issue #761, `stampJobData`): nothing while that is 1. Before the add, so
+	// input from a newer release throws here and nothing is queued.
+	const data = stampJobData({
 		kind: "local",
 		folder,
 		flow,
@@ -204,8 +287,12 @@ async function addLocalJob(queue, { folder, flow, task, command, provider, model
 		// Issue #502: the allowed-model list, conditional like `image`, so a job whose trigger named none keeps its
 		// data byte-identical. An outbox child carries its parent's (outbox.mjs), never one its request file names.
 		...(models !== undefined && { models }),
+		...(fallbackModels !== undefined && { fallbackModels }),
 		// Issue #501: the per-job dollar cap, passed through for the outbox child that inherits its parent's.
 		...(maxCostUsd !== undefined && { maxCostUsd }),
+		// Issue #800: the outbox child's `"packages": false`, off its parent's data. Not in this function's list before, so the collector could not have
+		// passed it. A key every worker already reads, so it asks for no `dataVersion`. Conditional like `image`, so no other job's data moves.
+		...(packages !== undefined && { packages }),
 		...(image !== undefined && { image }),
 		// #227. WHERE this job's container is built. Conditional like `image`, so a trigger that named no
 		// venue produces byte-identical job data -- and at JOB level, never inside `trigger`, for the reason
@@ -217,6 +304,10 @@ async function addLocalJob(queue, { folder, flow, task, command, provider, model
 		// same reason: `trigger` is copied verbatim into /job/event.json, and a permission boundary is the
 		// worker's business, never agent-visible input to reason about.
 		...(excludeTools !== undefined && { excludeTools }),
+		// Issue #800. Each its own conditional, and at JOB level, excludeTools' two reasons: a dropped list runs the job with what the
+		// file takes away, and what the session may not load is the worker's business, never agent-visible input.
+		...(excludeExtensions !== undefined && { excludeExtensions }),
+		...(excludeExtensionTools !== undefined && { excludeExtensionTools }),
 		...(mcp !== undefined && { mcp }),
 		// The host directory of operator-authored skills this trigger injects (REQ-PER-TRIGGER-SKILLS).
 		// Conditional like `image`, so an unflagged job's data stays byte-identical, and at JOB level rather
@@ -228,25 +319,65 @@ async function addLocalJob(queue, { folder, flow, task, command, provider, model
 		// this differs from `replicas` -- that one is refused on a local job and this one is not.
 		...(secrets !== undefined && { secrets }),
 		...(secretsProfile !== undefined && { secretsProfile }),
+		// Issue #645: durable runs, enqueueForgeJob's rule (only when armed, job level, never part of the id). No local
+		// producer passes it today, and no trigger can: the loader refuses run.recover on cron. It is accepted here so a
+		// local producer that gains it later cannot quietly move the minute-windowed dedup key by doing so.
+		...(recover === true && { recover: true }),
+		// Issue #646: steering a running job, the same rule (only when armed, job level, never part of the id). Unlike
+		// recover a local producer can pass it: the loader allows run.steer on cron, and an operator's local run may ask.
+		...(steer === true && { steer: true }),
 		...(chainDepth !== undefined && { chainDepth }),
 		...(parentJobId !== undefined && { parentJobId }),
 		// Issue #524: `enqueueLocalJobReporting`'s own mark, so it can tell whether ITS add is the one stored.
 		...(enqueueNonce !== undefined && { enqueueNonce }),
-	};
+	}, dataVersion, dataVersions);
 	await queue.add("local", data, {
 		jobId: id,
 		attempts: 2,
 		backoff: { type: "exponential", delay: 60_000 },
-		removeOnComplete: { age: 24 * 3600 },
-		removeOnFail: { age: 7 * 24 * 3600 },
+		removeOnComplete: { age: LOCAL_JOB_KEPT_COMPLETED_S },
+		removeOnFail: { age: LOCAL_JOB_KEPT_FAILED_S },
 	});
 	return { id };
 }
 
-// Coalesces rapid re-label spam; the GUID jobId + 31d retention handle exact redelivery, so this
-// window only needs to absorb burst re-labels, not the full redelivery window.
+// Coalesces rapid re-label spam; the GUID jobId + its retention (`FORGE_JOB_RETENTION_S`) handle exact redelivery, so
+// this window only needs to absorb burst re-labels, not the full redelivery window. A Forgejo replay is the exception:
+// it carries a new delivery id, so this window is the only layer that coalesces one, and only inside it.
 const SEMANTIC_WINDOW_MS = 10 * 60 * 1000;
 
+/**
+ * Close the semantic window of a run that PARKED (REQ-AWAIT-INPUT, DES-AWAIT-INPUT-ENDS-THE-JOB). Never throws.
+ *
+ * The window exists to coalesce rapid duplicate deliveries into one ACTIVE job. A ttl-mode key outlives the job it
+ * was set for (bullmq 5.80.4, `removeDeduplicationKeyIfNeededOnFinalization.lua`: a finished job deletes its key only
+ * when it has no ttl), so without this a person's answer that lands within ten minutes of the delivery that STARTED
+ * the parked run is swallowed against that finished run: the receiver answers `deduplicated`, nothing runs, and the
+ * marker stays. Once a run has parked, the next delivery on its key is a deliberate answer, never a duplicate.
+ *
+ * It asks the BullMQ job itself, rather than rebuilding the key from the job data: `job.deduplicationId` is the id
+ * the add stored with the job (`deid` in its hash), so there is no second spelling of the key format to drift from
+ * `enqueueForgeJob`'s, and it covers every suffix (the `closed:`/`cmd:` slot, a replica's `:r<n>`) by construction.
+ * And `job.removeDeduplicationKey()` is a compare-and-delete (`removeDeduplicationKey-1.lua`: DEL only when the key
+ * still holds THIS job's id), where `queue.removeDeduplicationKey(id)` is a bare DEL that could free a key a later
+ * job now holds.
+ *
+ * Fails open with a named line: a failure only leaves today's behaviour (the answer waits out the window).
+ * `released` is false, logged as a release and not as a failure, when the key is already gone (the window lapsed
+ * during a long run) or now names another job.
+ */
+export async function releaseSemanticWindow(job, log = () => {}) {
+	if (!job?.deduplicationId || typeof job.removeDeduplicationKey !== "function") return false;
+	try {
+		const released = (await job.removeDeduplicationKey()) === true;
+		log("parked_window_released", { jobId: job.id ?? null, released });
+		return released;
+	} catch (error) {
+		log("parked_window_release_failed", { jobId: job.id ?? null, code: typeof error?.code === "string" ? error.code : "error" });
+		return false;
+	}
+}
+
 /**
  * Enqueue a GitHub-triggered job. Returns `{ jobId, deduplicated, survivingJobId? }` -- the computed
  * per-delivery id, plus whether the SEMANTIC window swallowed this delivery (issue #289; the GUID
@@ -311,14 +442,100 @@ export async function enqueueGitLabJob(queue, fields) {
  * window, replicas never coalesce against each other, and an unflagged job's dedup id is the same string it
  * has always been.
  */
-export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, models, maxCostUsd, packages, image, backend, excludeTools, mcp, skillsDir, instructions, resume, awaitInput, parkedReply, secrets, secretsProfile, waitFor, replica, replicas }) {
+export async function enqueueForgeJob(queue, kind, fields, { dataVersions } = {}) {
+	const { jobId, data: built, dedupId } = forgeJobOf(kind, fields, { dataVersions });
+	let data = built;
+	let routedName = null;
+	// Bind the delivery to the request visible at ingress: a delayed old reply must never decide a later park.
+	// A reply held before publication has no binding yet; its post-park enqueue reads again here.
+	if ((data.gates?.length > 0 && data.trigger?.comment) || data.approvalRequestId !== undefined) {
+		const found = await makeApprovalStore({ redis: await queue.client }).find(data);
+		if (!found.ok) throw new Error("approval-state-unavailable");
+		if (data.approvalRequestId !== undefined) {
+			if (found.request?.requestId !== data.approvalRequestId) throw new TypeError("approval-request-stale");
+		} else if (found.request?.state === "pending") {
+			data.approvalRequestId = found.request.requestId;
+		}
+		if (data.approvalRequestId !== undefined && found.request?.hostQueue) routedName = found.request.hostQueue;
+		// Issue #761: the binding above adds a field to built data, so the stamp is asked again for what is stored.
+		data = stampJobData(data, undefined, dataVersions);
+	}
+	// Only the host owning the retained session can consume this decision. Central routing also covers console
+	// answers and held replies, whose callers may have selected the shared queue before a request existed.
+	const routed = routedName !== null && routedName !== queue.name ? makeQueue(queue.opts.connection, { name: routedName }) : null;
+	let added;
+	try {
+		added = await (routed ?? queue).add(kind, data, {
+			jobId,
+			deduplication: { id: dedupId, ttl: SEMANTIC_WINDOW_MS }, // ttl in ms
+			...forgeJobOpts(),
+		});
+	} finally {
+		if (routed) await routed.close();
+	}
+	// Issue #289: WHICH of the two dedup layers spoke is readable off `queue.add`'s return, and only one
+	// of them is. Verified at bullmq 5.80.4's own Lua: a jobId collision (the GUID shield,
+	// REQ-DEDUP-BY-DELIVERY-GUID) returns THE SAME id -- structurally invisible here, and correctly so,
+	// because a resend of a delivery that IS queued (a hand redelivery, or Azure's own resend) deserves
+	// the answer "queued" -- while the `deduplication` option (the 10-minute semantic window) returns the
+	// EXISTING job's DIFFERENT id.
+	// So `added.id !== jobId` means this delivery was swallowed by the window and created nothing, which
+	// the receiver used to log as `enqueued` and answer as success. A defensive undefined (a fake queue
+	// predating the comparison) reads as not-deduplicated, today's behaviour exactly.
+	if (added?.id && added.id !== jobId) return { jobId, deduplicated: true, survivingJobId: added.id, ...(routed ? { queue: routedName } : {}) };
+	return { jobId, deduplicated: false, ...(routed ? { queue: routedName } : {}) };
+}
+
+// A forge job's retry policy and retention, one table for both of its enqueue paths (the window-deduplicated one above
+// and the late-message resume below). The retention is in SECONDS (`FORGE_JOB_RETENTION_S`, the delivery dedup window
+// of REQ-DEDUP-BY-DELIVERY-GUID, retention.mjs); the semantic window's ttl is in ms.
+// A function, so every add gets its own objects (bullmq keeps the options it is handed on the job).
+const forgeJobOpts = () => ({
+	attempts: 2,
+	backoff: { type: "exponential", delay: 60_000 },
+	removeOnComplete: { age: FORGE_JOB_RETENTION_S },
+	removeOnFail: { age: FORGE_JOB_RETENTION_S },
+});
+
+/**
+ * Enqueue the job a LATE forge message would have started (issue #646, DES-STEER-LATE-MESSAGE-RESUME-OR-REPLY, branch
+ * A): `fields` is what the receiver stored as the message's fallback (the object it would have handed
+ * `enqueueForgeJob`), rebuilt here through the same explicit field list, under the same delivery-GUID id and the same
+ * retry policy, on whichever queue the caller names. NO `deduplication` option: the run it follows held this target's
+ * semantic window, and the message is exactly the follow-up that window must not swallow. The GUID layer still holds,
+ * so two runs that both ended holding the message (replicas a comment fanned out to) make one job between them.
+ * Returns `{ jobId }` (an id bullmq already held answers the same, by design); throws what `queue.add` throws, and
+ * the caller falls back to a reply.
+ */
+export async function enqueueLateForgeJob(queue, kind, fields, { dataVersions } = {}) {
+	const { jobId, data } = forgeJobOf(kind, fields, { dataVersions });
+	await queue.add(kind, data, { jobId, ...forgeJobOpts() });
+	return { jobId };
+}
+
+/**
+ * The forge job's id, its data (an explicit field list, see `enqueueForgeJob`, stamped by `stampJobData`) and its
+ * semantic dedup id. Throws on a bad delivery id or console name, and on input a newer release stored with a
+ * `dataVersion` above this build's (issue #761; every caller that rebuilds stored input answers with a reply then).
+ */
+function forgeJobOf(kind, { repo, projectId, azure, target, flow, command, trigger, provider, model, maxTurns, models, fallbackModels, maxCostUsd, packages, image, backend, excludeTools, excludeExtensions, excludeExtensionTools, mcp, skillsDir, instructions, resume, awaitInput, gates, approvalRequestId, gateTimeout, gateBudgetRaiseUsd, progress, statusComment, recover, steer, fork, forkPolicy, forkReserveMicros, thinkingLevel, parkedReply, replyReceivedAt, console: consoleMeta, secrets, secretsProfile, waitFor, replica, replicas, dataVersion }, { dataVersions } = {}) {
 	const jobId = forgeDeliveryJobId(kind, trigger?.deliveryId, replica);
+	if (approvalRequestId !== undefined && (typeof approvalRequestId !== "string" || !/^[A-Za-z0-9._:-]{1,200}$/.test(approvalRequestId))) throw new TypeError("approval-request-invalid");
+	if (replyReceivedAt !== undefined && (!Number.isSafeInteger(replyReceivedAt) || replyReceivedAt < 0 || resume !== true || typeof trigger?.comment?.body !== "string")) throw new TypeError("reply-receipt-invalid");
+	// Issue #642: an answer typed into the operator console carries WHO typed it, `{ by: <token name> }`, and nothing
+	// else. Rebuilt from the one field rather than copied, `parkedReply`'s rule for durable job data, and refused when the
+	// name is not a console token's: the worker prints it on the forge in the answer's trail comment, so a name outside
+	// that charset (an `@`, markdown, a control byte) must never reach a queue.
+	const consoleBy = consoleMeta === undefined ? null : consoleAnswerBy(consoleMeta);
+	const consoleDraft = consoleDraftOf(consoleMeta);
+	if (consoleDraft?.invalid) throw new TypeError("invalid answer draft provenance");
+	if (consoleMeta !== undefined && consoleBy === null) throw new TypeError("enqueueForgeJob: console.by must be a console token name ([a-z0-9][a-z0-9._-]{0,31})");
 	// `packages` (whether to load the operator-staged pi packages) and `image` (which container image to run)
 	// come off the MATCHED trigger (INT-TRIGGERS-FILE-CONTRACT / REQ-GLOBAL-PI-OVERLAY) and land on `data`
 	// only when the filter resolved one, exactly like chainDepth/parentJobId above, so an unflagged trigger's
 	// job data is byte-identical. Both sit at JOB level, never inside `trigger` -- that object is descriptive
 	// and is copied verbatim into /job/event.json, where an execution knob has no business.
-	const data = {
+	const data = stampJobData({
 		kind,
 		repo,
 		...(projectId !== undefined && { projectId }),
@@ -340,6 +557,7 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 		// Issue #502: the allowed-model list, conditional like `packages` below so an unflagged trigger's job data is
 		// byte-identical, and at JOB level, never inside `trigger` (copied verbatim into /job/event.json).
 		...(models !== undefined && { models }),
+		...(fallbackModels !== undefined && { fallbackModels }),
 		...(packages !== undefined && { packages }),
 		...(image !== undefined && { image }),
 		// #227. WHERE this job's container is built. Conditional like `image`, so a trigger that named no
@@ -352,6 +570,10 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 		// same reason: `trigger` is copied verbatim into /job/event.json, and a permission boundary is the
 		// worker's business, never agent-visible input to reason about.
 		...(excludeTools !== undefined && { excludeTools }),
+		// Issue #800. Each its own conditional, and at JOB level, excludeTools' two reasons: a dropped list runs the job with what the
+		// file takes away, and what the session may not load is the worker's business, never agent-visible input.
+		...(excludeExtensions !== undefined && { excludeExtensions }),
+		...(excludeExtensionTools !== undefined && { excludeExtensionTools }),
 		...(mcp !== undefined && { mcp }),
 		// #501. The trigger's per-job dollar cap, as written. Conditional like `excludeTools`, at JOB level for its
 		// reason, and a narrowing like it: the worker runs the job under the smaller of this and the deployment cap.
@@ -369,9 +591,29 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 		// REQ-AWAIT-INPUT (issue #625): only when armed, so an unflagged job's data is byte-identical, and at JOB level
 		// for resume's reason: `trigger` is copied verbatim into /job/event.json.
 		...(awaitInput === true && { awaitInput: true }),
+		...(gates !== undefined && { gates }),
+		...(approvalRequestId !== undefined && { approvalRequestId }),
+		...(gateTimeout !== undefined && { gateTimeout }),
+		...(gateBudgetRaiseUsd !== undefined && { gateBudgetRaiseUsd }),
+		// Issue #644: the report_progress tool, and the forge status comment. Only when armed, awaitInput's rule and
+		// for its reasons (byte-identical unflagged data; job level, never inside `trigger`).
+		...(progress === true && { progress: true }),
+		...(statusComment === true && { statusComment: true }),
+		// Issue #645: durable runs. Only when armed, the same rule and reasons; never part of an id either, since whether a
+		// lost run continues from its checkpoint says nothing about which delivery this is.
+		...(recover === true && { recover: true }),
+		// Issue #646: steering a running job. Only when armed, the same rule and reasons; never part of an id, since whether
+		// a run accepts messages says nothing about which delivery started it.
+		...(steer === true && { steer: true }),
 		// A phrase-free reply to a parked issue (issue #625), gated by the worker on the key's `parked` mark. Never part
 		// of an id: it says how the delivery was routed, not what it is.
 		...(parkedReply === true && { parkedReply: true }),
+		// The held-reply settle supplies the receiver's original clock, not a timestamp from comment text.
+		// It is private accounting metadata, so /job/event.json's trigger remains byte-identical.
+		...(replyReceivedAt !== undefined && { replyReceivedAt }),
+		// Issue #642: the console answer's provenance, at JOB level and never inside `trigger` (copied verbatim into
+		// /job/event.json): it is for the worker's trail comment, not for the agent. Never part of an id.
+		...(consoleBy !== null && { console: { by: consoleBy, ...(consoleDraft ?? {}) } }),
 		// REQ-TRIGGER-SECRETS. The env variables this trigger binds and the profile whose resolver reads
 		// them. REFERENCES only: no value is ever enqueued, because a queued job is durable and a resolved
 		// credential in Redis would outlive the container it was scoped to. The worker resolves them at job
@@ -392,9 +634,11 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 		// Conditional for the same reason packages/image/resume are: an unflagged job's data must keep
 		// exactly the keys it has today. `replica` is this job's 1-based index and `replicas` the set size;
 		// both are integers, so the run record they land in stays PII-free by construction.
+		...(fork !== undefined && { fork: validateFork(fork), forkPolicy, forkReserveMicros }),
+		...(thinkingLevel !== undefined && { thinkingLevel }),
 		...(replica !== undefined && { replica }),
 		...(replicas !== undefined && { replicas }),
-	};
+	}, dataVersion, dataVersions);
 	// A close-triggered job (issue #231) leads the semantic key's flow slot with `closed:`. Without it,
 	// a label/comment/PR job on the same target and flow inside the 10-minute window silently swallows
 	// the close job -- and because a swallowed close job writes no run record, the once trigger it was
@@ -410,30 +654,14 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
 	// narrow to the matched action word, like the PR half already does.
 	const isCloseJob = matched?.type === "issue" || (matched?.type === "pull_request" && PR_CLOSE_WORDS.has(matched?.action));
 	const flowSlot = `${isCloseJob ? "closed:" : ""}${command !== undefined ? `cmd:${command}` : flow}`;
-	const added = await queue.add(kind, data, {
-		jobId,
-		// A command job (issue #189) fills the semantic key's flow slot with `cmd:<command>`: a command
-		// trigger carries no flow, so the slot would otherwise read `undefined` for every command and one
-		// command's 10-minute window would swallow a different command's delivery on the same target. The
-		// `cmd:` prefix keeps a command named X from coalescing against a flow named X -- `:` is outside
-		// the skill-name charset, so no real flow can spell the prefixed form -- and a flow job's key
-		// stays byte-identical to before the feature.
-		deduplication: { id: `${repo}${targetSeparator(kind, target?.type)}${target.number}:${flowSlot}${replica !== undefined ? `:r${replica}` : ""}`, ttl: SEMANTIC_WINDOW_MS }, // ttl in ms
-		attempts: 2,
-		backoff: { type: "exponential", delay: 60_000 },
-		removeOnComplete: { age: 31 * 24 * 3600 }, // age in seconds -- do not cross units with the ms ttl above
-		removeOnFail: { age: 31 * 24 * 3600 },
-	});
-	// Issue #289: WHICH of the two dedup layers spoke is readable off `queue.add`'s return, and only one
-	// of them is. Verified at bullmq 5.80.4's own Lua: a jobId collision (the GUID shield,
-	// REQ-DEDUP-BY-DELIVERY-GUID) returns THE SAME id -- structurally invisible here, and correctly so,
-	// because a forge retry of a delivery that IS queued deserves the answer "queued" -- while the
-	// `deduplication` option (the 10-minute semantic window) returns the EXISTING job's DIFFERENT id.
-	// So `added.id !== jobId` means this delivery was swallowed by the window and created nothing, which
-	// the receiver used to log as `enqueued` and answer as success. A defensive undefined (a fake queue
-	// predating the comparison) reads as not-deduplicated, today's behaviour exactly.
-	if (added?.id && added.id !== jobId) return { jobId, deduplicated: true, survivingJobId: added.id };
-	return { jobId, deduplicated: false };
+	// A command job (issue #189) fills the semantic key's flow slot with `cmd:<command>`: a command
+	// trigger carries no flow, so the slot would otherwise read `undefined` for every command and one
+	// command's 10-minute window would swallow a different command's delivery on the same target. The
+	// `cmd:` prefix keeps a command named X from coalescing against a flow named X -- `:` is outside
+	// the skill-name charset, so no real flow can spell the prefixed form -- and a flow job's key
+	// stays byte-identical to before the feature.
+	const dedupId = `${repo}${targetSeparator(kind, target?.type)}${target.number}:${flowSlot}${fork !== undefined ? `:fork:${fork.id}` : ""}${replica !== undefined ? `:r${replica}` : ""}`;
+	return { jobId, data, dedupId };
 }
 
 /**
@@ -458,12 +686,23 @@ export async function enqueueForgeJob(queue, kind, { repo, projectId, azure, tar
  * being wrong costs money, not for a reader that runs every second. Fails open to `[]`, so an unreadable
  * keyspace degrades to the registry's answer rather than refusing.
  */
-export async function discoverHostQueues(redis, { timeoutMs = 2_000, count = 500 } = {}) {
-	const prefix = `bull:${QUEUE}@`;
+export async function discoverHostQueues(redis, opts = {}) {
+	return (await discoverHostQueuesReporting(redis, opts)).names;
+}
+
+/**
+ * `discoverHostQueues`, and whether its answer is the whole keyspace (issue #710, review round 1): `{ names, incomplete }`,
+ * `incomplete` null for a scan that reached its end, else why it did not (the scan threw, or ran out of time with
+ * `names` the part it read). For a reader for which a queue it did not ask is not an answer: the console's run page
+ * asks whether a job's data is still held, and "no queue holds it" off a scan that never finished is a false refusal.
+ * The kill switch keeps `discoverHostQueues`, which fails open to what it read.
+ */
+export async function discoverHostQueuesReporting(redis, { timeoutMs = 2_000, count = 500 } = {}) {
+	const prefix = `${QUEUE}@`;
 	const names = new Set();
+	let cursor = "0";
 	try {
 		const deadline = Date.now() + timeoutMs;
-		let cursor = "0";
 		do {
 			// BOUNDED, because BullMQ's connections carry `maxRetriesPerRequest: null` and a command against
 			// an unreachable server therefore QUEUES FOREVER rather than rejecting -- so an unguarded await
@@ -476,24 +715,26 @@ export async function discoverHostQueues(redis, { timeoutMs = 2_000, count = 500
 			// thing on the loop, which is the one case it exists for.
 			let timer;
 			const [next, keys] = await Promise.race([
-				redis.scan(cursor, "MATCH", `${prefix}*:meta`, "COUNT", count),
+				redis.scan(cursor, "MATCH", metaKeyPattern(prefix), "COUNT", count),
 				new Promise((_, reject) => {
 					timer = setTimeout(() => reject(new Error("scan timed out")), Math.max(1, deadline - Date.now()));
 				}),
 			]).finally(() => clearTimeout(timer));
 			cursor = next;
 			for (const key of keys ?? []) {
-				const name = String(key).slice(prefix.length, -":meta".length);
+				const queueName = queueNameOfMetaKey(String(key));
+				if (queueName === null || !queueName.startsWith(prefix)) continue;
+				const name = queueName.slice(prefix.length);
 				// Validated like every other peer-derived name: a key an operator hand-created could hold
 				// anything, and `new Queue` throws on a `:`, which would take the kill switch out entirely.
 				if (WORKER_NAME_RE.test(name)) names.add(name);
 			}
 		} while (cursor !== "0" && Date.now() < deadline);
-	} catch {
-		// Fail open: the registry's answer alone is still better than refusing to pause.
-		return [];
+	} catch (err) {
+		// Fail open: the registry's answer alone is still better than refusing to pause. Nothing read is kept, as before.
+		return { names: [], incomplete: err?.message ?? String(err) };
 	}
-	return [...names].sort().map(hostQueueName);
+	return { names: [...names].sort().map(hostQueueName), incomplete: String(cursor) === "0" ? null : "the queue scan ran out of time" };
 }
 
 /**