@indigoai-us/hq-cli 5.126.0 → 5.127.0

package/dist/lib/lanes/watcher.js

@@ -0,0 +1,859 @@
+/**
+ * Per-lane watcher (spec §5.7).
+ *
+ * Armed at spawn, detached from the caller. Emits on progress, terminal, and
+ * silence (20 minutes with the process still alive). Terminal classification
+ * follows spec §5: an envelope decides done/ask/blocked; a process that exits
+ * with partial output and no envelope is `interrupted`; silence is never
+ * success.
+ *
+ * Progress has two sources. Output files (worker.out, worker.err,
+ * last-message.txt) cover codex and grok, which stream as they work. They do
+ * not cover Claude: `claude -p --output-format json` writes nothing until it
+ * exits, so a healthy Claude lane used to hit the silence deadline at exactly
+ * 20 minutes every time, and a hung one looked identical. The second source
+ * is logs/tool-activity.json, which the drain hook rewrites on every
+ * PreToolUse call (dropbox.ts). A worker that is working calls tools; a
+ * worker that has not called a tool or written output for the whole window
+ * is `stalled`, and the watcher says so on the lane record.
+ *
+ * What this cannot tell apart: a worker hung inside one tool call from a
+ * worker running one legitimately long tool call (a 25-minute build); a
+ * worker looping on tool calls without making progress from a working one;
+ * a model stuck mid-generation with no tool call from one that is writing a
+ * long answer. It also depends on the drain hook being registered in the
+ * provider home (hooks-install.ts): with no hook there is no touch, and the
+ * lane reads as stalled every window, which is the pre-existing behaviour.
+ * Distinguishing a hung tool from a slow one needs the provider to report
+ * per-call state, which none of the three CLIs expose today. `stalled` is
+ * therefore a flag for the senior to look at, never a kill.
+ *
+ * Queue admission of the next lane when a slot frees is owned by the
+ * periodic reconcile pass and is out of scope here.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { LanesError } from "./errors.js";
+import { raiseEnvelopeQuestions } from "./questions.js";
+import { isTerminalLaneState, parseLaneEnvelope, } from "./types.js";
+import { readLane, retryUpdateLane } from "./store.js";
+import { readWorkerRunResult } from "./delivery.js";
+import { readQuotaSignal, recordAccountQuota } from "./quota.js";
+import { readToolActivity, toolActivityPath } from "./dropbox.js";
+import { loadLanesConfig, providerProgressEventMinIntervalMs } from "./config.js";
+import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, progressEventDue, } from "./mesh-events.js";
+import { envelopeQuestionId } from "./questions.js";
+import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, } from "./spawn.js";
+/** Spec §5.7: no signal for 20 minutes while the process is still alive. */
+export const SILENCE_MS = 20 * 60 * 1000;
+export const WATCHER_POLL_MS = 5_000;
+/** Synced last_host_report_at cadence. Reconcile dead-host TTL is 20 minutes. */
+export const WATCHER_HEARTBEAT_MS = 60 * 1000;
+/**
+ * Cap on logs/watcher.out. A decision line is typically under 300 bytes, so
+ * 256 KiB holds hundreds of transitions. A stuck lane cannot fill the disk:
+ * once the file would exceed this, the oldest complete lines are dropped.
+ */
+export const WATCHER_OUT_MAX_BYTES = 256 * 1024;
+export const WATCHER_DECISION_EVENTS = [
+    "liveness_transition",
+    "silence_deadline",
+    "envelope_observed",
+    "envelope_invalid",
+    "envelope_repair",
+    "classification",
+    "terminal_write",
+    "mesh_event",
+    "mesh_event_failed",
+];
+/** Terminal reason when the envelope failed to parse after the repair turn (spec §4). */
+export const ENVELOPE_INVALID_REASON = "envelope_invalid";
+export const WATCHER_SIGNAL_KINDS = ["progress", "terminal", "silence"];
+export function parseWatcherSignalKind(value) {
+    if (typeof value === "string" &&
+        WATCHER_SIGNAL_KINDS.includes(value)) {
+        return value;
+    }
+    throw new LanesError("unknown_watcher_signal", `Unknown watcher signal ${JSON.stringify(value)}. Accepted: ${WATCHER_SIGNAL_KINDS.join(", ")}`, { accepted: [...WATCHER_SIGNAL_KINDS], value });
+}
+export function classifyLaneExit(input) {
+    if (input.envelope) {
+        if (input.envelope.decision === "done")
+            return { state: "done" };
+        if (input.envelope.decision === "ask")
+            return { state: "awaiting_input" };
+        if (input.envelope.decision === "blocked")
+            return { state: "blocked" };
+        throw new LanesError("unknown_envelope_decision", `Unknown envelope decision ${JSON.stringify(input.envelope.decision)}. Accepted: done, ask, blocked`, { accepted: ["done", "ask", "blocked"], value: input.envelope.decision });
+    }
+    if (input.invalidEnvelope) {
+        return { state: "failed", reason: ENVELOPE_INVALID_REASON };
+    }
+    if (input.quota) {
+        return {
+            state: input.hasPartialOutput ? "interrupted" : "failed",
+            reason: input.quota.reason,
+        };
+    }
+    if (input.hasPartialOutput) {
+        return { state: "interrupted", reason: "exited_with_partial_output" };
+    }
+    return { state: "failed", reason: "exited_without_envelope" };
+}
+export function tickWatcher(input) {
+    const silenceMs = input.silenceMs ?? SILENCE_MS;
+    const at = new Date(input.nowMs).toISOString();
+    if (!input.processAlive) {
+        const classification = classifyLaneExit({
+            envelope: input.envelope,
+            hasPartialOutput: input.hasPartialOutput,
+            quota: input.quota,
+            invalidEnvelope: input.invalidEnvelope,
+        });
+        return {
+            emit: { kind: "terminal", at, classification },
+            nextLastSignalMs: input.nowMs,
+            done: true,
+        };
+    }
+    if (input.progressed) {
+        return {
+            emit: { kind: "progress", at },
+            nextLastSignalMs: input.nowMs,
+            done: false,
+        };
+    }
+    if (input.nowMs - input.lastSignalMs >= silenceMs) {
+        return {
+            emit: { kind: "silence", at },
+            nextLastSignalMs: input.nowMs,
+            done: false,
+        };
+    }
+    return { nextLastSignalMs: input.lastSignalMs, done: false };
+}
+export function watcherEventsPath(hqRoot, laneId) {
+    return path.join(laneRunDir(hqRoot, laneId), "watcher", "events.jsonl");
+}
+export function watcherOutLogPath(hqRoot, laneId) {
+    return path.join(laneRunDir(hqRoot, laneId), "logs", "watcher.out");
+}
+export function appendCappedLog(filePath, line, maxBytes = WATCHER_OUT_MAX_BYTES) {
+    const payload = line.endsWith("\n") ? line : `${line}\n`;
+    const payloadBytes = Buffer.byteLength(payload, "utf8");
+    if (payloadBytes > maxBytes)
+        return;
+    let existing = "";
+    try {
+        existing = fs.readFileSync(filePath, "utf8");
+    }
+    catch (err) {
+        const code = err.code;
+        if (code !== "ENOENT")
+            throw err;
+    }
+    let body = existing;
+    while (Buffer.byteLength(body, "utf8") + payloadBytes > maxBytes) {
+        const nl = body.indexOf("\n");
+        if (nl === -1) {
+            body = "";
+            break;
+        }
+        body = body.slice(nl + 1);
+    }
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, body + payload, "utf8");
+}
+export function writeWatcherDecision(filePath, record, maxBytes = WATCHER_OUT_MAX_BYTES) {
+    const compact = {};
+    for (const [key, value] of Object.entries(record)) {
+        if (value !== undefined)
+            compact[key] = value;
+    }
+    appendCappedLog(filePath, JSON.stringify(compact), maxBytes);
+}
+export function watcherLastSignalPath(hqRoot, laneId) {
+    return path.join(laneRunDir(hqRoot, laneId), "watcher", "last-signal.json");
+}
+export function appendWatcherSignal(hqRoot, laneId, signal) {
+    const eventsPath = watcherEventsPath(hqRoot, laneId);
+    fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
+    fs.appendFileSync(eventsPath, `${JSON.stringify(signal)}\n`, "utf8");
+    fs.writeFileSync(watcherLastSignalPath(hqRoot, laneId), `${JSON.stringify(signal, null, 2)}\n`, "utf8");
+}
+/**
+ * Slot-free notification. The periodic reconcile pass admits the next queued
+ * lane; this watcher must not do it. Intentionally empty.
+ */
+export function notifySlotFreed(_input) {
+    // Reconcile lane owns admission when a running slot frees (spec §5.7).
+}
+/**
+ * On exit, read the worker's own result for a resumable provider and copy
+ * only the session id and the two cache counters onto the record. A missing
+ * or unparseable id is recorded as absent with its reason, never invented.
+ * Non-resumable providers return null and the record is left as it was.
+ */
+function runResultFields(hqRoot, lane) {
+    const runDir = laneRunDir(hqRoot, lane.lane_id);
+    const result = readWorkerRunResult(runDir, lane.provider, readSpawnManifest(runDir).worker_out_offset ?? 0);
+    if (result === null)
+        return {};
+    const fields = {
+        last_run_usage: {
+            cache_read_input_tokens: result.cache_read_input_tokens,
+            cache_creation_input_tokens: result.cache_creation_input_tokens,
+        },
+    };
+    if (result.session_id !== null) {
+        fields.provider_session_id = result.session_id;
+        fields.provider_session_id_absent = undefined;
+    }
+    else {
+        fields.provider_session_id = undefined;
+        fields.provider_session_id_absent = result.session_id_absent ?? "no_result_json";
+    }
+    return fields;
+}
+function maybeHeartbeatLane(hqRoot, laneId, nowMs) {
+    try {
+        retryUpdateLane(hqRoot, laneId, "watcher", (current) => {
+            if (current.state !== "running")
+                return current;
+            const last = Date.parse(current.last_host_report_at ?? "");
+            if (Number.isFinite(last) && nowMs - last < WATCHER_HEARTBEAT_MS) {
+                return current;
+            }
+            return { ...current, last_host_report_at: new Date(nowMs).toISOString() };
+        });
+    }
+    catch (err) {
+        process.stderr.write(`hq lanes watcher: heartbeat failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+    }
+}
+export function stalledReason(silenceMs) {
+    return `no_tool_call_or_output_for_${Math.round(silenceMs / 1000)}s`;
+}
+/**
+ * The consumer of the silence signal. A silence emit on a running lane sets
+ * `stalled` on the record so `hq lanes list` and `show` surface it; the next
+ * progress emit clears it. Nothing here changes `state`.
+ */
+export function markLaneStalled(hqRoot, laneId, at, reason) {
+    return retryUpdateLane(hqRoot, laneId, "watcher", (current) => {
+        if (current.state !== "running")
+            return current;
+        if (current.stalled === true && current.stalled_reason === reason)
+            return current;
+        return { ...current, stalled: true, stalled_at: at, stalled_reason: reason };
+    });
+}
+export function clearLaneStalled(hqRoot, laneId) {
+    return retryUpdateLane(hqRoot, laneId, "watcher", (current) => {
+        if (current.stalled === undefined && current.stalled_at === undefined)
+            return current;
+        return { ...current, stalled: undefined, stalled_at: undefined, stalled_reason: undefined };
+    });
+}
+export function applyTerminalToLane(hqRoot, laneId, classification, opts = {}) {
+    return applyTerminalToLaneWithResult(hqRoot, laneId, classification, opts).record;
+}
+/** Terminal write outcome for emitters: only the CAS winner may publish mesh effects. */
+export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, opts = {}) {
+    const at = opts.at ?? new Date().toISOString();
+    const existing = readLane(hqRoot, laneId);
+    if (!isTerminalLaneState(existing.state) &&
+        classification.state === "awaiting_input" &&
+        opts.envelope &&
+        (opts.envelope.questions?.length ?? 0) > 0) {
+        raiseEnvelopeQuestions(hqRoot, existing, opts.envelope, opts.envelopeRef ?? "envelopes/unknown.json", at);
+    }
+    let changed = false;
+    const updated = retryUpdateLane(hqRoot, laneId, "watcher", (current) => {
+        const terminal = current.state === "done" ||
+            current.state === "failed" ||
+            current.state === "interrupted" ||
+            current.state === "cancelled" ||
+            current.state === "killed";
+        if (terminal)
+            return current;
+        if (current.state === "awaiting_input" || current.state === "blocked") {
+            if (current.state === classification.state)
+                return current;
+        }
+        const next = {
+            ...current,
+            ...runResultFields(hqRoot, current),
+            state: classification.state,
+            queued_reason: undefined,
+            stalled: undefined,
+            stalled_at: undefined,
+            stalled_reason: undefined,
+        };
+        changed = true;
+        if (classification.reason) {
+            next.terminal_reason =
+                classification.reason;
+        }
+        if (opts.quota?.quota_reset_at) {
+            next.quota_reset_at = opts.quota.quota_reset_at;
+        }
+        if (opts.quota?.marker) {
+            next.model_quota_marker = opts.quota.marker;
+        }
+        if (opts.envelope && opts.envelopeRef) {
+            next.last_envelope = {
+                ref: opts.envelopeRef,
+                decision: opts.envelope.decision,
+                at,
+            };
+            next.story_status = opts.envelope.story_status;
+            if (opts.envelope.story_complete !== undefined) {
+                next.story_complete = opts.envelope.story_complete;
+                next.story_complete_reason = opts.envelope.story_complete_reason;
+            }
+        }
+        return next;
+    });
+    notifySlotFreed({
+        hqRoot,
+        provider: updated.provider,
+        account: updated.account,
+    });
+    return { record: updated, changed };
+}
+export function isPidAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ESRCH")
+            return false;
+        if (code === "EPERM")
+            return true;
+        process.stderr.write(`hq lanes watcher: liveness probe for pid ${pid} failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        throw err;
+    }
+}
+function fileMtimeMs(filePath) {
+    try {
+        return fs.statSync(filePath).mtimeMs;
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return null;
+        process.stderr.write(`hq lanes watcher: failed to stat ${filePath}: ${err instanceof Error ? err.message : String(err)}\n`);
+        throw err;
+    }
+}
+function fileNonEmpty(filePath) {
+    try {
+        return fs.statSync(filePath).size > 0;
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return false;
+        process.stderr.write(`hq lanes watcher: failed to stat ${filePath}: ${err instanceof Error ? err.message : String(err)}\n`);
+        throw err;
+    }
+}
+export function readLatestEnvelope(hqRoot, laneId, minName) {
+    return readLatestEnvelopeRecord(hqRoot, laneId, minName)?.envelope ?? null;
+}
+/** Valid-only view over `readLatestEnvelopeFile`; an unparseable file reads as absent. */
+export function readLatestEnvelopeRecord(hqRoot, laneId, minName) {
+    const latest = readLatestEnvelopeFile(hqRoot, laneId, minName);
+    if (latest === null)
+        return null;
+    if (latest.envelope === null) {
+        process.stderr.write(`hq lanes watcher: envelope ${latest.path} is not a valid hq-lane-envelope.v1: ${latest.error}\n`);
+        return null;
+    }
+    return { envelope: latest.envelope, ref: latest.ref };
+}
+/**
+ * `minName` is this run's envelope slot (basename from spawn.json). Envelopes
+ * that sort before it belong to an earlier run of the same lane and must not
+ * classify this exit: a resumed run that dies before writing its own envelope
+ * would otherwise be marked done again from the previous one.
+ */
+export function readLatestEnvelopeFile(hqRoot, laneId, minName) {
+    const dir = path.join(laneRunDir(hqRoot, laneId), "envelopes");
+    let names;
+    try {
+        names = fs
+            .readdirSync(dir)
+            .filter((name) => name.endsWith(".json"))
+            .filter((name) => minName === undefined || name >= minName)
+            .sort();
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return null;
+        process.stderr.write(`hq lanes watcher: failed to list envelopes for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+        throw err;
+    }
+    if (names.length === 0)
+        return null;
+    const latest = names[names.length - 1];
+    const filePath = path.join(dir, latest);
+    let text;
+    try {
+        text = fs.readFileSync(filePath, "utf8");
+    }
+    catch (err) {
+        process.stderr.write(`hq lanes watcher: failed to read envelope ${filePath}: ${err instanceof Error ? err.message : String(err)}\n`);
+        throw err;
+    }
+    const ref = path.posix.join("envelopes", latest);
+    try {
+        return {
+            ref,
+            path: filePath,
+            text,
+            envelope: parseLaneEnvelope(JSON.parse(text)),
+            error: null,
+        };
+    }
+    catch (err) {
+        return {
+            ref,
+            path: filePath,
+            text,
+            envelope: null,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+/**
+ * Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
+ * the event name and the error message, and never propagates: the lane's
+ * state is decided by the envelope and the process, not by the network.
+ */
+export async function emitMeshSoft(event, logDecision, at, fn, timeoutMs = 10_000) {
+    let timer;
+    try {
+        const controller = new AbortController();
+        await Promise.race([
+            fn(controller.signal),
+            new Promise((_resolve, reject) => {
+                timer = setTimeout(() => { controller.abort(); reject(new Error(`mesh request timed out after ${timeoutMs}ms`)); }, timeoutMs);
+            }),
+        ]);
+        logDecision({ event: "mesh_event", at, mesh_event: event });
+        return true;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        logDecision({ event: "mesh_event_failed", at, mesh_event: event, error: message });
+        process.stderr.write(`hq lanes watcher: mesh_event_failed ${event}: ${message}\n`);
+        return false;
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+function readProgressIntervalMs(hqRoot, lane) {
+    try {
+        const config = loadLanesConfig(hqRoot);
+        if (lane)
+            return providerProgressEventMinIntervalMs(config.providers[lane.provider]);
+    }
+    catch (err) {
+        process.stderr.write(`hq lanes watcher: lanes config unreadable, using default progress interval: ${err instanceof Error ? err.message : String(err)}\n`);
+    }
+    return providerProgressEventMinIntervalMs({ max_running: 1, max_queued: 1, accounts: {} });
+}
+function readLaneSoft(hqRoot, laneId) {
+    try {
+        return readLane(hqRoot, laneId);
+    }
+    catch (err) {
+        process.stderr.write(`hq lanes watcher: lane read failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+        return null;
+    }
+}
+/** Question ids an ask envelope raised, as `raiseEnvelopeQuestions` mints them. */
+export function envelopeQuestionIds(laneId, envelopeRef, envelope) {
+    return (envelope.questions ?? []).map((_q, i) => envelopeQuestionId(laneId, envelopeRef, i));
+}
+function defaultSleep(ms) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
+export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
+    const sleep = deps.sleep ?? defaultSleep;
+    const now = deps.now ?? (() => Date.now());
+    const isAlive = deps.isAlive ?? isPidAlive;
+    const readStarttime = deps.readStarttime ?? readProcStarttime;
+    const pollMs = deps.pollMs ?? WATCHER_POLL_MS;
+    const silenceMs = deps.silenceMs ?? SILENCE_MS;
+    const runDir = laneRunDir(hqRoot, laneId);
+    const outPath = deps.watcherOutPath ?? watcherOutLogPath(hqRoot, laneId);
+    const outMax = deps.watcherOutMaxBytes ?? WATCHER_OUT_MAX_BYTES;
+    const workerPidPath = path.join(runDir, "worker.pid");
+    const manifest = readSpawnManifest(runDir);
+    const minEnvelopeName = manifest.envelope_path !== undefined
+        ? path.basename(manifest.envelope_path)
+        : undefined;
+    const logFiles = [
+        path.join(runDir, "logs", "worker.out"),
+        path.join(runDir, "logs", "worker.err"),
+        path.join(runDir, "logs", "last-message.txt"),
+    ];
+    // Progress only. A tool call is not partial output: a worker that called
+    // tools and exited with no envelope and no output is still `failed`.
+    const activityFiles = [toolActivityPath(hqRoot, laneId)];
+    const progressFiles = [...logFiles, ...activityFiles];
+    // Mesh seam (spec §8). Non-interactive: nobody is at this process's tty.
+    const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
+    const progressIntervalMs = deps.progressEventMinIntervalMs ?? readProgressIntervalMs(hqRoot, readLaneSoft(hqRoot, laneId));
+    const spawnRepair = deps.spawnRepair ?? spawnEnvelopeRepair;
+    const meshRequestTimeoutMs = deps.meshRequestTimeoutMs ?? 10_000;
+    let lastSignalMs = now();
+    let lastSeenMtime = 0;
+    let lastAlive = null;
+    let lastEnvelopeRef = null;
+    let lastInvalidRef = null;
+    let lastProgressEmitMs = (() => {
+        const stamp = readLaneSoft(hqRoot, laneId)?.last_progress_event_at;
+        return stamp ? Date.parse(stamp) || null : null;
+    })();
+    for (const file of progressFiles) {
+        const mtime = fileMtimeMs(file);
+        if (mtime !== null && mtime > lastSeenMtime)
+            lastSeenMtime = mtime;
+    }
+    const logDecision = (record) => {
+        writeWatcherDecision(outPath, record, outMax);
+    };
+    while (true) {
+        const worker = readPidRecord(workerPidPath);
+        const probe = worker === null
+            ? missingPidProbe()
+            : probePidRecord(worker, isAlive, readStarttime);
+        const processAlive = probe.alive;
+        if (lastAlive !== false && !processAlive) {
+            logDecision({
+                event: "liveness_transition",
+                at: new Date(now()).toISOString(),
+                pid: probe.pid,
+                recorded_starttime: probe.recorded_starttime,
+                live_starttime: probe.live_starttime,
+                starttime_error: probe.starttime_error,
+                decided_by: probe.decided_by,
+            });
+        }
+        lastAlive = processAlive;
+        let progressed = false;
+        let newestMtime = lastSeenMtime;
+        let newestFile = null;
+        for (const file of progressFiles) {
+            const mtime = fileMtimeMs(file);
+            if (mtime !== null && mtime > newestMtime) {
+                newestMtime = mtime;
+                newestFile = file;
+                progressed = true;
+            }
+        }
+        const latestFile = readLatestEnvelopeFile(hqRoot, laneId, minEnvelopeName);
+        const latest = latestFile && latestFile.envelope
+            ? { envelope: latestFile.envelope, ref: latestFile.ref }
+            : null;
+        const envelope = latest?.envelope ?? null;
+        if (latest && latest.ref !== lastEnvelopeRef) {
+            logDecision({
+                event: "envelope_observed",
+                at: new Date(now()).toISOString(),
+                filename: path.posix.basename(latest.ref),
+                decision: latest.envelope.decision,
+            });
+            lastEnvelopeRef = latest.ref;
+        }
+        if (latestFile && latestFile.envelope === null && latestFile.ref !== lastInvalidRef) {
+            logDecision({
+                event: "envelope_invalid",
+                at: new Date(now()).toISOString(),
+                filename: path.posix.basename(latestFile.ref),
+                error: latestFile.error,
+            });
+            lastInvalidRef = latestFile.ref;
+        }
+        const hasPartialOutput = logFiles.some((file) => fileNonEmpty(file));
+        let quota = null;
+        let quotaLane = null;
+        if (!processAlive && !envelope) {
+            try {
+                quotaLane = readLane(hqRoot, laneId);
+                quota = readQuotaSignal(runDir, quotaLane.model, new Date(now()), readSpawnManifest(runDir).worker_out_offset ?? 0);
+            }
+            catch (err) {
+                process.stderr.write(`hq lanes watcher: quota read failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        }
+        // Spec §4: the process exited and its envelope file does not parse.
+        // Exactly one repair turn, then failed with both errors.
+        let invalidEnvelope = null;
+        if (!processAlive && !envelope && latestFile && latestFile.envelope === null) {
+            const repair = decideEnvelopeRepair(hqRoot, laneId, latestFile, now(), spawnRepair, logDecision);
+            if (repair.action === "spawned") {
+                lastSignalMs = now();
+                lastAlive = null;
+                if (progressed)
+                    lastSeenMtime = newestMtime;
+                await sleep(pollMs);
+                continue;
+            }
+            invalidEnvelope = { errors: repair.errors };
+        }
+        if (processAlive === false && envelope && latestFile) {
+            markRepairSucceeded(hqRoot, laneId, latestFile.ref);
+        }
+        const tick = tickWatcher({
+            nowMs: now(),
+            processAlive,
+            progressed,
+            lastSignalMs,
+            envelope,
+            hasPartialOutput,
+            silenceMs,
+            quota,
+            invalidEnvelope,
+        });
+        if (processAlive) {
+            maybeHeartbeatLane(hqRoot, laneId, now());
+        }
+        if (tick.emit) {
+            parseWatcherSignalKind(tick.emit.kind);
+            appendWatcherSignal(hqRoot, laneId, tick.emit);
+            if (tick.emit.kind === "silence") {
+                const reason = stalledReason(silenceMs);
+                logDecision({
+                    event: "silence_deadline",
+                    at: tick.emit.at,
+                    last_activity_at: new Date(lastSignalMs).toISOString(),
+                    last_tool_call_at: readToolActivity(hqRoot, laneId)?.at ?? null,
+                    deadline_at: new Date(lastSignalMs + silenceMs).toISOString(),
+                    silence_ms: silenceMs,
+                    stalled_reason: reason,
+                });
+                try {
+                    markLaneStalled(hqRoot, laneId, tick.emit.at, reason);
+                }
+                catch (err) {
+                    process.stderr.write(`hq lanes watcher: stalled write failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+                }
+            }
+            if (tick.emit.kind === "progress") {
+                const progressEmitAt = tick.emit.at;
+                try {
+                    clearLaneStalled(hqRoot, laneId);
+                }
+                catch (err) {
+                    process.stderr.write(`hq lanes watcher: stalled clear failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+                }
+                // Spec §8 progress row. Only a tick that saw new tool activity or
+                // new output gets here (tickWatcher emits progress on `progressed`
+                // alone), and the throttle bounds the rate. Silence never reaches
+                // this branch.
+                if (mesh &&
+                    progressEventDue({
+                        lastEmitMs: lastProgressEmitMs,
+                        nowMs: progressEmitAt ? Date.parse(progressEmitAt) : now(),
+                        minIntervalMs: progressIntervalMs,
+                    })) {
+                    const lane = readLaneSoft(hqRoot, laneId);
+                    if (lane) {
+                        const detail = newestFile && activityFiles.includes(newestFile)
+                            ? `tool call at ${readToolActivity(hqRoot, laneId)?.at ?? tick.emit.at}`
+                            : envelope
+                                ? "envelope written"
+                                : "new output";
+                        const sent = await emitMeshSoft("lane_progress", logDecision, progressEmitAt, (signal) => emitLaneProgressEvent(mesh, lane, detail, signal), meshRequestTimeoutMs);
+                        if (sent) {
+                            lastProgressEmitMs = Date.parse(progressEmitAt);
+                            try {
+                                retryUpdateLane(hqRoot, laneId, "watcher", (current) => ({
+                                    ...current,
+                                    last_progress_event_at: progressEmitAt,
+                                }));
+                            }
+                            catch (err) {
+                                process.stderr.write(`hq lanes watcher: progress timestamp write failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+                            }
+                        }
+                    }
+                }
+            }
+            if (tick.emit.kind === "terminal" && tick.emit.classification) {
+                const classification = tick.emit.classification;
+                logDecision({
+                    event: "classification",
+                    at: tick.emit.at,
+                    envelope_present: envelope !== null,
+                    has_partial_output: hasPartialOutput,
+                    state: classification.state,
+                    reason: classification.reason,
+                    ...(invalidEnvelope ? { envelope_errors: invalidEnvelope.errors } : {}),
+                });
+                if (quota &&
+                    quotaLane?.account &&
+                    (classification.reason === "account_quota_exhausted" ||
+                        classification.reason === "account_model_unavailable")) {
+                    try {
+                        recordAccountQuota(hqRoot, quotaLane.provider, quotaLane.account, quota);
+                    }
+                    catch (err) {
+                        process.stderr.write(`hq lanes watcher: quota record failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+                    }
+                }
+                const terminalWrite = applyTerminalToLaneWithResult(hqRoot, laneId, classification, {
+                    envelope,
+                    envelopeRef: latest?.ref,
+                    at: tick.emit.at,
+                    quota,
+                });
+                const updated = terminalWrite.record;
+                logDecision({
+                    event: "terminal_write",
+                    at: tick.emit.at,
+                    state: updated.state,
+                    version: updated.version,
+                });
+                // Spec §8 terminal rows: done → lane_done + story review; blocked →
+                // lane_blocked; ask → lane_ask with the queue ids. After the state
+                // write, so a mesh failure can never leave the lane unclassified.
+                if (mesh && envelope && latest && terminalWrite.changed) {
+                    const questionIds = envelope.decision === "ask"
+                        ? envelopeQuestionIds(laneId, latest.ref, envelope)
+                        : undefined;
+                    await emitMeshSoft(`lane_${envelope.decision}`, logDecision, tick.emit.at, (signal) => emitLaneDecisionEvents(mesh, updated, envelope, { questionIds, signal }), meshRequestTimeoutMs);
+                }
+            }
+        }
+        lastSignalMs = tick.nextLastSignalMs;
+        if (progressed)
+            lastSeenMtime = newestMtime;
+        if (tick.done)
+            return;
+        await sleep(pollMs);
+    }
+}
+/**
+ * Spec §4 repair decision for an exited worker whose envelope file does not
+ * parse. First failure on a resumable provider with a recorded session id:
+ * record `envelope_repair` on the lane and spawn the repair turn. A provider
+ * with no session resume gets the same single attempt as a fresh turn. Anything
+ * else (already attempted, no session id on a resumable provider, spawn
+ * failure): give up with every error collected, and the caller classifies
+ * the lane `failed` with reason `envelope_invalid`.
+ */
+export function decideEnvelopeRepair(hqRoot, laneId, latest, nowMs, spawnRepair, logDecision) {
+    const at = new Date(nowMs).toISOString();
+    const error = latest.error ?? "unparseable envelope";
+    const lane = readLaneSoft(hqRoot, laneId);
+    if (!lane)
+        return { action: "give_up", errors: [error] };
+    const prior = lane.envelope_repair;
+    if (prior && prior.attempted) {
+        // The repair turn ran and its output does not parse either.
+        if (prior.attempted) {
+            const errors = [prior.first_error, error];
+            try {
+                retryUpdateLane(hqRoot, laneId, "watcher", (current) => ({
+                    ...current,
+                    envelope_repair: { ...prior, succeeded: false, second_error: error },
+                }));
+            }
+            catch (err) {
+                process.stderr.write(`hq lanes watcher: envelope_repair write failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+            }
+            logDecision({ event: "envelope_repair", at, outcome: "second_failure", errors });
+            return { action: "give_up", errors };
+        }
+        return { action: "give_up", errors: [prior.first_error] };
+    }
+    const runDir = laneRunDir(hqRoot, laneId);
+    // `resume` (claude, codex) continues the recorded session; `fresh_turn`
+    // (grok) runs the repair prompt as a new single turn on the same account.
+    const mode = envelopeRepairMode(lane.provider);
+    let skipped;
+    let sessionId = null;
+    if (mode === "resume") {
+        const result = readWorkerRunResult(runDir, lane.provider, readSpawnManifest(runDir).worker_out_offset ?? 0);
+        sessionId = result?.session_id ?? null;
+        if (sessionId === null)
+            skipped = result?.session_id_absent ?? "no_result_json";
+    }
+    const base = {
+        first_error: error,
+        envelope_ref: latest.ref,
+        at,
+    };
+    if (skipped !== undefined) {
+        writeRepairRecord(hqRoot, laneId, { ...base, attempted: false, succeeded: false, skipped_reason: skipped });
+        logDecision({ event: "envelope_repair", at, outcome: "skipped", skipped_reason: skipped, error });
+        return { action: "give_up", errors: [error] };
+    }
+    writeRepairRecord(hqRoot, laneId, { ...base, mode, attempted: true, succeeded: false });
+    try {
+        const pid = spawnRepair(hqRoot, lane, {
+            session_id: sessionId,
+            envelopePath: latest.path,
+            error,
+            rawText: latest.text,
+        });
+        logDecision({ event: "envelope_repair", at, outcome: "spawned", mode, pid, session_id: sessionId, error });
+        return { action: "spawned" };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`hq lanes watcher: envelope repair spawn failed for ${laneId}: ${message}\n`);
+        writeRepairRecord(hqRoot, laneId, {
+            ...base,
+            mode,
+            attempted: true,
+            succeeded: false,
+            second_error: `repair_spawn_failed: ${message}`,
+        });
+        logDecision({ event: "envelope_repair", at, outcome: "spawn_failed", error: message });
+        return { action: "give_up", errors: [error, `repair_spawn_failed: ${message}`] };
+    }
+}
+function writeRepairRecord(hqRoot, laneId, record) {
+    retryUpdateLane(hqRoot, laneId, "watcher", (current) => ({ ...current, envelope_repair: record }));
+}
+/** The repair run wrote a parseable envelope: close the record as succeeded. */
+function markRepairSucceeded(hqRoot, laneId, _envelopeRef) {
+    try {
+        retryUpdateLane(hqRoot, laneId, "watcher", (current) => {
+            const prior = current.envelope_repair;
+            if (!prior || !prior.attempted || prior.succeeded) {
+                return current;
+            }
+            return { ...current, envelope_repair: { ...prior, succeeded: true, second_error: undefined } };
+        });
+    }
+    catch (err) {
+        process.stderr.write(`hq lanes watcher: envelope_repair success write failed for ${laneId}: ${err instanceof Error ? err.message : String(err)}\n`);
+    }
+}
+/** Exported so tests can prove the watcher reads the live record, not spawn env. */
+export function laneIdentityFromRecord(hqRoot, laneId) {
+    const lane = readLane(hqRoot, laneId);
+    return {
+        lane_id: lane.lane_id,
+        company: lane.company,
+        project_id: lane.project_id,
+        story_id: lane.story_id,
+        senior: lane.senior,
+    };
+}
+//# sourceMappingURL=watcher.js.map