@indigoai-us/hq-cli 5.126.0 → 5.127.0

package/dist/lib/lanes/reconcile.js

@@ -0,0 +1,985 @@
+/**
+ * Periodic reconcile pass (spec §5.7, §6.8, §9).
+ *
+ * Owned by the machine, not by any lane. Reads never mutate; every write is a
+ * compare-and-set. The per-lane watcher owns progress/terminal/silence of a
+ * live process — this pass owns the transitions no single lane can own.
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { parseUserYaml } from "../../utils/user-yaml-error.js";
+import { accountMaxRunning, loadLanesConfig } from "./config.js";
+import { LanesError, isLaneVersionConflict } from "./errors.js";
+import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
+import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
+import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
+import { applyThrottleDecision, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
+import { readQuotaSignal, recordAccountQuota } from "./quota.js";
+import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
+import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
+import { createLaneMesh, emitLaneDecisionEvents } from "./mesh-events.js";
+import { decideEnvelopeRepair, envelopeQuestionIds } from "./watcher.js";
+/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
+export const QUESTION_AGE_MS = 60 * 60 * 1000;
+/** Spec §6.6 + §9: v1 permits exactly one aged advance. */
+export const MAX_AGED_ADVANCES = 1;
+/** Aligns with watcher silence: a host that has not reported for this long is dead. */
+export const DEAD_HOST_MS = SILENCE_MS;
+export const RECONCILE_LOCK_STALE_MS = 30_000;
+export const RECONCILE_LOCK_TIMEOUT_MS = 5_000;
+export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
+export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
+function logSkip(skip) {
+    process.stderr.write(`hq lanes reconcile: skipped ${skip.kind}${skip.id ? ` ${skip.id}` : ""}: ${skip.why}\n`);
+}
+function emptySummary(dryRun) {
+    return {
+        ok: true,
+        action: "reconcile",
+        dry_run: dryRun,
+        admitted: [],
+        overdue: [],
+        aged: [],
+        advanced: [],
+        resumed: [],
+        dead_host: [],
+        heartbeats: [],
+        skipped: [],
+        errors: [],
+        heartbeat_at: null,
+    };
+}
+function markDirty(summary, skip) {
+    summary.skipped.push(skip);
+    summary.ok = false;
+    logSkip(skip);
+}
+function markError(summary, err) {
+    const message = err instanceof Error ? err.message : String(err);
+    const code = err instanceof LanesError ? err.error : undefined;
+    summary.errors.push(code ? { message, code } : { message });
+    summary.ok = false;
+    process.stderr.write(`hq lanes reconcile: ${message}${err instanceof Error && err.stack ? `\n${err.stack}` : ""}\n`);
+}
+export function reconcileHeartbeatPath(hqRoot) {
+    return path.join(hqRoot, "workspace", "lanes-runs", "reconcile-heartbeat.json");
+}
+export function timeoutPath(hqRoot, laneId) {
+    return path.join(laneRunDir(hqRoot, laneId), "timeout.json");
+}
+export function laneRunsHere(lane, host) {
+    if (host.outpostId && lane.host.kind === "outpost") {
+        return lane.host.id === host.outpostId;
+    }
+    return lane.host.kind === "local" && lane.host.id === host.hostname;
+}
+export function agedAdvanceCount(question) {
+    return question.routing_history.filter((entry) => entry.reason === "unanswered_timeout").length;
+}
+export function currentHopStartedAtMs(question) {
+    const atCurrent = [...question.routing_history]
+        .reverse()
+        .find((entry) => entry.hop === question.hop);
+    const raw = atCurrent?.at ?? question.asked_at;
+    const ms = Date.parse(raw);
+    return Number.isFinite(ms) ? ms : Number.NEGATIVE_INFINITY;
+}
+export function shouldAgeQuestion(question, audience, nowMs) {
+    if (question.status !== "pending")
+        return false;
+    if (audience.kind === "user")
+        return false;
+    if (agedAdvanceCount(question) >= MAX_AGED_ADVANCES)
+        return false;
+    const started = currentHopStartedAtMs(question);
+    if (started === Number.NEGATIVE_INFINITY)
+        return false;
+    return nowMs - started >= QUESTION_AGE_MS;
+}
+export function shouldAdvanceAudienceEnded(audience, liveness) {
+    if (audience.kind === "user")
+        return false;
+    if (audience.kind !== "session")
+        return false;
+    return liveness === "ended" || liveness === "lost" || liveness === null;
+}
+export function readTimeoutDeadline(hqRoot, laneId) {
+    const filePath = timeoutPath(hqRoot, laneId);
+    let text;
+    try {
+        text = fs.readFileSync(filePath, "utf8");
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return null;
+        throw err;
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch (err) {
+        throw new LanesError("timeout_unreadable", `Timeout file for ${laneId} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId, path: filePath });
+    }
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+        throw new LanesError("timeout_unreadable", `Timeout file for ${laneId} is not an object`, { lane_id: laneId, path: filePath });
+    }
+    const deadline = parsed.deadline_at;
+    if (typeof deadline !== "string" || !Number.isFinite(Date.parse(deadline))) {
+        return null;
+    }
+    const timeoutSecs = parsed.timeout_secs;
+    return {
+        deadline_at: deadline,
+        timeout_secs: typeof timeoutSecs === "number" && Number.isInteger(timeoutSecs)
+            ? timeoutSecs
+            : undefined,
+    };
+}
+export function isDeadlinePassed(deadlineAt, nowMs) {
+    const at = Date.parse(deadlineAt);
+    if (!Number.isFinite(at))
+        return false;
+    return nowMs > at;
+}
+function compareQueueOrder(a, b) {
+    if (b.priority !== a.priority)
+        return b.priority - a.priority;
+    return a.created_at.localeCompare(b.created_at);
+}
+function readDirJsonIds(dir, label) {
+    let names;
+    try {
+        names = fs.readdirSync(dir);
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return [];
+        throw new LanesError("store_unreadable", `Failed to read ${label} directory ${dir}: ${err instanceof Error ? err.message : String(err)}`, { path: dir });
+    }
+    return names.filter((name) => typeof name === "string" &&
+        name.endsWith(".json") &&
+        !name.startsWith(".") &&
+        !name.includes(".conflict-"));
+}
+function loadLaneRecords(hqRoot, summary) {
+    const dir = path.join(lanesStoreRoot(hqRoot), "lanes");
+    const names = readDirJsonIds(dir, "lane");
+    const lanes = [];
+    for (const name of names) {
+        const filePath = path.join(dir, name);
+        const id = name.slice(0, -".json".length);
+        try {
+            const text = fs.readFileSync(filePath, "utf8");
+            const parsed = JSON.parse(text);
+            lanes.push(parseLaneRecord(parsed, filePath));
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "lane",
+                id,
+                why: `unreadable: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
+    }
+    return lanes;
+}
+function loadQuestionRecords(hqRoot, summary) {
+    const dir = path.join(lanesStoreRoot(hqRoot), "questions");
+    const names = readDirJsonIds(dir, "question");
+    const questions = [];
+    for (const name of names) {
+        const filePath = path.join(dir, name);
+        const id = name.slice(0, -".json".length);
+        try {
+            const text = fs.readFileSync(filePath, "utf8");
+            const parsed = JSON.parse(text);
+            questions.push(parseQuestionRecord(parsed, filePath));
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "question",
+                id,
+                why: `unreadable: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
+    }
+    return questions;
+}
+function defaultReadSessionSenior(hqRoot, sessionId) {
+    const filePath = path.join(hqRoot, "workspace", "sessions", sessionId, "meta.yaml");
+    let text;
+    try {
+        text = fs.readFileSync(filePath, "utf8");
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return { kind: "user" };
+        throw err;
+    }
+    const doc = parseUserYaml(text, filePath);
+    if (doc === null || typeof doc !== "object" || Array.isArray(doc)) {
+        return { kind: "user" };
+    }
+    const senior = doc.senior;
+    if (senior === "user" || senior === undefined || senior === null) {
+        return { kind: "user" };
+    }
+    if (typeof senior === "string") {
+        const colon = senior.indexOf(":");
+        if (colon <= 0)
+            return { kind: "user" };
+        const kind = senior.slice(0, colon);
+        const id = senior.slice(colon + 1);
+        if (kind === "session" || kind === "lane")
+            return { kind, id };
+        if (kind === "user")
+            return { kind: "user" };
+    }
+    if (typeof senior === "object" && !Array.isArray(senior)) {
+        const rec = senior;
+        if (rec.kind === "user")
+            return { kind: "user" };
+        if ((rec.kind === "session" || rec.kind === "lane") &&
+            typeof rec.id === "string" &&
+            rec.id.length > 0) {
+            return { kind: rec.kind, id: rec.id };
+        }
+    }
+    return { kind: "user" };
+}
+export function resolveQuestionAudience(question, readLaneSenior, readSessionSenior) {
+    let current = { kind: "lane", id: question.origin.id };
+    for (let hop = 0; hop < question.hop; hop += 1) {
+        if (current.kind === "user")
+            return current;
+        if (current.kind === "lane") {
+            if (!current.id)
+                return current;
+            current = readLaneSenior(current.id);
+            continue;
+        }
+        if (!current.id)
+            return current;
+        current = readSessionSenior(current.id);
+    }
+    return current;
+}
+function casLaneIf(hqRoot, laneId, shouldWrite, updater, dryRun) {
+    for (let i = 0; i < 5; i += 1) {
+        const current = readLane(hqRoot, laneId);
+        if (!shouldWrite(current))
+            return { changed: false, record: current };
+        if (dryRun)
+            return { changed: true, record: updater(current) };
+        try {
+            return {
+                changed: true,
+                record: updateLane(hqRoot, laneId, current.version, "reconcile", updater),
+            };
+        }
+        catch (err) {
+            if (!isLaneVersionConflict(err) || i === 4)
+                throw err;
+        }
+    }
+    throw new LanesError("version_conflict", `Exhausted CAS retries for lane ${laneId}`, { lane_id: laneId });
+}
+function casQuestionIf(hqRoot, questionId, shouldWrite, updater, dryRun) {
+    for (let i = 0; i < 5; i += 1) {
+        const current = readQuestion(hqRoot, questionId);
+        if (!shouldWrite(current))
+            return { changed: false, record: current };
+        if (dryRun)
+            return { changed: true, record: updater(current) };
+        try {
+            return {
+                changed: true,
+                record: updateQuestion(hqRoot, questionId, current.version, updater),
+            };
+        }
+        catch (err) {
+            if (!isLaneVersionConflict(err) || i === 4)
+                throw err;
+        }
+    }
+    throw new LanesError("version_conflict", `Exhausted CAS retries for question ${questionId}`, { question_id: questionId });
+}
+function fileNonEmpty(filePath) {
+    try {
+        return fs.statSync(filePath).size > 0;
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return false;
+        throw err;
+    }
+}
+function lastHostReportMs(lane) {
+    const raw = lane.last_host_report_at ?? lane.updated_at;
+    const ms = Date.parse(raw);
+    return Number.isFinite(ms) ? ms : 0;
+}
+function writeHeartbeatFile(hqRoot, summary, at) {
+    const filePath = reconcileHeartbeatPath(hqRoot);
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    const tmp = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+    const body = `${JSON.stringify({ at, ok: summary.ok, action: "reconcile" }, null, 2)}\n`;
+    try {
+        fs.writeFileSync(tmp, body, { encoding: "utf8" });
+        fs.renameSync(tmp, filePath);
+    }
+    catch (err) {
+        try {
+            fs.unlinkSync(tmp);
+        }
+        catch (unlinkErr) {
+            const code = unlinkErr.code;
+            if (code !== "ENOENT") {
+                process.stderr.write(`hq lanes reconcile: unlink heartbeat tmp failed: ${unlinkErr instanceof Error ? unlinkErr.message : String(unlinkErr)}\n`);
+            }
+        }
+        throw err;
+    }
+}
+function writeAnsweredQuestions(hqRoot, lane, questions) {
+    const filePath = path.join(laneRunDir(hqRoot, lane.lane_id), ANSWERED_QUESTIONS_FILE);
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, `${JSON.stringify(answeredQuestionEntries(hqRoot, lane, questions), null, 2)}\n`, "utf8");
+}
+async function withReconcileLock(hqRoot, fn) {
+    const lockPath = path.join(hqRoot, "workspace", "lanes-runs", ".admission-locks", "reconcile.lock");
+    fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+    const started = Date.now();
+    while (true) {
+        let fd;
+        try {
+            fd = fs.openSync(lockPath, "wx");
+            try {
+                fs.writeSync(fd, Buffer.from(`${process.pid}\n${Date.now()}\n`, "utf8"));
+            }
+            catch (err) {
+                fs.closeSync(fd);
+                fd = undefined;
+                throw err;
+            }
+            try {
+                return await fn();
+            }
+            finally {
+                try {
+                    fs.closeSync(fd);
+                }
+                catch (err) {
+                    process.stderr.write(`hq lanes reconcile: lock close failed: ${err instanceof Error ? err.message : String(err)}\n`);
+                }
+                try {
+                    fs.unlinkSync(lockPath);
+                }
+                catch (err) {
+                    const code = err.code;
+                    if (code !== "ENOENT") {
+                        process.stderr.write(`hq lanes reconcile: lock unlink failed: ${err instanceof Error ? err.message : String(err)}\n`);
+                    }
+                }
+            }
+        }
+        catch (err) {
+            const code = err.code;
+            if (code !== "EEXIST")
+                throw err;
+            if (Date.now() - started > RECONCILE_LOCK_TIMEOUT_MS) {
+                throw new LanesError("store_lock_timeout", `Timed out waiting for reconcile lock ${lockPath}`, { path: lockPath });
+            }
+            try {
+                const raw = fs.readFileSync(lockPath, "utf8");
+                const tsLine = raw.split("\n")[1];
+                const ts = tsLine ? Number(tsLine) : NaN;
+                if (Number.isFinite(ts) && Date.now() - ts > RECONCILE_LOCK_STALE_MS) {
+                    try {
+                        fs.unlinkSync(lockPath);
+                    }
+                    catch (unlinkErr) {
+                        const unlinkCode = unlinkErr.code;
+                        if (unlinkCode !== "ENOENT")
+                            throw unlinkErr;
+                    }
+                    continue;
+                }
+            }
+            catch (statErr) {
+                const statCode = statErr.code;
+                if (statCode !== "ENOENT") {
+                    process.stderr.write(`hq lanes reconcile: stale-lock probe failed: ${statErr instanceof Error ? statErr.message : String(statErr)}\n`);
+                }
+            }
+            Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10);
+        }
+    }
+}
+function classifyDeadLocalLane(hqRoot, laneId) {
+    const latest = readLatestEnvelopeRecord(hqRoot, laneId);
+    const envelope = latest?.envelope ?? readLatestEnvelope(hqRoot, laneId);
+    const runDir = laneRunDir(hqRoot, laneId);
+    const hasPartialOutput = [
+        path.join(runDir, "logs", "worker.out"),
+        path.join(runDir, "logs", "worker.err"),
+        path.join(runDir, "logs", "last-message.txt"),
+    ].some((file) => fileNonEmpty(file));
+    const lane = readLane(hqRoot, laneId);
+    const quota = envelope
+        ? null
+        : readQuotaSignal(runDir, lane.model, new Date(), readSpawnManifest(runDir).worker_out_offset ?? 0);
+    return {
+        classification: classifyLaneExit({ envelope, hasPartialOutput, quota }),
+        latest,
+        quota,
+        lane,
+    };
+}
+/** Unlocked pass body. `runReconcilePass` takes the process lock around this. */
+export async function runPassBody(hqRoot, deps) {
+    const now = deps.now();
+    const nowMs = now.getTime();
+    const nowIso = now.toISOString();
+    const summary = emptySummary(deps.dryRun);
+    const livenessDeps = deps.livenessDeps ?? defaultLivenessDeps();
+    let lanes;
+    let questions;
+    try {
+        lanes = loadLaneRecords(hqRoot, summary);
+        questions = loadQuestionRecords(hqRoot, summary);
+    }
+    catch (err) {
+        markError(summary, err);
+        return summary;
+    }
+    let presenceConnected;
+    try {
+        presenceConnected =
+            deps.presenceConnected ?? readDaemonPresenceConnected(livenessDeps);
+    }
+    catch (err) {
+        markError(summary, err);
+        return summary;
+    }
+    let livenessIndex;
+    try {
+        livenessIndex = loadLivenessIndex(hqRoot, undefined, livenessDeps, !deps.dryRun);
+    }
+    catch (err) {
+        markError(summary, err);
+        return summary;
+    }
+    const config = loadLanesConfig(hqRoot);
+    const laneById = new Map(lanes.map((lane) => [lane.lane_id, lane]));
+    const readLaneSenior = (laneId) => {
+        const lane = laneById.get(laneId);
+        if (lane)
+            return lane.senior;
+        try {
+            return readLane(hqRoot, laneId).senior;
+        }
+        catch (err) {
+            throw new LanesError("lane_not_found", `Cannot resolve senior for missing lane ${laneId}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId });
+        }
+    };
+    const readSessionSenior = deps.readSessionSenior ??
+        ((sessionId) => defaultReadSessionSenior(hqRoot, sessionId));
+    const sessionState = (sessionId) => {
+        const entry = livenessIndex.sessions[sessionId];
+        if (!entry)
+            return null;
+        return classifySessionLiveness(entry, nowMs, presenceConnected, config.liveness);
+    };
+    if (deps.afterLoad)
+        await deps.afterLoad();
+    markOverdueLanes(hqRoot, lanes, summary, deps, now);
+    await heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now);
+    for (const question of questions) {
+        if (question.status !== "pending")
+            continue;
+        let audience;
+        try {
+            audience = resolveQuestionAudience(question, readLaneSenior, readSessionSenior);
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "question",
+                id: question.question_id,
+                why: `audience unreadable: ${err instanceof Error ? err.message : String(err)}`,
+            });
+            continue;
+        }
+        const liveness = audience.kind === "session" && audience.id
+            ? sessionState(audience.id)
+            : audience.kind === "user"
+                ? "live"
+                : null;
+        try {
+            if (shouldAdvanceAudienceEnded(audience, liveness)) {
+                const nextHop = question.hop + 1;
+                const nextAudience = resolveQuestionAudience({ ...question, hop: nextHop }, readLaneSenior, readSessionSenior);
+                const result = casQuestionIf(hqRoot, question.question_id, (current) => current.status === "pending" && current.hop === question.hop, (current) => {
+                    const entry = {
+                        hop: nextHop,
+                        audience: nextAudience,
+                        at: nowIso,
+                        reason: "audience_ended",
+                    };
+                    return {
+                        ...current,
+                        hop: nextHop,
+                        routing_history: appendCappedHistory(current.routing_history, entry),
+                    };
+                }, deps.dryRun);
+                if (result.changed) {
+                    summary.advanced.push({
+                        id: question.question_id,
+                        from: String(question.hop),
+                        to: String(nextHop),
+                        reason: "audience_ended",
+                    });
+                }
+                continue;
+            }
+            if (shouldAgeQuestion(question, audience, nowMs)) {
+                const nextHop = question.hop + 1;
+                const nextAudience = resolveQuestionAudience({ ...question, hop: nextHop }, readLaneSenior, readSessionSenior);
+                const result = casQuestionIf(hqRoot, question.question_id, (current) => current.status === "pending" &&
+                    agedAdvanceCount(current) < MAX_AGED_ADVANCES &&
+                    shouldAgeQuestion(current, resolveQuestionAudience(current, readLaneSenior, readSessionSenior), nowMs), (current) => {
+                    const entry = {
+                        hop: current.hop + 1,
+                        audience: nextAudience,
+                        at: nowIso,
+                        reason: "unanswered_timeout",
+                    };
+                    return {
+                        ...current,
+                        hop: current.hop + 1,
+                        routing_history: appendCappedHistory(current.routing_history, entry),
+                    };
+                }, deps.dryRun);
+                if (result.changed) {
+                    summary.aged.push({
+                        id: question.question_id,
+                        from: String(question.hop),
+                        to: String(nextHop),
+                        reason: "unanswered_timeout",
+                    });
+                }
+            }
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "question",
+                id: question.question_id,
+                why: `advance failed: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
+    }
+    const freshLanes = loadLaneRecords(hqRoot, summary);
+    const freshQuestions = loadQuestionRecords(hqRoot, summary);
+    await admitQueued(hqRoot, freshLanes, summary, deps, now);
+    await resumeAwaiting(hqRoot, freshLanes, freshQuestions, summary, deps);
+    if (!deps.dryRun && summary.errors.length === 0) {
+        try {
+            writeHeartbeatFile(hqRoot, summary, nowIso);
+            summary.heartbeat_at = nowIso;
+        }
+        catch (err) {
+            markError(summary, err);
+        }
+    }
+    return summary;
+}
+function markOverdueLanes(hqRoot, lanes, summary, deps, now) {
+    const nowMs = now.getTime();
+    const nowIso = now.toISOString();
+    for (const lane of lanes) {
+        if (lane.overdue === true)
+            continue;
+        if (!laneRunsHere(lane, deps.host))
+            continue;
+        let deadline;
+        try {
+            deadline = readTimeoutDeadline(hqRoot, lane.lane_id);
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "lane",
+                id: lane.lane_id,
+                why: `timeout unreadable: ${err instanceof Error ? err.message : String(err)}`,
+            });
+            continue;
+        }
+        if (!deadline || !isDeadlinePassed(deadline.deadline_at, nowMs))
+            continue;
+        try {
+            const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.overdue !== true, (current) => ({
+                ...current,
+                overdue: true,
+                overdue_at: nowIso,
+            }), deps.dryRun);
+            if (result.changed) {
+                summary.overdue.push({
+                    id: lane.lane_id,
+                    from: "ok",
+                    to: "overdue",
+                });
+            }
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "lane",
+                id: lane.lane_id,
+                why: `overdue CAS failed: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
+    }
+}
+async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
+    const nowMs = now.getTime();
+    const nowIso = now.toISOString();
+    const isAlive = deps.isAlive ?? isPidAlive;
+    for (const lane of lanes) {
+        if (lane.state !== "running")
+            continue;
+        const local = laneRunsHere(lane, deps.host);
+        if (local) {
+            const runDir = laneRunDir(hqRoot, lane.lane_id);
+            let workerPid;
+            let watcherPid;
+            let workerAlive;
+            let watcherAlive;
+            try {
+                const workerRec = readPidRecord(path.join(runDir, "worker.pid"));
+                const watcherRec = readPidRecord(path.join(runDir, "watcher.pid"));
+                workerPid = workerRec?.pid ?? null;
+                watcherPid = watcherRec?.pid ?? null;
+                workerAlive = workerRec !== null && isPidRecordAlive(workerRec, isAlive);
+                watcherAlive = watcherRec !== null && isPidRecordAlive(watcherRec, isAlive);
+            }
+            catch (err) {
+                markDirty(summary, {
+                    kind: "lane",
+                    id: lane.lane_id,
+                    why: `pid file unreadable: ${err instanceof Error ? err.message : String(err)}`,
+                });
+                continue;
+            }
+            if (workerAlive) {
+                try {
+                    const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.last_host_report_at !== nowIso, (current) => ({ ...current, last_host_report_at: nowIso }), deps.dryRun);
+                    if (result.changed)
+                        summary.heartbeats.push(lane.lane_id);
+                }
+                catch (err) {
+                    markDirty(summary, {
+                        kind: "lane",
+                        id: lane.lane_id,
+                        why: `heartbeat CAS failed: ${err instanceof Error ? err.message : String(err)}`,
+                    });
+                }
+                if (!watcherAlive && !deps.dryRun) {
+                    try {
+                        armLaneWatcher(hqRoot, lane.lane_id, deps.spawnLaneDeps ?? {});
+                        summary.heartbeats.push(`${lane.lane_id}:watcher-rearmed`);
+                    }
+                    catch (err) {
+                        markDirty(summary, {
+                            kind: "lane",
+                            id: lane.lane_id,
+                            why: `watcher re-arm failed: ${err instanceof Error ? err.message : String(err)}`,
+                        });
+                    }
+                }
+                continue;
+            }
+            if (watcherAlive) {
+                // Watcher owns terminal classification of a process that has just exited.
+                continue;
+            }
+            const hadPids = workerPid !== null || watcherPid !== null;
+            if (!hadPids && nowMs - lastHostReportMs(lane) < DEAD_HOST_MS) {
+                try {
+                    const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.last_host_report_at !== nowIso, (current) => ({ ...current, last_host_report_at: nowIso }), deps.dryRun);
+                    if (result.changed)
+                        summary.heartbeats.push(lane.lane_id);
+                }
+                catch (err) {
+                    markDirty(summary, {
+                        kind: "lane",
+                        id: lane.lane_id,
+                        why: `heartbeat CAS failed: ${err instanceof Error ? err.message : String(err)}`,
+                    });
+                }
+                continue;
+            }
+            try {
+                const { classification, latest, quota, lane: classified } = classifyDeadLocalLane(hqRoot, lane.lane_id);
+                // Match the watcher's one-repair decision before classifying an
+                // unparseable envelope after both processes have died.
+                const rawLatest = readLatestEnvelopeFile(hqRoot, lane.lane_id);
+                let invalidEnvelope = null;
+                if (!deps.dryRun && rawLatest && rawLatest.envelope === null) {
+                    const repair = decideEnvelopeRepair(hqRoot, lane.lane_id, rawLatest, nowMs, deps.spawnRepair ?? spawnEnvelopeRepair, () => undefined);
+                    if (repair.action === "spawned") {
+                        armLaneWatcher(hqRoot, lane.lane_id, deps.spawnLaneDeps);
+                        summary.heartbeats.push(`${lane.lane_id}:envelope-repair`);
+                        continue;
+                    }
+                    invalidEnvelope = { errors: repair.errors };
+                }
+                const repairedClassification = invalidEnvelope
+                    ? classifyLaneExit({ envelope: null, hasPartialOutput: false, invalidEnvelope })
+                    : classification;
+                if (!deps.dryRun &&
+                    quota &&
+                    classified.account &&
+                    (classification.reason === "account_quota_exhausted" ||
+                        classification.reason === "account_model_unavailable")) {
+                    recordAccountQuota(hqRoot, classified.provider, classified.account, quota);
+                }
+                if (!deps.dryRun &&
+                    classification.state === "awaiting_input" &&
+                    latest &&
+                    (latest.envelope.questions?.length ?? 0) > 0) {
+                    raiseEnvelopeQuestions(hqRoot, readLane(hqRoot, lane.lane_id), latest.envelope, latest.ref, nowIso);
+                }
+                const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "running", (current) => ({
+                    ...current,
+                    state: repairedClassification.state,
+                    queued_reason: undefined,
+                    terminal_reason: repairedClassification.reason ?? "host_lost",
+                    ...(quota?.quota_reset_at ? { quota_reset_at: quota.quota_reset_at } : {}),
+                    ...(quota?.marker ? { model_quota_marker: quota.marker } : {}),
+                    last_host_report_at: nowIso,
+                }), deps.dryRun);
+                if (result.changed) {
+                    summary.dead_host.push({
+                        id: lane.lane_id,
+                        from: "running",
+                        to: repairedClassification.state,
+                        reason: repairedClassification.reason ?? "host_lost",
+                    });
+                    if (latest?.envelope) {
+                        const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
+                        if (mesh) {
+                            const meshEmitTimeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
+                            let timer;
+                            try {
+                                const questionIds = latest.envelope.decision === "ask"
+                                    ? envelopeQuestionIds(lane.lane_id, latest.ref, latest.envelope)
+                                    : undefined;
+                                await Promise.race([
+                                    emitLaneDecisionEvents(mesh, result.record, latest.envelope, { questionIds }),
+                                    new Promise((_resolve, reject) => {
+                                        timer = setTimeout(() => reject(new Error(`terminal mesh emit timed out after ${meshEmitTimeoutMs}ms`)), meshEmitTimeoutMs);
+                                    }),
+                                ]);
+                            }
+                            catch (err) {
+                                markDirty(summary, {
+                                    kind: "lane",
+                                    id: lane.lane_id,
+                                    why: `terminal mesh emit failed: ${err instanceof Error ? err.message : String(err)}`,
+                                });
+                            }
+                            finally {
+                                if (timer)
+                                    clearTimeout(timer);
+                            }
+                        }
+                    }
+                }
+            }
+            catch (err) {
+                markDirty(summary, {
+                    kind: "lane",
+                    id: lane.lane_id,
+                    why: `dead-host classify failed: ${err instanceof Error ? err.message : String(err)}`,
+                });
+            }
+            continue;
+        }
+        if (nowMs - lastHostReportMs(lane) < DEAD_HOST_MS)
+            continue;
+        try {
+            const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "running" &&
+                nowMs - lastHostReportMs(current) >= DEAD_HOST_MS, (current) => ({
+                ...current,
+                state: "interrupted",
+                queued_reason: undefined,
+                terminal_reason: "host_lost",
+            }), deps.dryRun);
+            if (result.changed) {
+                summary.dead_host.push({
+                    id: lane.lane_id,
+                    from: "running",
+                    to: "interrupted",
+                    reason: "host_lost",
+                });
+            }
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "lane",
+                id: lane.lane_id,
+                why: `dead-host CAS failed: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
+    }
+}
+async function admitQueued(hqRoot, lanes, summary, deps, now) {
+    const localQueued = lanes
+        .filter((lane) => lane.state === "queued" && laneRunsHere(lane, deps.host))
+        .sort(compareQueueOrder);
+    if (localQueued.length === 0)
+        return;
+    const config = loadLanesConfig(hqRoot);
+    const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
+    const filePath = throttleStatePath(hqRoot);
+    const loaded = readThrottleState(filePath, throttleDeps);
+    const throttleState = loaded === "unreadable"
+        ? "unreadable"
+        : loaded === "missing"
+            ? emptyThrottleState()
+            : loaded;
+    const busy = await resolveCpuBusyFraction(throttleDeps);
+    const availableMemMb = readAvailableMemMb(throttleDeps);
+    const nowMs = now.getTime();
+    const decision = evaluateThrottle({
+        busy,
+        availableMemMb,
+        cpuCheckEnabled: isCpuCheckEnabled(throttleDeps.env),
+        state: throttleState,
+        throttle: config.throttle,
+        nowMs,
+    });
+    const spawnFn = deps.spawn ??
+        ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
+    const running = lanes.filter((lane) => lane.state === "running");
+    const starvation = decision.admit && decision.reason === "starvation_bound";
+    const candidates = starvation ? localQueued.slice(0, 1) : localQueued;
+    const spawned = [];
+    if (!decision.admit) {
+        if (!deps.dryRun && throttleState !== "unreadable") {
+            writeThrottleState(filePath, applyThrottleDecision(throttleState, decision, nowMs), throttleDeps);
+        }
+        return;
+    }
+    for (const lane of candidates) {
+        const provider = lane.provider;
+        const providerCfg = config.providers[provider];
+        const providerRunning = running.filter((row) => row.provider === provider).length +
+            spawned.filter((row) => row.provider === provider).length;
+        if (providerRunning >= providerCfg.max_running && !starvation)
+            continue;
+        if (lane.account) {
+            const accountRunning = running.filter((row) => row.provider === provider && row.account === lane.account).length +
+                spawned.filter((row) => row.provider === provider && row.account === lane.account).length;
+            if (accountRunning >= accountMaxRunning(providerCfg, lane.account) &&
+                !starvation) {
+                continue;
+            }
+        }
+        try {
+            if (deps.dryRun) {
+                summary.admitted.push({
+                    id: lane.lane_id,
+                    from: "queued",
+                    to: "running",
+                    reason: decision.reason,
+                });
+                spawned.push({ ...lane, state: "running" });
+                if (starvation)
+                    break;
+                continue;
+            }
+            const started = spawnFn(hqRoot, lane);
+            if (decision.reason === "starvation_bound") {
+                casLaneIf(hqRoot, lane.lane_id, (current) => current.admitted_reason !== "starvation_bound", (current) => ({ ...current, admitted_reason: "starvation_bound" }), false);
+            }
+            summary.admitted.push({
+                id: started.lane_id,
+                from: "queued",
+                to: started.state,
+                reason: decision.reason,
+                ...(started.queued_reason === "provider_binary_missing" &&
+                    typeof started.terminal_details?.searched_path === "string"
+                    ? {
+                        searched_path: started.terminal_details.searched_path,
+                    }
+                    : {}),
+            });
+            spawned.push(started);
+            if (starvation)
+                break;
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "lane",
+                id: lane.lane_id,
+                why: `admit failed: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
+    }
+    if (!deps.dryRun) {
+        const admittedAny = spawned.length > 0;
+        const applied = admittedAny
+            ? { admit: true, reason: decision.reason }
+            : decision;
+        const base = throttleState === "unreadable" ? emptyThrottleState() : throttleState;
+        writeThrottleState(filePath, applyThrottleDecision(base, applied, nowMs), throttleDeps);
+    }
+}
+async function resumeAwaiting(hqRoot, lanes, questions, summary, deps) {
+    const spawnFn = deps.spawn ??
+        ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
+    const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" && laneRunsHere(lane, deps.host));
+    for (const lane of awaiting) {
+        const pending = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "pending");
+        const answered = questions.filter((q) => q.origin.id === lane.lane_id && q.status === "answered");
+        if (pending.length > 0 || answered.length === 0)
+            continue;
+        try {
+            if (deps.dryRun) {
+                summary.resumed.push({
+                    id: lane.lane_id,
+                    from: "awaiting_input",
+                    to: "running",
+                });
+                continue;
+            }
+            writeAnsweredQuestions(hqRoot, lane, questions);
+            const started = spawnFn(hqRoot, lane);
+            summary.resumed.push({
+                id: started.lane_id,
+                from: "awaiting_input",
+                to: started.state,
+            });
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "lane",
+                id: lane.lane_id,
+                why: `resume failed: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
+    }
+}
+/**
+ * One idempotent reconcile pass. Throws only when the store directory itself
+ * cannot be listed; per-record failures are logged onto the summary and set
+ * `ok: false` rather than being swallowed.
+ */
+export async function runReconcilePass(hqRoot, deps) {
+    return withReconcileLock(hqRoot, async () => runPassBody(hqRoot, deps));
+}
+//# sourceMappingURL=reconcile.js.map