@indigoai-us/hq-cli 5.215.0 → 5.217.0

package/dist/lib/lanes/reconcile.js

@@ -21,11 +21,11 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
 import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
 import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
 import { readLanesRegistryFlag } from "./registry-flag.js";
-import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
+import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
 import { providerSupportsSessionResume } from "./delivery.js";
 import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
 import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
-import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, writeWatcherDecision, } from "./watcher.js";
+import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
 import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
 import { laneStories, storyStatusForEnvelope } from "./stories.js";
 import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
@@ -800,7 +800,7 @@ export async function runPassBody(hqRoot, deps) {
     const keepAliveFlagCompanies = new Map();
     const keepAliveHireGrantFlagCompanies = new Map();
     for (const lane of lanes) {
-        if (!(lane.state === "done" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
+        if (!(lane.state === "done" || lane.state === "interrupted" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
             (lane.keep_alive !== true && lane.worker_profile === undefined) ||
             !laneRunsHere(lane, deps.host) ||
             (lane.state === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
@@ -838,7 +838,7 @@ export async function runPassBody(hqRoot, deps) {
     ]);
     await withAllLaneAdmissionLocks(hqRoot, async () => {
         const freshLanes = loadLaneRecords(hqRoot, summary);
-        resumeDoneKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
+        resumeKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
         const drainLanes = loadLaneRecords(hqRoot, summary);
         const freshQuestions = loadQuestionRecords(hqRoot, summary);
         await admitQueued(hqRoot, drainLanes, summary, deps, now);
@@ -1059,6 +1059,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                         state: repairedClassification.state,
                         queued_reason: undefined,
                         terminal_reason: repairedClassification.reason ?? "host_lost",
+                        interrupted_at: repairedClassification.state === "interrupted" ? nowIso : undefined,
                         ...(providerRejection
                             ? { terminal_details: providerRejectionDetails(providerRejection, current) }
                             : {}),
@@ -1175,6 +1176,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                 state: "interrupted",
                 queued_reason: undefined,
                 terminal_reason: "host_lost",
+                interrupted_at: nowIso,
             }), deps.dryRun);
             if (result.changed) {
                 summary.dead_host.push({
@@ -1392,7 +1394,8 @@ function keepAliveProgressResetByHuman(lane) {
     if (count > 0 && !Number.isFinite(lastAutoMs))
         return true;
     const messageMs = Date.parse(lane.last_message_at ?? "");
-    const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT
+    const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT ||
+        lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
         ? Number.NaN
         : Date.parse(lane.last_resume?.at ?? "");
     return Number.isFinite(lastAutoMs) &&
@@ -1433,6 +1436,36 @@ function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
         });
     }
 }
+function readLatestInterruptedClassificationAt(hqRoot, lane) {
+    const logPath = watcherOutLogPath(hqRoot, lane.lane_id);
+    let body;
+    try {
+        body = fs.readFileSync(logPath, "utf8");
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return undefined;
+        throw err;
+    }
+    let latestClassification;
+    for (const line of body.split(/\r?\n/).filter(Boolean)) {
+        const parsed = JSON.parse(line);
+        if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+            throw new Error("watcher log contains a non-object decision record");
+        }
+        const record = parsed;
+        if (record.event === "classification")
+            latestClassification = record;
+    }
+    const at = latestClassification?.at;
+    if (latestClassification?.state !== "interrupted" ||
+        latestClassification.reason !== lane.terminal_reason ||
+        typeof at !== "string" ||
+        !Number.isFinite(Date.parse(at)) ||
+        new Date(Date.parse(at)).toISOString() !== at)
+        return undefined;
+    return at;
+}
 function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
     const filePath = path.join(laneRunDir(hqRoot, lane.lane_id), ref);
     try {
@@ -1442,7 +1475,7 @@ function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
         throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
     }
 }
-function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
+function resumeKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
     const nowMs = now.getTime();
     const nowIso = now.toISOString();
     const config = loadLanesConfig(hqRoot);
@@ -1456,34 +1489,97 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
             markError(summary, err);
         }
     }
-    for (const lane of lanes) {
-        if (!laneRunsHere(lane, deps.host))
+    for (const originalLane of lanes) {
+        if (!laneRunsHere(originalLane, deps.host))
             continue;
-        if (lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") {
-            if (lane.keep_alive === true &&
-                flagByCompany.get(keepAliveCompanyKey(lane.company)) === true)
-                recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps);
+        if (originalLane.state === "killed" ||
+            originalLane.state === "cancelled" ||
+            originalLane.state === "failed") {
+            if (originalLane.keep_alive === true &&
+                flagByCompany.get(keepAliveCompanyKey(originalLane.company)) === true)
+                recordKeepAliveSkip(hqRoot, originalLane, "terminal_state", summary, deps);
             continue;
         }
-        if (lane.state !== "done" ||
-            (lane.keep_alive !== true && lane.worker_profile === undefined))
+        if ((originalLane.state !== "done" && originalLane.state !== "interrupted") ||
+            (originalLane.keep_alive !== true && originalLane.worker_profile === undefined))
             continue;
-        const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
-        const envelope = latest?.envelope;
-        if (!envelope || envelope.decision !== "done")
-            continue;
-        if (envelope.story_complete === true) {
-            if (hasKeepAliveProgress(lane)) {
-                casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
-                    current.state === "done" &&
-                    current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
+        const initialState = originalLane.state;
+        let lane = originalLane;
+        if (initialState === "done") {
+            const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
+            if (!envelope || envelope.decision !== "done")
+                continue;
+            if (envelope.story_complete === true) {
+                if (hasKeepAliveProgress(lane)) {
+                    casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
+                        current.state === "done" &&
+                        current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
+                }
+                continue;
             }
-            continue;
         }
         if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
             continue;
-        if (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
-            lane.terminal_reason === "account_not_logged_in") {
+        // Persist a stable timestamp for legacy interrupted rows before a later
+        // keep-alive status write can advance their general updated_at clock. The
+        // watcher classification is the interruption time; updated_at is only an
+        // upper bound and can postdate explicit opt-in.
+        if (initialState === "interrupted" && lane.keep_alive === true && lane.interrupted_at === undefined) {
+            let interruptedAt;
+            try {
+                interruptedAt = readLatestInterruptedClassificationAt(hqRoot, lane);
+            }
+            catch (err) {
+                const detail = `watcher classification read failed: ${err instanceof Error ? err.message : String(err)}`;
+                recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, detail);
+                const skip = summary.skipped[summary.skipped.length - 1];
+                if (skip) {
+                    summary.ok = false;
+                    logSkip(skip);
+                }
+                continue;
+            }
+            if (interruptedAt === undefined) {
+                recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "no latest matching watcher classification event");
+                const skip = summary.skipped[summary.skipped.length - 1];
+                if (skip) {
+                    summary.ok = false;
+                    logSkip(skip);
+                }
+                continue;
+            }
+            try {
+                const backfill = casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
+                    current.state === "interrupted" &&
+                    current.keep_alive === true &&
+                    current.terminal_reason === lane.terminal_reason &&
+                    current.interrupted_at === undefined, (current) => ({ ...current, interrupted_at: interruptedAt }), deps.dryRun);
+                lane = backfill.record;
+                if (lane.interrupted_at === undefined) {
+                    recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "lane changed before watcher classification timestamp could be recorded");
+                    const skip = summary.skipped[summary.skipped.length - 1];
+                    if (skip) {
+                        summary.ok = false;
+                        logSkip(skip);
+                    }
+                    continue;
+                }
+            }
+            catch (err) {
+                markDirty(summary, {
+                    kind: "lane",
+                    id: lane.lane_id,
+                    why: `keep-alive interruption timestamp backfill failed: ${err instanceof Error ? err.message : String(err)}`,
+                });
+                continue;
+            }
+        }
+        if (initialState === "interrupted" && lane.terminal_reason !== "exited_with_partial_output") {
+            recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason ?? "missing interruption reason");
+            continue;
+        }
+        if (initialState === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
+            lane.terminal_reason === "account_not_logged_in")) {
             recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason);
             continue;
         }
@@ -1496,20 +1592,35 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
             recordKeepAliveSkip(hqRoot, lane, "missing_opt_in_time", summary, deps);
             continue;
         }
-        let envelopeAtMs;
-        try {
-            envelopeAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
+        let eligibilityAtMs;
+        if (initialState === "done") {
+            const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
+            if (!latest || latest.envelope.decision !== "done" || latest.envelope.story_complete === true)
+                continue;
+            try {
+                eligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
+            }
+            catch (err) {
+                markDirty(summary, {
+                    kind: "lane",
+                    id: lane.lane_id,
+                    why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
+                });
+                continue;
+            }
         }
-        catch (err) {
-            markDirty(summary, {
-                kind: "lane",
-                id: lane.lane_id,
-                why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
-            });
-            continue;
+        else {
+            eligibilityAtMs = Date.parse(lane.interrupted_at ?? "");
+            if (!Number.isFinite(eligibilityAtMs)) {
+                recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps);
+                continue;
+            }
         }
-        if (envelopeAtMs <= optedInAtMs) {
-            recordKeepAliveSkip(hqRoot, lane, "envelope_before_opt_in", summary, deps);
+        const initialTimeBlockReason = initialState === "done"
+            ? "envelope_before_opt_in"
+            : "interruption_before_opt_in";
+        if (eligibilityAtMs <= optedInAtMs) {
+            recordKeepAliveSkip(hqRoot, lane, initialTimeBlockReason, summary, deps);
             continue;
         }
         if (retiredLaneIds === null) {
@@ -1521,33 +1632,54 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
             recordKeepAliveSkip(hqRoot, lane, exclusion, summary, deps);
             continue;
         }
+        const providerSessionId = lane.provider_session_id ?? lane.last_resume?.session_id;
         if (!providerSupportsSessionResume(lane.provider) ||
-            !(lane.provider_session_id ?? lane.last_resume?.session_id))
+            typeof providerSessionId !== "string" ||
+            providerSessionId.trim().length === 0) {
+            if (initialState === "interrupted") {
+                recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, "provider session cannot be resumed");
+            }
             continue;
-        const keepAliveStillEnabled = (candidate) => candidate.keep_alive === true &&
-            candidate.keep_alive_opted_in_at === lane.keep_alive_opted_in_at;
-        const current = readLane(hqRoot, lane.lane_id);
-        if (current.version !== lane.version ||
-            current.state !== "done" ||
-            !keepAliveStillEnabled(current) ||
-            !laneRunsHere(current, deps.host))
+        }
+        // This second read is intentionally independent of the pass snapshot. The
+        // final timestamp comparison below closes the opt-in race before the CAS.
+        const rechecked = deps.readKeepAliveRecheckLane?.(hqRoot, lane.lane_id) ?? readLane(hqRoot, lane.lane_id);
+        // Dry-run backfills only in memory. Carry that simulated legacy timestamp
+        // through the fresh read when the stored lane version has not changed.
+        const current = deps.dryRun &&
+            initialState === "interrupted" &&
+            originalLane.interrupted_at === undefined &&
+            lane.interrupted_at !== undefined &&
+            rechecked.version === lane.version &&
+            rechecked.interrupted_at === undefined
+            ? { ...rechecked, interrupted_at: lane.interrupted_at }
+            : rechecked;
+        if (current.state !== initialState || !laneRunsHere(current, deps.host))
             continue;
-        const currentEnvelopeRecord = readLatestEnvelopeRecord(hqRoot, current.lane_id);
-        const currentEnvelope = currentEnvelopeRecord?.envelope;
-        if (!currentEnvelope || currentEnvelope.decision !== "done")
+        if (current.keep_alive !== true) {
+            recordKeepAliveSkip(hqRoot, current, "not_opted_in", summary, deps);
             continue;
-        if (currentEnvelope.story_complete === true) {
-            if (hasKeepAliveProgress(current)) {
-                casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
-                    latestLane.state === "done" &&
-                    keepAliveStillEnabled(latestLane), (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
-            }
+        }
+        if (initialState === "interrupted" && current.terminal_reason !== "exited_with_partial_output") {
+            recordKeepAliveSkip(hqRoot, current, "terminal_state", summary, deps, current.terminal_reason ?? "missing interruption reason");
             continue;
         }
-        if (currentEnvelopeRecord !== null) {
-            let currentEnvelopeAtMs;
+        let currentEligibilityAtMs;
+        if (initialState === "done") {
+            const currentEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id);
+            if (!currentEnvelope || currentEnvelope.envelope.decision !== "done")
+                continue;
+            if (currentEnvelope.envelope.story_complete === true) {
+                if (hasKeepAliveProgress(current)) {
+                    casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
+                        latestLane.state === "done" &&
+                        latestLane.keep_alive === true &&
+                        latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
+                }
+                continue;
+            }
             try {
-                currentEnvelopeAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelopeRecord.ref);
+                currentEligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelope.ref);
             }
             catch (err) {
                 markDirty(summary, {
@@ -1557,11 +1689,26 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
                 });
                 continue;
             }
-            if (currentEnvelopeAtMs <= Date.parse(current.keep_alive_opted_in_at ?? "")) {
-                recordKeepAliveSkip(hqRoot, current, "envelope_before_opt_in", summary, deps);
+        }
+        else {
+            currentEligibilityAtMs = Date.parse(current.interrupted_at ?? "");
+            if (!Number.isFinite(currentEligibilityAtMs)) {
+                recordKeepAliveSkip(hqRoot, current, "missing_interruption_time", summary, deps);
                 continue;
             }
         }
+        const currentOptedInAtMs = Date.parse(current.keep_alive_opted_in_at ?? "");
+        if (!Number.isFinite(currentOptedInAtMs)) {
+            recordKeepAliveSkip(hqRoot, current, "missing_opt_in_time", summary, deps);
+            continue;
+        }
+        const currentTimeBlockReason = initialState === "done"
+            ? "envelope_before_opt_in"
+            : "interruption_before_opt_in";
+        if (currentEligibilityAtMs <= currentOptedInAtMs) {
+            recordKeepAliveSkip(hqRoot, current, currentTimeBlockReason, summary, deps);
+            continue;
+        }
         if (retiredLaneIds === null) {
             recordKeepAliveSkip(hqRoot, current, "retired_list_invalid", summary, deps);
             continue;
@@ -1572,11 +1719,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
             recordKeepAliveSkip(hqRoot, current, currentExclusion, summary, deps);
             continue;
         }
-        const providerSessionId = current.provider_session_id ?? current.last_resume?.session_id;
-        if (current.terminal_reason === PROVIDER_REJECTED_REASON ||
-            current.terminal_reason === "account_not_logged_in" ||
-            typeof providerSessionId !== "string" ||
-            providerSessionId.trim().length === 0)
+        const currentSessionId = current.provider_session_id ?? current.last_resume?.session_id;
+        if (!providerSupportsSessionResume(current.provider) ||
+            typeof currentSessionId !== "string" ||
+            currentSessionId.trim().length === 0)
             continue;
         const resetByHuman = keepAliveProgressResetByHuman(current);
         const attempts = resetByHuman ? 0 : current.keep_alive_resume_count ?? 0;
@@ -1584,8 +1730,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
             if (current.keep_alive_exhausted_at !== undefined)
                 continue;
             const exhausted = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
-                latestLane.state === "done" &&
-                keepAliveStillEnabled(latestLane) &&
+                latestLane.state === current.state &&
+                latestLane.keep_alive === true &&
+                latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
                 (latestLane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES &&
                 latestLane.keep_alive_exhausted_at === undefined, (latestLane) => ({ ...latestLane, keep_alive_exhausted_at: nowIso }), deps.dryRun);
             if (exhausted.changed && !deps.dryRun) {
@@ -1593,13 +1740,16 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
                     event: "keep_alive_exhausted",
                     at: nowIso,
                     attempt: attempts,
+                    resumed_from: current.state,
                 });
             }
             continue;
         }
         const attempt = attempts + 1;
-        const doneAtMs = Date.parse(current.updated_at);
-        const attemptEligibleMs = doneAtMs + leadLaneKeepAliveBackoffMs(attempt);
+        const terminalAtMs = initialState === "done"
+            ? Date.parse(current.updated_at)
+            : Date.parse(current.interrupted_at ?? "");
+        const attemptEligibleMs = terminalAtMs + leadLaneKeepAliveBackoffMs(attempt);
         const storedNextMs = resetByHuman
             ? Number.NaN
             : Date.parse(current.keep_alive_next_eligible_at ?? "");
@@ -1609,8 +1759,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
         if (nowMs < nextEligibleMs) {
             if (resetByHuman && hasKeepAliveProgress(current)) {
                 casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
-                    latestLane.state === "done" &&
-                    keepAliveStillEnabled(latestLane), (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
+                    latestLane.state === current.state &&
+                    latestLane.keep_alive === true &&
+                    latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
             }
             continue;
         }
@@ -1628,9 +1779,14 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
             });
             continue;
         }
+        const resumedFrom = current.state;
+        const resumeText = resumedFrom === "interrupted"
+            ? LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
+            : LEAD_LANE_KEEP_ALIVE_RESUME_TEXT;
         const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
-            latestLane.state === "done" &&
-            keepAliveStillEnabled(latestLane) &&
+            latestLane.state === current.state &&
+            latestLane.keep_alive === true &&
+            latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
             (expectedSeniorId === undefined ||
                 (latestLane.senior.kind === "lane" && latestLane.senior.id === expectedSeniorId)), (latestLane) => ({
             ...latestLane,
@@ -1641,10 +1797,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
             terminal_reason: undefined,
             terminal_details: undefined,
             pending_resume: {
-                session_id: providerSessionId,
+                session_id: currentSessionId,
                 trigger: "resume",
-                resumed_from: "done",
-                text: LEAD_LANE_KEEP_ALIVE_RESUME_TEXT,
+                resumed_from: resumedFrom,
+                text: resumeText,
                 compact: true,
                 at: nowIso,
             },
@@ -1693,7 +1849,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
             continue;
         summary.resumed.push({
             id: current.lane_id,
-            from: "done",
+            from: resumedFrom,
             to: "queued",
             reason: "keep_alive",
         });
@@ -1703,6 +1859,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
                 at: nowIso,
                 attempt,
                 next_eligible_at: nextEligibleAt,
+                resumed_from: resumedFrom,
             });
         }
     }

package/dist/lib/lanes/spawn.d.ts

@@ -305,6 +305,8 @@ export interface EnvelopeRepairSpawnInput {
     rawText: string;
     /** True when this run never wrote its expected envelope file. */
     missing?: boolean;
+    /** Repair-only warning hook for profile metadata ignored during recovery. */
+    onProfileWarning?: (fields: readonly string[]) => void;
 }
 /** How the spec §4 repair turn reaches the provider (recorded as `envelope_repair.mode`). */
 export type EnvelopeRepairMode = "resume" | "fresh_turn";

package/dist/lib/lanes/spawn.js

@@ -2152,7 +2152,12 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
     ensureDir(logsDir);
     const config = loadLanesConfig(hqRoot);
     const profile = lane.worker_profile
-        ? loadWorkerProfile(hqRoot, lane.worker_profile)
+        ? loadWorkerProfile(hqRoot, lane.worker_profile, {
+            onUnknownMetaFields: (fields) => {
+                process.stderr.write(`hq lanes watcher: envelope repair ignored unknown worker profile meta field(s) for ${lane.worker_profile}: ${fields.join(", ")}\n`);
+                input.onProfileWarning?.(fields);
+            },
+        })
         : undefined;
     const profileDefaultsApply = profile !== undefined && profile.meta.provider === lane.provider;
     const profileModel = profileDefaultsApply ? profile.meta.model : undefined;

package/dist/lib/lanes/store.js

@@ -414,6 +414,8 @@ export function parseLaneRecord(raw, filePath) {
         "not_opted_in",
         "missing_opt_in_time",
         "envelope_before_opt_in",
+        "interruption_before_opt_in",
+        "missing_interruption_time",
         "retired",
         "superseded",
         "retired_list_invalid",
@@ -443,6 +445,7 @@ export function parseLaneRecord(raw, filePath) {
         "keep_alive_last_resume_at",
         "keep_alive_next_eligible_at",
         "keep_alive_exhausted_at",
+        "interrupted_at",
     ]) {
         if (raw[field] !== undefined)
             requireTimestamp(raw[field], field, filePath);
@@ -470,6 +473,11 @@ export function parseLaneRecord(raw, filePath) {
         new Set(raw.pending_question_ids).size !== raw.pending_question_ids.length)) {
         throw new LaneSchemaError("pending_question_ids must be an array of unique non-empty ids", { path: filePath, field: "pending_question_ids" });
     }
+    if (raw.envelope_validation_errors !== undefined && (!Array.isArray(raw.envelope_validation_errors) ||
+        raw.envelope_validation_errors.length === 0 ||
+        raw.envelope_validation_errors.some((error) => typeof error !== "string" || error.trim() === ""))) {
+        throw new LaneSchemaError("envelope_validation_errors must be a non-empty array of non-empty strings", { path: filePath, field: "envelope_validation_errors" });
+    }
     requireTimestamp(raw.created_at, "created_at", filePath);
     requireTimestamp(raw.updated_at, "updated_at", filePath);
     if (raw.queued_reason !== undefined && !isQueuedReason(raw.queued_reason)) {

package/dist/lib/lanes/types.d.ts

@@ -359,7 +359,7 @@ export interface LaneRecord {
     /** Time this lane most recently explicitly opted into keep-alive. */
     keep_alive_opted_in_at?: string;
     /** Why reconcile declined to resume this lane most recently. */
-    keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
+    keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "interruption_before_opt_in" | "missing_interruption_time" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
     /** Consecutive automatic keep-alive resumes since the last human reset. */
     keep_alive_resume_count?: number;
     /** Time the most recent automatic keep-alive resume was queued. */
@@ -432,6 +432,8 @@ export interface LaneRecord {
     story_complete_reason?: string;
     /** Copied from the last ingested envelope when present. */
     story_status?: StoryStatus;
+    /** Envelope parser failures retained when a usable ask is recovered. */
+    envelope_validation_errors?: string[];
     /** Set by the reconcile pass when `--timeout` has passed (spec §5.7). */
     overdue?: boolean;
     overdue_at?: string;
@@ -449,6 +451,8 @@ export interface LaneRecord {
     /** Set when the reconcile pass force-admits under the starvation bound. */
     admitted_reason?: string;
     terminal_reason?: string;
+    /** Classification time when this lane entered interrupted; stable across later metadata writes. */
+    interrupted_at?: string;
     /**
      * Structured detail behind a named `terminal_reason` (dispatch, pickup and
      * spawn failures), the missing-binary search path behind

package/dist/lib/lanes/types.js

@@ -230,9 +230,9 @@ export const LANE_FIELD_WRITERS = {
     last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
     last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
     overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
-    admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
+    admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], interrupted_at: ["watcher", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
     quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
-    provider_session_id: ["watcher", "command", "spawn"], provider_session_id_absent: ["watcher", "command"], provider_home_at_spawn: ["spawn"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], last_progress_event_at: ["watcher"],
+    provider_session_id: ["watcher", "command", "spawn"], provider_session_id_absent: ["watcher", "command"], provider_home_at_spawn: ["spawn"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], envelope_validation_errors: ["watcher"], last_progress_event_at: ["watcher"],
 };
 function isPlainObject(value) {
     return value !== null && typeof value === "object" && !Array.isArray(value);

package/dist/lib/lanes/watcher.d.ts

@@ -143,6 +143,8 @@ export declare function notifySlotFreed(_input: {
 export interface ApplyTerminalOpts {
     envelope?: LaneEnvelope | null;
     envelopeRef?: string;
+    envelopeValidationErrors?: string[];
+    questionIndexes?: readonly number[];
     at?: string;
     quota?: QuotaSignal | null;
     /** Written to `terminal_details` with the model and account the run used. */
@@ -225,7 +227,7 @@ export interface WatcherLoopDeps {
  */
 export declare function emitMeshSoft(event: string, logDecision: (record: Record<string, unknown>) => void, at: string, fn: (signal: AbortSignal) => Promise<void>, timeoutMs?: number): Promise<boolean>;
 /** Question ids an ask envelope raised, as `raiseEnvelopeQuestions` mints them. */
-export declare function envelopeQuestionIds(laneId: string, envelopeRef: string, envelope: LaneEnvelope): string[];
+export declare function envelopeQuestionIds(laneId: string, envelopeRef: string, envelope: LaneEnvelope, questionIndexes?: readonly number[]): string[];
 export declare function runWatcherLoop(hqRoot: string, laneId: string, deps?: WatcherLoopDeps): Promise<void>;
 export type EnvelopeRepairDecision = {
     action: "spawned";