@indigoai-us/hq-cli 5.301.0 → 5.303.0

package/dist/lib/lanes/reconcile.js

@@ -18,14 +18,14 @@ import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./err
 import { loadWorkerProfile } from "./profile.js";
 import { terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
 import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
-import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
+import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
 import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
 import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
 import { readLanesRegistryFlag } from "./registry-flag.js";
 import { resolveLaneCompany } from "./lane-company.js";
 import { appendDropboxMessage } from "./dropbox.js";
 import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
-import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
+import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
 import { providerSupportsSessionResume } from "./delivery.js";
 import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, readQuotaSignal, recordAccountQuota, } from "./quota.js";
 import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
@@ -33,6 +33,7 @@ import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingEx
 import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
 import { activeLaneStories, laneStories, terminalEnvelopeStoryStatus } from "./stories.js";
 import { applyStorySwitchEnvelope } from "./story-switch.js";
+import { readProjectStoryStates } from "./story-ledger.js";
 import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
 import { PROVIDER_REJECTED_REASON, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
 /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
@@ -46,6 +47,8 @@ export const RECONCILE_LOCK_TIMEOUT_MS = 5_000;
 /** A pickup probe that has not cleared its marker within this window is abandoned. */
 export const PICKUP_VALIDATION_MAX_AGE_MS = 15 * 60 * 1000;
 export const PICKUP_VALIDATION_TIMEOUT_REASON = "pickup_validation_timeout";
+export const CLOSE_INTERRUPTED_DONE_STORY_WORKERS_FLAG = "lanes.close-interrupted-done-story-workers";
+const INTERRUPTED_DONE_STORY_MIN_AGE_MS = 24 * 60 * 60 * 1000;
 export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
 export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
 function spawnDepsForHost(deps) {
@@ -324,6 +327,129 @@ function casLaneIf(hqRoot, laneId, shouldWrite, updater, dryRun) {
     }
     throw new LanesError("version_conflict", `Exhausted CAS retries for lane ${laneId}`, { lane_id: laneId });
 }
+function laneStoryLedgerIsClosed(hqRoot, lane) {
+    const stories = laneStories(lane);
+    if (stories.length === 0)
+        return { closed: false, states: [] };
+    const states = [];
+    for (const projectId of new Set(stories.map((story) => story.project_id))) {
+        const projectStories = stories.filter((story) => story.project_id === projectId);
+        const ledgerStates = readProjectStoryStates(hqRoot, projectId, projectStories.map((story) => story.story_id), lane.company?.slug);
+        for (const story of projectStories) {
+            const state = ledgerStates.get(story.story_id);
+            if (state === undefined)
+                return { closed: false, states: [...states, "missing"] };
+            states.push(state);
+        }
+    }
+    return { closed: states.length > 0 && states.every((state) => state === "done" || state === "dropped"), states };
+}
+async function closeInterruptedDoneStoryWorkers(hqRoot, lanes, questions, summary, deps, now, enabledByCompany) {
+    if (deps.dryRun)
+        return;
+    const nowMs = now.getTime();
+    const at = now.toISOString();
+    for (const lane of lanes) {
+        if (lane.depth < 1 || lane.state !== "interrupted" || lane.keep_alive === true ||
+            !laneRunsHere(lane, deps.host) || !lane.interrupted_at ||
+            !Number.isFinite(Date.parse(lane.interrupted_at)) ||
+            nowMs - Date.parse(lane.interrupted_at) < INTERRUPTED_DONE_STORY_MIN_AGE_MS ||
+            hasPendingKeepAliveWork(hqRoot, lane, questions))
+            continue;
+        let company;
+        try {
+            company = resolveLaneCompany(hqRoot, lane).company;
+        }
+        catch (err) {
+            markDirty(summary, { kind: "lane", id: lane.lane_id, why: `story close company lookup failed: ${err instanceof Error ? err.message : String(err)}` });
+            continue;
+        }
+        if (!company || enabledByCompany.get(keepAliveCompanyKey(company)) !== true)
+            continue;
+        let ledger;
+        try {
+            ledger = laneStoryLedgerIsClosed(hqRoot, lane);
+        }
+        catch (err) {
+            markDirty(summary, { kind: "lane", id: lane.lane_id, why: `story close ledger read failed: ${err instanceof Error ? err.message : String(err)}` });
+            continue;
+        }
+        if (!ledger.closed)
+            continue;
+        try {
+            const result = casLaneIf(hqRoot, lane.lane_id, (current) => {
+                const currentCompany = resolveLaneCompany(hqRoot, current).company;
+                if (current.state !== "interrupted" || current.depth < 1 || current.keep_alive === true ||
+                    !currentCompany || enabledByCompany.get(keepAliveCompanyKey(currentCompany)) !== true ||
+                    !laneRunsHere(current, deps.host) || !current.interrupted_at ||
+                    !Number.isFinite(Date.parse(current.interrupted_at)) ||
+                    nowMs - Date.parse(current.interrupted_at) < INTERRUPTED_DONE_STORY_MIN_AGE_MS ||
+                    hasPendingKeepAliveWork(hqRoot, current, questions))
+                    return false;
+                return laneStoryLedgerIsClosed(hqRoot, current).closed;
+            }, (current) => ({
+                ...current,
+                state: "killed",
+                queued_reason: undefined,
+                queued_details: undefined,
+                updated_at: at,
+                terminal_reason: "story_closed",
+                terminal_details: { ...(current.terminal_details ?? {}), operator_reason: "story_closed" },
+                stop_history: [...(current.stop_history ?? []), { at, reason: "story_closed" }],
+            }), false);
+            if (!result.changed)
+                continue;
+            stopLaneProcesses(hqRoot, lane.lane_id, {
+                killProcess: deps.spawnLaneDeps?.killProcess,
+                isAlive: deps.isAlive,
+            });
+            for (const question of questions) {
+                if (question.origin.id !== lane.lane_id || question.status !== "pending")
+                    continue;
+                casQuestionIf(hqRoot, question.question_id, (current) => current.status === "pending", (current) => ({
+                    ...current,
+                    status: "cancelled",
+                    routing_history: appendCappedHistory(current.routing_history, {
+                        hop: current.hop,
+                        audience: { kind: "user" },
+                        at,
+                        reason: "cancelled",
+                    }),
+                }), false);
+            }
+            appendWatcherEvent(watcherEventsPath(hqRoot, lane.lane_id), {
+                event: "story_closed",
+                at,
+                lane_id: lane.lane_id,
+                story_state: [...new Set(ledger.states)].join(","),
+                terminal_reason: "story_closed",
+            });
+            const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
+            if (mesh) {
+                const timeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
+                let timer;
+                try {
+                    await Promise.race([
+                        emitLaneTerminalStateEvent(mesh, result.record, "killed", "story_closed"),
+                        new Promise((_resolve, reject) => {
+                            timer = setTimeout(() => reject(new Error(`terminal mesh emit timed out after ${timeoutMs}ms`)), timeoutMs);
+                        }),
+                    ]);
+                }
+                catch (err) {
+                    markDirty(summary, { kind: "lane", id: lane.lane_id, why: `story close mesh emit failed: ${err instanceof Error ? err.message : String(err)}` });
+                }
+                finally {
+                    if (timer)
+                        clearTimeout(timer);
+                }
+            }
+        }
+        catch (err) {
+            markDirty(summary, { kind: "lane", id: lane.lane_id, why: `story close failed: ${err instanceof Error ? err.message : String(err)}` });
+        }
+    }
+}
 function writeQueueReason(hqRoot, lane, reason, dryRun) {
     if (lane.queued_reason === reason)
         return;
@@ -929,7 +1055,39 @@ export async function runPassBody(hqRoot, deps) {
         }
     }
     const readRegistryFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
+    const storyCloseCompanies = new Map();
+    for (const lane of lanes) {
+        if (lane.depth < 1 || lane.state !== "interrupted" || lane.keep_alive === true ||
+            !laneRunsHere(lane, deps.host) || !lane.interrupted_at ||
+            !Number.isFinite(Date.parse(lane.interrupted_at)) ||
+            nowMs - Date.parse(lane.interrupted_at) < INTERRUPTED_DONE_STORY_MIN_AGE_MS ||
+            hasPendingKeepAliveWork(hqRoot, lane, questions) || laneStories(lane).length === 0)
+            continue;
+        try {
+            const company = resolveLaneCompany(hqRoot, lane).company;
+            if (company)
+                storyCloseCompanies.set(keepAliveCompanyKey(company), company);
+        }
+        catch (err) {
+            markDirty(summary, { kind: "lane", id: lane.lane_id, why: `story close company lookup failed: ${err instanceof Error ? err.message : String(err)}` });
+        }
+    }
+    const storyCloseEnabledByCompany = new Map();
+    const env = deps.env ?? process.env;
+    await Promise.all([...storyCloseCompanies.entries()].map(async ([key, company]) => {
+        try {
+            storyCloseEnabledByCompany.set(key, await readRegistryFlag(CLOSE_INTERRUPTED_DONE_STORY_WORKERS_FLAG, {
+                company: company ?? undefined,
+                env,
+            }));
+        }
+        catch (err) {
+            storyCloseEnabledByCompany.set(key, false);
+            markDirty(summary, { kind: "lane", why: `story close flag read failed for ${company?.slug ?? "unresolved company"}: ${err instanceof Error ? err.message : String(err)}` });
+        }
+    }));
     const keepAliveFlagCompanies = new Map();
+    const nextCheckAtFlagCompanyKeys = new Set();
     for (const lane of lanes) {
         if (!(lane.state === "done" || lane.state === "interrupted" || lane.state === "blocked" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
             (lane.keep_alive !== true && lane.worker_profile === undefined) ||
@@ -940,6 +1098,7 @@ export async function runPassBody(hqRoot, deps) {
                 !providerSupportsSessionResume(lane.provider) ||
                 !(lane.provider_session_id ?? lane.last_resume?.session_id))))
             continue;
+        let hasNextCheckAt = false;
         if (lane.state === "done") {
             const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
             if (!envelope || envelope.decision !== "done")
@@ -947,17 +1106,20 @@ export async function runPassBody(hqRoot, deps) {
             const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, lane, questions);
             if (doneKeepAliveVerdict(lane, envelope, hasQueuedWork) !== "eligible")
                 continue;
+            hasNextCheckAt = envelope.next_check_at !== undefined;
         }
         const company = resolveLaneCompany(hqRoot, lane).company;
         if (company === null)
             continue;
         const key = keepAliveCompanyKey(company);
         keepAliveFlagCompanies.set(key, company);
+        if (hasNextCheckAt)
+            nextCheckAtFlagCompanyKeys.add(key);
     }
     const keepAliveFlagByCompany = new Map();
+    const nextCheckAtFlagByCompany = new Map();
     const blockedResumeFlagByCompany = new Map();
     const supersessionLineageAwareByCompany = new Map();
-    const env = deps.env ?? process.env;
     await Promise.all([...keepAliveFlagCompanies.entries()].map(async ([key, company]) => {
         const options = {
             company: company ?? undefined,
@@ -967,14 +1129,18 @@ export async function runPassBody(hqRoot, deps) {
             lane.keep_alive === true &&
             laneRunsHere(lane, startDeps.host) &&
             keepAliveCompanyKey(resolveLaneCompany(hqRoot, lane).company) === key);
-        const [keepAliveEnabled, lineageAware, blockedResumeEnabled] = await Promise.all([
+        const [keepAliveEnabled, nextCheckAtEnabled, lineageAware, blockedResumeEnabled] = await Promise.all([
             readRegistryFlag(LEAD_LANE_KEEP_ALIVE_FLAG, options),
+            nextCheckAtFlagCompanyKeys.has(key)
+                ? readRegistryFlag(LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_FLAG, options)
+                : Promise.resolve(false),
             readRegistryFlag(LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, options),
             hasBlockedKeepAliveLane
                 ? readRegistryFlag(LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, options)
                 : Promise.resolve(false),
         ]);
         keepAliveFlagByCompany.set(key, keepAliveEnabled);
+        nextCheckAtFlagByCompany.set(key, nextCheckAtEnabled);
         blockedResumeFlagByCompany.set(key, blockedResumeEnabled);
         supersessionLineageAwareByCompany.set(key, lineageAware);
     }));
@@ -995,7 +1161,8 @@ export async function runPassBody(hqRoot, deps) {
     await withAllLaneAdmissionLocks(hqRoot, async () => {
         const freshLanes = loadLaneRecords(hqRoot, summary);
         const freshQuestions = loadQuestionRecords(hqRoot, summary);
-        await resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, admissionDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany, blockedResumeFlagByCompany);
+        await closeInterruptedDoneStoryWorkers(hqRoot, freshLanes, freshQuestions, summary, admissionDeps, now, storyCloseEnabledByCompany);
+        await resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, admissionDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany, blockedResumeFlagByCompany, nextCheckAtFlagByCompany);
         const drainLanes = loadLaneRecords(hqRoot, summary);
         await admitQueued(hqRoot, drainLanes, summary, admissionDeps, now);
         await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, admissionDeps);
@@ -1741,25 +1908,119 @@ function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
         summary.keep_alive_blocked.push({ id: lane.lane_id, reason });
         process.stderr.write(`hq lanes reconcile: keep-alive blocked for lane ${lane.lane_id} (${reason}); ${keepAliveSkipGuidance(lane.lane_id, reason)}\n`);
     }
-    if (deps.dryRun || lane.keep_alive_blocked_reason === reason)
+    if (deps.dryRun)
         return;
-    try {
-        casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version && current.state === lane.state, (current) => ({ ...current, keep_alive_blocked_reason: reason }), false);
+    let laneVersion = lane.version;
+    if (lane.keep_alive_blocked_reason !== reason) {
+        try {
+            const update = casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version && current.state === lane.state, (current) => ({ ...current, keep_alive_blocked_reason: reason }), false);
+            laneVersion = update.record.version;
+        }
+        catch (err) {
+            markDirty(summary, {
+                kind: "lane",
+                id: lane.lane_id,
+                why: `keep-alive skip state write failed: ${err instanceof Error ? err.message : String(err)}`,
+            });
+        }
     }
-    catch (err) {
-        markDirty(summary, {
-            kind: "lane",
-            id: lane.lane_id,
-            why: `keep-alive skip state write failed: ${err instanceof Error ? err.message : String(err)}`,
-        });
+    if (lane.keep_alive === true) {
+        recordKeepAliveSkipEvent(hqRoot, lane, reason, laneVersion, summary, deps, detail);
     }
 }
-function recordDoneKeepAlivePolicySkip(summary, lane, verdict) {
+function recordDoneKeepAlivePolicySkip(hqRoot, summary, lane, verdict, deps) {
     summary.skipped.push({
         kind: "lane",
         id: lane.lane_id,
         why: `keep_alive_${verdict}`,
     });
+    if (lane.keep_alive === true && !deps.dryRun) {
+        recordKeepAliveSkipEvent(hqRoot, lane, verdict, lane.version, summary, deps);
+    }
+}
+function recordKeepAliveSkipEvent(hqRoot, lane, reason, laneVersion, summary, deps, detail) {
+    if (lane.keep_alive !== true || deps.dryRun)
+        return;
+    const eventsPath = watcherEventsPath(hqRoot, lane.lane_id);
+    try {
+        let lastSkip;
+        try {
+            const scanBackwards = (body, targetReason, targetVersion) => {
+                const lines = body.split(/\r?\n/);
+                for (let index = lines.length - 1; index >= 0; index -= 1) {
+                    if (!lines[index])
+                        continue;
+                    try {
+                        const value = JSON.parse(lines[index]);
+                        if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+                            const event = value;
+                            if (event.event === "keep_alive_skip" &&
+                                (targetReason === undefined || (event.reason === targetReason && event.lane_version === targetVersion))) {
+                                return event;
+                            }
+                        }
+                    }
+                    catch {
+                        // A partial concurrent append must not prevent future skip events.
+                    }
+                }
+                return undefined;
+            };
+            const maxTailBytes = 64 * 1024;
+            const fd = fs.openSync(eventsPath, "r");
+            let fileSize;
+            let tail;
+            let tailOffset;
+            try {
+                fileSize = fs.fstatSync(fd).size;
+                tailOffset = Math.max(0, fileSize - maxTailBytes);
+                const buffer = Buffer.alloc(fileSize - tailOffset);
+                let bytesRead = 0;
+                while (bytesRead < buffer.length) {
+                    const count = fs.readSync(fd, buffer, bytesRead, buffer.length - bytesRead, tailOffset + bytesRead);
+                    if (count === 0)
+                        break;
+                    bytesRead += count;
+                }
+                tail = buffer.subarray(0, bytesRead).toString("utf8");
+            }
+            finally {
+                fs.closeSync(fd);
+            }
+            // When the bounded read starts mid-line, ignore that incomplete prefix.
+            if (tailOffset > 0) {
+                const firstNewline = tail.indexOf("\n");
+                tail = firstNewline < 0 ? "" : tail.slice(firstNewline + 1);
+            }
+            lastSkip = scanBackwards(tail, reason === "next_check_at" ? reason : undefined, laneVersion);
+            // For small logs the tail is not a complete record boundary if a single
+            // record exceeded the bound, so allow one full-file scan as a fallback.
+            if (!lastSkip && fileSize < maxTailBytes) {
+                lastSkip = scanBackwards(fs.readFileSync(eventsPath, "utf8"), reason === "next_check_at" ? reason : undefined, laneVersion);
+            }
+        }
+        catch (err) {
+            if (err.code !== "ENOENT")
+                throw err;
+        }
+        if (lastSkip?.reason === reason && lastSkip.lane_version === laneVersion)
+            return;
+        recordKeepAliveEvent(hqRoot, lane.lane_id, summary, {
+            event: "keep_alive_skip",
+            at: deps.now().toISOString(),
+            reason,
+            ...(detail === undefined ? {} : { detail }),
+            lane_version: laneVersion,
+            lane_state: lane.state,
+        });
+    }
+    catch (err) {
+        markDirty(summary, {
+            kind: "lane",
+            id: lane.lane_id,
+            why: `keep-alive skip event dedupe failed: ${err instanceof Error ? err.message : String(err)}`,
+        });
+    }
 }
 function readLatestInterruptedClassificationAt(hqRoot, lane) {
     const logPath = watcherOutLogPath(hqRoot, lane.lane_id);
@@ -1774,10 +2035,15 @@ function readLatestInterruptedClassificationAt(hqRoot, lane) {
     }
     let latestClassification;
     for (const line of body.split(/\r?\n/).filter(Boolean)) {
-        const parsed = JSON.parse(line);
-        if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
-            throw new Error("watcher log contains a non-object decision record");
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch {
+            continue;
         }
+        if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
+            continue;
         const record = parsed;
         if (record.event === "classification")
             latestClassification = record;
@@ -1975,7 +2241,7 @@ function appendBlockedKeepAliveHistory(hqRoot, lane, terminalAt, at, reason, dep
         }),
     }), false);
 }
-async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCompany, supersessionLineageAwareByCompany, blockedResumeFlagByCompany) {
+async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCompany, supersessionLineageAwareByCompany, blockedResumeFlagByCompany, nextCheckAtFlagByCompany) {
     const nowMs = now.getTime();
     const nowIso = now.toISOString();
     const config = loadLanesConfig(hqRoot);
@@ -2022,16 +2288,26 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
             const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
             if (!envelope || envelope.decision !== "done")
                 continue;
-            const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, lane, questions);
-            const verdict = doneKeepAliveVerdict(lane, envelope, hasQueuedWork);
+            const nextCheckAtEnabled = nextCheckAtFlagByCompany.get(keepAliveCompanyKey(companyForLane(lane))) === true;
+            const hasPendingWork = lane.keep_alive_standing === true || nextCheckAtEnabled
+                ? hasPendingKeepAliveWork(hqRoot, lane, questions)
+                : false;
+            const hasQueuedWork = lane.keep_alive_standing === true && hasPendingWork;
+            const verdict = doneKeepAliveVerdict(lane, envelope, hasQueuedWork, {
+                nextCheckAtEnabled,
+                hasPendingWork,
+                nowMs,
+                nextCheckAnchorMs: Date.parse(lane.last_envelope?.at ?? lane.updated_at),
+            });
             if (verdict !== "eligible") {
-                if (verdict !== null)
-                    recordDoneKeepAlivePolicySkip(summary, lane, verdict);
+                let skipLane = lane;
                 if (envelope.story_complete === true && lane.keep_alive_standing !== true && hasKeepAliveProgress(lane)) {
-                    casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
+                    skipLane = casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
                         current.state === "done" &&
-                        current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
+                        current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun).record;
                 }
+                if (verdict !== null)
+                    recordDoneKeepAlivePolicySkip(hqRoot, summary, skipLane, verdict, deps);
                 continue;
             }
         }
@@ -2043,6 +2319,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                     id: lane.lane_id,
                     why: "keep_alive_company_unresolved",
                 });
+                recordKeepAliveSkipEvent(hqRoot, lane, "company_unresolved", lane.version, summary, deps);
             }
             continue;
         }
@@ -2054,6 +2331,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                     id: lane.lane_id,
                     why: "keep_alive_flag_off",
                 });
+                recordKeepAliveSkipEvent(hqRoot, lane, "keep_alive_flag_off", lane.version, summary, deps);
             }
             continue;
         }
@@ -2134,14 +2412,24 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
             const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
             if (!latest || latest.envelope.decision !== "done")
                 continue;
-            const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, lane, questions);
-            const verdict = doneKeepAliveVerdict(lane, latest.envelope, hasQueuedWork);
+            const nextCheckAtEnabled = nextCheckAtFlagByCompany.get(keepAliveCompanyKey(companyForLane(lane))) === true;
+            const hasPendingWork = lane.keep_alive_standing === true || nextCheckAtEnabled
+                ? hasPendingKeepAliveWork(hqRoot, lane, questions)
+                : false;
+            const hasQueuedWork = lane.keep_alive_standing === true && hasPendingWork;
+            const verdict = doneKeepAliveVerdict(lane, latest.envelope, hasQueuedWork, {
+                nextCheckAtEnabled,
+                hasPendingWork,
+                nowMs,
+                nextCheckAnchorMs: Date.parse(lane.last_envelope?.at ?? lane.updated_at),
+            });
             if (verdict !== "eligible") {
-                if (verdict !== null)
-                    recordDoneKeepAlivePolicySkip(summary, lane, verdict);
+                let skipLane = lane;
                 if (latest.envelope.story_complete === true && lane.keep_alive_standing !== true && hasKeepAliveProgress(lane)) {
-                    casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version && current.state === "done", (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
+                    skipLane = casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version && current.state === "done", (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun).record;
                 }
+                if (verdict !== null)
+                    recordDoneKeepAlivePolicySkip(hqRoot, summary, skipLane, verdict, deps);
                 continue;
             }
             try {
@@ -2216,17 +2504,27 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
             const currentEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id);
             if (!currentEnvelope || currentEnvelope.envelope.decision !== "done")
                 continue;
-            const hasQueuedWork = current.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, current, questions);
-            const verdict = doneKeepAliveVerdict(current, currentEnvelope.envelope, hasQueuedWork);
+            const nextCheckAtEnabled = nextCheckAtFlagByCompany.get(keepAliveCompanyKey(companyForLane(current))) === true;
+            const hasPendingWork = current.keep_alive_standing === true || nextCheckAtEnabled
+                ? hasPendingKeepAliveWork(hqRoot, current, questions)
+                : false;
+            const hasQueuedWork = current.keep_alive_standing === true && hasPendingWork;
+            const verdict = doneKeepAliveVerdict(current, currentEnvelope.envelope, hasQueuedWork, {
+                nextCheckAtEnabled,
+                hasPendingWork,
+                nowMs,
+                nextCheckAnchorMs: Date.parse(current.last_envelope?.at ?? current.updated_at),
+            });
             if (verdict !== "eligible") {
-                if (verdict !== null)
-                    recordDoneKeepAlivePolicySkip(summary, current, verdict);
+                let skipLane = current;
                 if (currentEnvelope.envelope.story_complete === true && current.keep_alive_standing !== true && hasKeepAliveProgress(current)) {
-                    casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
+                    skipLane = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
                         latestLane.state === "done" &&
                         latestLane.keep_alive === true &&
-                        latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
+                        latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun).record;
                 }
+                if (verdict !== null)
+                    recordDoneKeepAlivePolicySkip(hqRoot, summary, skipLane, verdict, deps);
                 continue;
             }
             try {
@@ -2272,6 +2570,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                 id: current.lane_id,
                 why: "keep_alive_company_unresolved",
             });
+            recordKeepAliveSkipEvent(hqRoot, current, "company_unresolved", current.version, summary, deps);
             continue;
         }
         const currentCompanyKey = keepAliveCompanyKey(currentCompany);
@@ -2281,6 +2580,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                 id: current.lane_id,
                 why: "keep_alive_flag_off",
             });
+            recordKeepAliveSkipEvent(hqRoot, current, "keep_alive_flag_off", current.version, summary, deps);
             continue;
         }
         const currentExclusion = keepAliveExclusion(current, currentLanes, retiredLaneIds, supersessionLineageAwareByCompany.get(currentCompanyKey) === true, companyForLane);
@@ -2342,12 +2642,15 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
             ? Math.max(attemptEligibleMs, storedNextMs)
             : attemptEligibleMs;
         if (nowMs < nextEligibleMs) {
+            let skipLane = current;
             if (resetByHuman && hasKeepAliveProgress(current)) {
-                casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
+                const clearedProgress = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
                     latestLane.state === current.state &&
                     latestLane.keep_alive === true &&
                     latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
+                skipLane = clearedProgress.record;
             }
+            recordKeepAliveSkipEvent(hqRoot, skipLane, "backoff", skipLane.version, summary, deps, new Date(nextEligibleMs).toISOString());
             continue;
         }
         const nextEligibleAt = new Date(nowMs + leadLaneKeepAliveBackoffMs(attempt + 1)).toISOString();
@@ -2362,6 +2665,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                 id: current.lane_id,
                 why: `lane_queue_full: ${current.provider} queue has ${queuedCount}/${maxQueued} lanes`,
             });
+            recordKeepAliveSkipEvent(hqRoot, current, "lane_queue_full", current.version, summary, deps, `${current.provider} queue has ${queuedCount}/${maxQueued} lanes`);
             continue;
         }
         const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
@@ -2399,6 +2703,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                     id: current.lane_id,
                     why: `keep_alive_hire_admission: ${grantPreflight.error}: ${grantPreflight.message}`,
                 });
+                recordKeepAliveSkipEvent(hqRoot, current, "hire_admission", current.version, summary, deps, `${grantPreflight.error}: ${grantPreflight.message}`);
                 continue;
             }
         }
@@ -2481,8 +2786,22 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                 const envelope = readLatestEnvelopeRecord(hqRoot, latest.lane_id);
                 if (!envelope || envelope.envelope.decision !== "done")
                     continue;
-                if (doneKeepAliveVerdict(latest, envelope.envelope, latest.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, latest, questions)) !== "eligible")
+                const nextCheckAtEnabled = nextCheckAtFlagByCompany.get(keepAliveCompanyKey(companyForLane(latest))) === true;
+                const hasPendingWork = latest.keep_alive_standing === true || nextCheckAtEnabled
+                    ? hasPendingKeepAliveWork(hqRoot, latest, questions)
+                    : false;
+                const finalDoneVerdict = doneKeepAliveVerdict(latest, envelope.envelope, latest.keep_alive_standing === true && hasPendingWork, {
+                    nextCheckAtEnabled,
+                    hasPendingWork,
+                    nowMs,
+                    nextCheckAnchorMs: Date.parse(latest.last_envelope?.at ?? latest.updated_at),
+                });
+                if (finalDoneVerdict !== "eligible") {
+                    if (finalDoneVerdict === "next_check_at") {
+                        recordDoneKeepAlivePolicySkip(hqRoot, summary, latest, finalDoneVerdict, deps);
+                    }
                     continue;
+                }
                 try {
                     latestEligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, latest, envelope.ref);
                 }
@@ -2514,6 +2833,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                 why: `keep_alive_account_selection_failed: ${accountSelectionFailure}`,
             });
             if (!deps.dryRun) {
+                recordKeepAliveSkipEvent(hqRoot, current, "account_selection_failed", current.version, summary, deps, accountSelectionFailure);
                 recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
                     event: "keep_alive_account_selection_failed",
                     at: nowIso,

package/dist/lib/lanes/spawn.d.ts

@@ -48,6 +48,7 @@ export interface ProviderProfileLaunch {
 export type DetachedStdio = ["ignore", number, number];
 export interface DetachedSpawnOptions {
     detached: true;
+    windowsHide: true;
     stdio: DetachedStdio;
     env: NodeJS.ProcessEnv;
     cwd: string;