@indigoai-us/hq-cli 5.281.0 → 5.283.0

package/dist/lib/lanes/outpost-snapshot-cache.d.ts

@@ -0,0 +1,26 @@
+export declare const OUTPOST_SNAPSHOT_CACHE_TTL_MS = 3000;
+export declare const OUTPOST_SNAPSHOT_CACHE_STALE_LOCK_MS = 45000;
+export interface OutpostSnapshotCachePolicy {
+    ttlMs?: number;
+    staleLockMs?: number;
+    lockWaitMs?: number;
+    pollMs?: number;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+    acquireLock?: (lockPath: string, token: string, nowMs: number) => void;
+}
+/** Deduplicates identical remote snapshot queries within one command invocation. */
+export declare class OutpostSnapshotInvocationMemo {
+    private readonly requests;
+    getOrCreate<T>(outpostId: string, query: unknown, fetch: () => Promise<T>): Promise<T>;
+}
+export declare function outpostLaneListSnapshotKey(outpostId: string, normalizedQuery: unknown): string;
+export declare function outpostSnapshotCachePath(hqRoot: string, key: string): string;
+export declare function getOrFetchOutpostSnapshot<T>(input: {
+    hqRoot: string;
+    key: string;
+    isValue: (value: unknown) => value is T;
+    fetch: () => Promise<T>;
+    policy?: OutpostSnapshotCachePolicy;
+}): Promise<T>;
+//# sourceMappingURL=outpost-snapshot-cache.d.ts.map

package/dist/lib/lanes/outpost-snapshot-cache.js

@@ -0,0 +1,215 @@
+import { createHash, randomBytes } from "node:crypto";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { LanesError } from "./errors.js";
+export const OUTPOST_SNAPSHOT_CACHE_TTL_MS = 3_000;
+export const OUTPOST_SNAPSHOT_CACHE_STALE_LOCK_MS = 45_000;
+const CACHE_RELATIVE_PATH = path.join("workspace", "lanes-runs", ".cache", "outpost-snapshots");
+const CACHE_SCHEMA_VERSION = "hq-lanes-outpost-snapshot-cache.v1";
+const LOCK_POLL_MS = 20;
+const LOCK_WAIT_MS = 60_000;
+/** Deduplicates identical remote snapshot queries within one command invocation. */
+export class OutpostSnapshotInvocationMemo {
+    requests = new Map();
+    getOrCreate(outpostId, query, fetch) {
+        const key = JSON.stringify([outpostId, query]);
+        const existing = this.requests.get(key);
+        if (existing)
+            return existing;
+        const request = fetch();
+        this.requests.set(key, request);
+        return request;
+    }
+}
+export function outpostLaneListSnapshotKey(outpostId, normalizedQuery) {
+    return JSON.stringify(["lane-list", outpostId, normalizedQuery]);
+}
+export function outpostSnapshotCachePath(hqRoot, key) {
+    const digest = createKeyDigest(key);
+    return path.join(hqRoot, CACHE_RELATIVE_PATH, `${digest}.json`);
+}
+export async function getOrFetchOutpostSnapshot(input) {
+    const policy = input.policy ?? {};
+    const now = policy.now ?? Date.now;
+    const ttlMs = policy.ttlMs ?? OUTPOST_SNAPSHOT_CACHE_TTL_MS;
+    const staleLockMs = policy.staleLockMs ?? OUTPOST_SNAPSHOT_CACHE_STALE_LOCK_MS;
+    const lockWaitMs = policy.lockWaitMs ?? LOCK_WAIT_MS;
+    const pollMs = policy.pollMs ?? LOCK_POLL_MS;
+    const sleep = policy.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const filePath = outpostSnapshotCachePath(input.hqRoot, input.key);
+    const cached = readFreshCache(filePath, input.key, now(), ttlMs, input.isValue);
+    if (cached !== undefined)
+        return cached;
+    const lockPath = `${filePath}.lock`;
+    const token = `${process.pid}:${randomBytes(12).toString("hex")}`;
+    const startedAt = now();
+    const acquireLock = policy.acquireLock ?? createCacheLock;
+    while (true) {
+        try {
+            acquireLock(lockPath, token, now());
+            break;
+        }
+        catch (err) {
+            const code = err.code;
+            if (code !== "EEXIST") {
+                process.stderr.write(`${JSON.stringify({
+                    event: "lanes.outpost_snapshot_cache_lock_unavailable",
+                    error: typeof code === "string" ? code : "UnknownError",
+                    fallback: "uncached",
+                })}\n`);
+                const value = await input.fetch();
+                if (!input.isValue(value)) {
+                    throw new LanesError("outpost_snapshot_cache_value_invalid", "Outpost snapshot fetch returned an invalid value", { key: input.key });
+                }
+                return value;
+            }
+            if (recoverStaleLock(lockPath, staleLockMs, now()))
+                continue;
+            const afterWait = readFreshCache(filePath, input.key, now(), ttlMs, input.isValue);
+            if (afterWait !== undefined)
+                return afterWait;
+            if (now() - startedAt >= lockWaitMs) {
+                throw new LanesError("outpost_snapshot_cache_lock_timeout", "Timed out waiting for an Outpost snapshot refresh", { path: lockPath });
+            }
+            await sleep(pollMs);
+        }
+    }
+    try {
+        const afterLock = readFreshCache(filePath, input.key, now(), ttlMs, input.isValue);
+        if (afterLock !== undefined)
+            return afterLock;
+        const value = await input.fetch();
+        if (!input.isValue(value)) {
+            throw new LanesError("outpost_snapshot_cache_value_invalid", "Outpost snapshot fetch returned an invalid value", { key: input.key });
+        }
+        writeCache(filePath, input.key, value, now());
+        return value;
+    }
+    finally {
+        releaseLock(lockPath, token);
+    }
+}
+function createCacheLock(lockPath, token, nowMs) {
+    fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+    const lockFd = fs.openSync(lockPath, "wx", 0o600);
+    try {
+        fs.writeSync(lockFd, `${token}\n${nowMs}\n`);
+    }
+    catch (err) {
+        try {
+            fs.closeSync(lockFd);
+        }
+        catch (closeErr) {
+            process.stderr.write(`lanes snapshot cache: lock close failed: ${errorText(closeErr)}\n`);
+        }
+        releaseLock(lockPath, token);
+        throw err;
+    }
+    fs.closeSync(lockFd);
+}
+function createKeyDigest(key) {
+    // A lane-list query can contain lane ids. Keep raw ids out of cache filenames.
+    return createHash("sha256").update(key).digest("hex");
+}
+function readFreshCache(filePath, key, nowMs, ttlMs, isValue) {
+    let contents;
+    try {
+        contents = fs.readFileSync(filePath, "utf8");
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            return undefined;
+        process.stderr.write(`lanes snapshot cache: read failed: ${errorText(err)}\n`);
+        return undefined;
+    }
+    let raw;
+    try {
+        raw = JSON.parse(contents);
+    }
+    catch (err) {
+        process.stderr.write(`lanes snapshot cache: ignoring malformed cache: ${errorText(err)}\n`);
+        return undefined;
+    }
+    if (raw === null || typeof raw !== "object" || Array.isArray(raw))
+        return undefined;
+    const entry = raw;
+    if (Object.keys(entry).length !== 4 ||
+        entry.schema_version !== CACHE_SCHEMA_VERSION ||
+        entry.key !== key ||
+        !Number.isSafeInteger(entry.cached_at) ||
+        entry.cached_at > nowMs ||
+        nowMs - entry.cached_at >= ttlMs ||
+        !isValue(entry.value))
+        return undefined;
+    return entry.value;
+}
+function writeCache(filePath, key, value, nowMs) {
+    const temporaryPath = `${filePath}.${process.pid}.${randomBytes(8).toString("hex")}.tmp`;
+    try {
+        const fd = fs.openSync(temporaryPath, "wx", 0o600);
+        try {
+            fs.writeFileSync(fd, `${JSON.stringify({
+                schema_version: CACHE_SCHEMA_VERSION,
+                key,
+                cached_at: nowMs,
+                value,
+            })}\n`);
+            fs.fsyncSync(fd);
+        }
+        finally {
+            fs.closeSync(fd);
+        }
+        fs.renameSync(temporaryPath, filePath);
+    }
+    catch (err) {
+        try {
+            fs.unlinkSync(temporaryPath);
+        }
+        catch (cleanupErr) {
+            if (cleanupErr.code !== "ENOENT") {
+                process.stderr.write(`lanes snapshot cache: temp cleanup failed: ${errorText(cleanupErr)}\n`);
+            }
+        }
+        // The cache is an optimization; a successful, validated remote result stays usable.
+        process.stderr.write(`lanes snapshot cache: write failed: ${errorText(err)}\n`);
+    }
+}
+function recoverStaleLock(lockPath, staleLockMs, nowMs) {
+    let stat;
+    try {
+        stat = fs.statSync(lockPath);
+    }
+    catch (err) {
+        return err.code === "ENOENT";
+    }
+    if (nowMs - stat.mtimeMs <= staleLockMs)
+        return false;
+    const stalePath = `${lockPath}.stale.${process.pid}.${randomBytes(6).toString("hex")}`;
+    try {
+        fs.renameSync(lockPath, stalePath);
+        fs.unlinkSync(stalePath);
+        return true;
+    }
+    catch (err) {
+        const code = err.code;
+        if (code === "ENOENT")
+            return true;
+        throw new LanesError("outpost_snapshot_cache_lock_recovery_failed", `Could not recover a stale Outpost snapshot lock: ${errorText(err)}`, { path: lockPath });
+    }
+}
+function releaseLock(lockPath, token) {
+    try {
+        const contents = fs.readFileSync(lockPath, "utf8");
+        if (contents.split("\n", 1)[0] === token)
+            fs.unlinkSync(lockPath);
+    }
+    catch (err) {
+        if (err.code !== "ENOENT") {
+            process.stderr.write(`lanes snapshot cache: lock release failed: ${errorText(err)}\n`);
+        }
+    }
+}
+function errorText(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+//# sourceMappingURL=outpost-snapshot-cache.js.map

package/dist/lib/lanes/reconcile.d.ts

@@ -7,7 +7,7 @@
  */
 import { type LivenessDeps } from "./liveness.js";
 import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
-import { type ThrottleDeps } from "./throttle.js";
+import { type ThrottleDeps, type CpuBusyObservation } from "./throttle.js";
 import { readLanesRegistryFlag } from "./registry-flag.js";
 import { type RunawayProcessInfo } from "./runaway-tool-guard.js";
 import { type KeepAliveBlockedReason } from "./keep-alive.js";
@@ -78,6 +78,12 @@ export interface ReconcileDeps {
     readSessionSenior?: (sessionId: string) => SeniorRef;
     presenceConnected?: boolean;
     env?: NodeJS.ProcessEnv;
+    /** CPU sample taken before acquiring provider admission locks. */
+    cpuBusyFraction?: number | null;
+    /** CPU observation and cumulative counters taken before provider locks. */
+    cpuObservation?: CpuBusyObservation;
+    /** Skip admission this pass if candidates appeared after the pre-sample snapshot. */
+    skipAdmissionWithoutCpuSample?: boolean;
     /** Flag reader seam used to assert rollout reads happen before admission locks. */
     readRegistryFlag?: typeof readLanesRegistryFlag;
     /** Pre-read manager gate decisions, scoped to each lane for this pass. */

package/dist/lib/lanes/reconcile.js

@@ -20,7 +20,7 @@ import { terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./gr
 import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
 import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
 import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
-import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
+import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
 import { readLanesRegistryFlag } from "./registry-flag.js";
 import { resolveLaneCompany } from "./lane-company.js";
 import { appendDropboxMessage } from "./dropbox.js";
@@ -358,7 +358,7 @@ function casLaneIf(hqRoot, laneId, shouldWrite, updater, dryRun) {
 function writeQueueReason(hqRoot, lane, reason, dryRun) {
     if (lane.queued_reason === reason)
         return;
-    casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" && current.queued_reason !== reason, (current) => ({ ...current, queued_reason: reason, admission_reserved_at: undefined }), dryRun);
+    casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" && current.queued_reason !== reason, (current) => ({ ...current, queued_reason: reason, queued_details: undefined, admission_reserved_at: undefined }), dryRun);
 }
 function availableBalancedAccount(hqRoot, lane, occupying, config, host, profileMaxConcurrent, nowMs) {
     if (!lane.account || !lane.admission_accounts?.includes(lane.account))
@@ -482,6 +482,7 @@ function expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now) {
                     ...current,
                     state: "failed",
                     queued_reason: undefined,
+                    queued_details: undefined,
                     pickup_validation_pending: undefined,
                     pickup_validation_pending_since: undefined,
                     terminal_reason: PICKUP_VALIDATION_TIMEOUT_REASON,
@@ -998,14 +999,40 @@ export async function runPassBody(hqRoot, deps) {
         keepAliveFlagByCompany.set(key, keepAliveEnabled);
         supersessionLineageAwareByCompany.set(key, lineageAware);
     }));
+    const admissionThrottleDeps = startDeps.throttleDeps ?? defaultThrottleDeps(startDeps.env ?? process.env);
+    const hasAdmissionCandidates = lanes.some((lane) => laneRunsHere(lane, startDeps.host) &&
+        (lane.state === "queued" || lane.state === "awaiting_input")) || [...keepAliveFlagByCompany.values()].some(Boolean);
+    const priorThrottleState = readThrottleState(throttleStatePath(hqRoot), admissionThrottleDeps);
+    const priorThrottleSnapshot = priorThrottleState === "missing" || priorThrottleState === "unreadable"
+        ? undefined
+        : priorThrottleState.cpu_snapshot;
+    const cpuObservation = await sampleCpuBusyObservation(admissionThrottleDeps, priorThrottleSnapshot, { measureBusy: hasAdmissionCandidates });
+    const admissionDeps = {
+        ...startDeps,
+        cpuBusyFraction: cpuObservation.busy,
+        cpuObservation,
+        skipAdmissionWithoutCpuSample: !hasAdmissionCandidates,
+    };
     await withAllLaneAdmissionLocks(hqRoot, async () => {
         const freshLanes = loadLaneRecords(hqRoot, summary);
         const freshQuestions = loadQuestionRecords(hqRoot, summary);
-        await resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, startDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
+        await resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, admissionDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
         const drainLanes = loadLaneRecords(hqRoot, summary);
-        await admitQueued(hqRoot, drainLanes, summary, startDeps, now);
-        await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, startDeps);
+        await admitQueued(hqRoot, drainLanes, summary, admissionDeps, now);
+        await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, admissionDeps);
     });
+    if (!deps.dryRun && cpuObservation.snapshot !== undefined) {
+        const latestThrottleState = readThrottleState(throttleStatePath(hqRoot), admissionThrottleDeps);
+        if (latestThrottleState === "unreadable") {
+            markError(summary, new LanesError("throttle_state_unreadable", "Throttle state became unreadable while reconciling"));
+        }
+        else {
+            const currentThrottleState = latestThrottleState === "missing"
+                ? emptyThrottleState()
+                : latestThrottleState;
+            writeThrottleState(throttleStatePath(hqRoot), recordCpuBusyObservation(currentThrottleState, cpuObservation), admissionThrottleDeps);
+        }
+    }
     if (!deps.dryRun && summary.errors.length === 0) {
         try {
             writeHeartbeatFile(hqRoot, summary, nowIso);
@@ -1246,6 +1273,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                         ...current,
                         state: terminalClassification.state,
                         queued_reason: undefined,
+                        queued_details: undefined,
                         terminal_reason: terminalClassification.reason ?? "host_lost",
                         interrupted_at: terminalClassification.state === "interrupted" ? nowIso : undefined,
                         ...(providerRejection
@@ -1390,6 +1418,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                 ...current,
                 state: "interrupted",
                 queued_reason: undefined,
+                queued_details: undefined,
                 terminal_reason: "host_lost",
                 interrupted_at: nowIso,
             }), deps.dryRun);
@@ -1448,7 +1477,7 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
         laneRunsHere(lane, deps.host))
         .filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()))
         .sort(compareLaneQueueOrder);
-    if (localQueued.length === 0)
+    if (localQueued.length === 0 || deps.skipAdmissionWithoutCpuSample === true)
         return;
     const config = loadLanesConfig(hqRoot);
     const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
@@ -1459,7 +1488,9 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
         : loaded === "missing"
             ? emptyThrottleState()
             : loaded;
-    const busy = await resolveCpuBusyFraction(throttleDeps);
+    const busy = deps.cpuBusyFraction !== undefined
+        ? deps.cpuBusyFraction
+        : await resolveCpuBusyFraction(throttleDeps);
     const availableMemMb = readAvailableMemMb(throttleDeps);
     const nowMs = now.getTime();
     const decision = evaluateThrottle({
@@ -1476,7 +1507,6 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
         hasFreshAdmissionReservation(lane, now.getTime()) ||
         hasActivePendingResume(lane, now.getTime())) &&
         laneRunsHere(lane, deps.host));
-    const starvation = decision.admit && decision.reason === "starvation_bound";
     const spawned = [];
     if (!decision.admit) {
         for (const lane of localQueued) {
@@ -1543,8 +1573,6 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
                     reason: decision.reason,
                 });
                 spawned.push({ ...admittedLane, state: "running" });
-                if (starvation)
-                    break;
                 continue;
             }
             const started = spawnFn(hqRoot, admittedLane);
@@ -1564,10 +1592,38 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
                     : {}),
             });
             spawned.push(started);
-            if (starvation)
-                break;
         }
         catch (err) {
+            if (err instanceof LanesError && err.error === "worker_profile_hash_mismatch") {
+                const expectedHash = typeof err.details.expected_hash === "string"
+                    ? err.details.expected_hash
+                    : null;
+                const actualHash = typeof err.details.actual_hash === "string"
+                    ? err.details.actual_hash
+                    : undefined;
+                const workerProfile = typeof err.details.worker === "string"
+                    ? err.details.worker
+                    : lane.worker_profile;
+                if (actualHash !== undefined && workerProfile !== undefined) {
+                    casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued", (current) => ({
+                        ...current,
+                        queued_reason: "worker_profile_changed",
+                        queued_details: {
+                            reason: "worker_profile_changed",
+                            expected_hash: expectedHash,
+                            actual_hash: actualHash,
+                            recovery_command: `hq lanes resume ${lane.lane_id} --worker ${workerProfile}`,
+                        },
+                        admission_reserved_at: undefined,
+                    }), false);
+                    summary.skipped.push({
+                        kind: "lane",
+                        id: lane.lane_id,
+                        why: "worker_profile_changed",
+                    });
+                    continue;
+                }
+            }
             terminalizeModelUnsupported(hqRoot, lane, err);
             markDirty(summary, {
                 kind: "lane",
@@ -2273,6 +2329,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
             ...latestLane,
             state: "queued",
             queued_reason: undefined,
+            queued_details: undefined,
             queued_at: nowIso,
             admission_reserved_at: undefined,
             terminal_reason: undefined,
@@ -2348,7 +2405,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
     const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" &&
         laneRunsHere(lane, deps.host) &&
         (onlyLaneIds === undefined || onlyLaneIds.has(lane.lane_id)));
-    if (awaiting.length === 0)
+    if (awaiting.length === 0 || deps.skipAdmissionWithoutCpuSample === true)
         return;
     const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
     let throttleDecision;
@@ -2356,7 +2413,9 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
         const loaded = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
         const state = loaded === "missing" ? emptyThrottleState() : loaded;
         throttleDecision = evaluateThrottle({
-            busy: await resolveCpuBusyFraction(throttleDeps),
+            busy: deps.cpuBusyFraction !== undefined
+                ? deps.cpuBusyFraction
+                : deps.cpuObservation?.busy ?? await resolveCpuBusyFraction(throttleDeps),
             availableMemMb: readAvailableMemMb(throttleDeps),
             cpuCheckEnabled: isCpuCheckEnabled(throttleDeps.env),
             state,
@@ -2365,7 +2424,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
         });
     }
     else {
-        const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
+        const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps, deps.cpuBusyFraction, deps.cpuObservation);
         throttleDecision = throttle.decision;
     }
     const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps, lane)));
@@ -2444,6 +2503,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
                     ...latest,
                     state: "queued",
                     queued_reason: queueReason,
+                    queued_details: undefined,
                     queued_at: answeredAt,
                     pending_resume: {
                         ...(latest.provider_session_id
@@ -2503,14 +2563,26 @@ export async function runReconcilePass(hqRoot, deps) {
 /** Run the reconcile answer-resume path for one lane before questions answer returns. */
 export async function wakeAnsweredLaneNow(hqRoot, laneId, deps) {
     const summary = emptySummary(deps.dryRun);
+    const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
     await withReconcileLock(hqRoot, async () => {
+        const previousState = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
+        const previousSnapshot = previousState === "missing" || previousState === "unreadable"
+            ? undefined
+            : previousState.cpu_snapshot;
+        const cpuObservation = await sampleCpuBusyObservation(throttleDeps, previousSnapshot);
+        const admissionDeps = {
+            ...deps,
+            cpuBusyFraction: cpuObservation.busy,
+            cpuObservation,
+            skipAdmissionWithoutCpuSample: false,
+        };
         await withAllLaneAdmissionLocks(hqRoot, async () => {
             const lanes = loadLaneRecords(hqRoot, summary);
             const questions = loadQuestionRecords(hqRoot, summary);
             if (!lanes.some((lane) => lane.lane_id === laneId)) {
                 throw new LanesError("lane_not_found", `Lane ${laneId} was not found`, { lane_id: laneId });
             }
-            await resumeAwaiting(hqRoot, lanes, questions, summary, deps, new Set([laneId]));
+            await resumeAwaiting(hqRoot, lanes, questions, summary, admissionDeps, new Set([laneId]));
         });
     });
     const latest = readLane(hqRoot, laneId);

package/dist/lib/lanes/spawn.js

@@ -476,6 +476,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
                 ...row,
                 state: "queued",
                 queued_reason: "account_retry",
+                queued_details: undefined,
                 queued_at: at,
                 admission_reserved_at: undefined,
                 terminal_reason: undefined,
@@ -1360,6 +1361,7 @@ export function markSpawnFailure(hqRoot, laneId, reasonRaw, details) {
             ...current,
             state: "failed",
             queued_reason: undefined,
+            queued_details: undefined,
             queued_at: undefined,
             admission_reserved_at: undefined,
             terminal_reason: reason,
@@ -1425,6 +1427,7 @@ function markOutpostSelfTarget(hqRoot, laneId, details) {
             ...current,
             state: "cancelled",
             queued_reason: undefined,
+            queued_details: undefined,
             terminal_reason: "outpost_self_target",
             terminal_details: details,
         };
@@ -1948,6 +1951,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
             ...current,
             state: "queued",
             queued_reason: "provider_binary_missing",
+            queued_details: undefined,
             terminal_reason: undefined,
             terminal_details: {
                 provider: lane.provider,
@@ -2024,6 +2028,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
             ...current,
             state: "running",
             queued_reason: undefined,
+            queued_details: undefined,
             queued_at: undefined,
             admission_reserved_at: undefined,
             terminal_reason: undefined,

package/dist/lib/lanes/store.d.ts

@@ -22,6 +22,8 @@ export interface StoreDeps {
     unlinkSync: typeof fs.unlinkSync;
     existsSync: typeof fs.existsSync;
     readdirSync: typeof fs.readdirSync;
+    /** Bulk readers may provide a pre-indexed conflict-sidecar lookup. */
+    conflictSidecarPathsFor?: (recordPath: string) => readonly string[];
     openSync: typeof fs.openSync;
     closeSync: typeof fs.closeSync;
     fsyncSync: typeof fs.fsyncSync;
@@ -50,6 +52,8 @@ export declare function mergeLaneRecords(live: LaneRecord, sidecars: readonly La
 export declare function mergeQuestionRecords(live: QuestionRecord, sidecars: readonly QuestionRecord[]): QuestionRecord;
 export declare function tryReadLane(hqRoot: string, laneId: string, deps?: StoreDeps): LaneRecord | null;
 export declare function readLane(hqRoot: string, laneId: string, deps?: StoreDeps): LaneRecord;
+/** Reads every lane while scanning the lane directory for conflict sidecars once. */
+export declare function readAllLanes(hqRoot: string, deps?: StoreDeps): LaneRecord[];
 export declare function tryReadQuestion(hqRoot: string, questionId: string, deps?: StoreDeps): QuestionRecord | null;
 export declare function readQuestion(hqRoot: string, questionId: string, deps?: StoreDeps): QuestionRecord;
 /** Semver precedence includes prerelease identifiers (for example, rc.2 > rc.1). */

package/dist/lib/lanes/store.js

@@ -528,6 +528,33 @@ export function parseLaneRecord(raw, filePath) {
             value: raw.queued_reason,
         });
     }
+    if (raw.queued_details !== undefined) {
+        const details = raw.queued_details;
+        if (details === null || typeof details !== "object" || Array.isArray(details) ||
+            typeof details.reason !== "string") {
+            throw new LaneSchemaError("queued_details must describe a worker_profile_changed queue reason", {
+                path: filePath,
+                field: "queued_details",
+            });
+        }
+        const detailsRecord = details;
+        if (detailsRecord.reason !== "worker_profile_changed" ||
+            (detailsRecord.expected_hash !== null && typeof detailsRecord.expected_hash !== "string") ||
+            typeof detailsRecord.actual_hash !== "string" || detailsRecord.actual_hash === "" ||
+            typeof detailsRecord.recovery_command !== "string" || detailsRecord.recovery_command.trim() === "") {
+            throw new LaneSchemaError("queued_details must describe a worker_profile_changed queue reason", {
+                path: filePath,
+                field: "queued_details",
+            });
+        }
+        if (raw.queued_reason !== "worker_profile_changed") {
+            throw new LaneSchemaError("queued_details requires queued_reason worker_profile_changed", {
+                path: filePath,
+                field: "queued_details",
+                queued_reason: raw.queued_reason,
+            });
+        }
+    }
     if (raw.queued_at !== undefined)
         requireTimestamp(raw.queued_at, "queued_at", filePath);
     if (raw.admission_reserved_at !== undefined) {
@@ -927,6 +954,9 @@ export function isConflictSidecarName(recordBasename, name) {
     return name.startsWith(`${recordBasename}.conflict-`);
 }
 function sidecarPathsFor(recordPath, deps) {
+    if (deps.conflictSidecarPathsFor) {
+        return [...deps.conflictSidecarPathsFor(recordPath)];
+    }
     const dir = path.dirname(recordPath);
     const base = path.basename(recordPath);
     if (!deps.existsSync(dir))
@@ -1130,6 +1160,42 @@ export function readLane(hqRoot, laneId, deps = defaultStoreDeps()) {
         throw new LaneNotFoundError("lane", laneId);
     return record;
 }
+/** Reads every lane while scanning the lane directory for conflict sidecars once. */
+export function readAllLanes(hqRoot, deps = defaultStoreDeps()) {
+    const laneIds = listLaneIds(hqRoot, deps);
+    if (laneIds.length === 0)
+        return [];
+    const laneDir = path.join(lanesStoreRoot(hqRoot), LANE_RECORDS_DIR);
+    let names;
+    try {
+        names = deps.readdirSync(laneDir);
+    }
+    catch (err) {
+        if (err.code === "ENOENT")
+            names = [];
+        else
+            throw err;
+    }
+    const sidecarsByRecord = new Map();
+    for (const name of names) {
+        if (typeof name !== "string")
+            continue;
+        const marker = name.indexOf(".json.conflict-");
+        if (marker < 0 || name.startsWith("."))
+            continue;
+        const recordBasename = name.slice(0, marker + ".json".length);
+        if (!isConflictSidecarName(recordBasename, name))
+            continue;
+        const sidecars = sidecarsByRecord.get(recordBasename) ?? [];
+        sidecars.push(path.join(laneDir, name));
+        sidecarsByRecord.set(recordBasename, sidecars);
+    }
+    const indexedDeps = {
+        ...deps,
+        conflictSidecarPathsFor: (recordPath) => sidecarsByRecord.get(path.basename(recordPath)) ?? [],
+    };
+    return laneIds.map((laneId) => readLane(hqRoot, laneId, indexedDeps));
+}
 export function tryReadQuestion(hqRoot, questionId, deps = defaultStoreDeps()) {
     const filePath = questionRecordPath(hqRoot, questionId);
     return readMergedRecord(filePath, parseQuestionRecord, mergeQuestionRecords, deps);