@indigoai-us/hq-cli 5.280.0 → 5.282.0

package/dist/lib/lanes/outpost-dispatch.d.ts

@@ -27,7 +27,7 @@ export interface OutpostDispatchResult {
     stderr: string;
 }
 export interface OutpostLaneCommand {
-    action: "message" | "resume" | "stop" | "show" | "grant" | "grant-target" | "reparent-target" | "approve-target";
+    action: "message" | "resume" | "stop" | "show" | "logs" | "grant" | "grant-target" | "reparent-target" | "approve-target";
     laneId: string;
     hqBinary?: string;
     text?: string;
@@ -47,6 +47,8 @@ export interface OutpostLaneCommand {
     senior?: string;
     reason?: string;
     compact?: boolean;
+    tail?: string;
+    json?: boolean;
 }
 export declare function buildOutpostQuestionAnswerCommand(input: {
     questionId: string;

package/dist/lib/lanes/outpost-dispatch.js

@@ -174,6 +174,10 @@ export function buildOutpostLaneCommand(input) {
         args.push("--keep-alive");
     if (input.compact === false)
         args.push("--no-compact");
+    if (input.action === "logs" && input.tail !== undefined)
+        args.push("--tail", input.tail);
+    if (input.action === "logs" && input.json === true)
+        args.push("--json");
     return withRemoteHqDir(joinCommandParts(args));
 }
 /** True when a remote CLI exited after rejecting the forwarded effort flag. */

package/dist/lib/lanes/reconcile.d.ts

@@ -7,7 +7,7 @@
  */
 import { type LivenessDeps } from "./liveness.js";
 import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
-import { type ThrottleDeps } from "./throttle.js";
+import { type ThrottleDeps, type CpuBusyObservation } from "./throttle.js";
 import { readLanesRegistryFlag } from "./registry-flag.js";
 import { type RunawayProcessInfo } from "./runaway-tool-guard.js";
 import { type KeepAliveBlockedReason } from "./keep-alive.js";
@@ -78,6 +78,12 @@ export interface ReconcileDeps {
     readSessionSenior?: (sessionId: string) => SeniorRef;
     presenceConnected?: boolean;
     env?: NodeJS.ProcessEnv;
+    /** CPU sample taken before acquiring provider admission locks. */
+    cpuBusyFraction?: number | null;
+    /** CPU observation and cumulative counters taken before provider locks. */
+    cpuObservation?: CpuBusyObservation;
+    /** Skip admission this pass if candidates appeared after the pre-sample snapshot. */
+    skipAdmissionWithoutCpuSample?: boolean;
     /** Flag reader seam used to assert rollout reads happen before admission locks. */
     readRegistryFlag?: typeof readLanesRegistryFlag;
     /** Pre-read manager gate decisions, scoped to each lane for this pass. */

package/dist/lib/lanes/reconcile.js

@@ -20,7 +20,7 @@ import { terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./gr
 import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
 import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
 import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
-import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
+import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
 import { readLanesRegistryFlag } from "./registry-flag.js";
 import { resolveLaneCompany } from "./lane-company.js";
 import { appendDropboxMessage } from "./dropbox.js";
@@ -358,7 +358,7 @@ function casLaneIf(hqRoot, laneId, shouldWrite, updater, dryRun) {
 function writeQueueReason(hqRoot, lane, reason, dryRun) {
     if (lane.queued_reason === reason)
         return;
-    casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" && current.queued_reason !== reason, (current) => ({ ...current, queued_reason: reason, admission_reserved_at: undefined }), dryRun);
+    casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" && current.queued_reason !== reason, (current) => ({ ...current, queued_reason: reason, queued_details: undefined, admission_reserved_at: undefined }), dryRun);
 }
 function availableBalancedAccount(hqRoot, lane, occupying, config, host, profileMaxConcurrent, nowMs) {
     if (!lane.account || !lane.admission_accounts?.includes(lane.account))
@@ -482,6 +482,7 @@ function expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now) {
                     ...current,
                     state: "failed",
                     queued_reason: undefined,
+                    queued_details: undefined,
                     pickup_validation_pending: undefined,
                     pickup_validation_pending_since: undefined,
                     terminal_reason: PICKUP_VALIDATION_TIMEOUT_REASON,
@@ -998,14 +999,40 @@ export async function runPassBody(hqRoot, deps) {
         keepAliveFlagByCompany.set(key, keepAliveEnabled);
         supersessionLineageAwareByCompany.set(key, lineageAware);
     }));
+    const admissionThrottleDeps = startDeps.throttleDeps ?? defaultThrottleDeps(startDeps.env ?? process.env);
+    const hasAdmissionCandidates = lanes.some((lane) => laneRunsHere(lane, startDeps.host) &&
+        (lane.state === "queued" || lane.state === "awaiting_input")) || [...keepAliveFlagByCompany.values()].some(Boolean);
+    const priorThrottleState = readThrottleState(throttleStatePath(hqRoot), admissionThrottleDeps);
+    const priorThrottleSnapshot = priorThrottleState === "missing" || priorThrottleState === "unreadable"
+        ? undefined
+        : priorThrottleState.cpu_snapshot;
+    const cpuObservation = await sampleCpuBusyObservation(admissionThrottleDeps, priorThrottleSnapshot, { measureBusy: hasAdmissionCandidates });
+    const admissionDeps = {
+        ...startDeps,
+        cpuBusyFraction: cpuObservation.busy,
+        cpuObservation,
+        skipAdmissionWithoutCpuSample: !hasAdmissionCandidates,
+    };
     await withAllLaneAdmissionLocks(hqRoot, async () => {
         const freshLanes = loadLaneRecords(hqRoot, summary);
         const freshQuestions = loadQuestionRecords(hqRoot, summary);
-        await resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, startDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
+        await resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, admissionDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
         const drainLanes = loadLaneRecords(hqRoot, summary);
-        await admitQueued(hqRoot, drainLanes, summary, startDeps, now);
-        await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, startDeps);
+        await admitQueued(hqRoot, drainLanes, summary, admissionDeps, now);
+        await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, admissionDeps);
     });
+    if (!deps.dryRun && cpuObservation.snapshot !== undefined) {
+        const latestThrottleState = readThrottleState(throttleStatePath(hqRoot), admissionThrottleDeps);
+        if (latestThrottleState === "unreadable") {
+            markError(summary, new LanesError("throttle_state_unreadable", "Throttle state became unreadable while reconciling"));
+        }
+        else {
+            const currentThrottleState = latestThrottleState === "missing"
+                ? emptyThrottleState()
+                : latestThrottleState;
+            writeThrottleState(throttleStatePath(hqRoot), recordCpuBusyObservation(currentThrottleState, cpuObservation), admissionThrottleDeps);
+        }
+    }
     if (!deps.dryRun && summary.errors.length === 0) {
         try {
             writeHeartbeatFile(hqRoot, summary, nowIso);
@@ -1246,6 +1273,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                         ...current,
                         state: terminalClassification.state,
                         queued_reason: undefined,
+                        queued_details: undefined,
                         terminal_reason: terminalClassification.reason ?? "host_lost",
                         interrupted_at: terminalClassification.state === "interrupted" ? nowIso : undefined,
                         ...(providerRejection
@@ -1390,6 +1418,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                 ...current,
                 state: "interrupted",
                 queued_reason: undefined,
+                queued_details: undefined,
                 terminal_reason: "host_lost",
                 interrupted_at: nowIso,
             }), deps.dryRun);
@@ -1448,7 +1477,7 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
         laneRunsHere(lane, deps.host))
         .filter((lane) => !hasFreshAdmissionReservation(lane, now.getTime()))
         .sort(compareLaneQueueOrder);
-    if (localQueued.length === 0)
+    if (localQueued.length === 0 || deps.skipAdmissionWithoutCpuSample === true)
         return;
     const config = loadLanesConfig(hqRoot);
     const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
@@ -1459,7 +1488,9 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
         : loaded === "missing"
             ? emptyThrottleState()
             : loaded;
-    const busy = await resolveCpuBusyFraction(throttleDeps);
+    const busy = deps.cpuBusyFraction !== undefined
+        ? deps.cpuBusyFraction
+        : await resolveCpuBusyFraction(throttleDeps);
     const availableMemMb = readAvailableMemMb(throttleDeps);
     const nowMs = now.getTime();
     const decision = evaluateThrottle({
@@ -1476,7 +1507,6 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
         hasFreshAdmissionReservation(lane, now.getTime()) ||
         hasActivePendingResume(lane, now.getTime())) &&
         laneRunsHere(lane, deps.host));
-    const starvation = decision.admit && decision.reason === "starvation_bound";
     const spawned = [];
     if (!decision.admit) {
         for (const lane of localQueued) {
@@ -1543,8 +1573,6 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
                     reason: decision.reason,
                 });
                 spawned.push({ ...admittedLane, state: "running" });
-                if (starvation)
-                    break;
                 continue;
             }
             const started = spawnFn(hqRoot, admittedLane);
@@ -1564,10 +1592,38 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
                     : {}),
             });
             spawned.push(started);
-            if (starvation)
-                break;
         }
         catch (err) {
+            if (err instanceof LanesError && err.error === "worker_profile_hash_mismatch") {
+                const expectedHash = typeof err.details.expected_hash === "string"
+                    ? err.details.expected_hash
+                    : null;
+                const actualHash = typeof err.details.actual_hash === "string"
+                    ? err.details.actual_hash
+                    : undefined;
+                const workerProfile = typeof err.details.worker === "string"
+                    ? err.details.worker
+                    : lane.worker_profile;
+                if (actualHash !== undefined && workerProfile !== undefined) {
+                    casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued", (current) => ({
+                        ...current,
+                        queued_reason: "worker_profile_changed",
+                        queued_details: {
+                            reason: "worker_profile_changed",
+                            expected_hash: expectedHash,
+                            actual_hash: actualHash,
+                            recovery_command: `hq lanes resume ${lane.lane_id} --worker ${workerProfile}`,
+                        },
+                        admission_reserved_at: undefined,
+                    }), false);
+                    summary.skipped.push({
+                        kind: "lane",
+                        id: lane.lane_id,
+                        why: "worker_profile_changed",
+                    });
+                    continue;
+                }
+            }
             terminalizeModelUnsupported(hqRoot, lane, err);
             markDirty(summary, {
                 kind: "lane",
@@ -2273,6 +2329,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
             ...latestLane,
             state: "queued",
             queued_reason: undefined,
+            queued_details: undefined,
             queued_at: nowIso,
             admission_reserved_at: undefined,
             terminal_reason: undefined,
@@ -2348,7 +2405,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
     const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" &&
         laneRunsHere(lane, deps.host) &&
         (onlyLaneIds === undefined || onlyLaneIds.has(lane.lane_id)));
-    if (awaiting.length === 0)
+    if (awaiting.length === 0 || deps.skipAdmissionWithoutCpuSample === true)
         return;
     const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
     let throttleDecision;
@@ -2356,7 +2413,9 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
         const loaded = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
         const state = loaded === "missing" ? emptyThrottleState() : loaded;
         throttleDecision = evaluateThrottle({
-            busy: await resolveCpuBusyFraction(throttleDeps),
+            busy: deps.cpuBusyFraction !== undefined
+                ? deps.cpuBusyFraction
+                : deps.cpuObservation?.busy ?? await resolveCpuBusyFraction(throttleDeps),
             availableMemMb: readAvailableMemMb(throttleDeps),
             cpuCheckEnabled: isCpuCheckEnabled(throttleDeps.env),
             state,
@@ -2365,7 +2424,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
         });
     }
     else {
-        const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
+        const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps, deps.cpuBusyFraction, deps.cpuObservation);
         throttleDecision = throttle.decision;
     }
     const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps, lane)));
@@ -2444,6 +2503,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
                     ...latest,
                     state: "queued",
                     queued_reason: queueReason,
+                    queued_details: undefined,
                     queued_at: answeredAt,
                     pending_resume: {
                         ...(latest.provider_session_id
@@ -2503,14 +2563,26 @@ export async function runReconcilePass(hqRoot, deps) {
 /** Run the reconcile answer-resume path for one lane before questions answer returns. */
 export async function wakeAnsweredLaneNow(hqRoot, laneId, deps) {
     const summary = emptySummary(deps.dryRun);
+    const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
     await withReconcileLock(hqRoot, async () => {
+        const previousState = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
+        const previousSnapshot = previousState === "missing" || previousState === "unreadable"
+            ? undefined
+            : previousState.cpu_snapshot;
+        const cpuObservation = await sampleCpuBusyObservation(throttleDeps, previousSnapshot);
+        const admissionDeps = {
+            ...deps,
+            cpuBusyFraction: cpuObservation.busy,
+            cpuObservation,
+            skipAdmissionWithoutCpuSample: false,
+        };
         await withAllLaneAdmissionLocks(hqRoot, async () => {
             const lanes = loadLaneRecords(hqRoot, summary);
             const questions = loadQuestionRecords(hqRoot, summary);
             if (!lanes.some((lane) => lane.lane_id === laneId)) {
                 throw new LanesError("lane_not_found", `Lane ${laneId} was not found`, { lane_id: laneId });
             }
-            await resumeAwaiting(hqRoot, lanes, questions, summary, deps, new Set([laneId]));
+            await resumeAwaiting(hqRoot, lanes, questions, summary, admissionDeps, new Set([laneId]));
         });
     });
     const latest = readLane(hqRoot, laneId);

package/dist/lib/lanes/spawn.js

@@ -476,6 +476,7 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
                 ...row,
                 state: "queued",
                 queued_reason: "account_retry",
+                queued_details: undefined,
                 queued_at: at,
                 admission_reserved_at: undefined,
                 terminal_reason: undefined,
@@ -1360,6 +1361,7 @@ export function markSpawnFailure(hqRoot, laneId, reasonRaw, details) {
             ...current,
             state: "failed",
             queued_reason: undefined,
+            queued_details: undefined,
             queued_at: undefined,
             admission_reserved_at: undefined,
             terminal_reason: reason,
@@ -1425,6 +1427,7 @@ function markOutpostSelfTarget(hqRoot, laneId, details) {
             ...current,
             state: "cancelled",
             queued_reason: undefined,
+            queued_details: undefined,
             terminal_reason: "outpost_self_target",
             terminal_details: details,
         };
@@ -1948,6 +1951,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
             ...current,
             state: "queued",
             queued_reason: "provider_binary_missing",
+            queued_details: undefined,
             terminal_reason: undefined,
             terminal_details: {
                 provider: lane.provider,
@@ -2024,6 +2028,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
             ...current,
             state: "running",
             queued_reason: undefined,
+            queued_details: undefined,
             queued_at: undefined,
             admission_reserved_at: undefined,
             terminal_reason: undefined,

package/dist/lib/lanes/store.js

@@ -416,6 +416,17 @@ export function parseLaneRecord(raw, filePath) {
     requireInt(raw.version, "version");
     requireInt(raw.depth, "depth");
     requireInt(raw.priority, "priority");
+    if (raw.ended_at !== undefined) {
+        requireTimestamp(raw.ended_at, "ended_at", filePath);
+    }
+    if (raw.envelope_summary !== undefined &&
+        (typeof raw.envelope_summary !== "string" || raw.envelope_summary.trim() === "")) {
+        throw new LaneSchemaError("envelope_summary must be a non-empty string when present", {
+            path: filePath,
+            field: "envelope_summary",
+            value: raw.envelope_summary,
+        });
+    }
     if (raw.hire_grant !== undefined &&
         (typeof raw.hire_grant !== "number" || !Number.isSafeInteger(raw.hire_grant) || raw.hire_grant < 0)) {
         throw new LaneSchemaError("Expected non-negative safe integer for hire_grant", {
@@ -517,6 +528,33 @@ export function parseLaneRecord(raw, filePath) {
             value: raw.queued_reason,
         });
     }
+    if (raw.queued_details !== undefined) {
+        const details = raw.queued_details;
+        if (details === null || typeof details !== "object" || Array.isArray(details) ||
+            typeof details.reason !== "string") {
+            throw new LaneSchemaError("queued_details must describe a worker_profile_changed queue reason", {
+                path: filePath,
+                field: "queued_details",
+            });
+        }
+        const detailsRecord = details;
+        if (detailsRecord.reason !== "worker_profile_changed" ||
+            (detailsRecord.expected_hash !== null && typeof detailsRecord.expected_hash !== "string") ||
+            typeof detailsRecord.actual_hash !== "string" || detailsRecord.actual_hash === "" ||
+            typeof detailsRecord.recovery_command !== "string" || detailsRecord.recovery_command.trim() === "") {
+            throw new LaneSchemaError("queued_details must describe a worker_profile_changed queue reason", {
+                path: filePath,
+                field: "queued_details",
+            });
+        }
+        if (raw.queued_reason !== "worker_profile_changed") {
+            throw new LaneSchemaError("queued_details requires queued_reason worker_profile_changed", {
+                path: filePath,
+                field: "queued_details",
+                queued_reason: raw.queued_reason,
+            });
+        }
+    }
     if (raw.queued_at !== undefined)
         requireTimestamp(raw.queued_at, "queued_at", filePath);
     if (raw.admission_reserved_at !== undefined) {

package/dist/lib/lanes/throttle.d.ts

@@ -12,7 +12,8 @@
 import * as fs from "node:fs";
 import * as os from "node:os";
 import type { ThrottleConfig } from "./types.js";
-export declare const CPU_SAMPLE_MS = 200;
+export declare const CPU_SAMPLE_MS = 5000;
+export declare const CPU_CUMULATIVE_MAX_AGE_MS: number;
 export declare const STARVATION_MAX_DEFERRALS = 10;
 export declare const STARVATION_MAX_MS: number;
 export declare const PROC_MEMINFO = "/proc/meminfo";
@@ -39,12 +40,34 @@ export interface ThrottleDeps {
     renameSync: typeof fs.renameSync;
     unlinkSync: typeof fs.unlinkSync;
 }
+export type CpuBusySource = "interval" | "window" | "override";
+export interface CpuCumulativeSnapshot {
+    idle: number;
+    total: number;
+    sampled_at: string;
+}
+export interface CpuBusyObservation {
+    busy: number | null;
+    source: CpuBusySource | undefined;
+    snapshot: CpuCumulativeSnapshot | undefined;
+}
 export declare function defaultThrottleDeps(env?: NodeJS.ProcessEnv): ThrottleDeps;
 export declare function cpuTimesSnapshot(cpus: os.CpuInfo[]): CpuTimes;
 export declare function busyFractionFromDeltas(before: CpuTimes, after: CpuTimes): number | null;
 export declare function sampleCpuBusyFraction(deps?: ThrottleDeps, sampleMs?: number): Promise<number | null>;
 export declare function isCpuCheckEnabled(env?: NodeJS.ProcessEnv): boolean;
 export declare function resolveCpuBusyFraction(deps?: ThrottleDeps): Promise<number | null>;
+/**
+ * Measure CPU busy against the previous reconcile snapshot when it is fresh.
+ * A short stable window is used only to establish a baseline or recover from
+ * stale/backwards counters (for example, after a reboot).
+ */
+export declare function sampleCpuBusyObservation(deps: ThrottleDeps, previous: CpuCumulativeSnapshot | undefined, options?: {
+    measureBusy?: boolean;
+}): Promise<CpuBusyObservation>;
+export declare function recordCpuBusyObservation(state: ThrottleState, observation: CpuBusyObservation, options?: {
+    advanceSnapshot?: boolean;
+}): ThrottleState;
 /** Parse MemAvailable kB from /proc/meminfo. */
 export declare function parseMemAvailableKb(contents: string): number | null;
 /**
@@ -56,6 +79,10 @@ export interface ThrottleState {
     consecutiveDeferrals: number;
     lastAdmissionAt: string | null;
     cpuPaused: boolean;
+    cpu_snapshot?: CpuCumulativeSnapshot;
+    last_busy?: number | null;
+    busy_source?: CpuBusySource;
+    sampled_at?: string;
 }
 export type ThrottleAdmitReason = "ok" | "starvation_bound" | "unreadable" | "cpu_check_disabled";
 export type ThrottleDeferReason = "cpu" | "memory";
@@ -82,7 +109,7 @@ export declare function applyThrottleDecision(state: ThrottleState, decision: Th
 export declare function throttleStatePath(hqRoot: string): string;
 export declare function readThrottleState(filePath: string, deps?: ThrottleDeps): ThrottleState | "unreadable" | "missing";
 export declare function writeThrottleState(filePath: string, state: ThrottleState, deps?: ThrottleDeps): void;
-export declare function decideAdmission(hqRoot: string, throttle?: ThrottleConfig, deps?: ThrottleDeps): Promise<{
+export declare function decideAdmission(hqRoot: string, throttle?: ThrottleConfig, deps?: ThrottleDeps, cpuBusyFraction?: number | null, cpuObservation?: CpuBusyObservation): Promise<{
     decision: ThrottleDecision;
     state: ThrottleState;
 }>;

package/dist/lib/lanes/throttle.js

@@ -13,7 +13,10 @@ import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import { DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_MIN_AVAILABLE_MEM_MB, } from "./config.js";
-export const CPU_SAMPLE_MS = 200;
+// Reconcile normally measures the interval between cumulative CPU snapshots.
+// A five-second window establishes a baseline or recovers from stale counters.
+export const CPU_SAMPLE_MS = 5_000;
+export const CPU_CUMULATIVE_MAX_AGE_MS = 5 * 60 * 1000;
 export const STARVATION_MAX_DEFERRALS = 10;
 export const STARVATION_MAX_MS = 30 * 60 * 1000;
 export const PROC_MEMINFO = "/proc/meminfo";
@@ -83,6 +86,93 @@ export async function resolveCpuBusyFraction(deps = defaultThrottleDeps()) {
     }
     return sampleCpuBusyFraction(deps);
 }
+function snapshotFromDeps(deps) {
+    return {
+        ...cpuTimesSnapshot(deps.cpus()),
+        sampled_at: new Date(deps.now()).toISOString(),
+    };
+}
+/**
+ * Measure CPU busy against the previous reconcile snapshot when it is fresh.
+ * A short stable window is used only to establish a baseline or recover from
+ * stale/backwards counters (for example, after a reboot).
+ */
+export async function sampleCpuBusyObservation(deps, previous, options = {}) {
+    let start;
+    try {
+        start = snapshotFromDeps(deps);
+    }
+    catch (err) {
+        process.stderr.write(`lanes throttle: CPU counters failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        return { busy: null, source: undefined, snapshot: previous };
+    }
+    if (options.measureBusy === false || !isCpuCheckEnabled(deps.env)) {
+        return { busy: null, source: undefined, snapshot: start };
+    }
+    const rawOverride = deps.env[LANES_CPU_BUSY_OVERRIDE];
+    if (rawOverride !== undefined && rawOverride !== "") {
+        const override = Number(rawOverride);
+        if (Number.isFinite(override)) {
+            return {
+                busy: Math.max(0, Math.min(1, override)),
+                source: "override",
+                snapshot: start,
+            };
+        }
+        process.stderr.write(`lanes throttle: ${LANES_CPU_BUSY_OVERRIDE} is not a finite number (${JSON.stringify(rawOverride)}); sampling instead\n`);
+    }
+    const nowMs = Date.parse(start.sampled_at);
+    const previousAtMs = previous ? Date.parse(previous.sampled_at) : Number.NaN;
+    if (previous !== undefined &&
+        Number.isFinite(previousAtMs) &&
+        nowMs >= previousAtMs &&
+        nowMs - previousAtMs <= CPU_CUMULATIVE_MAX_AGE_MS &&
+        start.idle >= previous.idle &&
+        start.total >= previous.total) {
+        return {
+            busy: busyFractionFromDeltas(previous, start),
+            source: "interval",
+            snapshot: start,
+        };
+    }
+    try {
+        await deps.sleep(CPU_SAMPLE_MS);
+        const end = snapshotFromDeps(deps);
+        return {
+            busy: busyFractionFromDeltas(start, end),
+            source: "window",
+            snapshot: end,
+        };
+    }
+    catch (err) {
+        process.stderr.write(`lanes throttle: CPU window sample failed: ${err instanceof Error ? err.message : String(err)}\n`);
+        return { busy: null, source: "window", snapshot: start };
+    }
+}
+export function recordCpuBusyObservation(state, observation, options = {}) {
+    const observationAt = observation.snapshot === undefined
+        ? Number.NaN
+        : Date.parse(observation.snapshot.sampled_at);
+    const currentSnapshotAt = state.cpu_snapshot === undefined
+        ? Number.NaN
+        : Date.parse(state.cpu_snapshot.sampled_at);
+    const snapshotIsCurrent = options.advanceSnapshot !== false && observation.snapshot !== undefined && (!Number.isFinite(currentSnapshotAt) ||
+        observationAt > currentSnapshotAt ||
+        (observationAt === currentSnapshotAt && observation.snapshot.total >= (state.cpu_snapshot?.total ?? 0)));
+    const currentMetricAt = state.sampled_at === undefined ? Number.NaN : Date.parse(state.sampled_at);
+    const metricIsCurrent = observation.source !== undefined && Number.isFinite(observationAt) && (!Number.isFinite(currentMetricAt) || observationAt >= currentMetricAt);
+    return {
+        ...state,
+        ...(snapshotIsCurrent ? { cpu_snapshot: observation.snapshot } : {}),
+        ...(metricIsCurrent
+            ? {
+                last_busy: observation.busy,
+                busy_source: observation.source,
+                sampled_at: observation.snapshot.sampled_at,
+            }
+            : {}),
+    };
+}
 /** Parse MemAvailable kB from /proc/meminfo. */
 export function parseMemAvailableKb(contents) {
     for (const line of contents.split("\n")) {
@@ -184,12 +274,14 @@ export function evaluateThrottle(input) {
 export function applyThrottleDecision(state, decision, nowMs) {
     if (decision.admit) {
         return {
+            ...state,
             consecutiveDeferrals: 0,
             lastAdmissionAt: new Date(nowMs).toISOString(),
             cpuPaused: false,
         };
     }
     return {
+        ...state,
         consecutiveDeferrals: state.consecutiveDeferrals + 1,
         lastAdmissionAt: state.lastAdmissionAt,
         cpuPaused: decision.reason === "cpu" ? true : state.cpuPaused,
@@ -231,10 +323,47 @@ export function readThrottleState(filePath, deps = defaultThrottleDeps()) {
             process.stderr.write(`lanes throttle: state ${filePath} has invalid cpuPaused\n`);
             return "unreadable";
         }
+        const optionalTimestamp = (value) => typeof value === "string" && value.trim() !== "" && Number.isFinite(Date.parse(value));
+        let cpuSnapshot;
+        if (obj.cpu_snapshot !== undefined) {
+            const rawSnapshot = obj.cpu_snapshot;
+            if (rawSnapshot === null || typeof rawSnapshot !== "object" || Array.isArray(rawSnapshot)) {
+                process.stderr.write(`lanes throttle: state ${filePath} has invalid cpu_snapshot\n`);
+                return "unreadable";
+            }
+            const snapshot = rawSnapshot;
+            if (typeof snapshot.idle !== "number" || !Number.isFinite(snapshot.idle) || snapshot.idle < 0 ||
+                typeof snapshot.total !== "number" || !Number.isFinite(snapshot.total) || snapshot.total < 0 ||
+                !optionalTimestamp(snapshot.sampled_at)) {
+                process.stderr.write(`lanes throttle: state ${filePath} has invalid cpu_snapshot fields\n`);
+                return "unreadable";
+            }
+            cpuSnapshot = {
+                idle: snapshot.idle,
+                total: snapshot.total,
+                sampled_at: snapshot.sampled_at,
+            };
+        }
+        if (obj.last_busy !== undefined && obj.last_busy !== null && (typeof obj.last_busy !== "number" || !Number.isFinite(obj.last_busy) || obj.last_busy < 0 || obj.last_busy > 1)) {
+            process.stderr.write(`lanes throttle: state ${filePath} has invalid last_busy\n`);
+            return "unreadable";
+        }
+        if (obj.busy_source !== undefined && !["interval", "window", "override"].includes(String(obj.busy_source))) {
+            process.stderr.write(`lanes throttle: state ${filePath} has invalid busy_source\n`);
+            return "unreadable";
+        }
+        if (obj.sampled_at !== undefined && !optionalTimestamp(obj.sampled_at)) {
+            process.stderr.write(`lanes throttle: state ${filePath} has invalid sampled_at\n`);
+            return "unreadable";
+        }
         return {
             consecutiveDeferrals: obj.consecutiveDeferrals,
             lastAdmissionAt: obj.lastAdmissionAt,
             cpuPaused: obj.cpuPaused,
+            ...(cpuSnapshot !== undefined ? { cpu_snapshot: cpuSnapshot } : {}),
+            ...(obj.last_busy !== undefined ? { last_busy: obj.last_busy } : {}),
+            ...(obj.busy_source !== undefined ? { busy_source: obj.busy_source } : {}),
+            ...(obj.sampled_at !== undefined ? { sampled_at: obj.sampled_at } : {}),
         };
     }
     catch (err) {
@@ -269,7 +398,7 @@ export async function decideAdmission(hqRoot, throttle = {
     cpu_busy_pause: DEFAULT_CPU_BUSY_PAUSE,
     cpu_busy_resume: DEFAULT_CPU_BUSY_RESUME,
     min_available_mem_mb: DEFAULT_MIN_AVAILABLE_MEM_MB,
-}, deps = defaultThrottleDeps()) {
+}, deps = defaultThrottleDeps(), cpuBusyFraction, cpuObservation) {
     const filePath = throttleStatePath(hqRoot);
     const loaded = readThrottleState(filePath, deps);
     if (loaded === "unreadable") {
@@ -279,18 +408,29 @@ export async function decideAdmission(hqRoot, throttle = {
         };
     }
     const state = loaded === "missing" ? emptyThrottleState() : loaded;
-    const busy = await resolveCpuBusyFraction(deps);
+    const cpuCheckEnabled = isCpuCheckEnabled(deps.env);
+    const observation = cpuObservation ?? (cpuBusyFraction === undefined && cpuCheckEnabled
+        ? await sampleCpuBusyObservation(deps, state.cpu_snapshot)
+        : undefined);
+    const busy = cpuBusyFraction === undefined
+        ? observation?.busy ?? null
+        : cpuBusyFraction;
     const availableMemMb = readAvailableMemMb(deps);
     const nowMs = deps.now();
     const decision = evaluateThrottle({
         busy,
         availableMemMb,
-        cpuCheckEnabled: isCpuCheckEnabled(deps.env),
+        cpuCheckEnabled,
         state,
         throttle,
         nowMs,
     });
-    const next = applyThrottleDecision(state, decision, nowMs);
+    const observedState = observation
+        ? recordCpuBusyObservation(state, observation, {
+            advanceSnapshot: cpuObservation !== undefined,
+        })
+        : state;
+    const next = applyThrottleDecision(observedState, decision, nowMs);
     writeThrottleState(filePath, next, deps);
     return { decision, state: next };
 }