@indigoai-us/hq-cli 5.312.1 → 5.313.0

package/dist/lib/lanes/disk-guard.js

@@ -0,0 +1,172 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { createGzip } from "node:zlib";
+import { createReadStream, createWriteStream } from "node:fs";
+import { pipeline } from "node:stream/promises";
+import { readLanesRegistryFlag } from "./registry-flag.js";
+import { resolveLaneCompany } from "./lane-company.js";
+import { effectiveAccountHome } from "./account-home.js";
+import { lanesConfigPath } from "./config.js";
+export const LOW_DISK_GUARD_FLAG = "lanes.low-disk-guard";
+export const CODEX_ROLLOUT_RETENTION_FLAG = "lanes.codex-rollout-retention";
+export const DEFAULT_LOW_DISK_FREE_BYTES = 5 * 1024 ** 3;
+export const DEFAULT_LOW_DISK_FRACTION = 0.02;
+const EXCLUDED_SESSION_FILES = new Set(["auth.json", ".credentials.json", "thread_history.sqlite"]);
+export async function lowDiskGuardEnabledForLane(hqRoot, lane, readFlag = readLanesRegistryFlag) {
+    const company = resolveLaneCompany(hqRoot, lane).company;
+    return readFlag(LOW_DISK_GUARD_FLAG, { company, companyUidSource: "resolved_lane_company" });
+}
+export function measureDiskSpace(target, statfs = (p) => fs.statfsSync(p), minFreeBytes = DEFAULT_LOW_DISK_FREE_BYTES) {
+    const stats = statfs(target);
+    if (![stats.bsize, stats.blocks, stats.bavail].every(Number.isFinite) || stats.bsize <= 0 || stats.blocks < 0 || stats.bavail < 0) {
+        throw new Error(`invalid statfs result for ${target}`);
+    }
+    const totalBytes = stats.bsize * stats.blocks;
+    const freeBytes = stats.bsize * stats.bavail;
+    const thresholdBytes = Math.floor(Math.min(minFreeBytes, totalBytes * DEFAULT_LOW_DISK_FRACTION));
+    return {
+        path: target,
+        free_bytes: freeBytes,
+        total_bytes: totalBytes,
+        threshold_bytes: thresholdBytes,
+        low: freeBytes < thresholdBytes,
+    };
+}
+export function checkLaneDiskSpace(hqRoot, providerHome, statfs = (p) => fs.statfsSync(p), minFreeBytes = DEFAULT_LOW_DISK_FREE_BYTES) {
+    const hq = measureDiskSpace(hqRoot, statfs, minFreeBytes);
+    const provider = measureDiskSpace(providerHome, statfs, minFreeBytes);
+    return { low: hq.low || provider.low, hq, provider };
+}
+const DAY_MS = 24 * 60 * 60 * 1000;
+function rolloutSessionId(name) {
+    const match = /^rollout-.+-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:\.jsonl(?:\.gz)?)$/.exec(name);
+    return match?.[1];
+}
+function listRollouts(root, io, depth = 0) {
+    if (depth > 4)
+        return [];
+    let entries;
+    try {
+        entries = (io.readdirSync ?? fs.readdirSync)(root, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return [];
+        throw error;
+    }
+    const found = [];
+    for (const entry of entries) {
+        if (entry.isDirectory())
+            found.push(...listRollouts(path.join(root, entry.name), io, depth + 1));
+        else if (entry.isFile() &&
+            !EXCLUDED_SESSION_FILES.has(entry.name) &&
+            /^rollout-.+\.jsonl(?:\.gz)?$/.test(entry.name))
+            found.push(path.join(root, entry.name));
+    }
+    return found;
+}
+/**
+ * Codex homes that rollout retention may touch: enabled accounts in the
+ * operator's lanes config file. Without that file the documented defaults
+ * point at the real `~/.codex`, so retention gets no homes at all.
+ */
+export function configuredCodexRolloutHomes(hqRoot, config, hostname) {
+    if (!fs.existsSync(lanesConfigPath(hqRoot)))
+        return [];
+    return [...new Set(Object.values(config.providers.codex.accounts)
+            .filter((account) => account.enabled)
+            .map((account) => effectiveAccountHome(account, hostname).home))];
+}
+/** Operates only on rollout files under configured Codex `sessions/` trees. */
+export async function retainCodexRollouts(options) {
+    const now = options.nowMs ?? Date.now();
+    const compressAfter = options.compressAfterMs ?? DAY_MS;
+    const deleteAfter = options.deleteAfterMs ?? 7 * DAY_MS;
+    if (!(compressAfter > 0 && deleteAfter >= compressAfter))
+        throw new Error("invalid rollout retention ages");
+    const io = options;
+    const candidates = [];
+    for (const home of new Set(options.homes)) {
+        const sessions = path.join(home, "sessions");
+        for (const file of listRollouts(sessions, io)) {
+            const session = rolloutSessionId(path.basename(file));
+            if (!session)
+                continue;
+            const stat = (io.statSync ?? fs.statSync)(file);
+            candidates.push({ file, session, mtimeMs: stat.mtimeMs, size: stat.size });
+        }
+    }
+    const protectedSessions = new Set();
+    const mappedSessions = new Set();
+    for (const lane of options.lanes) {
+        if (lane.provider !== "codex")
+            continue;
+        const sessions = new Set([lane.provider_session_id, lane.last_resume?.session_id].filter((session) => Boolean(session)));
+        for (const session of sessions)
+            mappedSessions.add(session);
+        if (["running", "queued"].includes(lane.state)) {
+            for (const session of sessions)
+                protectedSessions.add(session);
+        }
+        else if (!["killed", "cancelled"].includes(lane.state)) {
+            for (const session of sessions) {
+                const newest = candidates.filter((candidate) => candidate.session === session).sort((a, b) => b.mtimeMs - a.mtimeMs)[0];
+                if (newest)
+                    protectedSessions.add(`${session}:${newest.file}`);
+            }
+        }
+    }
+    const result = { scanned: candidates.length, compressed: [], deleted: [], bytes_freed: 0 };
+    for (const candidate of candidates) {
+        const isProtected = protectedSessions.has(candidate.session) || protectedSessions.has(`${candidate.session}:${candidate.file}`);
+        if (isProtected)
+            continue;
+        const age = now - candidate.mtimeMs;
+        if (age < compressAfter)
+            continue;
+        if (age >= deleteAfter && mappedSessions.has(candidate.session)) {
+            result.deleted.push(candidate.file);
+            if (!options.dryRun) {
+                result.bytes_freed += candidate.size;
+                (io.unlinkSync ?? fs.unlinkSync)(candidate.file);
+            }
+            continue;
+        }
+        if (candidate.file.endsWith(".gz"))
+            continue;
+        const compressedPath = `${candidate.file}.gz`;
+        result.compressed.push(compressedPath);
+        if (options.dryRun)
+            continue;
+        const temp = `${compressedPath}.${process.pid}.tmp`;
+        const compressedBytes = options.compressFile
+            ? await options.compressFile(candidate.file, compressedPath)
+            : await gzipRolloutStream(candidate.file, temp, compressedPath, io);
+        // Keep the source age on the compressed file so the seven-day deletion
+        // policy measures from the original rollout, not from compression time.
+        (io.utimesSync ?? fs.utimesSync)(compressedPath, new Date(candidate.mtimeMs), new Date(candidate.mtimeMs));
+        (io.unlinkSync ?? fs.unlinkSync)(candidate.file);
+        result.bytes_freed += Math.max(0, candidate.size - compressedBytes);
+    }
+    return result;
+}
+async function gzipRolloutStream(source, temp, target, io) {
+    try {
+        await pipeline(createReadStream(source), createGzip(), createWriteStream(temp, { flags: "wx" }));
+        const compressedBytes = (io.statSync ?? fs.statSync)(temp).size;
+        (io.renameSync ?? fs.renameSync)(temp, target);
+        return compressedBytes;
+    }
+    catch (error) {
+        try {
+            (io.unlinkSync ?? fs.unlinkSync)(temp);
+        }
+        catch (cleanupError) {
+            if (cleanupError.code !== "ENOENT") {
+                process.stderr.write(`hq lanes: rollout compression temp cleanup failed for ${temp}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}\n`);
+            }
+        }
+        throw error;
+    }
+}
+//# sourceMappingURL=disk-guard.js.map

package/dist/lib/lanes/flag-keys.d.ts

@@ -2,6 +2,8 @@
 export declare const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
 /** Enables one automatic continuation after a missing-envelope repair. */
 export declare const LANE_ENVELOPE_REPAIR_CONTINUE_FLAG = "lanes.envelope-repair-continue";
+/** Count only children with live work against their senior's hire grant. */
+export declare const LANE_HIRE_GRANT_FINISHED_LANES_FLAG = "lanes.hire-grant-count-live-only";
 /** Enables missed-deadline lead nudges and second-miss operator escalation. */
 export declare const LANE_AUTOPILOT_MISSED_DEADLINE_ESCALATION_FLAG = "lanes.autopilot-missed-deadline-escalation";
 /** Enables company-scoped lookup of a hostless lane ULID across Outpost indexes. */

package/dist/lib/lanes/flag-keys.js

@@ -2,6 +2,8 @@
 export const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
 /** Enables one automatic continuation after a missing-envelope repair. */
 export const LANE_ENVELOPE_REPAIR_CONTINUE_FLAG = "lanes.envelope-repair-continue";
+/** Count only children with live work against their senior's hire grant. */
+export const LANE_HIRE_GRANT_FINISHED_LANES_FLAG = "lanes.hire-grant-count-live-only";
 /** Enables missed-deadline lead nudges and second-miss operator escalation. */
 export const LANE_AUTOPILOT_MISSED_DEADLINE_ESCALATION_FLAG = "lanes.autopilot-missed-deadline-escalation";
 /** Enables company-scoped lookup of a hostless lane ULID across Outpost indexes. */

package/dist/lib/lanes/grants.d.ts

@@ -1,7 +1,8 @@
 import { type LaneRecord } from "./types.js";
 import { LanesError } from "./errors.js";
 export type LaneHireGrantRow = Pick<LaneRecord, "lane_id" | "state" | "senior" | "hire_grant">;
-export declare function aliveDirectChildren<T extends LaneHireGrantRow>(lanes: readonly T[], parentLaneId: string): T[];
+export declare function isFinishedResumableHireChild(state: LaneRecord["state"]): boolean;
+export declare function aliveDirectChildren<T extends LaneHireGrantRow>(lanes: readonly T[], parentLaneId: string, finishedChildrenDoNotCount?: boolean): T[];
 export declare function assignedChildHireGrants(children: readonly LaneHireGrantRow[], exceptLaneId?: string): number;
 export interface HireGrantAdmissionSnapshot {
     parent: LaneHireGrantRow;
@@ -10,10 +11,10 @@ export interface HireGrantAdmissionSnapshot {
     availableChildGrant: number;
 }
 /** Shared refusal logic for resuming a terminal lane under an active senior. */
-export declare function terminalChildResumeHireGrantError(current: LaneRecord, snapshot: HireGrantAdmissionSnapshot): LanesError | null;
+export declare function terminalChildResumeHireGrantError(current: LaneRecord, snapshot: HireGrantAdmissionSnapshot, finishedChildrenDoNotCount?: boolean): LanesError | null;
 export declare function hireGrantCapacityError(parent: LaneHireGrantRow, children: readonly LaneHireGrantRow[]): LanesError;
 /** Keep the occupancy snapshot and the resulting lane write in one parent-lock scope. */
-export declare function withHireGrantAdmission<T>(hqRoot: string, parentLaneId: string, fn: (snapshot: HireGrantAdmissionSnapshot) => T, suppliedHierarchy?: readonly LaneHireGrantRow[], preferSuppliedLaneIds?: ReadonlySet<string>): T;
+export declare function withHireGrantAdmission<T>(hqRoot: string, parentLaneId: string, fn: (snapshot: HireGrantAdmissionSnapshot) => T, suppliedHierarchy?: readonly LaneHireGrantRow[], preferSuppliedLaneIds?: ReadonlySet<string>, finishedChildrenDoNotCount?: boolean): T;
 export interface HireGrantBackfillRow {
     lane_id: string;
     existing_grant: number | null;

package/dist/lib/lanes/grants.js

@@ -1,17 +1,22 @@
 import { isTerminalLaneState } from "./types.js";
 import { listLaneIds, readLane, withLaneHireLocks } from "./store.js";
 import { LanesError } from "./errors.js";
-export function aliveDirectChildren(lanes, parentLaneId) {
+export function isFinishedResumableHireChild(state) {
+    return state === "awaiting_input" || state === "blocked" || state === "done";
+}
+export function aliveDirectChildren(lanes, parentLaneId, finishedChildrenDoNotCount = false) {
     return lanes.filter((lane) => lane.senior.kind === "lane" &&
         lane.senior.id === parentLaneId &&
-        !isTerminalLaneState(lane.state));
+        !isTerminalLaneState(lane.state) &&
+        !(finishedChildrenDoNotCount && isFinishedResumableHireChild(lane.state)));
 }
 export function assignedChildHireGrants(children, exceptLaneId) {
     return children.reduce((sum, child) => child.lane_id === exceptLaneId ? sum : sum + (child.hire_grant ?? 0), 0);
 }
 /** Shared refusal logic for resuming a terminal lane under an active senior. */
-export function terminalChildResumeHireGrantError(current, snapshot) {
-    if (!isTerminalLaneState(current.state))
+export function terminalChildResumeHireGrantError(current, snapshot, finishedChildrenDoNotCount = false) {
+    const terminal = isTerminalLaneState(current.state);
+    if (!terminal && !(finishedChildrenDoNotCount && isFinishedResumableHireChild(current.state)))
         return null;
     const { parent, children, parentGrant, availableChildGrant } = snapshot;
     if (isTerminalLaneState(parent.state)) {
@@ -21,7 +26,7 @@ export function terminalChildResumeHireGrantError(current, snapshot) {
         return hireGrantCapacityError(parent, children);
     const requestedGrant = current.hire_grant ?? 0;
     if (requestedGrant > availableChildGrant) {
-        return new LanesError("hire_grant_carve_exceeded", `Senior lane ${parent.lane_id} has ${availableChildGrant} unassigned hire grant remaining, but terminal child ${current.lane_id} needs ${requestedGrant} to resume. Ask its senior for more: write an ask envelope with the desired total grant and why; the senior runs hq lanes grant ${parent.lane_id} --hire <n>.`, {
+        return new LanesError("hire_grant_carve_exceeded", `Senior lane ${parent.lane_id} has ${availableChildGrant} unassigned hire grant remaining, but ${terminal ? "terminal" : "finished"} child ${current.lane_id} needs ${requestedGrant} to resume. Ask its senior for more: write an ask envelope with the desired total grant and why; the senior runs hq lanes grant ${parent.lane_id} --hire <n>.`, {
             senior_lane_id: parent.lane_id,
             parent_grant: parentGrant,
             alive_children: children.map((child) => child.lane_id),
@@ -72,7 +77,7 @@ export function hireGrantCapacityError(parent, children) {
     });
 }
 /** Keep the occupancy snapshot and the resulting lane write in one parent-lock scope. */
-export function withHireGrantAdmission(hqRoot, parentLaneId, fn, suppliedHierarchy, preferSuppliedLaneIds = new Set()) {
+export function withHireGrantAdmission(hqRoot, parentLaneId, fn, suppliedHierarchy, preferSuppliedLaneIds = new Set(), finishedChildrenDoNotCount = false) {
     return withLaneHireLocks(hqRoot, [parentLaneId], () => {
         const ownerLanes = listLaneIds(hqRoot).map((id) => readLane(hqRoot, id));
         let allLanes;
@@ -96,7 +101,7 @@ export function withHireGrantAdmission(hqRoot, parentLaneId, fn, suppliedHierarc
             }
             parent = suppliedParent;
         }
-        const children = aliveDirectChildren(allLanes, parentLaneId);
+        const children = aliveDirectChildren(allLanes, parentLaneId, finishedChildrenDoNotCount);
         const parentGrant = parent.hire_grant ?? 0;
         const availableChildGrant = Math.max(0, parentGrant - assignedChildHireGrants(children));
         return fn({ parent, children, parentGrant, availableChildGrant });

package/dist/lib/lanes/hire-grant-dispatch.js

@@ -1,4 +1,3 @@
-import { aliveDirectChildren } from "./grants.js";
 import { LanesError } from "./errors.js";
 import { isLaneState, isSeniorKind, } from "./types.js";
 const HIRE_GRANT_CONTEXT_SCHEMA = "hq-lane-hire-grant-context.v1";
@@ -92,11 +91,11 @@ export function createOutpostHireGrantContext(target, lanes, supersededMessageCa
             throw new LanesError("hire_grant_senior_invalid", `Lane ${target.lane_id} has no senior lane in the primary hierarchy`, { lane_id: target.lane_id, senior_lane_id: parentId ?? null });
         }
         relevant.set(parent.lane_id, parent);
-        for (const sibling of aliveDirectChildren(lanes, parent.lane_id)) {
+        for (const sibling of directChildren(lanes, parent.lane_id)) {
             relevant.set(sibling.lane_id, sibling);
         }
     }
-    for (const child of aliveDirectChildren(lanes, current.lane_id)) {
+    for (const child of directChildren(lanes, current.lane_id)) {
         relevant.set(child.lane_id, child);
     }
     if (supersededMessageCallerId !== undefined) {

package/dist/lib/lanes/index.d.ts

@@ -3,7 +3,7 @@
  */
 export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
 export { MAX_LANE_DEPTH } from "./constants.js";
-export { assignedChildHireGrants, aliveDirectChildren, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
+export { assignedChildHireGrants, aliveDirectChildren, isFinishedResumableHireChild, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
 export type { HireGrantBackfillRow } from "./grants.js";
 export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneOccupiesRunningSlot, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
 export type { AdmissionCapReason, AdmissionHost } from "./admission.js";

package/dist/lib/lanes/index.js

@@ -3,7 +3,7 @@
  */
 export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
 export { MAX_LANE_DEPTH } from "./constants.js";
-export { assignedChildHireGrants, aliveDirectChildren, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
+export { assignedChildHireGrants, aliveDirectChildren, isFinishedResumableHireChild, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
 export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneOccupiesRunningSlot, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
 export { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "./host-capacity.js";
 export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";

package/dist/lib/lanes/reconcile.d.ts

@@ -61,6 +61,12 @@ export interface ReconcileSummary {
         code?: string;
     }>;
     heartbeat_at: string | null;
+    rollout_retention?: {
+        scanned: number;
+        compressed: string[];
+        deleted: string[];
+        bytes_freed: number;
+    };
 }
 export interface ReconcileHost {
     hostname: string;
@@ -144,7 +150,7 @@ export declare function runPassBody(hqRoot: string, deps: ReconcileDeps): Promis
  */
 export declare function runReconcilePass(hqRoot: string, deps: ReconcileDeps): Promise<ReconcileSummary>;
 export interface AnsweredLaneWakeResult {
-    status: "resumed" | "queued" | "awaiting_answers" | "failed";
+    status: "resumed" | "queued" | "awaiting_answers" | "deferred" | "failed";
     lane_state: LaneRecord["state"];
     queued_reason?: QueuedReason;
     remaining_question_ids?: string[];

package/dist/lib/lanes/reconcile.js

@@ -21,8 +21,10 @@ import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDa
 import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
 import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
 import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
-import { readLanesRegistryFlag } from "./registry-flag.js";
-import { resolveLaneCompany } from "./lane-company.js";
+import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
+import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG } from "./flag-keys.js";
+import { checkLaneDiskSpace, configuredCodexRolloutHomes, lowDiskGuardEnabledForLane, retainCodexRollouts } from "./disk-guard.js";
+import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
 import { appendDropboxMessage } from "./dropbox.js";
 import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
 import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
@@ -916,6 +918,55 @@ export async function runPassBody(hqRoot, deps) {
         return summary;
     }
     const config = loadLanesConfig(hqRoot);
+    try {
+        const companyRefs = new Map();
+        for (const lane of lanes) {
+            const company = resolveLaneCompany(hqRoot, lane).company;
+            if (company)
+                companyRefs.set(company.uid ?? company.slug, company);
+        }
+        const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
+        const homes = configuredCodexRolloutHomes(hqRoot, config, deps.host.hostname);
+        let retentionEnabled = false;
+        for (const company of homes.length > 0 ? companyRefs.values() : []) {
+            retentionEnabled ||= await readFlag("lanes.codex-rollout-retention", { company });
+            if (retentionEnabled)
+                break;
+        }
+        if (retentionEnabled) {
+            const statePath = path.join(hqRoot, "workspace", "lanes-runs", `.rollout-retention-${deps.host.hostname.replaceAll(/[^a-zA-Z0-9_-]/g, "_")}.json`);
+            let lastRunAt;
+            try {
+                const state = JSON.parse(fs.readFileSync(statePath, "utf8"));
+                if (typeof state.last_run_at !== "string" || !Number.isFinite(Date.parse(state.last_run_at))) {
+                    throw new LanesError("rollout_retention_state_invalid", `Invalid rollout-retention state at ${statePath}`, { path: statePath });
+                }
+                lastRunAt = Date.parse(state.last_run_at);
+            }
+            catch (err) {
+                if (err.code !== "ENOENT")
+                    throw err;
+            }
+            const lowDisk = homes.some((home) => checkLaneDiskSpace(hqRoot, home, deps.spawnLaneDeps?.statfs, config.low_disk_min_free_bytes).low);
+            if (lowDisk || lastRunAt === undefined || nowMs - lastRunAt >= 60 * 60 * 1000) {
+                const result = await retainCodexRollouts({
+                    homes,
+                    lanes,
+                    nowMs,
+                    compressAfterMs: config.codex_rollout_compress_after_hours * 60 * 60 * 1000,
+                    deleteAfterMs: config.codex_rollout_delete_after_days * 24 * 60 * 60 * 1000,
+                });
+                summary.rollout_retention = result;
+                fs.mkdirSync(path.dirname(statePath), { recursive: true });
+                const tempPath = `${statePath}.${process.pid}.tmp`;
+                fs.writeFileSync(tempPath, `${JSON.stringify({ last_run_at: nowIso }, null, 2)}\n`, { flag: "wx" });
+                fs.renameSync(tempPath, statePath);
+            }
+        }
+    }
+    catch (err) {
+        markDirty(summary, { kind: "store", why: `Codex rollout retention failed: ${err instanceof Error ? err.message : String(err)}` });
+    }
     const laneById = new Map(lanes.map((lane) => [lane.lane_id, lane]));
     const readLaneSenior = (laneId) => {
         const lane = laneById.get(laneId);
@@ -1333,6 +1384,32 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
                     repairCandidate &&
                     repairCandidate.envelope === null &&
                     (!repairCandidate.missing || envelopeRepairMode(classified.provider) === "resume")) {
+                    if (await lowDiskGuardEnabledForLane(hqRoot, classified, deps.readRegistryFlag)) {
+                        const repairConfig = loadLanesConfig(hqRoot);
+                        const repairAccount = classified.account
+                            ? repairConfig.providers[classified.provider].accounts[classified.account]
+                            : undefined;
+                        if (!repairAccount) {
+                            throw new LanesError("low_disk_account_unavailable", `Cannot determine provider home for envelope repair of lane ${classified.lane_id}`, { lane_id: classified.lane_id });
+                        }
+                        const providerHome = effectiveAccountHome(repairAccount, deps.host.hostname).home;
+                        const free = checkLaneDiskSpace(hqRoot, providerHome, deps.spawnLaneDeps?.statfs, repairConfig.low_disk_min_free_bytes);
+                        if (free.low) {
+                            const measuredAt = now.toISOString();
+                            updateLane(hqRoot, classified.lane_id, classified.version, "reconcile", (current) => ({
+                                ...current,
+                                state: "queued",
+                                queued_reason: "low_disk",
+                                queued_details: { reason: "low_disk", hq_free_bytes: free.hq.free_bytes, provider_free_bytes: free.provider.free_bytes, measured_at: measuredAt },
+                                queued_at: current.queued_at ?? measuredAt,
+                                admission_reserved_at: undefined,
+                                terminal_reason: undefined,
+                                terminal_details: undefined,
+                            }));
+                            summary.skipped.push({ kind: "lane", id: classified.lane_id, why: `low_disk: hq=${free.hq.free_bytes} provider=${free.provider.free_bytes}` });
+                            continue;
+                        }
+                    }
                     const spawnRepair = deps.spawnRepair ??
                         ((root, repairLane, input) => spawnEnvelopeRepair(root, repairLane, input, spawnDepsForHost(deps)));
                     const repair = decideEnvelopeRepair(hqRoot, lane.lane_id, repairCandidate, nowMs, spawnRepair, () => undefined);
@@ -1676,7 +1753,10 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds) {
         nowMs,
     });
     const spawnFn = deps.spawn ??
-        ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
+        (async (root, lane) => spawnLaneWorker(root, lane, {
+            ...spawnDepsForHost(deps),
+            lowDiskGuardEnabled: await lowDiskGuardEnabledForLane(root, lane, deps.readRegistryFlag),
+        }));
     const running = lanes.filter((lane) => (lane.state === "running" ||
         hasFreshAdmissionReservation(lane, now.getTime()) ||
         hasActivePendingResume(lane, now.getTime())) &&
@@ -1749,7 +1829,7 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds) {
                 spawned.push({ ...admittedLane, state: "running" });
                 continue;
             }
-            const started = spawnFn(hqRoot, admittedLane);
+            const started = await spawnFn(hqRoot, admittedLane);
             if (decision.reason === "starvation_bound") {
                 casLaneIf(hqRoot, lane.lane_id, (current) => current.admitted_reason !== "starvation_bound", (current) => ({ ...current, admitted_reason: "starvation_bound" }), false);
             }
@@ -1757,7 +1837,7 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds) {
                 id: started.lane_id,
                 from: "queued",
                 to: started.state,
-                reason: decision.reason,
+                reason: started.queued_reason ?? decision.reason,
                 ...(started.queued_reason === "provider_binary_missing" &&
                     typeof started.terminal_details?.searched_path === "string"
                     ? {
@@ -2668,9 +2748,17 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
             continue;
         }
         const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
+        let finishedChildrenDoNotCount = false;
         if (seniorId !== undefined) {
             let grantPreflight;
             try {
+                const companyResolution = resolveLaneCompany(hqRoot, current);
+                const company = companyResolution.company ?? undefined;
+                finishedChildrenDoNotCount = await readLanesRegistryFlagFailOff(LANE_HIRE_GRANT_FINISHED_LANES_FLAG, {
+                    company,
+                    ...(company ? { companyUidSource: laneCompanyUidSource(companyResolution.source) } : {}),
+                    env: deps.env,
+                }, deps.readRegistryFlag ?? readLanesRegistryFlag);
                 grantPreflight = withHireGrantAdmission(hqRoot, seniorId, (snapshot) => {
                     const latestLane = readLane(hqRoot, current.lane_id);
                     if (latestLane.version !== current.version ||
@@ -2680,11 +2768,11 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                         latestLane.senior.kind !== "lane" ||
                         latestLane.senior.id !== seniorId)
                         return { kind: "stale" };
-                    const refusal = terminalChildResumeHireGrantError(latestLane, snapshot);
+                    const refusal = terminalChildResumeHireGrantError(latestLane, snapshot, finishedChildrenDoNotCount);
                     return refusal === null
                         ? { kind: "eligible" }
                         : { kind: "refused", error: refusal.error, message: refusal.message };
-                });
+                }, undefined, new Set(), finishedChildrenDoNotCount);
             }
             catch (err) {
                 markDirty(summary, {
@@ -2880,7 +2968,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                         latestLane.senior.kind !== "lane" ||
                         latestLane.senior.id !== seniorId)
                         return { changed: false, record: latestLane };
-                    const refusal = terminalChildResumeHireGrantError(latestLane, snapshot);
+                    const refusal = terminalChildResumeHireGrantError(latestLane, snapshot, finishedChildrenDoNotCount);
                     if (refusal !== null) {
                         summary.skipped.push({
                             kind: "lane",
@@ -2890,7 +2978,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                         return { changed: false, record: latestLane };
                     }
                     return queue(seniorId);
-                });
+                }, undefined, new Set(), finishedChildrenDoNotCount);
             }
             catch (err) {
                 markDirty(summary, {
@@ -2953,7 +3041,10 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
         const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps, deps.cpuBusyFraction, deps.cpuObservation);
         throttleDecision = throttle.decision;
     }
-    const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
+    const spawnFn = deps.spawn ?? (async (root, lane) => spawnLaneWorker(root, lane, {
+        ...spawnDepsForHost(deps),
+        lowDiskGuardEnabled: await lowDiskGuardEnabledForLane(root, lane, deps.readRegistryFlag),
+    }));
     const nowMs = deps.now().getTime();
     const fallbackAnswerAt = new Date(nowMs).toISOString();
     const answerAtByLane = new Map();
@@ -2992,6 +3083,39 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
         if (pending.length > 0 || answered.length === 0)
             continue;
         try {
+            if (current.senior.kind === "lane" && current.senior.id !== undefined) {
+                const companyResolution = resolveLaneCompany(hqRoot, current);
+                const company = companyResolution.company ?? undefined;
+                const finishedChildrenDoNotCount = await readLanesRegistryFlagFailOff(LANE_HIRE_GRANT_FINISHED_LANES_FLAG, {
+                    company,
+                    ...(company ? { companyUidSource: laneCompanyUidSource(companyResolution.source) } : {}),
+                    env: deps.env,
+                }, deps.readRegistryFlag ?? readLanesRegistryFlag);
+                if (finishedChildrenDoNotCount) {
+                    const admission = withHireGrantAdmission(hqRoot, current.senior.id, (snapshot) => {
+                        const latest = readLane(hqRoot, current.lane_id);
+                        if (latest.version !== current.version || latest.state !== "awaiting_input") {
+                            return { kind: "stale" };
+                        }
+                        const refusal = terminalChildResumeHireGrantError(latest, snapshot, true);
+                        return refusal === null
+                            ? { kind: "eligible" }
+                            : { kind: "refused", error: refusal.error, message: refusal.message };
+                    }, undefined, new Set(), true);
+                    if (admission.kind === "stale")
+                        continue;
+                    if (admission.kind === "refused") {
+                        if (!deps.dryRun)
+                            writeAnsweredQuestions(hqRoot, current, questions);
+                        summary.skipped.push({
+                            kind: "lane",
+                            id: current.lane_id,
+                            why: `${admission.error}: ${admission.message}`,
+                        });
+                        continue;
+                    }
+                }
+            }
             let profileMaxConcurrent;
             if (current.worker_profile !== undefined) {
                 profileMaxConcurrent = loadWorkerProfile(hqRoot, current.worker_profile).meta.max_concurrent;
@@ -3062,7 +3186,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
                 running.push({ ...current, state: "running" });
                 continue;
             }
-            const started = spawnFn(hqRoot, lane);
+            const started = await spawnFn(hqRoot, lane);
             if (started.state === "running")
                 running.push(started);
             summary.resumed.push({
@@ -3184,6 +3308,13 @@ export async function wakeAnsweredLaneNow(hqRoot, laneId, deps) {
         };
     }
     const failure = summary.skipped.find((skip) => skip.kind === "lane" && skip.id === laneId);
+    if (latest.state === "awaiting_input" && failure?.why.startsWith("hire_grant_exhausted:")) {
+        return {
+            status: "deferred",
+            lane_state: latest.state,
+            error: failure.why,
+        };
+    }
     return {
         status: "failed",
         lane_state: latest.state,

package/dist/lib/lanes/registry-flag.d.ts

@@ -25,5 +25,11 @@ export declare function readLanesFlagLocalOverride(flagKey: string, env?: NodeJS
  * written to disk. A local override is checked on every call and wins first.
  */
 export declare function readLanesRegistryFlag(flagKey: string, options?: ReadLanesFlagOptions): Promise<boolean>;
+/**
+ * Flag readers normally report their own failures and return the safe default.
+ * Keep enforcement callers safe if an injected or unexpected reader rejects.
+ */
+export declare function readLanesRegistryFlagFailOff(flagKey: string, options?: ReadLanesFlagOptions, reader?: typeof readLanesRegistryFlag): Promise<boolean>;
+export declare function _resetThrownFlagReadWarningsForTests(): void;
 export declare function _resetLanesRegistryFlagCacheForTests(): void;
 //# sourceMappingURL=registry-flag.d.ts.map

package/dist/lib/lanes/registry-flag.js

@@ -7,6 +7,7 @@ export function readLanesFlagLocalOverride(flagKey, env = process.env) {
     return readLocalOverride(flagKey, env);
 }
 const registryReads = new Map();
+const reportedThrownFlagReads = new Set();
 class MissingFlagTokenError extends Error {
     constructor() {
         super("No cached HQ identity token is available");
@@ -67,6 +68,30 @@ export async function readLanesRegistryFlag(flagKey, options = {}) {
     registryReads.set(key, pending);
     return pending;
 }
+/**
+ * Flag readers normally report their own failures and return the safe default.
+ * Keep enforcement callers safe if an injected or unexpected reader rejects.
+ */
+export async function readLanesRegistryFlagFailOff(flagKey, options = {}, reader = readLanesRegistryFlag) {
+    try {
+        return await reader(flagKey, options);
+    }
+    catch (error) {
+        if (!reportedThrownFlagReads.has(flagKey)) {
+            reportedThrownFlagReads.add(flagKey);
+            process.stderr.write(`${JSON.stringify({
+                event: "lanes.flag_read_threw",
+                flag: flagKey,
+                error: error instanceof Error ? "Error" : "NonError",
+                fallback: false,
+            })}\n`);
+        }
+        return false;
+    }
+}
+export function _resetThrownFlagReadWarningsForTests() {
+    reportedThrownFlagReads.clear();
+}
 async function readRegistryValue(flagKey, endpoint, options) {
     const company = options.company;
     const companyUid = company?.uid?.trim();