@indigoai-us/hq-cli 5.345.38 → 5.345.39

package/dist/commands/lanes.js

@@ -63,7 +63,7 @@ import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, lan
 import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
 import { listLaneReviews } from "../lib/lanes/reviews.js";
 import { effectiveAccountHome } from "../lib/lanes/account-home.js";
-import { AccountResolutionError, compareLaneQueueOrder, LANE_PROVIDERS, LANE_STATES, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, isFinishedResumableHireChild, childrenHoldingHireGrantCarve, laneHoldsHireGrantCarve, assignedChildHireGrants, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, syncLaneToProjectStoryLedger, removeLaneFromProjectStoryLedger, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listLaneIds, listQuestionIds, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseQuestionRecord, parseSeniorKind, readAllLanes, readLane, sanitizeHostComponent, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithRegistry, registryWorkerRecord, runReconcilePass, DEAD_HOST_MS, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, dropboxDir, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, readWorkerRunResult, resumeCommandRefusal, withLaneHireLocks, raiseLaneQuestion, sha256Text, } from "../lib/lanes/index.js";
+import { AccountResolutionError, compareLaneQueueOrder, LANE_PROVIDERS, LANE_STATES, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, isFinishedResumableHireChild, childrenHoldingHireGrantCarve, laneHoldsHireGrantCarve, assignedChildHireGrants, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, syncLaneToProjectStoryLedger, removeLaneFromProjectStoryLedger, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listLaneIds, listQuestionIds, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseQuestionRecord, parseSeniorKind, readAllLanes, readLane, sanitizeHostComponent, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithRegistry, registryWorkerRecord, runReconcilePass, DEAD_HOST_MS, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, dropboxDir, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, isFailedQuotaKeepAliveLane, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, readWorkerRunResult, resumeCommandRefusal, withLaneHireLocks, raiseLaneQuestion, sha256Text, } from "../lib/lanes/index.js";
 import { laneStories, storyWarning, } from "../lib/lanes/stories.js";
 import { resolveLaneKeepAlive } from "../lib/lanes/profile.js";
 import { laneCompanyUidSource, resolveLaneCompany } from "../lib/lanes/lane-company.js";
@@ -6015,6 +6015,19 @@ async function runLaneDoctor(ctx, pendingQuestionsForLane) {
     const retiredEntries = loadRetiredLaneEntries(hqRoot);
     const findings = [];
     for (const lane of lanes) {
+        if (lane.keep_alive === true &&
+            lane.state === "failed" &&
+            (lane.terminal_reason === "account_quota_exhausted" || lane.terminal_reason === "account_model_unavailable")) {
+            findings.push({
+                kind: "failed_keep_alive_lane",
+                lane_id: lane.lane_id,
+                senior: lane.senior,
+                terminal_reason: lane.terminal_reason,
+                ...(isFailedQuotaKeepAliveLane(lane)
+                    ? { suggested_command: `hq lanes resume ${lane.lane_id}` }
+                    : { suggested_action: "Inspect the quota reset details before authorizing a failed-lane resume." }),
+            });
+        }
         if (lane.keep_alive !== true || isTerminalLaneState(lane.state))
             continue;
         const retiredEntry = retiredEntries.find((entry) => matchesRetiredLaneEntry(entry, lane.lane_id));
@@ -7206,6 +7219,11 @@ async function runReparentOwnerTarget(ctx, laneId, opts) {
         changed_descendant_ids: result.changedDescendantIds,
     });
 }
+function assertLaneNotRunningForReparent(lane) {
+    if (lane.state === "running") {
+        throw new LanesError("lane_running", `Cannot re-parent running lane ${lane.lane_id}; wait until it stops first`, { lane_id: lane.lane_id, state: lane.state });
+    }
+}
 function runReparentLocked(ctx, laneId, opts, finishedChildrenDoNotCount = false) {
     const hqRoot = ctx.hqRoot();
     if (!opts.senior) {
@@ -7215,9 +7233,6 @@ function runReparentLocked(ctx, laneId, opts, finishedChildrenDoNotCount = false
     assertSeniorExists(hqRoot, nextSenior, ctx);
     const lane = readLane(hqRoot, laneId);
     assertLaneRunsHereForOperatorMutation(ctx, lane, "reparent");
-    if (isTerminalLaneState(lane.state)) {
-        throw new LanesError("lane_terminal", `Cannot re-parent a lane in terminal state ${lane.state}`, { lane_id: laneId, state: lane.state });
-    }
     const seniorCompany = companyOfSenior(hqRoot, nextSenior, ctx);
     if (lane.company === null && seniorCompany.binding === "company") {
         const company = seniorCompany.slug ?? seniorCompany.uid ?? "<company>";
@@ -7257,6 +7272,7 @@ function runReparentLocked(ctx, laneId, opts, finishedChildrenDoNotCount = false
         chain.push({ lane_id: lane.lane_id, depth: newDepth });
         throw maxLaneDepthError(newDepth, chain, `Re-parent would put lane at depth ${newDepth}`, depthCap);
     }
+    assertLaneNotRunningForReparent(lane);
     const pending = pendingQuestionsForLane(hqRoot, laneId);
     const oldDepth = derivedLaneDepth(hqRoot, lane.senior, ctx);
     if (newDepth > oldDepth && pending.length > 0) {
@@ -7295,6 +7311,7 @@ function runReparentLocked(ctx, laneId, opts, finishedChildrenDoNotCount = false
     const at = ctx.now().toISOString();
     const updated = casUpdateLane(hqRoot, laneId, (current) => {
         assertLaneRunsHereForOperatorMutation(ctx, current, "reparent");
+        assertLaneNotRunningForReparent(current);
         const already = current.senior_history.some((entry) => entry.at === at && entry.reason === (opts.reason ?? "reparent"));
         if (already && current.depth === newDepth)
             return current;

package/dist/commands/report.js

@@ -10,6 +10,8 @@ import { listDeployApps } from "../lib/deploy-apps.js";
 import { listReportTemplates } from "../lib/report-templates.js";
 import { adoptReport, buildReportTarball, conventionReportDir, deployOrgContext, ReportPublishError, plainFailureReason, publishReport, } from "../lib/report-publish.js";
 const REPORT_CHANNEL_STATE_FILE = "report-channels.json";
+const REPORT_UPLOAD_TIMEOUT_MS = 120_000;
+const REPORT_CLEANUP_TIMEOUT_MS = 30_000;
 const MAX_REPORT_DISCOVERY_CHANNELS = 50;
 const JOB_EXECUTION_DISABLED_MESSAGE = "Scheduled jobs are turned off for this company. A company owner can turn them back on.";
 function reportPath(channelId, reportId) {
@@ -305,7 +307,7 @@ async function deployReportStatic(token, input) {
         throw new ReportPublishError("hq-deploy did not create an upload.");
     let upload;
     try {
-        upload = await fetch(presignedUrl, { method: "PUT", headers: { "Content-Type": "application/gzip" }, body: tarball.bytes });
+        upload = await fetch(presignedUrl, { method: "PUT", headers: { "Content-Type": "application/gzip" }, body: tarball.bytes, signal: AbortSignal.timeout(REPORT_UPLOAD_TIMEOUT_MS) });
     }
     catch {
         throw new ReportPublishError("Report upload failed.");
@@ -365,6 +367,7 @@ async function discardDraftAction(reportId, options) {
         const res = await fetch(`${DEPLOY_API_URL.replace(/\/+$/, "")}/api/apps/${encodeURIComponent(report.draft.appId)}`, {
             method: "DELETE",
             headers: deployHeaders(token, false, org.headers),
+            signal: AbortSignal.timeout(REPORT_CLEANUP_TIMEOUT_MS),
         });
         if (!res.ok && res.status !== 404)
             throw new ReportPublishError("Could not remove the draft deployment.");
@@ -487,6 +490,7 @@ async function decommissionAction(reportId, options) {
             const res = await fetch(`${DEPLOY_API_URL.replace(/\/+$/, "")}/api/apps/${encodeURIComponent(id)}`, {
                 method: "DELETE",
                 headers: deployHeaders(token, false, org.headers),
+                signal: AbortSignal.timeout(REPORT_CLEANUP_TIMEOUT_MS),
             });
             if (res.status === 404)
                 leftover.push("The published page was already gone.");

package/dist/lib/db/control-plane.d.ts

@@ -51,6 +51,12 @@ export interface RemoteAppSession {
         expiration?: string;
     };
 }
+export interface RemoteDestroyAppResponse {
+    ok: boolean;
+    appId: string;
+    mode: "cluster" | "schema";
+    destroyedAt: string;
+}
 export interface ControlPlaneClientOptions {
     baseUrl: string;
     /** Bearer access token */
@@ -73,5 +79,9 @@ export declare class ControlPlaneDbClient {
         companyUid: string;
         appId: string;
     }): Promise<RemoteAppSession>;
+    destroyApp(input: {
+        companyUid: string;
+        appId: string;
+    }): Promise<RemoteDestroyAppResponse>;
 }
 //# sourceMappingURL=control-plane.d.ts.map

package/dist/lib/db/control-plane.js

@@ -109,6 +109,24 @@ export class ControlPlaneDbClient {
             throw httpError("session", res.status, text);
         return JSON.parse(text);
     }
+    async destroyApp(input) {
+        const token = await this.getAccessToken();
+        const url = `${this.baseUrl}/v1/db/apps/${encodeURIComponent(input.appId)}/destroy`;
+        const res = await this.fetchImpl(url, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${token}`,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({ companyUid: input.companyUid }),
+            signal: AbortSignal.timeout(DB_PROVISION_TIMEOUT_MS),
+        });
+        const text = await res.text();
+        assertNoPostgresUrl("destroy", text);
+        if (!res.ok)
+            throw httpError("destroy", res.status, text);
+        return JSON.parse(text);
+    }
 }
 function httpError(label, status, text) {
     let msg = `${label} failed (${status})`;

package/dist/lib/deploy-api.js

@@ -1,5 +1,6 @@
 /** Shared authenticated request path for the hq-deploy control plane. */
 export const DEPLOY_API_URL = process.env.HQ_DEPLOY_API_URL?.trim() || "https://api.indigo-hq.com";
+const DEPLOY_API_REQUEST_TIMEOUT_MS = 30_000;
 export function deployHeaders(token, json = true, extra = {}) {
     return {
         Authorization: `Bearer ${token}`,
@@ -16,6 +17,7 @@ export async function deployApiRequest(token, route, method, body, extraHeaders
         method,
         headers: deployHeaders(token, body !== undefined, extraHeaders),
         ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+        signal: AbortSignal.timeout(DEPLOY_API_REQUEST_TIMEOUT_MS),
     });
 }
 //# sourceMappingURL=deploy-api.js.map

package/dist/lib/doctor/fix/apply.d.ts

@@ -127,6 +127,8 @@ export interface AppliedFixBackup {
 }
 /** A newly unlocked safe repair can be picked up for at most this many passes. */
 export declare const MAX_FIX_PASSES = 3;
+/** Two 90s pulls leave 120s for startup and check overhead under the hook's 300s ceiling. */
+export declare const MAX_REFRESH_SYNC_PLANS_PER_RUN = 2;
 /** The outcome of an {@link applyFixes} run. */
 export interface ApplyFixesResult {
     /** 0 on success, decline, or a clean no-op; 1 when a dirty tree is refused. */

package/dist/lib/doctor/fix/apply.js

@@ -47,6 +47,43 @@ export const HOOK_CONFIG_DIRS = [".claude", ".codex", ".grok"];
 export const REFRESH_SYNC_TIMEOUT_MS = 90_000;
 /** A newly unlocked safe repair can be picked up for at most this many passes. */
 export const MAX_FIX_PASSES = 3;
+/** Two 90s pulls leave 120s for startup and check overhead under the hook's 300s ceiling. */
+export const MAX_REFRESH_SYNC_PLANS_PER_RUN = 2;
+const REFRESH_SYNC_PROGRESS_RELATIVE_PATH = path.join("workspace", "doctor", "refresh-sync-progress.json");
+function readRefreshSyncProgress(hqRoot) {
+    let contents;
+    try {
+        contents = fs.readFileSync(path.join(hqRoot, REFRESH_SYNC_PROGRESS_RELATIVE_PATH), "utf8");
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return null;
+        throw error;
+    }
+    const parsed = JSON.parse(contents);
+    if (typeof parsed === "object" && parsed !== null &&
+        parsed.version === 1 &&
+        typeof parsed.lastAttemptedSlug === "string") {
+        return parsed;
+    }
+    throw new Error(`Invalid refresh-sync progress cursor at ${REFRESH_SYNC_PROGRESS_RELATIVE_PATH}`);
+}
+function writeRefreshSyncProgress(hqRoot, slug) {
+    const progressPath = path.join(hqRoot, REFRESH_SYNC_PROGRESS_RELATIVE_PATH);
+    const directory = path.dirname(progressPath);
+    fs.mkdirSync(directory, { recursive: true });
+    const temporaryPath = `${progressPath}.${process.pid}.tmp`;
+    fs.writeFileSync(temporaryPath, `${JSON.stringify({ version: 1, lastAttemptedSlug: slug })}\n`, { mode: 0o600 });
+    fs.renameSync(temporaryPath, progressPath);
+}
+function rotateRefreshSyncPlans(plans, afterSlug) {
+    const ordered = [...plans].sort((a, b) => a.target < b.target ? -1 : a.target > b.target ? 1 : 0);
+    if (!afterSlug || ordered.length < 2)
+        return ordered;
+    const next = ordered.findIndex((plan) => plan.target > afterSlug);
+    const start = next === -1 ? 0 : next;
+    return [...ordered.slice(start), ...ordered.slice(0, start)];
+}
 /**
  * Apply every auto-fixable finding, honouring the dirty-tree refusal, the
  * preview/confirmation gate, the pre-write backup, and the post-fix re-check.
@@ -87,6 +124,8 @@ export async function applyFixes(options) {
     //    Identity is deliberately the check id + target pair: an unsuccessful
     //    repair must not be applied repeatedly in one invocation.
     let findings = await collect(hqRoot);
+    let lastAttemptedRefreshSyncSlug = readRefreshSyncProgress(hqRoot)?.lastAttemptedSlug ?? null;
+    let refreshSyncAppliedThisRun = 0;
     const appliedKeys = new Set();
     const planned = [];
     const backups = [];
@@ -114,7 +153,11 @@ export async function applyFixes(options) {
             keysThisPass.add(key);
             plans.push(plan);
         }
-        return plans;
+        const refreshPlans = rotateRefreshSyncPlans(plans.filter((plan) => plan.fixClass === "refresh-sync"), lastAttemptedRefreshSyncSlug).slice(0, Math.max(0, MAX_REFRESH_SYNC_PLANS_PER_RUN - refreshSyncAppliedThisRun));
+        return [
+            ...plans.filter((plan) => plan.fixClass !== "refresh-sync"),
+            ...refreshPlans,
+        ];
     };
     for (let pass = 1; pass <= MAX_FIX_PASSES; pass += 1) {
         const plans = buildPlans(findings);
@@ -154,6 +197,14 @@ export async function applyFixes(options) {
         // 5. Apply each plan once. Later findings with this check id + target are
         //    excluded even if this repair did not make its check pass.
         for (const plan of plans) {
+            if (plan.fixClass === "refresh-sync") {
+                // Persist before starting the bounded pull. If this process is killed,
+                // the next invocation resumes after this company instead of starving
+                // every later journal behind it.
+                writeRefreshSyncProgress(hqRoot, plan.target);
+                lastAttemptedRefreshSyncSlug = plan.target;
+                refreshSyncAppliedThisRun += 1;
+            }
             appliedKeys.add(JSON.stringify([plan.checkId, plan.target]));
             planned.push({ plan, pass });
             await plan.apply();

package/dist/lib/flag-registry.d.ts

@@ -63,7 +63,7 @@ export declare function resolveFlagGate(reader: FlagReader | null | undefined, f
 /** Resolve one explicit company's flag without relying on process-wide binding. */
 export declare function resolveCompanyFlagGate(companyUid: string, idToken: string, flagKey: string, lookup: FlagLookupOptions, legacyFallback: () => boolean, dependencies?: CompanyFlagGateDependencies): Promise<boolean>;
 /** Resolve a process gate against the shared snapshot reader, when configured. */
-export declare function resolveProcessFlagGate(flagKey: string, lookup: FlagLookupOptions, legacyFallback: () => boolean): boolean;
+export declare function resolveProcessFlagGate(flagKey: string, lookup: FlagLookupOptions, legacyFallback: () => boolean, companyUid?: string): boolean;
 /**
  * The registry snapshot held by this process, or null when none has arrived
  * (no endpoint configured, offline, or still fetching). Callers that must

package/dist/lib/flag-registry.js

@@ -11,7 +11,7 @@ import { resolveFlagRegistryEndpoint, } from "./flag-registry-endpoint.js";
 import { FLAG_REGISTRY_WORKER_SOURCE } from "./flag-registry-worker-source.generated.js";
 import { resolveProcessFlagCompanyContext } from "./process-flag-company-context.js";
 import { registerFlagRegistryRefreshWaiter } from "./flag-registry-refresh-wait.js";
-import { flagSnapshotWriteFailureTags, writeFlagSnapshotFile } from "./flag-snapshot-file.js";
+import { flagSnapshotWriteFailureTags, readFlagSnapshotFile, writeFlagSnapshotFile, } from "./flag-snapshot-file.js";
 function captureFlagRegistryException(error, tags, sentry) {
     if (!sentry)
         return;
@@ -298,13 +298,26 @@ export async function resolveCompanyFlagGate(companyUid, idToken, flagKey, looku
     }
 }
 /** Resolve a process gate against the shared snapshot reader, when configured. */
-export function resolveProcessFlagGate(flagKey, lookup, legacyFallback) {
-    // The endpoint refresh is deliberately asynchronous. Until a validated
-    // snapshot arrives, preserve each caller's exact legacy fallback.
-    if (!processReader || !heldSnapshot) {
-        return localOverrideOrFallback(flagKey, legacyFallback);
+export function resolveProcessFlagGate(flagKey, lookup, legacyFallback, companyUid) {
+    const targetCompanyUid = companyUid ?? (processReader
+        ? refreshCompanyUid
+        : resolveProcessFlagCompanyContext().companyUid);
+    const processSnapshotMatchesTarget = refreshCompanyUid === targetCompanyUid;
+    if (processReader && heldSnapshot && processSnapshotMatchesTarget) {
+        return resolveFlagGate(processReader, flagKey, lookup, legacyFallback);
+    }
+    // A command can reach a synchronous gate before the startup worker returns.
+    // Use the last validated, fresh company snapshot in that window, and when
+    // the process reader was initialized for a different company scope.
+    if (targetCompanyUid) {
+        const cached = readFlagSnapshotFile(targetCompanyUid);
+        if (cached) {
+            const snapshot = { version: 0, flags: cached.flags };
+            const reader = createProcessSnapshotReader([targetCompanyUid], () => snapshot);
+            return resolveFlagGate(reader, flagKey, lookup, legacyFallback);
+        }
     }
-    return resolveFlagGate(processReader, flagKey, lookup, legacyFallback);
+    return localOverrideOrFallback(flagKey, legacyFallback);
 }
 /**
  * The registry snapshot held by this process, or null when none has arrived

package/dist/lib/lanes/accounts.js

@@ -233,9 +233,31 @@ export function selectBalancedAccount(provider, config, ctx = {}) {
         })
         : proven;
     if (quota && runnable.length === 0) {
+        const blocked = proven.map((id) => ({
+            id,
+            block: quotaBlockFor(quota, provider, id, modelForAccount(provider, id, config, ctx), now),
+        }));
+        const resetTimes = blocked
+            .map(({ block }) => block.quota_reset_at)
+            .filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value)))
+            .sort((a, b) => Date.parse(a) - Date.parse(b));
+        const earliest = blocked.find(({ block }) => block.reason === "account_quota_exhausted");
+        if (earliest !== undefined) {
+            const resetsByAccount = blocked.map(({ id, block }) => ({
+                account: id,
+                reason: block.reason,
+                quota_reset_at: block.quota_reset_at ?? null,
+            }));
+            throw new AccountResolutionError("account_quota_exhausted", `All logged-in accounts for ${provider} are quota-limited for the selected model`, {
+                provider,
+                account: earliest.id,
+                model: earliest.block.model ?? null,
+                quota_reset_at: resetTimes[0] ?? null,
+                account_reasons: resetsByAccount,
+            });
+        }
         const sample = proven[0];
-        const block = quotaBlockFor(quota, provider, sample, modelForAccount(provider, sample, config, ctx), now);
-        throwIfQuotaBlocked(block, provider, sample);
+        throwIfQuotaBlocked(blocked[0].block, provider, sample);
     }
     const underCap = runnable.filter((id) => {
         const cap = accountMaxRunning(providerCfg, id);

package/dist/lib/lanes/delivery.d.ts

@@ -68,14 +68,17 @@ export declare const LANE_DELIVERY_MECHANISMS: readonly ["dropbox", "resume", "n
 export type LaneDeliveryMechanism = (typeof LANE_DELIVERY_MECHANISMS)[number];
 /** Additional presentation-only result used by `hq lanes show`. */
 export type LaneShowDeliveryMechanism = LaneDeliveryMechanism | "questions";
-export type DeliveryLane = Pick<LaneRecord, "state" | "provider" | "provider_session_id" | "terminal_reason" | "envelope_repair">;
+export type DeliveryLane = Pick<LaneRecord, "state" | "provider" | "provider_session_id" | "terminal_reason" | "envelope_repair" | "keep_alive" | "quota_reset_at" | "terminal_details">;
+export declare function isFailedQuotaKeepAliveLane(lane: DeliveryLane): boolean;
 /**
  * `dropbox`: a process is (or is about to be) alive to drain the inbox.
  * `resume`: the process has exited, the provider can resume, the state allows
  * it, and a session id was recorded from the worker's own result.
  * `none`: nothing can deliver; the caller must refuse and write nothing.
  */
-export declare function laneDeliveryMechanism(lane: DeliveryLane): LaneDeliveryMechanism;
+export declare function laneDeliveryMechanism(lane: DeliveryLane, options?: {
+    forMessage?: boolean;
+}): LaneDeliveryMechanism;
 export declare function hasRecordedSessionId(lane: Pick<LaneRecord, "provider_session_id">): lane is Pick<LaneRecord, "provider_session_id"> & {
     provider_session_id: string;
 };

package/dist/lib/lanes/delivery.js

@@ -94,17 +94,25 @@ function isFailedEnvelopeRepairLane(lane) {
         lane.envelope_repair?.attempted === true &&
         lane.envelope_repair.succeeded === false);
 }
+export function isFailedQuotaKeepAliveLane(lane) {
+    const detailsReset = lane.terminal_details?.quota_reset_at;
+    const resetAt = lane.quota_reset_at ?? (typeof detailsReset === "string" ? detailsReset : undefined);
+    return lane.state === "failed" && lane.keep_alive === true &&
+        (lane.terminal_reason === "account_quota_exhausted" || lane.terminal_reason === "account_model_unavailable") &&
+        typeof resetAt === "string" && Number.isFinite(Date.parse(resetAt));
+}
 /**
  * `dropbox`: a process is (or is about to be) alive to drain the inbox.
  * `resume`: the process has exited, the provider can resume, the state allows
  * it, and a session id was recorded from the worker's own result.
  * `none`: nothing can deliver; the caller must refuse and write nothing.
  */
-export function laneDeliveryMechanism(lane) {
+export function laneDeliveryMechanism(lane, options = {}) {
     if (lane.state === "queued" || lane.state === "running")
         return "dropbox";
     if (providerSupportsSessionResume(lane.provider) &&
-        (isResumableLaneState(lane.state) || lane.state === "awaiting_input" || isFailedEnvelopeRepairLane(lane)) &&
+        (isResumableLaneState(lane.state) || lane.state === "awaiting_input" || isFailedEnvelopeRepairLane(lane) ||
+            (!options.forMessage && isFailedQuotaKeepAliveLane(lane))) &&
         hasRecordedSessionId(lane)) {
         return "resume";
     }
@@ -350,6 +358,7 @@ export function resumeCommandRefusal(lane, options = {}) {
     if (!isResumeCommandState(lane.state) &&
         !(options.allowAwaitingInput === true && lane.state === "awaiting_input") &&
         !isFailedEnvelopeRepairLane(lane) &&
+        !isFailedQuotaKeepAliveLane(lane) &&
         !(options.allowFailed === true && lane.state === "failed")) {
         return {
             error: "lane_not_resumable",
@@ -393,14 +402,14 @@ export function planLaneLaunch(lane, trigger) {
             }
             return { mode: "fresh" };
         case "message": {
-            if (laneDeliveryMechanism(lane) !== "resume" || !hasRecordedSessionId(lane)) {
+            if (laneDeliveryMechanism(lane, { forMessage: true }) !== "resume" || !hasRecordedSessionId(lane)) {
                 throw new LanesError("lane_not_accepting_messages", deliveryRefusalReason(lane) || `Lane is ${lane.state} and cannot be resumed`, { state: lane.state, provider: lane.provider });
             }
             return { mode: "resume", session_id: lane.provider_session_id };
         }
         case "resume": {
             const refusal = resumeCommandRefusal(lane, {
-                allowFailed: trigger.fromFailed === true,
+                allowFailed: trigger.fromFailed === true || isFailedQuotaKeepAliveLane(lane),
                 allowAwaitingInput: trigger.allowAwaitingInput === true,
             });
             if (refusal !== null || !hasRecordedSessionId(lane)) {
@@ -429,7 +438,7 @@ export function claimableStatesFor(trigger, lane) {
                     ? [...RESUMABLE_LANE_STATES, "awaiting_input"]
                     : RESUMABLE_LANE_STATES;
         case "resume": {
-            const states = trigger.fromFailed === true || envelopeRepairFailed
+            const states = trigger.fromFailed === true || envelopeRepairFailed || (lane !== undefined && isFailedQuotaKeepAliveLane(lane))
                 ? [...RESUME_COMMAND_STATES, "failed"]
                 : [...RESUME_COMMAND_STATES];
             return lane?.state === "awaiting_input" ? [...states, "awaiting_input"] : states;

package/dist/lib/lanes/index.d.ts

@@ -63,7 +63,7 @@ export type { RegistryPersona } from "./registry-persona.js";
 export type { RegistryWorkerEntry, ResolvedLaneWorker, WorkerTenancy, WorkerYaml, } from "./registry.js";
 export { bodyStoreForLane, envelopeQuestionId, parseQuestionBodyStore, questionBodyAbsPath, questionBodyRef, raiseEnvelopeQuestions, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "./questions.js";
 export type { QuestionBody, RaiseLaneQuestionInput } from "./questions.js";
-export { LANE_DELIVERY_MECHANISMS, LAUNCH_TRIGGERS, PROVIDER_SESSION_RESUME, PROVIDER_SESSION_PORTABLE, providerSessionPortable, sessionNotPortableMessage, parseGrokResultOutput, RESUMABLE_LANE_STATES, RESUME_COMMAND_STATES, SESSION_ID_ABSENT_REASONS, claimableStatesFor, deliveryRefusalReason, hasRecordedSessionId, isResumableLaneState, isResumeCommandState, laneDeliveryMechanism, parseClaudeResultOutput, parseCodexEventOutput, parseWorkerRunOutput, planLaneLaunch, providerSupportsSessionResume, readWorkerRunResult, resumeCommandRefusal, } from "./delivery.js";
+export { LANE_DELIVERY_MECHANISMS, LAUNCH_TRIGGERS, PROVIDER_SESSION_RESUME, PROVIDER_SESSION_PORTABLE, providerSessionPortable, sessionNotPortableMessage, parseGrokResultOutput, RESUMABLE_LANE_STATES, RESUME_COMMAND_STATES, SESSION_ID_ABSENT_REASONS, claimableStatesFor, deliveryRefusalReason, hasRecordedSessionId, isFailedQuotaKeepAliveLane, isResumableLaneState, isResumeCommandState, laneDeliveryMechanism, parseClaudeResultOutput, parseCodexEventOutput, parseWorkerRunOutput, planLaneLaunch, providerSupportsSessionResume, readWorkerRunResult, resumeCommandRefusal, } from "./delivery.js";
 export type { DeliveryLane, LaneDeliveryMechanism, LaneShowDeliveryMechanism, LaunchPlan, LaunchTrigger, LaunchTriggerKind, ResumableLaneState, ResumeCommandState, SessionIdAbsentReason, WorkerRunResult, } from "./delivery.js";
 export { DROPBOX_DIR, INBOX_NAME, appendDropboxMessage, answerDropboxMessage, drainDropbox, dropboxDir, dropboxInboxPath, formatOperatorInstruction, hookAdditionalContextJson, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneIdForAnswerDelivery, operatorMessageRefusalReason, readToolActivity, recordToolActivity, runDrainHook, toolActivityPath, TOOL_ACTIVITY_NAME, } from "./dropbox.js";
 export type { DrainDeps, DrainHookResult, DropboxMessage, ToolActivityRecord, } from "./dropbox.js";

package/dist/lib/lanes/index.js

@@ -34,7 +34,7 @@ export { WORKER_REGISTRY_REL, personalWorkerDir, personalWorkerExists, readWorke
 export { ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithRegistry, materializeWorkerProfile, personalWorkerProfileDir, REGISTRY_PERSONA_PROMPT_MAX_CHARS, workerProfileMarker, workerProfilePrompt, } from "./profile.js";
 export { loadRegistryPersonaFromRecord, registryWorkerRecord, resolveRegistryPersona, } from "./registry-persona.js";
 export { bodyStoreForLane, envelopeQuestionId, parseQuestionBodyStore, questionBodyAbsPath, questionBodyRef, raiseEnvelopeQuestions, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "./questions.js";
-export { LANE_DELIVERY_MECHANISMS, LAUNCH_TRIGGERS, PROVIDER_SESSION_RESUME, PROVIDER_SESSION_PORTABLE, providerSessionPortable, sessionNotPortableMessage, parseGrokResultOutput, RESUMABLE_LANE_STATES, RESUME_COMMAND_STATES, SESSION_ID_ABSENT_REASONS, claimableStatesFor, deliveryRefusalReason, hasRecordedSessionId, isResumableLaneState, isResumeCommandState, laneDeliveryMechanism, parseClaudeResultOutput, parseCodexEventOutput, parseWorkerRunOutput, planLaneLaunch, providerSupportsSessionResume, readWorkerRunResult, resumeCommandRefusal, } from "./delivery.js";
+export { LANE_DELIVERY_MECHANISMS, LAUNCH_TRIGGERS, PROVIDER_SESSION_RESUME, PROVIDER_SESSION_PORTABLE, providerSessionPortable, sessionNotPortableMessage, parseGrokResultOutput, RESUMABLE_LANE_STATES, RESUME_COMMAND_STATES, SESSION_ID_ABSENT_REASONS, claimableStatesFor, deliveryRefusalReason, hasRecordedSessionId, isFailedQuotaKeepAliveLane, isResumableLaneState, isResumeCommandState, laneDeliveryMechanism, parseClaudeResultOutput, parseCodexEventOutput, parseWorkerRunOutput, planLaneLaunch, providerSupportsSessionResume, readWorkerRunResult, resumeCommandRefusal, } from "./delivery.js";
 export { DROPBOX_DIR, INBOX_NAME, appendDropboxMessage, answerDropboxMessage, drainDropbox, dropboxDir, dropboxInboxPath, formatOperatorInstruction, hookAdditionalContextJson, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneIdForAnswerDelivery, operatorMessageRefusalReason, readToolActivity, recordToolActivity, runDrainHook, toolActivityPath, TOOL_ACTIVITY_NAME, } from "./dropbox.js";
 export { DRAIN_HOOK_BASENAME, DRAIN_HOOK_REGISTRATION_KINDS, DRAIN_HOOK_TIMEOUT_SECONDS, PROVIDER_REGISTRATION_KIND, drainHookCommand, drainHookRegistrationPath, drainHookShPath, installDrainHook, lanesHookAssetsDir, mergeClaudeSettingsHook, mergeCodexConfigHook, probeDrainHookRegistration, renderDrainHookJson, } from "./hooks-install.js";
 export { EFFORT_SOURCES, PROVIDER_EFFORT_LEVELS, appendEffortFlag, defaultEffortForModel, parseCliEffort, parseConfigEffort, resolveLaneEffort, } from "./effort.js";

package/dist/lib/lanes/reconcile.js

@@ -29,7 +29,7 @@ import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
 import { appendDropboxMessage } from "./dropbox.js";
 import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
 import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, dueKeepAliveNextCheckMs, hasPendingKeepAliveWork, keepAliveNextCheckAtMs, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
-import { providerSupportsSessionResume } from "./delivery.js";
+import { isFailedQuotaKeepAliveLane, providerSupportsSessionResume } from "./delivery.js";
 import { isIsoDateTime } from "./types.js";
 import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, loadQuotaState, readQuotaSignal, recordAccountQuota, } from "./quota.js";
 import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
@@ -2586,7 +2586,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
         dueAdmissionRan = true;
         await admitDueCandidates(new Set(queuedDueCandidateIds));
     };
-    for (const originalLane of candidates) {
+    for (let originalLane of candidates) {
         if (admitDueCandidates !== undefined && dueCandidateIds.size > 0 && !dueCandidateIds.has(originalLane.lane_id)) {
             await admitQueuedDueCandidates();
         }
@@ -2596,14 +2596,50 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
             continue;
         if (!laneRunsHere(originalLane, deps.host))
             continue;
+        const failedQuotaHold = isFailedQuotaKeepAliveLane(originalLane);
         if (originalLane.state === "killed" ||
             originalLane.state === "cancelled" ||
-            originalLane.state === "failed") {
+            (originalLane.state === "failed" && !failedQuotaHold)) {
             if (originalLane.keep_alive === true &&
                 flagByCompany.get(keepAliveCompanyKey(companyForLane(originalLane))) === true)
                 recordKeepAliveSkip(hqRoot, originalLane, "terminal_state", summary, deps);
             continue;
         }
+        if (failedQuotaHold) {
+            const detailsReset = originalLane.terminal_details?.quota_reset_at;
+            const resetAt = originalLane.quota_reset_at ??
+                (typeof detailsReset === "string" ? detailsReset : undefined);
+            const companyKey = keepAliveCompanyKey(companyForLane(originalLane));
+            if (resetAt === undefined ||
+                Date.parse(resetAt) > nowMs ||
+                flagByCompany.get(companyKey) !== true)
+                continue;
+            const resetAtMs = Date.parse(resetAt);
+            const priorInterruptedAtMs = Date.parse(originalLane.interrupted_at ?? "");
+            const optedInAtMs = Date.parse(originalLane.keep_alive_opted_in_at ?? "");
+            const recoveredInterruptedAt = new Date(Math.max(Number.isFinite(priorInterruptedAtMs) ? priorInterruptedAtMs : resetAtMs, Number.isFinite(optedInAtMs) ? optedInAtMs + 1 : resetAtMs)).toISOString();
+            const recovered = casLaneIf(hqRoot, originalLane.lane_id, (current) => current.version === originalLane.version &&
+                current.state === "failed" &&
+                current.keep_alive === true &&
+                current.terminal_reason === originalLane.terminal_reason &&
+                (current.quota_reset_at ?? current.terminal_details?.quota_reset_at) === resetAt, (current) => ({
+                ...current,
+                state: "interrupted",
+                interrupted_at: recoveredInterruptedAt,
+                quota_reset_at: resetAt,
+                keep_alive_next_eligible_at: resetAt,
+                keep_alive_resume_count: 0,
+                keep_alive_last_resume_at: undefined,
+                keep_alive_exhausted_at: undefined,
+                queued_reason: undefined,
+                queued_details: undefined,
+                queued_at: undefined,
+                admission_reserved_at: undefined,
+            }), deps.dryRun);
+            if (!recovered.changed)
+                continue;
+            originalLane = recovered.record;
+        }
         if (originalLane.state === "blocked" && originalLane.keep_alive === true) {
             await resumeBlockedKeepAlive(hqRoot, originalLane, questions, summary, deps, now, flagByCompany, blockedResumeFlagByCompany, companyForLane, lanes, retiredLaneIds, supersessionLineageAwareByCompany);
             continue;
@@ -2714,7 +2750,11 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
                 continue;
             }
         }
-        if (initialState === "interrupted" && lane.terminal_reason !== "exited_with_partial_output") {
+        const quotaHold = lane.keep_alive === true &&
+            (lane.terminal_reason === "account_quota_exhausted" || lane.terminal_reason === "account_model_unavailable") &&
+            typeof lane.quota_reset_at === "string" &&
+            Number.isFinite(Date.parse(lane.quota_reset_at));
+        if (initialState === "interrupted" && lane.terminal_reason !== "exited_with_partial_output" && !quotaHold) {
             recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason ?? "missing interruption reason");
             continue;
         }
@@ -2816,7 +2856,11 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
             recordKeepAliveSkip(hqRoot, current, "not_opted_in", summary, deps);
             continue;
         }
-        if (initialState === "interrupted" && current.terminal_reason !== "exited_with_partial_output") {
+        const currentQuotaHold = current.keep_alive === true &&
+            (current.terminal_reason === "account_quota_exhausted" || current.terminal_reason === "account_model_unavailable") &&
+            typeof current.quota_reset_at === "string" &&
+            Number.isFinite(Date.parse(current.quota_reset_at));
+        if (initialState === "interrupted" && current.terminal_reason !== "exited_with_partial_output" && !currentQuotaHold) {
             recordKeepAliveSkip(hqRoot, current, "terminal_state", summary, deps, current.terminal_reason ?? "missing interruption reason");
             continue;
         }

package/dist/lib/lanes/spawn.d.ts

@@ -15,11 +15,11 @@ import { type LaunchTrigger } from "./delivery.js";
 import { type MachineIdentity } from "./machine-identity.js";
 import { type AdmissionHost } from "./admission.js";
 import { type StatFs } from "./disk-guard.js";
-import { type LaneCompanyRef, type LaneProvider, type LaneRecord, type AccountHistoryEntry } from "./types.js";
+import { type LaneCompanyRef, type LaneProvider, type LaneRecord, type AccountHistoryEntry, type PendingResume } from "./types.js";
 export declare const LANES_RUNS_REL: string;
 /** Written by reconcile before a resume spawn; the worker prompt names this path. */
 export declare const ANSWERED_QUESTIONS_FILE = "answered-questions.json";
-export declare const SPAWN_TERMINAL_REASONS: readonly ["account_not_logged_in", "unknown_account", "account_disabled", "no_enabled_account", "spawn_failed", "not_detached", "watcher_not_armed", "model_unsupported", "account_quota_exhausted"];
+export declare const SPAWN_TERMINAL_REASONS: readonly ["account_not_logged_in", "unknown_account", "account_disabled", "no_enabled_account", "spawn_failed", "not_detached", "watcher_not_armed", "model_unsupported", "account_quota_exhausted", "account_model_unavailable"];
 export type SpawnTerminalReason = (typeof SPAWN_TERMINAL_REASONS)[number];
 declare const PROVIDER_COMMAND: Record<LaneProvider, string>;
 export declare function providerCommandOnPath(command: string, pathEnv: string | undefined): boolean;
@@ -244,7 +244,11 @@ export declare function workerEnv(base: NodeJS.ProcessEnv, resolved: {
 export declare const WORKER_POLICY_HOOK_CAPABILITY = "# hq-capability: worker-policy-dir v1";
 /** Grok does not run HQ hooks; Claude and Codex use the installed capability marker. */
 export declare function workerPolicyDeliveryMode(hqRoot: string, provider: LaneProvider): "hook" | "prompt";
-export declare function markSpawnFailure(hqRoot: string, laneId: string, reasonRaw: string, details: Record<string, unknown>): LaneRecord;
+export declare function markSpawnFailure(hqRoot: string, laneId: string, reasonRaw: string, details: Record<string, unknown>, options?: {
+    resumeTrigger?: "resume" | "message";
+    pendingResume?: PendingResume;
+    at?: string;
+}): LaneRecord;
 /**
  * Spec §5.2 executing-side check. Compares this machine's identity to
  * `origin_host` (never to `host.id`). Unresolvable identity is a cancel.