@indigoai-us/hq-cli 5.345.38 → 5.345.40

package/dist/lib/lanes/spawn.js

@@ -51,6 +51,7 @@ export const SPAWN_TERMINAL_REASONS = [
     "watcher_not_armed",
     "model_unsupported",
     "account_quota_exhausted",
+    "account_model_unavailable",
 ];
 const PROVIDER_COMMAND = {
     claude: "claude",
@@ -237,6 +238,7 @@ function selectSpawnAccount(input) {
         existsSync,
         accountDeps,
         hostname,
+        now,
         ...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
     };
     if (lane.account === undefined) {
@@ -374,6 +376,27 @@ function selectSpawnAccount(input) {
         };
     }
     catch (err) {
+        if (refusal instanceof AccountResolutionError &&
+            refusal.error === "account_quota_exhausted" &&
+            err instanceof AccountResolutionError &&
+            err.error === "account_quota_exhausted") {
+            const fallbackReasons = Array.isArray(err.details.account_reasons)
+                ? err.details.account_reasons.filter((candidate) => candidate !== null && typeof candidate === "object" && !Array.isArray(candidate))
+                : [{ account: err.details.account, reason: err.error, quota_reset_at: err.details.quota_reset_at }];
+            const accountReasons = [
+                { account: lane.account, reason: refusal.error, quota_reset_at: refusal.details.quota_reset_at },
+                ...fallbackReasons,
+            ];
+            const resetAt = accountReasons
+                .map((candidate) => candidate.quota_reset_at)
+                .filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value)))
+                .sort((a, b) => Date.parse(a) - Date.parse(b))[0];
+            throw new AccountResolutionError("account_quota_exhausted", err.message, {
+                ...err.details,
+                ...(resetAt === undefined ? {} : { quota_reset_at: resetAt }),
+                account_reasons: accountReasons,
+            });
+        }
         if (reason === "account_unavailable" || reason === "recorded_account_logged_out") {
             if (refusal instanceof Error)
                 throw refusal;
@@ -1458,22 +1481,56 @@ function killQuietly(kill, pid, label) {
         throw err;
     }
 }
-export function markSpawnFailure(hqRoot, laneId, reasonRaw, details) {
+export function markSpawnFailure(hqRoot, laneId, reasonRaw, details, options = {}) {
     const reason = parseSpawnTerminalReason(reasonRaw);
     return retryUpdateLane(hqRoot, laneId, "spawn", (current) => {
         const existing = current
             .terminal_reason;
-        if (current.state === "failed" && existing === reason)
+        const resetCandidates = [
+            details.quota_reset_at,
+            ...(Array.isArray(details.account_reasons)
+                ? details.account_reasons.flatMap((candidate) => candidate !== null && typeof candidate === "object" && !Array.isArray(candidate)
+                    ? [candidate.quota_reset_at]
+                    : [])
+                : []),
+        ].filter((value) => typeof value === "string" && Number.isFinite(Date.parse(value)));
+        const quotaResetAt = resetCandidates.sort((a, b) => Date.parse(a) - Date.parse(b))[0];
+        const resumableQuotaHold = current.keep_alive === true &&
+            (current.pending_resume !== undefined || options.resumeTrigger !== undefined) &&
+            (reason === "account_quota_exhausted" || reason === "account_model_unavailable") &&
+            typeof quotaResetAt === "string" &&
+            Number.isFinite(Date.parse(quotaResetAt));
+        if (current.state === "failed" && existing === reason && !resumableQuotaHold)
             return current;
+        const operatorQuotaHold = resumableQuotaHold && options.resumeTrigger !== undefined;
         return {
             ...current,
-            state: "failed",
+            state: resumableQuotaHold ? "interrupted" : "failed",
             queued_reason: undefined,
             queued_details: undefined,
             queued_at: undefined,
             admission_reserved_at: undefined,
             terminal_reason: reason,
             terminal_details: details,
+            ...(resumableQuotaHold
+                ? {
+                    ...(options.pendingResume !== undefined
+                        ? { pending_resume: options.pendingResume }
+                        : current.pending_resume === undefined && options.resumeTrigger !== undefined
+                            ? { pending_resume: { trigger: options.resumeTrigger, at: options.at ?? current.updated_at } }
+                            : {}),
+                    interrupted_at: options.at ?? current.updated_at,
+                    quota_reset_at: quotaResetAt,
+                    keep_alive_next_eligible_at: quotaResetAt,
+                    ...(operatorQuotaHold
+                        ? {
+                            keep_alive_resume_count: 0,
+                            keep_alive_last_resume_at: undefined,
+                            keep_alive_exhausted_at: undefined,
+                        }
+                        : {}),
+                }
+                : {}),
         };
     });
 }
@@ -1891,8 +1948,41 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
         if (err instanceof AccountResolutionError ||
             (err instanceof LanesError &&
                 (err.error === "account_not_logged_in" || err.error === "model_unsupported"))) {
-            if (markPreSpawnFailure) {
-                markSpawnFailure(hqRoot, lane.lane_id, err.error, err.details);
+            const resumableQuotaFailure = existing.keep_alive === true &&
+                (existing.pending_resume !== undefined || trigger.kind === "resume" || trigger.kind === "message") &&
+                err instanceof AccountResolutionError &&
+                (err.error === "account_quota_exhausted" || err.error === "account_model_unavailable") &&
+                typeof err.details.quota_reset_at === "string" &&
+                Number.isFinite(Date.parse(err.details.quota_reset_at));
+            if (markPreSpawnFailure || resumableQuotaFailure) {
+                const at = (deps.now ?? (() => new Date()))().toISOString();
+                const pendingResume = trigger.kind === "message"
+                    ? {
+                        ...(plan.mode === "resume" ? { session_id: plan.session_id } : {}),
+                        trigger: "message",
+                        caller: typeof env.HQ_LANE_ID === "string" ? env.HQ_LANE_ID.trim() || "user" : "user",
+                        text: trigger.text,
+                        ...(trigger.message_id !== undefined ? { message_id: trigger.message_id } : {}),
+                        ...(trigger.monitor_event === true ? { monitor_event: true } : {}),
+                        compact: launch.compact !== false,
+                        at,
+                    }
+                    : trigger.kind === "resume"
+                        ? {
+                            ...(plan.mode === "resume" ? { session_id: plan.session_id } : {}),
+                            trigger: "resume",
+                            resumed_from: trigger.from,
+                            ...(trigger.text !== undefined ? { text: trigger.text } : {}),
+                            ...(trigger.message_id !== undefined ? { message_id: trigger.message_id } : {}),
+                            compact: launch.compact !== false,
+                            at,
+                        }
+                        : existing.pending_resume;
+                markSpawnFailure(hqRoot, lane.lane_id, err.error, err.details, {
+                    ...(trigger.kind === "resume" || trigger.kind === "message" ? { resumeTrigger: trigger.kind } : {}),
+                    ...(pendingResume !== undefined ? { pendingResume } : {}),
+                    at,
+                });
             }
             throw err;
         }
@@ -2410,6 +2500,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
         const failed = markSpawnFailure(hqRoot, lane.lane_id, reason, {
             message: err instanceof Error ? err.message : String(err),
             ...(err instanceof LanesError ? err.details : {}),
+        }, {
+            ...(trigger.kind === "resume" || trigger.kind === "message" ? { resumeTrigger: trigger.kind } : {}),
         });
         if (err instanceof LanesError) {
             throw new LanesError(err.error, err.message, {

package/dist/lib/lanes/store.js

@@ -18,7 +18,7 @@ import { readOutpostIdentity } from "./admission-host.js";
 import { LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesError, isLaneVersionConflict, } from "./errors.js";
 import { HISTORY_CAP, HOST_KINDS, LANE_STORY_STATUSES, LANE_SCHEMA_VERSION, QUESTION_SCHEMA_VERSION, SENIOR_KINDS, isHostKind, isLaneProvider, isQueuedReason, isLaneState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseMajorSchema, REQUIRED_LANE_RECORD_FIELDS, LANE_FIELD_WRITERS, LANE_WRITERS, } from "./types.js";
 import { laneStories } from "./stories.js";
-import { laneStoryLedgerEnabled, syncLaneToProjectStoryLedger, } from "./story-ledger.js";
+import { laneStoryLedgerEnabled, syncLaneToProjectStoryLedger, validateProjectStories, } from "./story-ledger.js";
 export const LANES_STORE_REL = path.join("workspace", "lanes");
 export const LANE_RECORDS_DIR = "lanes";
 export const QUESTION_RECORDS_DIR = "questions";
@@ -1608,7 +1608,8 @@ export function updateLane(hqRoot, laneId, expectedVersion, writer, updater, dep
         beforeWrite: (current) => assertLaneStoreWriteAllowed(current, laneId, writer),
         stamp: (current) => ({ written_by_version: newestLaneStoreVersion(current.written_by_version) }),
     });
-    if (laneStoryLedgerEnabled() && before && laneLedgerRelevantChange(before, updated)) {
+    const ledgerRelevantChange = before !== undefined && laneLedgerRelevantChange(before, updated);
+    if (ledgerRelevantChange) {
         return syncStoryLedgerForLane(hqRoot, updated, deps);
     }
     return updated;
@@ -1620,21 +1621,54 @@ function laneLedgerRelevantChange(before, after) {
         JSON.stringify(before.stories) !== JSON.stringify(after.stories) ||
         JSON.stringify(before.last_envelope) !== JSON.stringify(after.last_envelope);
 }
-function syncStoryLedgerForLane(hqRoot, lane, deps) {
-    if (!laneStoryLedgerEnabled())
+function validateLaneStoriesInLedgers(hqRoot, lane) {
+    for (const story of laneStories(lane)) {
+        validateProjectStories(hqRoot, story.project_id, [story.story_id], lane.company?.slug);
+    }
+}
+function clearResolvedLedgerSyncError(hqRoot, lane, deps) {
+    if (lane.ledger_sync_error === undefined)
         return lane;
+    try {
+        validateLaneStoriesInLedgers(hqRoot, lane);
+    }
+    catch (error) {
+        if (error instanceof LanesError)
+            return lane;
+        throw error;
+    }
+    return retryUpdateLane(hqRoot, lane.lane_id, "story-ledger", (current) => {
+        if (current.ledger_sync_error === undefined)
+            return current;
+        try {
+            validateLaneStoriesInLedgers(hqRoot, current);
+        }
+        catch (error) {
+            if (error instanceof LanesError)
+                return current;
+            throw error;
+        }
+        return { ...current, ledger_sync_error: undefined };
+    }, deps);
+}
+function syncStoryLedgerForLane(hqRoot, lane, deps) {
+    const currentLane = lane.ledger_sync_error === undefined
+        ? lane
+        : clearResolvedLedgerSyncError(hqRoot, lane, deps);
+    if (!laneStoryLedgerEnabled(currentLane.company?.uid))
+        return currentLane;
     try {
         const questionRefs = [];
-        for (const questionId of lane.pending_question_ids ?? []) {
+        for (const questionId of currentLane.pending_question_ids ?? []) {
             const question = readQuestion(hqRoot, questionId, deps);
             const sameCompany = question.company === null
-                ? lane.company === null
-                : lane.company !== null &&
-                    question.company.uid === lane.company.uid &&
-                    question.company.slug === lane.company.slug;
-            if (question.lane_id !== lane.lane_id || !sameCompany) {
-                throw new LanesError("story_ledger_question_mismatch", `Pending question ${questionId} does not belong to lane ${lane.lane_id}`, {
-                    lane_id: lane.lane_id,
+                ? currentLane.company === null
+                : currentLane.company !== null &&
+                    question.company.uid === currentLane.company.uid &&
+                    question.company.slug === currentLane.company.slug;
+            if (question.lane_id !== currentLane.lane_id || !sameCompany) {
+                throw new LanesError("story_ledger_question_mismatch", `Pending question ${questionId} does not belong to lane ${currentLane.lane_id}`, {
+                    lane_id: currentLane.lane_id,
                     question_id: questionId,
                     question_lane_id: question.lane_id,
                     company: question.company?.slug ?? null,
@@ -1648,24 +1682,24 @@ function syncStoryLedgerForLane(hqRoot, lane, deps) {
                 });
             }
         }
-        syncLaneToProjectStoryLedger(hqRoot, lane, questionRefs);
-        if (lane.ledger_sync_error !== undefined) {
-            retryUpdateLane(hqRoot, lane.lane_id, "story-ledger", (current) => current.ledger_sync_error === undefined
+        syncLaneToProjectStoryLedger(hqRoot, currentLane, questionRefs);
+        if (currentLane.ledger_sync_error !== undefined) {
+            retryUpdateLane(hqRoot, currentLane.lane_id, "story-ledger", (current) => current.ledger_sync_error === undefined
                 ? current
                 : { ...current, ledger_sync_error: undefined }, deps);
         }
-        return readLane(hqRoot, lane.lane_id, deps);
+        return readLane(hqRoot, currentLane.lane_id, deps);
     }
     catch (error) {
         const message = (error instanceof Error ? error.message : String(error)).slice(0, 1000) || "Story ledger synchronization failed";
         try {
-            retryUpdateLane(hqRoot, lane.lane_id, "story-ledger", (current) => current.ledger_sync_error === message
+            retryUpdateLane(hqRoot, currentLane.lane_id, "story-ledger", (current) => current.ledger_sync_error === message
                 ? current
                 : { ...current, ledger_sync_error: message }, deps);
-            return readLane(hqRoot, lane.lane_id, deps);
+            return readLane(hqRoot, currentLane.lane_id, deps);
         }
         catch (recordError) {
-            throw new LanesError("story_ledger_sync_error_unrecorded", "Story ledger synchronization failed and the failure could not be recorded on lane " + lane.lane_id, { lane_id: lane.lane_id, cause: recordError instanceof Error ? recordError.message : String(recordError) });
+            throw new LanesError("story_ledger_sync_error_unrecorded", "Story ledger synchronization failed and the failure could not be recorded on lane " + currentLane.lane_id, { lane_id: currentLane.lane_id, cause: recordError instanceof Error ? recordError.message : String(recordError) });
         }
     }
 }
@@ -1833,7 +1867,7 @@ export function updateQuestion(hqRoot, questionId, expectedVersion, updater, dep
     return next;
 }
 function syncLaneQuestionReference(hqRoot, question, deps) {
-    if (!laneStoryLedgerEnabled())
+    if (!laneStoryLedgerEnabled(question.company?.uid))
         return;
     const lane = tryReadLane(hqRoot, question.lane_id, deps);
     if (!lane || question.company?.slug !== lane.company?.slug || question.company?.uid !== lane.company?.uid)

package/dist/lib/lanes/story-ledger.d.ts

@@ -60,7 +60,7 @@ export interface LaneStatusSnapshot {
         by_state: Record<(typeof STORY_STATES)[number], number>;
     };
 }
-export declare function laneStoryLedgerEnabled(): boolean;
+export declare function laneStoryLedgerEnabled(companyUid?: string): boolean;
 export declare function resolveProjectStoryLedger(hqRoot: string, projectId: string, companySlug?: string): ProjectStoryLedger;
 export declare function validateProjectStories(hqRoot: string, projectId: string, storyIds: readonly string[], companySlug?: string): ProjectStoryLedger;
 export declare function syncLaneToProjectStoryLedger(hqRoot: string, lane: LaneRecord, questionRefs?: readonly StoryLedgerQuestionRef[], deps?: StoryLedgerWriteDeps): void;

package/dist/lib/lanes/story-ledger.js

@@ -11,8 +11,8 @@ const LOCK_RETRY_MS = 10;
 const STORY_STATES = ["todo", "in_progress", "review", "waiting", "blocked", "done", "dropped"];
 const MANAGER_OWNED_STATES = new Set(["review", "done", "dropped"]);
 const TERMINAL_STATES = new Set(["done", "failed", "interrupted", "cancelled", "killed"]);
-export function laneStoryLedgerEnabled() {
-    return resolveProcessFlagGate(LANE_STORY_LEDGER_FLAG, { fallback: false }, () => false);
+export function laneStoryLedgerEnabled(companyUid) {
+    return resolveProcessFlagGate(LANE_STORY_LEDGER_FLAG, { fallback: false }, () => false, companyUid);
 }
 function isSafePathSegment(value) {
     return /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(value) &&
@@ -646,7 +646,7 @@ function updateOneStory(current, ledger, lane, storyId, projectId, questionRefs)
     return current;
 }
 export function syncLaneToProjectStoryLedger(hqRoot, lane, questionRefs = [], deps = {}) {
-    if (!laneStoryLedgerEnabled())
+    if (!laneStoryLedgerEnabled(lane.company?.uid))
         return;
     const stories = laneStoriesForLedger(lane);
     if (stories.length === 0)
@@ -664,7 +664,7 @@ export function syncLaneToProjectStoryLedger(hqRoot, lane, questionRefs = [], de
 }
 /** Remove only this lane's attribution from its former personal story ledgers. */
 export function removeLaneFromProjectStoryLedger(hqRoot, lane) {
-    if (!laneStoryLedgerEnabled())
+    if (!laneStoryLedgerEnabled(lane.company?.uid))
         return;
     const refs = [...new Map(laneStoriesForLedger(lane).map((story) => [
             story.project_id + "\u0000" + story.story_id,

package/dist/lib/lanes/types.js

@@ -232,14 +232,14 @@ export const LANE_FIELD_WRITERS = {
     pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
     pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
     priority: ["command"], worker: ["creator", "command"], worker_path: ["creator", "command"], worker_pack: ["creator", "command"], worker_profile: ["creator", "command"], worker_profile_hash: ["creator", "command", "spawn"], registry_worker: ["creator", "command", "spawn"], registry_worker_approval_history: ["command", "watcher", "reconcile"], worker_profile_repin_history: ["command"], worker_profile_ignored: ["creator", "command", "spawn"], worker_policy_delivery: ["command", "spawn"],
-    keep_alive: ["creator", "command"], keep_alive_standing: ["command"], keep_alive_opted_in_at: ["creator", "command"], keep_alive_blocked_reason: ["command", "reconcile"], keep_alive_resume_count: ["command", "reconcile"], keep_alive_history: ["reconcile"], keep_alive_blocked_resume_streak: ["reconcile", "spawn", "watcher"], keep_alive_last_resume_at: ["command", "reconcile"], keep_alive_next_eligible_at: ["command", "reconcile"], keep_alive_exhausted_at: ["command", "reconcile"],
+    keep_alive: ["creator", "command"], keep_alive_standing: ["command"], keep_alive_opted_in_at: ["creator", "command"], keep_alive_blocked_reason: ["command", "reconcile"], keep_alive_resume_count: ["command", "reconcile", "spawn"], keep_alive_history: ["reconcile"], keep_alive_blocked_resume_streak: ["reconcile", "spawn", "watcher"], keep_alive_last_resume_at: ["command", "reconcile", "spawn"], keep_alive_next_eligible_at: ["command", "reconcile", "spawn"], keep_alive_exhausted_at: ["command", "reconcile", "spawn"],
     provider: ["creator", "command"], provider_source: ["creator", "command"], account: ["creator", "spawn", "command", "watcher", "reconcile"], account_source: ["creator", "command"], profile_settings_pinned: ["command"], account_selection: ["creator", "command"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], failed_resume_history: ["command"], fallback_question_cancellation_failures: ["command"], pending_resume: ["command", "spawn", "watcher", "reconcile"], placement_default: ["creator"], host: ["creator", "command"], origin_host: ["creator"], origin_host_source: ["creator"],
     version: ["creator"], created_at: ["creator"], updated_at: ["creator"], ended_at: ["reconcile"], envelope_summary: ["reconcile"], attribution_history: ["command", "watcher", "reconcile"], story_switch_pending: ["command", "watcher", "reconcile"], story_switch_unacknowledged: ["watcher", "reconcile"],
     last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
     last_message_at: ["command"], status_gate: ["spawn", "watcher"], status_gate_history: ["watcher", "reconcile"], blocker_freshness_state: ["watcher"], kr_freshness_state: ["watcher"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
     overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
-    admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], stop_history: ["command", "reconcile"], settings_history: ["command"], pending_settings: ["command", "spawn"], interrupted_at: ["command", "watcher", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
-    quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
+    admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], stop_history: ["command", "reconcile"], settings_history: ["command"], pending_settings: ["command", "spawn"], interrupted_at: ["command", "watcher", "reconcile", "spawn"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
+    quota_reset_at: ["watcher", "reconcile", "spawn", "command"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
     provider_session_id: ["watcher", "command", "spawn"], provider_session_id_absent: ["watcher", "command"], provider_home_at_spawn: ["spawn"], last_resume: ["spawn"], resume_history: ["spawn", "watcher"], last_run_usage: ["watcher"], last_run_usage_at: ["watcher"], envelope_repair: ["watcher", "reconcile"], envelope_validation_errors: ["watcher"], last_progress_event_at: ["watcher"],
 };
 function isPlainObject(value) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.345.38",
+  "version": "5.345.40",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {