@indigoai-us/hq-cli 5.221.0 → 5.223.0

package/dist/lib/lanes/spawn.js

@@ -14,6 +14,8 @@ import * as path from "node:path";
 import { fileURLToPath } from "node:url";
 import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, selectBalancedAccount, } from "./accounts.js";
 import { effectiveAccountHome } from "./account-home.js";
+import { buildHostCapacitySnapshot } from "./host-capacity.js";
+import { selectClaudeAccount } from "./claude-account-balance.js";
 import { COMPACTION_PROMPT, compactionAfterUsage, compactionDue, compactLaneSession, } from "./compaction.js";
 import { installDrainHook } from "./hooks-install.js";
 import { loadLanesConfig, providerAutoCompactTokens, providerCompactAfterMs, } from "./config.js";
@@ -159,7 +161,7 @@ function spawnModelFailure(provider, accountId, config, pinnedModel, profileMode
     }
 }
 function selectSpawnAccount(input) {
-    const { hqRoot, lane, config, existsSync, accountDeps, modelDeps, pinnedModel, profileModel, currentModelValidated, hostname } = input;
+    const { hqRoot, lane, config, existsSync, accountDeps, modelDeps, pinnedModel, profileModel, currentModelValidated, hostname, repairRecordedAccount, recordedModel, now } = input;
     const context = {
         hqRoot,
         existsSync,
@@ -187,7 +189,7 @@ function selectSpawnAccount(input) {
     const operatorMovePending = lane.pending_resume !== undefined &&
         lane.account_history?.at(-1)?.reason === "operator_move";
     if (current === undefined || !current.enabled) {
-        reason = "account_unavailable";
+        reason = repairRecordedAccount ? "recorded_account_logged_out" : "account_unavailable";
         refusal = new AccountResolutionError(current === undefined ? "unknown_account" : "account_disabled", `Account ${JSON.stringify(lane.account)} is unavailable for ${lane.provider}`, { provider: lane.provider, account: lane.account });
     }
     else {
@@ -198,7 +200,7 @@ function selectSpawnAccount(input) {
             ...(accountDeps?.env ? { env: accountDeps.env } : {}),
         });
         if (isLoggedOut(status)) {
-            reason = "account_unavailable";
+            reason = repairRecordedAccount ? "recorded_account_logged_out" : "account_unavailable";
             refusal = new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in`, { provider: lane.provider, account: lane.account });
         }
         else {
@@ -242,6 +244,33 @@ function selectSpawnAccount(input) {
             account: lane.account,
         });
     }
+    if (reason === "recorded_account_logged_out" && lane.provider === "claude") {
+        const lanes = listLaneIds(hqRoot)
+            .map((laneId) => readLane(hqRoot, laneId))
+            .filter((record) => record.host.kind === lane.host.kind && record.host.id === lane.host.id);
+        const capacity = buildHostCapacitySnapshot({
+            host: lane.host,
+            lanes,
+            config,
+            existsSync,
+            probe: accountDeps?.probe,
+            env: accountDeps?.env,
+            hostname,
+            now,
+        });
+        const selection = selectClaudeAccount({
+            hosts: [{ capacity, lanes, laneListAvailable: true }],
+            targetHost: lane.host,
+            config,
+            ...((recordedModel ?? pinnedModel ?? profileModel) !== undefined
+                ? { model: recordedModel ?? pinnedModel ?? profileModel }
+                : {}),
+        });
+        return {
+            account: resolveAccount(lane.provider, selection.account, config, context),
+            changeReason: "recorded_account_logged_out",
+        };
+    }
     const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
         .filter(([id, account]) => id !== lane.account && account.enabled)
         .filter(([id]) => accountStatus(lane.provider, id, config, {
@@ -254,12 +283,15 @@ function selectSpawnAccount(input) {
         .map(([id]) => id);
     try {
         return {
-            account: selectBalancedAccount(lane.provider, config, { ...context, allowedAccounts }),
+            account: selectBalancedAccount(lane.provider, config, {
+                ...context,
+                allowedAccounts,
+            }),
             changeReason: reason,
         };
     }
     catch (err) {
-        if (reason === "account_unavailable") {
+        if (reason === "account_unavailable" || reason === "recorded_account_logged_out") {
             if (refusal instanceof Error)
                 throw refusal;
             throw new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in or available, and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account });
@@ -274,7 +306,8 @@ function previouslyUnavailableAccounts(lane) {
     for (const entry of [...(lane.account_history ?? [])].reverse()) {
         if (entry.reason === "first_assignment" || entry.reason === "operator_move")
             break;
-        if (entry.reason === "account_unavailable" && entry.from !== null)
+        if ((entry.reason === "account_unavailable" || entry.reason === "recorded_account_logged_out") &&
+            entry.from !== null)
             excluded.add(entry.from);
     }
     return excluded;
@@ -364,7 +397,9 @@ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer =
     };
     const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
     const profileModel = profile?.meta.provider === lane.provider ? profile.meta.model : undefined;
-    const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" ? lane.model : undefined;
+    const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
+        ? lane.model
+        : undefined;
     const unavailableAccounts = previouslyUnavailableAccounts(lane);
     const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
         .filter(([id, account]) => id !== lane.account && !unavailableAccounts.has(id) && account.enabled)
@@ -1649,7 +1684,12 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
         : undefined;
     const profileChanged = profile !== undefined && existing.worker_profile_hash !== profile.contentHash;
     const config = loadLanesConfig(hqRoot);
-    const pinnedModel = existing.model_source === undefined || existing.model_source === "cli"
+    const repairRecordedAccount = plan.mode === "resume" &&
+        (trigger.kind === "resume" || pendingResume?.trigger === "resume");
+    const recordedModel = repairRecordedAccount && existing.model_source === "account"
+        ? existing.model
+        : undefined;
+    let pinnedModel = existing.model_source === undefined || existing.model_source === "cli" || existing.model_source === "preserved"
         ? existing.model
         : undefined;
     const accountDeps = {
@@ -1669,6 +1709,9 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
             profileModel,
             currentModelValidated: deps.currentModelValidated === true,
             hostname,
+            repairRecordedAccount,
+            ...(recordedModel !== undefined ? { recordedModel } : {}),
+            now: (deps.now ?? (() => new Date()))(),
         });
     }
     catch (err) {
@@ -1683,6 +1726,9 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
         process.stderr.write(`hq lanes: account bind failed for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}\n`);
         throw err;
     }
+    const preservedAccountModel = selection.changeReason === "recorded_account_logged_out" ? recordedModel : undefined;
+    if (preservedAccountModel !== undefined)
+        pinnedModel = preservedAccountModel;
     const resolved = selection.account;
     const status = accountStatus(lane.provider, resolved.id, config, { existsSync, hostname });
     if (isLoggedOut(status)) {
@@ -1738,6 +1784,9 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
             return {
                 ...current,
                 account: resolved.id,
+                ...(preservedAccountModel !== undefined
+                    ? { model: preservedAccountModel, model_source: "preserved" }
+                    : {}),
                 account_history: appendCappedHistory(current.account_history ?? [], entry),
             };
         });
@@ -1934,7 +1983,9 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
             ...(model !== undefined
                 ? {
                     model_source: pinnedModel !== undefined
-                        ? "cli"
+                        ? preservedAccountModel !== undefined || existing.model_source === "preserved"
+                            ? "preserved"
+                            : "cli"
                         : profileModel !== undefined
                             ? "profile"
                             : accountCfg?.model !== undefined
@@ -2195,7 +2246,7 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
             ignoredFields: ignoredProfileFields,
         }
         : undefined;
-    const pinnedModel = lane.model_source === undefined || lane.model_source === "cli"
+    const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" || lane.model_source === "preserved"
         ? lane.model
         : undefined;
     const resolved = resolveAccount(lane.provider, lane.account, config, {
@@ -2307,7 +2358,7 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
                 if (model !== undefined) {
                     next.model = model;
                     next.model_source = pinnedModel !== undefined
-                        ? "cli"
+                        ? lane.model_source === "preserved" ? "preserved" : "cli"
                         : profileModel !== undefined
                             ? "profile"
                             : accountCfg?.model !== undefined

package/dist/lib/lanes/types.d.ts

@@ -412,8 +412,8 @@ export interface LaneRecord {
     last_envelope?: LastEnvelopeRef;
     /** Effective model id recorded at create/spawn. Omitted when the provider chooses. */
     model?: string;
-    /** Where the effective model came from; profile values are re-read on spawn. */
-    model_source?: "cli" | "profile" | "account" | "provider" | "none";
+    /** Where the effective model came from; repaired models stay pinned across the account move. */
+    model_source?: "cli" | "profile" | "account" | "provider" | "preserved" | "none";
     /** Effective effort level passed to the provider at the last spawn (effort.ts). */
     effort?: string;
     /** Where `effort` came from. `cli` pins it across later spawns. */
@@ -523,7 +523,7 @@ export interface AccountHistoryEntry {
     from: string | null;
     to: string;
     at: string;
-    reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "operator_move";
+    reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "operator_move";
 }
 export interface PendingResume {
     /** Fresh answered-question launches need no previous session id. */

package/dist/lib/lanes/watch.d.ts

@@ -1,16 +1,33 @@
-import { type LaneRecord, type QuestionRecord } from "./types.js";
+import { type LaneHost, type LaneProvider, type LaneRecord, type QuestionRecord } from "./types.js";
 export declare const DEFAULT_LANE_WATCH_INTERVAL_MS = 5000;
 export declare const MAX_LANE_WATCH_INTERVAL_MS = 30000;
 export declare const MAX_WATCHED_LANES = 64;
 export declare const MAX_WATCH_OUTPUT_LINES = 512;
+export interface LaneWatchAccountStatus {
+    lane_id: string;
+    provider: LaneProvider;
+    account: string;
+    host: LaneHost;
+    logged_in: boolean | null;
+}
+export interface LaneAccountLoggedOutEvent {
+    event: "account_logged_out";
+    provider: LaneProvider;
+    account: string;
+    host: LaneHost;
+    lane_ids: string[];
+}
 export interface LaneWatchDeps {
     readLane?: (hqRoot: string, laneId: string) => LaneRecord | null;
     listQuestionIds?: (hqRoot: string) => string[];
     readQuestion?: (hqRoot: string, questionId: string) => QuestionRecord | null;
+    readAccountStatuses?: (lanes: readonly LaneRecord[]) => Promise<readonly LaneWatchAccountStatus[]>;
+    keepAliveEnabled?: (lane: LaneRecord) => boolean | Promise<boolean>;
     sleep?: (ms: number) => Promise<void>;
     intervalMs?: number;
     maxOutputLines?: number;
     writeLine?: (line: string) => void;
+    writeAccountEvent?: (event: LaneAccountLoggedOutEvent) => void;
     writeError?: (line: string) => void;
 }
 export interface LaneWatchResult {

package/dist/lib/lanes/watch.js

@@ -1,6 +1,8 @@
 import { listQuestionIds, readLane, readQuestion, } from "./store.js";
 import { LaneNotFoundError, LanesError } from "./errors.js";
-import { isTerminalLaneState } from "./types.js";
+import { providerSupportsSessionResume } from "./delivery.js";
+import { LEAD_LANE_KEEP_ALIVE_MAX_RESUMES } from "./keep-alive.js";
+import { isTerminalLaneState, } from "./types.js";
 export const DEFAULT_LANE_WATCH_INTERVAL_MS = 5_000;
 export const MAX_LANE_WATCH_INTERVAL_MS = 30_000;
 export const MAX_WATCHED_LANES = 64;
@@ -30,6 +32,30 @@ function validateLaneIds(laneIds) {
     }
     return unique;
 }
+function canResumeKeepAliveLane(lane) {
+    if (lane.keep_alive !== true ||
+        lane.keep_alive_opted_in_at === undefined ||
+        !Number.isFinite(Date.parse(lane.keep_alive_opted_in_at)) ||
+        lane.keep_alive_exhausted_at !== undefined ||
+        (lane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES ||
+        lane.keep_alive_blocked_reason === "retired" ||
+        lane.keep_alive_blocked_reason === "superseded" ||
+        lane.keep_alive_blocked_reason === "retired_list_invalid" ||
+        lane.keep_alive_blocked_reason === "terminal_state" ||
+        !providerSupportsSessionResume(lane.provider) ||
+        !(lane.provider_session_id ?? lane.last_resume?.session_id))
+        return false;
+    if (lane.state === "interrupted") {
+        return lane.terminal_reason === "exited_with_partial_output";
+    }
+    if (lane.state === "done") {
+        return lane.last_envelope?.decision === "done" &&
+            lane.story_complete !== true &&
+            lane.terminal_reason !== "provider_rejected" &&
+            lane.terminal_reason !== "account_not_logged_in";
+    }
+    return false;
+}
 export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
     const watchedIds = validateLaneIds(laneIds);
     const readLaneRecord = deps.readLane ?? readLane;
@@ -37,11 +63,15 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
     const readQuestionRecord = deps.readQuestion ?? readQuestion;
     const sleep = deps.sleep ?? defaultSleep;
     const writeLine = deps.writeLine ?? ((line) => console.log(line));
+    const writeAccountEvent = deps.writeAccountEvent ?? ((event) => {
+        writeLine(`account_logged_out provider=${event.provider} account=${event.account} host=${event.host.kind}:${event.host.id} lane_ids=${event.lane_ids.join(",")}`);
+    });
     const writeError = deps.writeError ?? ((line) => process.stderr.write(`${line}\n`));
     const maxOutputLines = Math.max(1, Math.floor(deps.maxOutputLines ?? MAX_WATCH_OUTPUT_LINES));
     const intervalMs = boundedInterval(deps.intervalMs);
     const previousStates = new Map();
     const previousQuestionStatuses = new Map();
+    const previousAccountLogin = new Map();
     let polls = 0;
     let lines = 0;
     let suppressed = false;
@@ -57,6 +87,18 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
             suppressed = true;
         }
     };
+    const emitAccountEvent = (event) => {
+        if (lines < maxOutputLines - 1) {
+            writeAccountEvent(event);
+            lines += 1;
+            return;
+        }
+        if (!suppressed) {
+            writeLine("lane watch: output limit reached; further events suppressed");
+            lines = maxOutputLines;
+            suppressed = true;
+        }
+    };
     const unreadable = (kind, id, err) => {
         const message = `${kind} ${id}: unreadable record (${readableError(err)})`;
         writeError(`lane watch: ${message}`);
@@ -87,6 +129,15 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
                 previousStates.set(laneId, record.state);
             }
         }
+        const keepAliveWaiting = new Set();
+        if (deps.keepAliveEnabled !== undefined) {
+            for (const record of records) {
+                if (isTerminalLaneState(record.state) &&
+                    canResumeKeepAliveLane(record) &&
+                    await deps.keepAliveEnabled(record))
+                    keepAliveWaiting.add(record.lane_id);
+            }
+        }
         let questionIds = [];
         try {
             questionIds = listQuestions(hqRoot);
@@ -113,7 +164,76 @@ export async function runLaneWatch(hqRoot, laneIds, deps = {}) {
             }
             previousQuestionStatuses.set(questionId, record.status);
         }
-        if (records.every((record) => isTerminalLaneState(record.state))) {
+        if (deps.readAccountStatuses !== undefined) {
+            const accountLanes = records.filter((record) => record.account !== undefined &&
+                (!isTerminalLaneState(record.state) || keepAliveWaiting.has(record.lane_id)));
+            let statuses;
+            try {
+                statuses = await deps.readAccountStatuses(accountLanes);
+            }
+            catch (err) {
+                const message = `account login status: ${readableError(err)}`;
+                writeError(`lane watch: ${message}`);
+                throw new LanesError("watch_account_status_unavailable", message, {
+                    cause: readableError(err),
+                });
+            }
+            const statusLanes = new Map(accountLanes.map((record) => [record.lane_id, record]));
+            const grouped = new Map();
+            const seenLaneIds = new Set();
+            for (const status of statuses) {
+                const lane = statusLanes.get(status.lane_id);
+                if (!lane || lane.provider !== status.provider || lane.account !== status.account ||
+                    lane.host.kind !== status.host.kind || lane.host.id !== status.host.id ||
+                    (status.logged_in !== true && status.logged_in !== false && status.logged_in !== null) ||
+                    seenLaneIds.has(status.lane_id)) {
+                    const message = `malformed account login status for lane ${JSON.stringify(status.lane_id)}`;
+                    writeError(`lane watch: ${message}`);
+                    throw new LanesError("watch_account_status_invalid", message, {
+                        lane_id: status.lane_id,
+                    });
+                }
+                seenLaneIds.add(status.lane_id);
+                const key = JSON.stringify([status.provider, status.account, status.host.kind, status.host.id]);
+                const existing = grouped.get(key);
+                if (existing && existing.status.logged_in !== status.logged_in) {
+                    const message = `conflicting account login statuses for ${status.provider} account ${status.account} on ${status.host.kind}:${status.host.id}`;
+                    writeError(`lane watch: ${message}`);
+                    throw new LanesError("watch_account_status_invalid", message, {
+                        provider: status.provider,
+                        account: status.account,
+                        host: status.host,
+                    });
+                }
+                const group = existing ?? { status, laneIds: new Set() };
+                group.laneIds.add(status.lane_id);
+                grouped.set(key, group);
+            }
+            const missingLane = accountLanes.find((lane) => !seenLaneIds.has(lane.lane_id));
+            if (missingLane) {
+                const message = `account login status is missing for lane ${missingLane.lane_id}`;
+                writeError(`lane watch: ${message}`);
+                throw new LanesError("watch_account_status_invalid", message, {
+                    lane_id: missingLane.lane_id,
+                });
+            }
+            for (const [key, group] of grouped) {
+                const { status } = group;
+                const prior = previousAccountLogin.get(key);
+                if (prior === true && status.logged_in === false) {
+                    emitAccountEvent({
+                        event: "account_logged_out",
+                        provider: status.provider,
+                        account: status.account,
+                        host: status.host,
+                        lane_ids: [...group.laneIds].sort(),
+                    });
+                }
+                if (status.logged_in !== null)
+                    previousAccountLogin.set(key, status.logged_in);
+            }
+        }
+        if (records.every((record) => isTerminalLaneState(record.state) && !keepAliveWaiting.has(record.lane_id))) {
             return { polls, lines, suppressed };
         }
         await sleep(intervalMs);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.221.0",
+  "version": "5.223.0",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {