@indigoai-us/hq-cli 5.153.0 → 5.155.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,6 +24,25 @@ export const PROVIDER_REJECTED_REASON = "provider_rejected";
24
24
  export const PROVIDER_REJECTION_MESSAGE_CAP = 2000;
25
25
  /** Status the watcher routes to the quota path instead of `provider_rejected`. */
26
26
  export const QUOTA_STATUS = 429;
27
+ /** Exact auth failures observed from each installed CLI in an empty home. */
28
+ function isProviderLoginMessage(provider, message) {
29
+ switch (provider) {
30
+ case "claude":
31
+ return /\bnot logged in\b/i.test(message);
32
+ case "codex":
33
+ return /\bnot logged in\b/i.test(message) ||
34
+ /\bmissing bearer or basic authentication in header\b/i.test(message);
35
+ case "grok":
36
+ return /^\s*not signed in\b/i.test(message);
37
+ default: {
38
+ const _exhaustive = provider;
39
+ return _exhaustive;
40
+ }
41
+ }
42
+ }
43
+ export function isProviderLoginFailure(failure) {
44
+ return failure !== null && isProviderLoginMessage(failure.provider, failure.message);
45
+ }
27
46
  function isPlainObject(value) {
28
47
  return value !== null && typeof value === "object" && !Array.isArray(value);
29
48
  }
@@ -134,10 +153,9 @@ export function parseCodexTurnFailed(text) {
134
153
  }
135
154
  /**
136
155
  * Claude `-p --output-format json`: the last `type: "result"` object with
137
- * `is_error: true` and a numeric `api_error_status`. A result that is an
138
- * error with no status is not a provider rejection: it is either the quota
139
- * text shape quota.ts reads, or the worker's own failure, and keeps the
140
- * existing classification.
156
+ * `is_error: true` and a numeric `api_error_status`, or the CLI's explicit
157
+ * unauthenticated message. Other status-less errors keep the existing
158
+ * classification because they may be quota text or a worker failure.
141
159
  */
142
160
  export function parseClaudeResultRejection(text) {
143
161
  // The last `result` decides: a success or a status-less error there is
@@ -148,9 +166,9 @@ export function parseClaudeResultRejection(text) {
148
166
  if (rec.is_error !== true)
149
167
  return NOT_REJECTED;
150
168
  const status = statusOf(rec.api_error_status);
151
- if (status === null)
152
- return NOT_REJECTED;
153
169
  const message = typeof rec.result === "string" ? rec.result : "";
170
+ if (status === null && !isProviderLoginMessage("claude", message))
171
+ return NOT_REJECTED;
154
172
  return {
155
173
  reason: PROVIDER_REJECTED_REASON,
156
174
  provider: "claude",
@@ -15,7 +15,7 @@ import { MAX_LANE_DEPTH } from "./constants.js";
15
15
  import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
16
16
  import { loadWorkerProfile } from "./profile.js";
17
17
  import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
18
- import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
18
+ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
19
19
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
20
20
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
21
21
  import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
@@ -916,6 +916,11 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
916
916
  const repairedClassification = invalidEnvelope
917
917
  ? classifyLaneExit({ envelope: null, hasPartialOutput: false, invalidEnvelope })
918
918
  : classification;
919
+ const accountRetryReason = quota
920
+ ? "usage_limit"
921
+ : providerRejection && classification.reason === "account_not_logged_in"
922
+ ? "account_unavailable"
923
+ : null;
919
924
  if (!deps.dryRun &&
920
925
  quota &&
921
926
  classified.account &&
@@ -923,6 +928,30 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
923
928
  classification.reason === "account_model_unavailable")) {
924
929
  recordAccountQuota(hqRoot, classified.provider, classified.account, quota);
925
930
  }
931
+ if (!deps.dryRun && accountRetryReason && classified.account) {
932
+ try {
933
+ const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
934
+ ...(deps.spawnLaneDeps ?? {}),
935
+ admissionHost: deps.host,
936
+ });
937
+ summary.dead_host.push({
938
+ id: lane.lane_id,
939
+ from: "running",
940
+ to: recovered.state,
941
+ reason: accountRetryReason,
942
+ });
943
+ summary.heartbeats.push(`${lane.lane_id}:account-retry`);
944
+ continue;
945
+ }
946
+ catch (err) {
947
+ const message = err instanceof Error ? err.message : String(err);
948
+ process.stderr.write(`hq lanes reconcile: account failover failed for ${lane.lane_id}: ${message}\n`);
949
+ if (readLane(hqRoot, lane.lane_id).state !== "running") {
950
+ summary.errors.push({ message: `${lane.lane_id}: account failover left a non-running record after ${message}` });
951
+ continue;
952
+ }
953
+ }
954
+ }
926
955
  if (!deps.dryRun &&
927
956
  classification.state === "awaiting_input" &&
928
957
  latest &&
@@ -12,7 +12,8 @@ import { type RunCompactionFn } from "./compaction.js";
12
12
  import { type KnownModelsDeps } from "./models.js";
13
13
  import { type LaunchTrigger } from "./delivery.js";
14
14
  import { type MachineIdentity } from "./machine-identity.js";
15
- import { type LaneCompanyRef, type LaneProvider, type LaneRecord } from "./types.js";
15
+ import { type AdmissionHost } from "./admission.js";
16
+ import { type LaneCompanyRef, type LaneProvider, type LaneRecord, type AccountHistoryEntry } from "./types.js";
16
17
  export declare const LANES_RUNS_REL: string;
17
18
  /** Written by reconcile before a resume spawn; the worker prompt names this path. */
18
19
  export declare const ANSWERED_QUESTIONS_FILE = "answered-questions.json";
@@ -32,6 +33,9 @@ export declare function resolveLaneModel(input: {
32
33
  accountModel?: string;
33
34
  providerModel?: string;
34
35
  }): string | undefined;
36
+ type AutomaticAccountChangeReason = Exclude<AccountHistoryEntry["reason"], "operator_move">;
37
+ /** Change account only after a provider/runtime account failure, then re-admit. */
38
+ export declare function retryLaneOnAlternateAccount(hqRoot: string, lane: LaneRecord, reason: Extract<AutomaticAccountChangeReason, "usage_limit" | "account_unavailable">, writer?: "watcher" | "reconcile", deps?: SpawnLaneDeps): Promise<LaneRecord>;
35
39
  export declare function appendModelFlag(provider: LaneProvider, args: string[], model: string | undefined): string[];
36
40
  /** Add Codex fast-mode flags before the prompt/positional separator. */
37
41
  export declare function appendFastModeFlags(provider: LaneProvider, args: string[], enabled: boolean | undefined): string[];
@@ -63,8 +67,12 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
63
67
  export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
64
68
  export interface SpawnLaneDeps {
65
69
  env?: NodeJS.ProcessEnv;
70
+ /** The caller's host scope for admission checks; reconcile supplies its resolved host. */
71
+ admissionHost?: AdmissionHost;
66
72
  /** Provider model catalog dependencies, shared with create in tests. */
67
73
  modelDeps?: KnownModelsDeps;
74
+ /** Create validated this exact account/model candidate before its admission lock. */
75
+ currentModelValidated?: boolean;
68
76
  existsSync?: (p: string) => boolean;
69
77
  spawnProcess?: SpawnProcessFn;
70
78
  proveDetached?: ProveDetachedFn;
@@ -11,18 +11,21 @@ import { spawn, spawnSync } from "node:child_process";
11
11
  import * as fs from "node:fs";
12
12
  import * as path from "node:path";
13
13
  import { fileURLToPath } from "node:url";
14
- import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, } from "./accounts.js";
14
+ import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, selectBalancedAccount, } from "./accounts.js";
15
15
  import { COMPACTION_PROMPT, compactionAfterUsage, compactionDue, compactLaneSession, } from "./compaction.js";
16
16
  import { installDrainHook } from "./hooks-install.js";
17
17
  import { loadLanesConfig, providerAutoCompactTokens, providerCompactAfterMs, } from "./config.js";
18
18
  import { appendEffortFlag, resolveLaneEffort } from "./effort.js";
19
- import { validateProfileLaneModel, } from "./models.js";
19
+ import { defaultProbe, validateLaneModel, validateProfileLaneModel, } from "./models.js";
20
20
  import { claimableStatesFor, planLaneLaunch, providerSupportsSessionResume, } from "./delivery.js";
21
21
  import { AccountResolutionError, LanesError } from "./errors.js";
22
22
  import { ignoredWorkerProfileFields, loadWorkerProfile, materializeWorkerProfile, workerProfilePrompt, } from "./profile.js";
23
23
  import { resolveMachineIdentity, } from "./machine-identity.js";
24
- import { readLane, retryUpdateLane } from "./store.js";
24
+ import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./store.js";
25
+ import { copySessionFiles, planSessionCopy } from "./move-account.js";
25
26
  import { ensureSessionHome } from "./session-relink.js";
27
+ import { hasEarlierEligibleQueuedLane, laneAdmissionCapReason, withLaneAdmissionLock, } from "./admission.js";
28
+ import { resolveAdmissionHost } from "./admission-host.js";
26
29
  import { LANE_ENVELOPE_JSON_SCHEMA, parseLaneProvider, } from "./types.js";
27
30
  export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
28
31
  /** Written by reconcile before a resume spawn; the worker prompt names this path. */
@@ -101,6 +104,365 @@ export function resolveLaneModel(input) {
101
104
  }
102
105
  return undefined;
103
106
  }
107
+ function accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel) {
108
+ return resolveLaneModel({
109
+ cliModel: pinnedModel,
110
+ profileModel,
111
+ accountModel: config.providers[provider].accounts[accountId]?.model,
112
+ providerModel: config.providers[provider].model,
113
+ });
114
+ }
115
+ function supportsSpawnModel(provider, accountId, config, pinnedModel, profileModel, modelDeps) {
116
+ const account = config.providers[provider].accounts[accountId];
117
+ if (account === undefined)
118
+ return false;
119
+ const model = accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel);
120
+ if (model === undefined)
121
+ return true;
122
+ try {
123
+ validateLaneModel(provider, model, account.home, modelDeps);
124
+ return true;
125
+ }
126
+ catch (err) {
127
+ if (err instanceof LanesError && err.error === "model_unsupported")
128
+ return false;
129
+ throw err;
130
+ }
131
+ }
132
+ function spawnModelFailure(provider, accountId, config, pinnedModel, profileModel, profileId, modelDeps) {
133
+ const account = config.providers[provider].accounts[accountId];
134
+ if (account === undefined)
135
+ return new LanesError("unknown_account", `Unknown account ${accountId} for ${provider}`, { provider, account: accountId });
136
+ const model = accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel);
137
+ if (model === undefined)
138
+ return null;
139
+ try {
140
+ validateLaneModel(provider, model, account.home, modelDeps);
141
+ return null;
142
+ }
143
+ catch (err) {
144
+ if (!(err instanceof LanesError) || err.error !== "model_unsupported")
145
+ throw err;
146
+ if (profileModel !== undefined && pinnedModel === undefined && profileId !== undefined) {
147
+ return new LanesError(err.error, err.message, {
148
+ ...err.details,
149
+ profile_id: profileId,
150
+ worker_profile: profileId,
151
+ model_source: "profile",
152
+ });
153
+ }
154
+ return err;
155
+ }
156
+ }
157
+ function selectSpawnAccount(input) {
158
+ const { hqRoot, lane, config, existsSync, accountDeps, modelDeps, pinnedModel, profileModel, currentModelValidated } = input;
159
+ const context = {
160
+ hqRoot,
161
+ existsSync,
162
+ accountDeps,
163
+ ...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
164
+ };
165
+ if (lane.account === undefined) {
166
+ const modelSpecified = pinnedModel !== undefined || profileModel !== undefined;
167
+ const allowedAccounts = modelSpecified
168
+ ? Object.keys(config.providers[lane.provider].accounts).filter((id) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
169
+ : undefined;
170
+ return {
171
+ account: selectBalancedAccount(lane.provider, config, {
172
+ ...context,
173
+ ...(allowedAccounts !== undefined ? { allowedAccounts } : {}),
174
+ preferredAccounts: config.providers[lane.provider].preferred_accounts ?? [],
175
+ }),
176
+ changeReason: "first_assignment",
177
+ };
178
+ }
179
+ const current = config.providers[lane.provider].accounts[lane.account];
180
+ let reason;
181
+ let refusal;
182
+ const operatorMovePending = lane.pending_resume !== undefined &&
183
+ lane.account_history?.at(-1)?.reason === "operator_move";
184
+ if (current === undefined || !current.enabled) {
185
+ reason = "account_unavailable";
186
+ refusal = new AccountResolutionError(current === undefined ? "unknown_account" : "account_disabled", `Account ${JSON.stringify(lane.account)} is unavailable for ${lane.provider}`, { provider: lane.provider, account: lane.account });
187
+ }
188
+ else {
189
+ const status = accountStatus(lane.provider, lane.account, config, {
190
+ existsSync,
191
+ ...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
192
+ ...(accountDeps?.env ? { env: accountDeps.env } : {}),
193
+ });
194
+ if (isLoggedOut(status)) {
195
+ reason = "account_unavailable";
196
+ refusal = new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in`, { provider: lane.provider, account: lane.account });
197
+ }
198
+ else {
199
+ try {
200
+ resolveAccount(lane.provider, lane.account, config, context);
201
+ const modelFailure = currentModelValidated
202
+ ? null
203
+ : spawnModelFailure(lane.provider, lane.account, config, pinnedModel, profileModel, lane.worker_profile, modelDeps);
204
+ if (modelFailure !== null) {
205
+ reason = "model_change";
206
+ refusal = modelFailure;
207
+ }
208
+ else {
209
+ return { account: resolveAccount(lane.provider, lane.account, config, context) };
210
+ }
211
+ }
212
+ catch (err) {
213
+ if (err instanceof AccountResolutionError && err.error === "account_quota_exhausted") {
214
+ reason = "usage_limit";
215
+ refusal = err;
216
+ }
217
+ else if (err instanceof AccountResolutionError && err.error === "account_model_unavailable") {
218
+ reason = "usage_limit";
219
+ refusal = err;
220
+ }
221
+ else if (err instanceof LanesError && err.error === "model_unsupported") {
222
+ reason = "model_change";
223
+ refusal = err;
224
+ }
225
+ else {
226
+ throw err;
227
+ }
228
+ }
229
+ }
230
+ }
231
+ if (operatorMovePending) {
232
+ if (refusal instanceof Error)
233
+ throw refusal;
234
+ throw new LanesError("account_unavailable", `Explicit account move target ${lane.account} is unavailable`, {
235
+ provider: lane.provider,
236
+ account: lane.account,
237
+ });
238
+ }
239
+ const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
240
+ .filter(([id, account]) => id !== lane.account && account.enabled)
241
+ .filter(([id]) => accountStatus(lane.provider, id, config, {
242
+ existsSync,
243
+ ...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
244
+ ...(accountDeps?.env ? { env: accountDeps.env } : {}),
245
+ }).logged_in === true)
246
+ .filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
247
+ .map(([id]) => id);
248
+ try {
249
+ return {
250
+ account: selectBalancedAccount(lane.provider, config, { ...context, allowedAccounts }),
251
+ changeReason: reason,
252
+ };
253
+ }
254
+ catch (err) {
255
+ if (reason === "account_unavailable") {
256
+ if (refusal instanceof Error)
257
+ throw refusal;
258
+ throw new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in or available, and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account });
259
+ }
260
+ if (refusal instanceof Error)
261
+ throw refusal;
262
+ throw err;
263
+ }
264
+ }
265
+ function previouslyUnavailableAccounts(lane) {
266
+ const excluded = new Set();
267
+ for (const entry of [...(lane.account_history ?? [])].reverse()) {
268
+ if (entry.reason === "first_assignment" || entry.reason === "operator_move")
269
+ break;
270
+ if (entry.reason === "account_unavailable" && entry.from !== null)
271
+ excluded.add(entry.from);
272
+ }
273
+ return excluded;
274
+ }
275
+ function archivedResumeMessageText(hqRoot, laneId, messageId) {
276
+ if (!messageId || /[\\/\0]/.test(messageId))
277
+ return undefined;
278
+ const file = path.join(laneRunDir(hqRoot, laneId), "dropbox", "delivered", `${messageId}.resume.jsonl`);
279
+ let raw;
280
+ try {
281
+ raw = fs.readFileSync(file, "utf8");
282
+ }
283
+ catch (err) {
284
+ if (err.code === "ENOENT")
285
+ return undefined;
286
+ throw new LanesError("account_retry_context_unreadable", `Could not read the archived message needed to resume lane ${laneId}`, { lane_id: laneId, message_id: messageId });
287
+ }
288
+ const lines = raw.split(/\r?\n/).filter((line) => line.trim().length > 0);
289
+ if (lines.length !== 1) {
290
+ throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
291
+ }
292
+ let value;
293
+ try {
294
+ value = JSON.parse(lines[0]);
295
+ }
296
+ catch {
297
+ throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
298
+ }
299
+ if (value === null ||
300
+ typeof value !== "object" ||
301
+ value.message_id !== messageId ||
302
+ typeof value.text !== "string") {
303
+ throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
304
+ }
305
+ return value.text;
306
+ }
307
+ function pendingResumeForRetry(hqRoot, lane, sessionId, at) {
308
+ const last = lane.last_resume;
309
+ if (last === undefined || last.session_id !== sessionId) {
310
+ return { session_id: sessionId, trigger: "resume", resumed_from: "interrupted", at };
311
+ }
312
+ const compact = last.compact === undefined ? {} : { compact: last.compact };
313
+ if (last.trigger === "answered_questions") {
314
+ return { session_id: sessionId, trigger: "answered_questions", at, ...compact };
315
+ }
316
+ if (last.trigger === "message") {
317
+ if (last.message_id === undefined) {
318
+ throw new LanesError("account_retry_context_missing", `The message id needed to resume lane ${lane.lane_id} is missing`, { lane_id: lane.lane_id, trigger: last.trigger });
319
+ }
320
+ const text = last.text ?? archivedResumeMessageText(hqRoot, lane.lane_id, last.message_id);
321
+ if (text === undefined) {
322
+ throw new LanesError("account_retry_context_missing", `The operator message needed to resume lane ${lane.lane_id} is unavailable`, { lane_id: lane.lane_id, message_id: last.message_id });
323
+ }
324
+ return {
325
+ session_id: sessionId,
326
+ trigger: "message",
327
+ text,
328
+ message_id: last.message_id,
329
+ at,
330
+ ...compact,
331
+ };
332
+ }
333
+ return {
334
+ session_id: sessionId,
335
+ trigger: "resume",
336
+ resumed_from: last.resumed_from ?? "interrupted",
337
+ ...(last.text !== undefined ? { text: last.text } : {}),
338
+ ...(last.message_id !== undefined ? { message_id: last.message_id } : {}),
339
+ at,
340
+ ...compact,
341
+ };
342
+ }
343
+ /** Change account only after a provider/runtime account failure, then re-admit. */
344
+ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer = "watcher", deps = {}) {
345
+ const config = loadLanesConfig(hqRoot);
346
+ const existsSync = deps.existsSync ?? ((p) => fs.existsSync(p));
347
+ const env = deps.env ?? process.env;
348
+ const modelDeps = {
349
+ ...deps.modelDeps,
350
+ env: deps.modelDeps?.env ?? env,
351
+ };
352
+ const accountDeps = {
353
+ probe: modelDeps.probe ?? defaultProbe,
354
+ env,
355
+ };
356
+ const admissionHost = deps.admissionHost ?? resolveAdmissionHost();
357
+ const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
358
+ const profileModel = profile?.meta.provider === lane.provider ? profile.meta.model : undefined;
359
+ const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" ? lane.model : undefined;
360
+ const unavailableAccounts = previouslyUnavailableAccounts(lane);
361
+ const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
362
+ .filter(([id, account]) => id !== lane.account && !unavailableAccounts.has(id) && account.enabled)
363
+ .filter(([id]) => accountStatus(lane.provider, id, config, {
364
+ existsSync,
365
+ ...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
366
+ ...(accountDeps?.env ? { env: accountDeps.env } : {}),
367
+ }).logged_in === true)
368
+ .filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
369
+ .map(([id]) => id);
370
+ const modelContext = {
371
+ hqRoot,
372
+ existsSync,
373
+ accountDeps,
374
+ ...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
375
+ allowedAccounts,
376
+ };
377
+ let target;
378
+ try {
379
+ target = selectBalancedAccount(lane.provider, config, modelContext);
380
+ }
381
+ catch (err) {
382
+ if (reason === "account_unavailable" || unavailableAccounts.size > 0) {
383
+ throw new LanesError("account_not_logged_in", `Account ${lane.account ?? "(none)"} for ${lane.provider} is unavailable and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account ?? null });
384
+ }
385
+ throw err;
386
+ }
387
+ const source = lane.account ? config.providers[lane.provider].accounts[lane.account] : undefined;
388
+ const sessionId = providerSupportsSessionResume(lane.provider) &&
389
+ typeof lane.provider_session_id === "string" &&
390
+ lane.provider_session_id.trim().length > 0
391
+ ? lane.provider_session_id
392
+ : undefined;
393
+ if (source && sessionId) {
394
+ copySessionFiles(planSessionCopy(lane.provider, {
395
+ sourceHome: source.home,
396
+ targetHome: target.home,
397
+ cwd: hqRoot,
398
+ sessionId,
399
+ }));
400
+ }
401
+ const now = (deps.now ?? (() => new Date()))();
402
+ const at = now.toISOString();
403
+ const nowMs = now.getTime();
404
+ return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
405
+ const current = readLane(hqRoot, lane.lane_id);
406
+ if (current.state !== "running" ||
407
+ current.account !== lane.account ||
408
+ current.provider_session_id !== lane.provider_session_id) {
409
+ throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
410
+ lane_id: lane.lane_id,
411
+ state: current.state,
412
+ account: current.account ?? null,
413
+ provider_session_id_changed: current.provider_session_id !== lane.provider_session_id,
414
+ });
415
+ }
416
+ const moveToRetryQueue = (row) => {
417
+ const entry = {
418
+ from: row.account ?? null,
419
+ to: target.id,
420
+ at,
421
+ reason,
422
+ };
423
+ return {
424
+ ...row,
425
+ state: "queued",
426
+ queued_reason: "account_retry",
427
+ queued_at: at,
428
+ admission_reserved_at: undefined,
429
+ terminal_reason: undefined,
430
+ terminal_details: undefined,
431
+ account: target.id,
432
+ account_history: appendCappedHistory(row.account_history ?? [], entry),
433
+ pending_resume: sessionId
434
+ ? pendingResumeForRetry(hqRoot, row, sessionId, at)
435
+ : undefined,
436
+ };
437
+ };
438
+ const candidate = moveToRetryQueue(current);
439
+ const admissionLanes = listLaneIds(hqRoot)
440
+ .filter((id) => id !== current.lane_id)
441
+ .map((id) => readLane(hqRoot, id));
442
+ admissionLanes.push(candidate);
443
+ const profileMaxFor = (row) => row.worker_profile === undefined
444
+ ? undefined
445
+ : loadWorkerProfile(hqRoot, row.worker_profile).meta.max_concurrent;
446
+ const capacityReason = laneAdmissionCapReason(admissionLanes, candidate, config, admissionHost, profile?.meta.max_concurrent, nowMs);
447
+ const earlierEligibleLane = hasEarlierEligibleQueuedLane(admissionLanes, candidate, config, admissionHost, profileMaxFor, nowMs);
448
+ const moved = retryUpdateLane(hqRoot, lane.lane_id, writer, (row) => {
449
+ if (row.state !== "running" ||
450
+ row.account !== lane.account ||
451
+ row.provider_session_id !== lane.provider_session_id) {
452
+ throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
453
+ lane_id: lane.lane_id,
454
+ state: row.state,
455
+ account: row.account ?? null,
456
+ provider_session_id_changed: row.provider_session_id !== lane.provider_session_id,
457
+ });
458
+ }
459
+ return moveToRetryQueue(row);
460
+ });
461
+ if (capacityReason !== undefined || earlierEligibleLane)
462
+ return moved;
463
+ return spawnLaneWorker(hqRoot, moved, deps);
464
+ });
465
+ }
104
466
  export function appendModelFlag(provider, args, model) {
105
467
  if (model === undefined)
106
468
  return args;
@@ -1157,16 +1519,28 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1157
1519
  const pinnedModel = existing.model_source === undefined || existing.model_source === "cli"
1158
1520
  ? existing.model
1159
1521
  : undefined;
1160
- let resolved;
1522
+ const accountDeps = {
1523
+ probe: modelDeps.probe ?? defaultProbe,
1524
+ env,
1525
+ };
1526
+ let selection;
1161
1527
  try {
1162
- resolved = resolveAccount(lane.provider, lane.account, config, {
1528
+ selection = selectSpawnAccount({
1163
1529
  hqRoot,
1530
+ lane: existing,
1531
+ config,
1164
1532
  existsSync,
1165
- model: pinnedModel ?? profileModel,
1533
+ accountDeps,
1534
+ modelDeps,
1535
+ pinnedModel,
1536
+ profileModel,
1537
+ currentModelValidated: deps.currentModelValidated === true,
1166
1538
  });
1167
1539
  }
1168
1540
  catch (err) {
1169
- if (err instanceof AccountResolutionError) {
1541
+ if (err instanceof AccountResolutionError ||
1542
+ (err instanceof LanesError &&
1543
+ (err.error === "account_not_logged_in" || err.error === "model_unsupported"))) {
1170
1544
  if (markPreSpawnFailure) {
1171
1545
  markSpawnFailure(hqRoot, lane.lane_id, err.error, err.details);
1172
1546
  }
@@ -1175,6 +1549,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1175
1549
  process.stderr.write(`hq lanes: account bind failed for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}\n`);
1176
1550
  throw err;
1177
1551
  }
1552
+ const resolved = selection.account;
1178
1553
  const status = accountStatus(lane.provider, resolved.id, config, { existsSync });
1179
1554
  if (isLoggedOut(status)) {
1180
1555
  const failed = markPreSpawnFailure
@@ -1185,8 +1560,20 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1185
1560
  : existing;
1186
1561
  throw new LanesError("account_not_logged_in", `Account ${resolved.id} for ${lane.provider} is not logged in (no credential file)`, { provider: lane.provider, account: resolved.id, lane_id: lane.lane_id, lane: failed });
1187
1562
  }
1188
- if (profileDefaultsApply && profileModel !== undefined && pinnedModel === undefined) {
1189
- validateProfileLaneModel(existing.provider, profileModel, resolved.home, profile.id, modelDeps);
1563
+ if (existing.account !== resolved.id) {
1564
+ const source = existing.account
1565
+ ? config.providers[existing.provider].accounts[existing.account]
1566
+ : undefined;
1567
+ if (source &&
1568
+ existing.provider_session_id &&
1569
+ providerSupportsSessionResume(existing.provider)) {
1570
+ copySessionFiles(planSessionCopy(existing.provider, {
1571
+ sourceHome: source.home,
1572
+ targetHome: resolved.home,
1573
+ cwd: hqRoot,
1574
+ sessionId: existing.provider_session_id,
1575
+ }));
1576
+ }
1190
1577
  }
1191
1578
  if (plan.mode === "resume" && existing.provider !== "grok") {
1192
1579
  ensureSessionHome({
@@ -1199,11 +1586,26 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1199
1586
  providerHomeAtSpawn: existing.provider_home_at_spawn,
1200
1587
  });
1201
1588
  }
1202
- if (lane.account !== resolved.id) {
1589
+ if (existing.account !== resolved.id) {
1590
+ const at = (deps.now ?? (() => new Date()))().toISOString();
1591
+ const reason = selection.changeReason ?? "first_assignment";
1203
1592
  retryUpdateLane(hqRoot, lane.lane_id, "spawn", (current) => {
1204
1593
  if (current.account === resolved.id)
1205
1594
  return current;
1206
- return { ...current, account: resolved.id };
1595
+ if (current.account !== existing.account) {
1596
+ throw new LanesError("account_bind_conflict", `Lane ${lane.lane_id} changed accounts while its worker spawn was prepared`, { lane_id: lane.lane_id, expected_account: existing.account ?? null, account: current.account ?? null });
1597
+ }
1598
+ const entry = {
1599
+ from: current.account ?? null,
1600
+ to: resolved.id,
1601
+ at,
1602
+ reason,
1603
+ };
1604
+ return {
1605
+ ...current,
1606
+ account: resolved.id,
1607
+ account_history: appendCappedHistory(current.account_history ?? [], entry),
1608
+ };
1207
1609
  });
1208
1610
  }
1209
1611
  try {
@@ -1255,10 +1657,10 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1255
1657
  ? trigger.text
1256
1658
  : pendingResume?.text,
1257
1659
  resumedFrom: trigger.kind === "resume" ? trigger.from : pendingResume?.resumed_from,
1258
- answersPath: trigger.kind === "answered_questions" && existsSync(answersPath)
1660
+ answersPath: (trigger.kind === "answered_questions" || pendingResume?.trigger === "answered_questions") && existsSync(answersPath)
1259
1661
  ? answersPath
1260
1662
  : undefined,
1261
- answers: trigger.kind === "answered_questions" && existsSync(answersPath)
1663
+ answers: (trigger.kind === "answered_questions" || pendingResume?.trigger === "answered_questions") && existsSync(answersPath)
1262
1664
  ? readAnsweredQuestionLines(answersPath)
1263
1665
  : undefined,
1264
1666
  }),
@@ -1425,6 +1827,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1425
1827
  ...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
1426
1828
  ...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
1427
1829
  ...(pendingResume.text !== undefined ? { text: pendingResume.text } : {}),
1830
+ ...(pendingResume.compact !== undefined ? { compact: pendingResume.compact } : {}),
1428
1831
  };
1429
1832
  }
1430
1833
  else if (resume && trigger.kind !== "admit") {
@@ -1433,7 +1836,10 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1433
1836
  at: nowIso,
1434
1837
  account: resolved.id,
1435
1838
  trigger: trigger.kind,
1436
- ...(trigger.kind === "message" ? { message_id: trigger.message_id } : {}),
1839
+ ...(trigger.kind === "message"
1840
+ ? { message_id: trigger.message_id, text: trigger.text }
1841
+ : {}),
1842
+ ...(launch.compact !== undefined ? { compact: launch.compact } : {}),
1437
1843
  ...(trigger.kind === "resume"
1438
1844
  ? {
1439
1845
  resumed_from: trigger.from,