@indigoai-us/hq-cli 5.154.0 → 5.155.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ import { MAX_LANE_DEPTH } from "./constants.js";
15
15
  import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
16
16
  import { loadWorkerProfile } from "./profile.js";
17
17
  import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
18
- import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
18
+ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
19
19
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
20
20
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
21
21
  import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
@@ -916,6 +916,11 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
916
916
  const repairedClassification = invalidEnvelope
917
917
  ? classifyLaneExit({ envelope: null, hasPartialOutput: false, invalidEnvelope })
918
918
  : classification;
919
+ const accountRetryReason = quota
920
+ ? "usage_limit"
921
+ : providerRejection && classification.reason === "account_not_logged_in"
922
+ ? "account_unavailable"
923
+ : null;
919
924
  if (!deps.dryRun &&
920
925
  quota &&
921
926
  classified.account &&
@@ -923,6 +928,30 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
923
928
  classification.reason === "account_model_unavailable")) {
924
929
  recordAccountQuota(hqRoot, classified.provider, classified.account, quota);
925
930
  }
931
+ if (!deps.dryRun && accountRetryReason && classified.account) {
932
+ try {
933
+ const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
934
+ ...(deps.spawnLaneDeps ?? {}),
935
+ admissionHost: deps.host,
936
+ });
937
+ summary.dead_host.push({
938
+ id: lane.lane_id,
939
+ from: "running",
940
+ to: recovered.state,
941
+ reason: accountRetryReason,
942
+ });
943
+ summary.heartbeats.push(`${lane.lane_id}:account-retry`);
944
+ continue;
945
+ }
946
+ catch (err) {
947
+ const message = err instanceof Error ? err.message : String(err);
948
+ process.stderr.write(`hq lanes reconcile: account failover failed for ${lane.lane_id}: ${message}\n`);
949
+ if (readLane(hqRoot, lane.lane_id).state !== "running") {
950
+ summary.errors.push({ message: `${lane.lane_id}: account failover left a non-running record after ${message}` });
951
+ continue;
952
+ }
953
+ }
954
+ }
926
955
  if (!deps.dryRun &&
927
956
  classification.state === "awaiting_input" &&
928
957
  latest &&
@@ -12,7 +12,8 @@ import { type RunCompactionFn } from "./compaction.js";
12
12
  import { type KnownModelsDeps } from "./models.js";
13
13
  import { type LaunchTrigger } from "./delivery.js";
14
14
  import { type MachineIdentity } from "./machine-identity.js";
15
- import { type LaneCompanyRef, type LaneProvider, type LaneRecord } from "./types.js";
15
+ import { type AdmissionHost } from "./admission.js";
16
+ import { type LaneCompanyRef, type LaneProvider, type LaneRecord, type AccountHistoryEntry } from "./types.js";
16
17
  export declare const LANES_RUNS_REL: string;
17
18
  /** Written by reconcile before a resume spawn; the worker prompt names this path. */
18
19
  export declare const ANSWERED_QUESTIONS_FILE = "answered-questions.json";
@@ -32,6 +33,9 @@ export declare function resolveLaneModel(input: {
32
33
  accountModel?: string;
33
34
  providerModel?: string;
34
35
  }): string | undefined;
36
+ type AutomaticAccountChangeReason = Exclude<AccountHistoryEntry["reason"], "operator_move">;
37
+ /** Change account only after a provider/runtime account failure, then re-admit. */
38
+ export declare function retryLaneOnAlternateAccount(hqRoot: string, lane: LaneRecord, reason: Extract<AutomaticAccountChangeReason, "usage_limit" | "account_unavailable">, writer?: "watcher" | "reconcile", deps?: SpawnLaneDeps): Promise<LaneRecord>;
35
39
  export declare function appendModelFlag(provider: LaneProvider, args: string[], model: string | undefined): string[];
36
40
  /** Add Codex fast-mode flags before the prompt/positional separator. */
37
41
  export declare function appendFastModeFlags(provider: LaneProvider, args: string[], enabled: boolean | undefined): string[];
@@ -63,8 +67,12 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
63
67
  export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
64
68
  export interface SpawnLaneDeps {
65
69
  env?: NodeJS.ProcessEnv;
70
+ /** The caller's host scope for admission checks; reconcile supplies its resolved host. */
71
+ admissionHost?: AdmissionHost;
66
72
  /** Provider model catalog dependencies, shared with create in tests. */
67
73
  modelDeps?: KnownModelsDeps;
74
+ /** Create validated this exact account/model candidate before its admission lock. */
75
+ currentModelValidated?: boolean;
68
76
  existsSync?: (p: string) => boolean;
69
77
  spawnProcess?: SpawnProcessFn;
70
78
  proveDetached?: ProveDetachedFn;
@@ -11,18 +11,21 @@ import { spawn, spawnSync } from "node:child_process";
11
11
  import * as fs from "node:fs";
12
12
  import * as path from "node:path";
13
13
  import { fileURLToPath } from "node:url";
14
- import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, } from "./accounts.js";
14
+ import { PROVIDER_HOME_ENV, accountStatus, isLoggedOut, resolveAccount, selectBalancedAccount, } from "./accounts.js";
15
15
  import { COMPACTION_PROMPT, compactionAfterUsage, compactionDue, compactLaneSession, } from "./compaction.js";
16
16
  import { installDrainHook } from "./hooks-install.js";
17
17
  import { loadLanesConfig, providerAutoCompactTokens, providerCompactAfterMs, } from "./config.js";
18
18
  import { appendEffortFlag, resolveLaneEffort } from "./effort.js";
19
- import { validateProfileLaneModel, } from "./models.js";
19
+ import { defaultProbe, validateLaneModel, validateProfileLaneModel, } from "./models.js";
20
20
  import { claimableStatesFor, planLaneLaunch, providerSupportsSessionResume, } from "./delivery.js";
21
21
  import { AccountResolutionError, LanesError } from "./errors.js";
22
22
  import { ignoredWorkerProfileFields, loadWorkerProfile, materializeWorkerProfile, workerProfilePrompt, } from "./profile.js";
23
23
  import { resolveMachineIdentity, } from "./machine-identity.js";
24
- import { readLane, retryUpdateLane } from "./store.js";
24
+ import { appendCappedHistory, listLaneIds, readLane, retryUpdateLane } from "./store.js";
25
+ import { copySessionFiles, planSessionCopy } from "./move-account.js";
25
26
  import { ensureSessionHome } from "./session-relink.js";
27
+ import { hasEarlierEligibleQueuedLane, laneAdmissionCapReason, withLaneAdmissionLock, } from "./admission.js";
28
+ import { resolveAdmissionHost } from "./admission-host.js";
26
29
  import { LANE_ENVELOPE_JSON_SCHEMA, parseLaneProvider, } from "./types.js";
27
30
  export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
28
31
  /** Written by reconcile before a resume spawn; the worker prompt names this path. */
@@ -101,6 +104,365 @@ export function resolveLaneModel(input) {
101
104
  }
102
105
  return undefined;
103
106
  }
107
+ function accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel) {
108
+ return resolveLaneModel({
109
+ cliModel: pinnedModel,
110
+ profileModel,
111
+ accountModel: config.providers[provider].accounts[accountId]?.model,
112
+ providerModel: config.providers[provider].model,
113
+ });
114
+ }
115
+ function supportsSpawnModel(provider, accountId, config, pinnedModel, profileModel, modelDeps) {
116
+ const account = config.providers[provider].accounts[accountId];
117
+ if (account === undefined)
118
+ return false;
119
+ const model = accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel);
120
+ if (model === undefined)
121
+ return true;
122
+ try {
123
+ validateLaneModel(provider, model, account.home, modelDeps);
124
+ return true;
125
+ }
126
+ catch (err) {
127
+ if (err instanceof LanesError && err.error === "model_unsupported")
128
+ return false;
129
+ throw err;
130
+ }
131
+ }
132
+ function spawnModelFailure(provider, accountId, config, pinnedModel, profileModel, profileId, modelDeps) {
133
+ const account = config.providers[provider].accounts[accountId];
134
+ if (account === undefined)
135
+ return new LanesError("unknown_account", `Unknown account ${accountId} for ${provider}`, { provider, account: accountId });
136
+ const model = accountModelForSpawn(provider, accountId, config, pinnedModel, profileModel);
137
+ if (model === undefined)
138
+ return null;
139
+ try {
140
+ validateLaneModel(provider, model, account.home, modelDeps);
141
+ return null;
142
+ }
143
+ catch (err) {
144
+ if (!(err instanceof LanesError) || err.error !== "model_unsupported")
145
+ throw err;
146
+ if (profileModel !== undefined && pinnedModel === undefined && profileId !== undefined) {
147
+ return new LanesError(err.error, err.message, {
148
+ ...err.details,
149
+ profile_id: profileId,
150
+ worker_profile: profileId,
151
+ model_source: "profile",
152
+ });
153
+ }
154
+ return err;
155
+ }
156
+ }
157
+ function selectSpawnAccount(input) {
158
+ const { hqRoot, lane, config, existsSync, accountDeps, modelDeps, pinnedModel, profileModel, currentModelValidated } = input;
159
+ const context = {
160
+ hqRoot,
161
+ existsSync,
162
+ accountDeps,
163
+ ...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
164
+ };
165
+ if (lane.account === undefined) {
166
+ const modelSpecified = pinnedModel !== undefined || profileModel !== undefined;
167
+ const allowedAccounts = modelSpecified
168
+ ? Object.keys(config.providers[lane.provider].accounts).filter((id) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
169
+ : undefined;
170
+ return {
171
+ account: selectBalancedAccount(lane.provider, config, {
172
+ ...context,
173
+ ...(allowedAccounts !== undefined ? { allowedAccounts } : {}),
174
+ preferredAccounts: config.providers[lane.provider].preferred_accounts ?? [],
175
+ }),
176
+ changeReason: "first_assignment",
177
+ };
178
+ }
179
+ const current = config.providers[lane.provider].accounts[lane.account];
180
+ let reason;
181
+ let refusal;
182
+ const operatorMovePending = lane.pending_resume !== undefined &&
183
+ lane.account_history?.at(-1)?.reason === "operator_move";
184
+ if (current === undefined || !current.enabled) {
185
+ reason = "account_unavailable";
186
+ refusal = new AccountResolutionError(current === undefined ? "unknown_account" : "account_disabled", `Account ${JSON.stringify(lane.account)} is unavailable for ${lane.provider}`, { provider: lane.provider, account: lane.account });
187
+ }
188
+ else {
189
+ const status = accountStatus(lane.provider, lane.account, config, {
190
+ existsSync,
191
+ ...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
192
+ ...(accountDeps?.env ? { env: accountDeps.env } : {}),
193
+ });
194
+ if (isLoggedOut(status)) {
195
+ reason = "account_unavailable";
196
+ refusal = new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in`, { provider: lane.provider, account: lane.account });
197
+ }
198
+ else {
199
+ try {
200
+ resolveAccount(lane.provider, lane.account, config, context);
201
+ const modelFailure = currentModelValidated
202
+ ? null
203
+ : spawnModelFailure(lane.provider, lane.account, config, pinnedModel, profileModel, lane.worker_profile, modelDeps);
204
+ if (modelFailure !== null) {
205
+ reason = "model_change";
206
+ refusal = modelFailure;
207
+ }
208
+ else {
209
+ return { account: resolveAccount(lane.provider, lane.account, config, context) };
210
+ }
211
+ }
212
+ catch (err) {
213
+ if (err instanceof AccountResolutionError && err.error === "account_quota_exhausted") {
214
+ reason = "usage_limit";
215
+ refusal = err;
216
+ }
217
+ else if (err instanceof AccountResolutionError && err.error === "account_model_unavailable") {
218
+ reason = "usage_limit";
219
+ refusal = err;
220
+ }
221
+ else if (err instanceof LanesError && err.error === "model_unsupported") {
222
+ reason = "model_change";
223
+ refusal = err;
224
+ }
225
+ else {
226
+ throw err;
227
+ }
228
+ }
229
+ }
230
+ }
231
+ if (operatorMovePending) {
232
+ if (refusal instanceof Error)
233
+ throw refusal;
234
+ throw new LanesError("account_unavailable", `Explicit account move target ${lane.account} is unavailable`, {
235
+ provider: lane.provider,
236
+ account: lane.account,
237
+ });
238
+ }
239
+ const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
240
+ .filter(([id, account]) => id !== lane.account && account.enabled)
241
+ .filter(([id]) => accountStatus(lane.provider, id, config, {
242
+ existsSync,
243
+ ...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
244
+ ...(accountDeps?.env ? { env: accountDeps.env } : {}),
245
+ }).logged_in === true)
246
+ .filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
247
+ .map(([id]) => id);
248
+ try {
249
+ return {
250
+ account: selectBalancedAccount(lane.provider, config, { ...context, allowedAccounts }),
251
+ changeReason: reason,
252
+ };
253
+ }
254
+ catch (err) {
255
+ if (reason === "account_unavailable") {
256
+ if (refusal instanceof Error)
257
+ throw refusal;
258
+ throw new LanesError("account_not_logged_in", `Account ${lane.account} for ${lane.provider} is not logged in or available, and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account });
259
+ }
260
+ if (refusal instanceof Error)
261
+ throw refusal;
262
+ throw err;
263
+ }
264
+ }
265
+ function previouslyUnavailableAccounts(lane) {
266
+ const excluded = new Set();
267
+ for (const entry of [...(lane.account_history ?? [])].reverse()) {
268
+ if (entry.reason === "first_assignment" || entry.reason === "operator_move")
269
+ break;
270
+ if (entry.reason === "account_unavailable" && entry.from !== null)
271
+ excluded.add(entry.from);
272
+ }
273
+ return excluded;
274
+ }
275
+ function archivedResumeMessageText(hqRoot, laneId, messageId) {
276
+ if (!messageId || /[\\/\0]/.test(messageId))
277
+ return undefined;
278
+ const file = path.join(laneRunDir(hqRoot, laneId), "dropbox", "delivered", `${messageId}.resume.jsonl`);
279
+ let raw;
280
+ try {
281
+ raw = fs.readFileSync(file, "utf8");
282
+ }
283
+ catch (err) {
284
+ if (err.code === "ENOENT")
285
+ return undefined;
286
+ throw new LanesError("account_retry_context_unreadable", `Could not read the archived message needed to resume lane ${laneId}`, { lane_id: laneId, message_id: messageId });
287
+ }
288
+ const lines = raw.split(/\r?\n/).filter((line) => line.trim().length > 0);
289
+ if (lines.length !== 1) {
290
+ throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
291
+ }
292
+ let value;
293
+ try {
294
+ value = JSON.parse(lines[0]);
295
+ }
296
+ catch {
297
+ throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
298
+ }
299
+ if (value === null ||
300
+ typeof value !== "object" ||
301
+ value.message_id !== messageId ||
302
+ typeof value.text !== "string") {
303
+ throw new LanesError("account_retry_context_invalid", `The archived message needed to resume lane ${laneId} is malformed`, { lane_id: laneId, message_id: messageId });
304
+ }
305
+ return value.text;
306
+ }
307
+ function pendingResumeForRetry(hqRoot, lane, sessionId, at) {
308
+ const last = lane.last_resume;
309
+ if (last === undefined || last.session_id !== sessionId) {
310
+ return { session_id: sessionId, trigger: "resume", resumed_from: "interrupted", at };
311
+ }
312
+ const compact = last.compact === undefined ? {} : { compact: last.compact };
313
+ if (last.trigger === "answered_questions") {
314
+ return { session_id: sessionId, trigger: "answered_questions", at, ...compact };
315
+ }
316
+ if (last.trigger === "message") {
317
+ if (last.message_id === undefined) {
318
+ throw new LanesError("account_retry_context_missing", `The message id needed to resume lane ${lane.lane_id} is missing`, { lane_id: lane.lane_id, trigger: last.trigger });
319
+ }
320
+ const text = last.text ?? archivedResumeMessageText(hqRoot, lane.lane_id, last.message_id);
321
+ if (text === undefined) {
322
+ throw new LanesError("account_retry_context_missing", `The operator message needed to resume lane ${lane.lane_id} is unavailable`, { lane_id: lane.lane_id, message_id: last.message_id });
323
+ }
324
+ return {
325
+ session_id: sessionId,
326
+ trigger: "message",
327
+ text,
328
+ message_id: last.message_id,
329
+ at,
330
+ ...compact,
331
+ };
332
+ }
333
+ return {
334
+ session_id: sessionId,
335
+ trigger: "resume",
336
+ resumed_from: last.resumed_from ?? "interrupted",
337
+ ...(last.text !== undefined ? { text: last.text } : {}),
338
+ ...(last.message_id !== undefined ? { message_id: last.message_id } : {}),
339
+ at,
340
+ ...compact,
341
+ };
342
+ }
343
+ /** Change account only after a provider/runtime account failure, then re-admit. */
344
+ export async function retryLaneOnAlternateAccount(hqRoot, lane, reason, writer = "watcher", deps = {}) {
345
+ const config = loadLanesConfig(hqRoot);
346
+ const existsSync = deps.existsSync ?? ((p) => fs.existsSync(p));
347
+ const env = deps.env ?? process.env;
348
+ const modelDeps = {
349
+ ...deps.modelDeps,
350
+ env: deps.modelDeps?.env ?? env,
351
+ };
352
+ const accountDeps = {
353
+ probe: modelDeps.probe ?? defaultProbe,
354
+ env,
355
+ };
356
+ const admissionHost = deps.admissionHost ?? resolveAdmissionHost();
357
+ const profile = lane.worker_profile ? loadWorkerProfile(hqRoot, lane.worker_profile) : undefined;
358
+ const profileModel = profile?.meta.provider === lane.provider ? profile.meta.model : undefined;
359
+ const pinnedModel = lane.model_source === undefined || lane.model_source === "cli" ? lane.model : undefined;
360
+ const unavailableAccounts = previouslyUnavailableAccounts(lane);
361
+ const allowedAccounts = Object.entries(config.providers[lane.provider].accounts)
362
+ .filter(([id, account]) => id !== lane.account && !unavailableAccounts.has(id) && account.enabled)
363
+ .filter(([id]) => accountStatus(lane.provider, id, config, {
364
+ existsSync,
365
+ ...(accountDeps?.probe ? { probe: accountDeps.probe } : {}),
366
+ ...(accountDeps?.env ? { env: accountDeps.env } : {}),
367
+ }).logged_in === true)
368
+ .filter(([id]) => supportsSpawnModel(lane.provider, id, config, pinnedModel, profileModel, modelDeps))
369
+ .map(([id]) => id);
370
+ const modelContext = {
371
+ hqRoot,
372
+ existsSync,
373
+ accountDeps,
374
+ ...(pinnedModel ?? profileModel ? { model: pinnedModel ?? profileModel } : {}),
375
+ allowedAccounts,
376
+ };
377
+ let target;
378
+ try {
379
+ target = selectBalancedAccount(lane.provider, config, modelContext);
380
+ }
381
+ catch (err) {
382
+ if (reason === "account_unavailable" || unavailableAccounts.size > 0) {
383
+ throw new LanesError("account_not_logged_in", `Account ${lane.account ?? "(none)"} for ${lane.provider} is unavailable and no other logged-in account can serve this lane`, { provider: lane.provider, account: lane.account ?? null });
384
+ }
385
+ throw err;
386
+ }
387
+ const source = lane.account ? config.providers[lane.provider].accounts[lane.account] : undefined;
388
+ const sessionId = providerSupportsSessionResume(lane.provider) &&
389
+ typeof lane.provider_session_id === "string" &&
390
+ lane.provider_session_id.trim().length > 0
391
+ ? lane.provider_session_id
392
+ : undefined;
393
+ if (source && sessionId) {
394
+ copySessionFiles(planSessionCopy(lane.provider, {
395
+ sourceHome: source.home,
396
+ targetHome: target.home,
397
+ cwd: hqRoot,
398
+ sessionId,
399
+ }));
400
+ }
401
+ const now = (deps.now ?? (() => new Date()))();
402
+ const at = now.toISOString();
403
+ const nowMs = now.getTime();
404
+ return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
405
+ const current = readLane(hqRoot, lane.lane_id);
406
+ if (current.state !== "running" ||
407
+ current.account !== lane.account ||
408
+ current.provider_session_id !== lane.provider_session_id) {
409
+ throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
410
+ lane_id: lane.lane_id,
411
+ state: current.state,
412
+ account: current.account ?? null,
413
+ provider_session_id_changed: current.provider_session_id !== lane.provider_session_id,
414
+ });
415
+ }
416
+ const moveToRetryQueue = (row) => {
417
+ const entry = {
418
+ from: row.account ?? null,
419
+ to: target.id,
420
+ at,
421
+ reason,
422
+ };
423
+ return {
424
+ ...row,
425
+ state: "queued",
426
+ queued_reason: "account_retry",
427
+ queued_at: at,
428
+ admission_reserved_at: undefined,
429
+ terminal_reason: undefined,
430
+ terminal_details: undefined,
431
+ account: target.id,
432
+ account_history: appendCappedHistory(row.account_history ?? [], entry),
433
+ pending_resume: sessionId
434
+ ? pendingResumeForRetry(hqRoot, row, sessionId, at)
435
+ : undefined,
436
+ };
437
+ };
438
+ const candidate = moveToRetryQueue(current);
439
+ const admissionLanes = listLaneIds(hqRoot)
440
+ .filter((id) => id !== current.lane_id)
441
+ .map((id) => readLane(hqRoot, id));
442
+ admissionLanes.push(candidate);
443
+ const profileMaxFor = (row) => row.worker_profile === undefined
444
+ ? undefined
445
+ : loadWorkerProfile(hqRoot, row.worker_profile).meta.max_concurrent;
446
+ const capacityReason = laneAdmissionCapReason(admissionLanes, candidate, config, admissionHost, profile?.meta.max_concurrent, nowMs);
447
+ const earlierEligibleLane = hasEarlierEligibleQueuedLane(admissionLanes, candidate, config, admissionHost, profileMaxFor, nowMs);
448
+ const moved = retryUpdateLane(hqRoot, lane.lane_id, writer, (row) => {
449
+ if (row.state !== "running" ||
450
+ row.account !== lane.account ||
451
+ row.provider_session_id !== lane.provider_session_id) {
452
+ throw new LanesError("account_retry_conflict", `Lane ${lane.lane_id} changed while account recovery was prepared`, {
453
+ lane_id: lane.lane_id,
454
+ state: row.state,
455
+ account: row.account ?? null,
456
+ provider_session_id_changed: row.provider_session_id !== lane.provider_session_id,
457
+ });
458
+ }
459
+ return moveToRetryQueue(row);
460
+ });
461
+ if (capacityReason !== undefined || earlierEligibleLane)
462
+ return moved;
463
+ return spawnLaneWorker(hqRoot, moved, deps);
464
+ });
465
+ }
104
466
  export function appendModelFlag(provider, args, model) {
105
467
  if (model === undefined)
106
468
  return args;
@@ -1157,16 +1519,28 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1157
1519
  const pinnedModel = existing.model_source === undefined || existing.model_source === "cli"
1158
1520
  ? existing.model
1159
1521
  : undefined;
1160
- let resolved;
1522
+ const accountDeps = {
1523
+ probe: modelDeps.probe ?? defaultProbe,
1524
+ env,
1525
+ };
1526
+ let selection;
1161
1527
  try {
1162
- resolved = resolveAccount(lane.provider, lane.account, config, {
1528
+ selection = selectSpawnAccount({
1163
1529
  hqRoot,
1530
+ lane: existing,
1531
+ config,
1164
1532
  existsSync,
1165
- model: pinnedModel ?? profileModel,
1533
+ accountDeps,
1534
+ modelDeps,
1535
+ pinnedModel,
1536
+ profileModel,
1537
+ currentModelValidated: deps.currentModelValidated === true,
1166
1538
  });
1167
1539
  }
1168
1540
  catch (err) {
1169
- if (err instanceof AccountResolutionError) {
1541
+ if (err instanceof AccountResolutionError ||
1542
+ (err instanceof LanesError &&
1543
+ (err.error === "account_not_logged_in" || err.error === "model_unsupported"))) {
1170
1544
  if (markPreSpawnFailure) {
1171
1545
  markSpawnFailure(hqRoot, lane.lane_id, err.error, err.details);
1172
1546
  }
@@ -1175,6 +1549,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1175
1549
  process.stderr.write(`hq lanes: account bind failed for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}\n`);
1176
1550
  throw err;
1177
1551
  }
1552
+ const resolved = selection.account;
1178
1553
  const status = accountStatus(lane.provider, resolved.id, config, { existsSync });
1179
1554
  if (isLoggedOut(status)) {
1180
1555
  const failed = markPreSpawnFailure
@@ -1185,8 +1560,20 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1185
1560
  : existing;
1186
1561
  throw new LanesError("account_not_logged_in", `Account ${resolved.id} for ${lane.provider} is not logged in (no credential file)`, { provider: lane.provider, account: resolved.id, lane_id: lane.lane_id, lane: failed });
1187
1562
  }
1188
- if (profileDefaultsApply && profileModel !== undefined && pinnedModel === undefined) {
1189
- validateProfileLaneModel(existing.provider, profileModel, resolved.home, profile.id, modelDeps);
1563
+ if (existing.account !== resolved.id) {
1564
+ const source = existing.account
1565
+ ? config.providers[existing.provider].accounts[existing.account]
1566
+ : undefined;
1567
+ if (source &&
1568
+ existing.provider_session_id &&
1569
+ providerSupportsSessionResume(existing.provider)) {
1570
+ copySessionFiles(planSessionCopy(existing.provider, {
1571
+ sourceHome: source.home,
1572
+ targetHome: resolved.home,
1573
+ cwd: hqRoot,
1574
+ sessionId: existing.provider_session_id,
1575
+ }));
1576
+ }
1190
1577
  }
1191
1578
  if (plan.mode === "resume" && existing.provider !== "grok") {
1192
1579
  ensureSessionHome({
@@ -1199,11 +1586,26 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1199
1586
  providerHomeAtSpawn: existing.provider_home_at_spawn,
1200
1587
  });
1201
1588
  }
1202
- if (lane.account !== resolved.id) {
1589
+ if (existing.account !== resolved.id) {
1590
+ const at = (deps.now ?? (() => new Date()))().toISOString();
1591
+ const reason = selection.changeReason ?? "first_assignment";
1203
1592
  retryUpdateLane(hqRoot, lane.lane_id, "spawn", (current) => {
1204
1593
  if (current.account === resolved.id)
1205
1594
  return current;
1206
- return { ...current, account: resolved.id };
1595
+ if (current.account !== existing.account) {
1596
+ throw new LanesError("account_bind_conflict", `Lane ${lane.lane_id} changed accounts while its worker spawn was prepared`, { lane_id: lane.lane_id, expected_account: existing.account ?? null, account: current.account ?? null });
1597
+ }
1598
+ const entry = {
1599
+ from: current.account ?? null,
1600
+ to: resolved.id,
1601
+ at,
1602
+ reason,
1603
+ };
1604
+ return {
1605
+ ...current,
1606
+ account: resolved.id,
1607
+ account_history: appendCappedHistory(current.account_history ?? [], entry),
1608
+ };
1207
1609
  });
1208
1610
  }
1209
1611
  try {
@@ -1255,10 +1657,10 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1255
1657
  ? trigger.text
1256
1658
  : pendingResume?.text,
1257
1659
  resumedFrom: trigger.kind === "resume" ? trigger.from : pendingResume?.resumed_from,
1258
- answersPath: trigger.kind === "answered_questions" && existsSync(answersPath)
1660
+ answersPath: (trigger.kind === "answered_questions" || pendingResume?.trigger === "answered_questions") && existsSync(answersPath)
1259
1661
  ? answersPath
1260
1662
  : undefined,
1261
- answers: trigger.kind === "answered_questions" && existsSync(answersPath)
1663
+ answers: (trigger.kind === "answered_questions" || pendingResume?.trigger === "answered_questions") && existsSync(answersPath)
1262
1664
  ? readAnsweredQuestionLines(answersPath)
1263
1665
  : undefined,
1264
1666
  }),
@@ -1425,6 +1827,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1425
1827
  ...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
1426
1828
  ...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
1427
1829
  ...(pendingResume.text !== undefined ? { text: pendingResume.text } : {}),
1830
+ ...(pendingResume.compact !== undefined ? { compact: pendingResume.compact } : {}),
1428
1831
  };
1429
1832
  }
1430
1833
  else if (resume && trigger.kind !== "admit") {
@@ -1433,7 +1836,10 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
1433
1836
  at: nowIso,
1434
1837
  account: resolved.id,
1435
1838
  trigger: trigger.kind,
1436
- ...(trigger.kind === "message" ? { message_id: trigger.message_id } : {}),
1839
+ ...(trigger.kind === "message"
1840
+ ? { message_id: trigger.message_id, text: trigger.text }
1841
+ : {}),
1842
+ ...(launch.compact !== undefined ? { compact: launch.compact } : {}),
1437
1843
  ...(trigger.kind === "resume"
1438
1844
  ? {
1439
1845
  resumed_from: trigger.from,
@@ -166,7 +166,7 @@ export declare const TERMINAL_LANE_STATES: readonly ["done", "failed", "interrup
166
166
  export type TerminalLaneState = (typeof TERMINAL_LANE_STATES)[number];
167
167
  export declare const LANE_PROVIDERS: readonly ["claude", "codex", "grok"];
168
168
  export type LaneProvider = (typeof LANE_PROVIDERS)[number];
169
- export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing"];
169
+ export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry"];
170
170
  export type QueuedReason = (typeof QUEUED_REASONS)[number];
171
171
  export declare function isQueuedReason(value: unknown): value is QueuedReason;
172
172
  export declare const HOST_KINDS: readonly ["local", "outpost"];
@@ -436,11 +436,11 @@ export interface AccountHistoryEntry {
436
436
  from: string | null;
437
437
  to: string;
438
438
  at: string;
439
- reason: "operator_move";
439
+ reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "operator_move";
440
440
  }
441
441
  export interface PendingResume {
442
442
  session_id: string;
443
- trigger: "message" | "resume";
443
+ trigger: "answered_questions" | "message" | "resume";
444
444
  resumed_from?: "interrupted" | "done" | "blocked";
445
445
  text?: string;
446
446
  message_id?: string;
@@ -461,6 +461,8 @@ export interface LaneResumeRecord {
461
461
  */
462
462
  resumed_from?: "interrupted" | "done" | "blocked";
463
463
  text?: string;
464
+ /** False when the continued turn explicitly skipped compaction. */
465
+ compact?: boolean;
464
466
  }
465
467
  export interface LaneRunUsage {
466
468
  cache_read_input_tokens: number | null;
@@ -575,6 +577,8 @@ export interface ProviderConfig {
575
577
  * Account pick is least-loaded among enabled, logged-in accounts.
576
578
  */
577
579
  default_account?: string;
580
+ /** Ordered account preferences used only for a lane's first assignment. */
581
+ preferred_accounts?: string[];
578
582
  /** Default model for this provider when the account does not set one. */
579
583
  model?: string;
580
584
  /** Default effort for this provider when the account does not set one. */
@@ -103,6 +103,7 @@ export const QUEUED_REASONS = [
103
103
  "queue_position",
104
104
  "throttle",
105
105
  "provider_binary_missing",
106
+ "account_retry",
106
107
  ];
107
108
  export function isQueuedReason(value) {
108
109
  return typeof value === "string" && QUEUED_REASONS.includes(value);
@@ -215,11 +216,11 @@ export const LANE_FIELD_WRITERS = {
215
216
  schema_version: ["creator"], lane_id: ["creator"], company: ["creator"],
216
217
  project_id: ["command"], story_id: ["command"], stories: ["creator", "command", "watcher", "reconcile", "spawn"], thread_id: ["creator", "command", "spawn"], mesh_company: ["creator"],
217
218
  depth: ["command"], senior: ["command"], senior_history: ["command"],
218
- state: ["creator", "command", "watcher", "reconcile", "spawn"], queued_reason: ["creator", "command", "spawn", "reconcile"], queued_at: ["creator", "command", "spawn"], admission_reserved_at: ["creator", "command", "reconcile", "spawn"],
219
+ state: ["creator", "command", "watcher", "reconcile", "spawn"], queued_reason: ["creator", "command", "spawn", "watcher", "reconcile"], queued_at: ["creator", "command", "spawn", "watcher", "reconcile"], admission_reserved_at: ["creator", "command", "reconcile", "spawn", "watcher"],
219
220
  pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
220
221
  pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
221
222
  priority: ["command"], worker: ["creator"], worker_path: ["creator"], worker_pack: ["creator"], worker_profile: ["creator"], worker_profile_hash: ["creator", "spawn"], worker_profile_ignored: ["creator", "spawn"],
222
- provider: ["creator"], account: ["creator", "spawn", "command", "reconcile"], admission_accounts: ["creator", "command"], account_history: ["command"], pending_resume: ["command", "spawn"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
223
+ provider: ["creator"], account: ["creator", "spawn", "command", "watcher", "reconcile"], admission_accounts: ["creator", "command"], account_history: ["command", "spawn", "watcher", "reconcile"], pending_resume: ["command", "spawn", "watcher", "reconcile"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
223
224
  version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
224
225
  last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn"], effort_source: ["creator", "spawn"], fast_mode: ["spawn"],
225
226
  last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],