@indigoai-us/hq-cli 5.278.0 → 5.280.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,24 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [5.280.0] — 2026-09-29
6
+
7
+ ### Fixed
8
+
9
+ - Single-lane commands for Outpost-hosted lanes use a configurable pickup wait,
10
+ defaulting to 120 seconds. `hq lanes list` capacity probes use a separate
11
+ 15-second timeout. A lane ID with a registered host suffix routes to that
12
+ Outpost first.
13
+
14
+ ## [5.279.0] — 2026-09-29
15
+
16
+ ### Fixed
17
+
18
+ - Claude keep-alive resumes use the same account selection as manual resumes,
19
+ including profile pins and the optional recent-use tiebreak. If no eligible
20
+ account can be selected, the lane resumes on its current account and records
21
+ the failure.
22
+
5
23
  ## [5.278.0] — 2026-09-29
6
24
 
7
25
  ### Changed
@@ -195,6 +195,7 @@ export interface Ctx {
195
195
  resolveCompany: (company?: string) => Promise<LaneCompanyRef>;
196
196
  resolveActiveCompany: (company: string) => Promise<LaneCompanyRef | null>;
197
197
  listOutpostIds: () => Promise<string[]>;
198
+ listOutpostRoutes: () => Promise<OutpostRouteIdentity[]>;
198
199
  readOutpostIdentity: () => OutpostIdentity;
199
200
  spawnLogin: (opts: {
200
201
  command: string;
@@ -244,6 +245,10 @@ export interface Ctx {
244
245
  mergeHoldEnabled?: (company: LaneCompanyRef) => boolean | Promise<boolean>;
245
246
  mergeReviewEnabled?: (company: LaneCompanyRef) => boolean | Promise<boolean>;
246
247
  }
248
+ export interface OutpostRouteIdentity {
249
+ outpostId: string;
250
+ instanceName?: string;
251
+ }
247
252
  export declare const LANE_OUTPOST_LOAD_BALANCING_FLAG = "lanes.outpost-load-balancing";
248
253
  export declare const CLAUDE_ACCOUNT_BALANCE_SIGNALS_FLAG = "lanes.claude-account-balance-signals";
249
254
  export declare function defaultLanePlacementEnabled(company: LaneCompanyRef | undefined): Promise<boolean>;
@@ -6,7 +6,7 @@
6
6
  * `watch` and the monitor reminder are the line-oriented text surfaces.
7
7
  */
8
8
  import { Command } from "commander";
9
- import { type LanesMesh, type SpawnLoginResult } from "./lanes-shared.js";
9
+ import { type LanesMesh, type OutpostRouteIdentity, type SpawnLoginResult } from "./lanes-shared.js";
10
10
  export type { LanesMesh, SpawnLoginResult } from "./lanes-shared.js";
11
11
  export { CLAUDE_ACCOUNT_BALANCE_SIGNALS_FLAG, LANE_OUTPOST_LOAD_BALANCING_FLAG } from "./lanes-shared.js";
12
12
  export { resolveAnswerChoice } from "./lanes-questions.js";
@@ -38,6 +38,7 @@ export interface LanesCommandDeps {
38
38
  /** Membership-backed explicit company selection for account-home writers. */
39
39
  resolveActiveCompany?: (company: string) => Promise<LaneCompanyRef | null>;
40
40
  listOutpostIds?: () => Promise<string[]>;
41
+ listOutpostRoutes?: () => Promise<OutpostRouteIdentity[]>;
41
42
  readOutpostIdentity?: () => OutpostIdentity;
42
43
  spawnLogin?: (opts: {
43
44
  command: string;
@@ -36,7 +36,7 @@ import { createOutpostProfileRepinContext, parseOutpostProfileRepinContext, } fr
36
36
  import { createOutpostRegistryWorkerApprovalContext, parseOutpostRegistryWorkerApprovalContext, } from "../lib/lanes/outpost-registry-worker-approval.js";
37
37
  import { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity, resolveAdmissionHost, } from "../lib/lanes/admission-host.js";
38
38
  export { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity } from "../lib/lanes/admission-host.js";
39
- import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostQuestionListCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
39
+ import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostQuestionListCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, PICKUP_WAIT_MS, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
40
40
  import { outpostHireSummaryCacheKey, readOutpostHireSummaryCache, writeOutpostHireSummaryCache, } from "../lib/lanes/outpost-hire-cache.js";
41
41
  import { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "../lib/lanes/host-capacity.js";
42
42
  import { BALANCE_MOVABLE_STATES, selectClaudeAccount, } from "../lib/lanes/claude-balance.js";
@@ -159,6 +159,10 @@ async function defaultListOutpostIds() {
159
159
  const rows = await defaultOutpostsClient().list();
160
160
  return rows.map((row) => row.outpostId);
161
161
  }
162
+ async function defaultListOutpostRoutes() {
163
+ const rows = await defaultOutpostsClient().list();
164
+ return rows.map(({ outpostId, instanceName }) => ({ outpostId, instanceName }));
165
+ }
162
166
  async function defaultStageOutpostPayload(outpostId, bytes) {
163
167
  return stageOutpostPayload(defaultOutpostsClient(), outpostId, bytes);
164
168
  }
@@ -218,6 +222,9 @@ function resolveDeps(deps = {}) {
218
222
  };
219
223
  const bindProcessCwd = deps.hqRoot === undefined;
220
224
  const resolveRoot = deps.hqRoot ?? (() => resolveLanesHqRoot({ env, cwd: cwd() }));
225
+ const listOutpostRoutes = deps.listOutpostRoutes ?? (deps.listOutpostIds
226
+ ? async () => (await deps.listOutpostIds()).map((outpostId) => ({ outpostId }))
227
+ : defaultListOutpostRoutes);
221
228
  const hqRoot = () => {
222
229
  callerCwd();
223
230
  const root = resolveRoot();
@@ -236,6 +243,7 @@ function resolveDeps(deps = {}) {
236
243
  resolveCompany: deps.resolveCompany ?? defaultResolveCompany,
237
244
  resolveActiveCompany: deps.resolveActiveCompany ?? defaultResolveActiveCompany,
238
245
  listOutpostIds: deps.listOutpostIds ?? defaultListOutpostIds,
246
+ listOutpostRoutes,
239
247
  readOutpostIdentity: deps.readOutpostIdentity ?? (() => readOutpostIdentity()),
240
248
  spawnLogin: deps.spawnLogin ?? defaultSpawnLogin,
241
249
  throttleDeps: deps.throttleDeps ?? defaultThrottleDeps(env),
@@ -611,6 +619,7 @@ async function fetchOwnerLaneHireFields(ctx, lane) {
611
619
  }
612
620
  const outpostId = lane.host.id;
613
621
  const hqBinary = outpostCliOverride(ctx, outpostId);
622
+ const waitMs = outpostSingleLaneWaitMs(ctx.env);
614
623
  let dispatched;
615
624
  try {
616
625
  dispatched = await ctx.dispatchOutpostExec({
@@ -620,12 +629,15 @@ async function fetchOwnerLaneHireFields(ctx, lane) {
620
629
  laneId: lane.lane_id,
621
630
  hqBinary,
622
631
  }),
623
- timeoutSeconds: OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS,
624
- waitMs: OUTPOST_HIRE_SUMMARY_WAIT_MS,
632
+ timeoutSeconds: Math.ceil(waitMs / 1000),
633
+ waitMs,
625
634
  });
626
635
  }
627
636
  catch (err) {
628
- throw new LanesError("outpost_lane_hire_summary_unavailable", `Could not read hire summary for lane ${lane.lane_id} from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, outpost_id: outpostId });
637
+ const message = dispatchErrorMessage(err);
638
+ throw new LanesError("outpost_lane_hire_summary_unavailable", /timed out waiting for outpost pickup/i.test(message)
639
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `reading hire summary for lane ${lane.lane_id}`)
640
+ : `Could not read hire summary for lane ${lane.lane_id} from Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId });
629
641
  }
630
642
  if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
631
643
  throw new LanesError("outpost_lane_hire_summary_unavailable", `Outpost ${outpostId} did not confirm the hire summary for lane ${lane.lane_id}`, { lane_id: lane.lane_id, outpost_id: outpostId, command_id: dispatched.commandId });
@@ -808,8 +820,6 @@ async function batchOwnerLaneHireFields(ctx, lanes) {
808
820
  writeOutpostHireSummaryCache(ctx.hqRoot(), cacheUpdates, ctx.now().getTime(), cached);
809
821
  return outcomes;
810
822
  }
811
- const OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS = 20;
812
- const OUTPOST_HIRE_SUMMARY_WAIT_MS = 15_000;
813
823
  const OUTPOST_HIRE_SUMMARY_CONCURRENCY = 4;
814
824
  async function laneHirePresentation(ctx, lane, lanes) {
815
825
  const local = laneHireFields(lane, lanes);
@@ -886,6 +896,28 @@ async function withResumeHireAdmission(ctx, lane, fn, grantContext) {
886
896
  }
887
897
  const CAPACITY_PROBE_TIMEOUT_MS = 15_000;
888
898
  const MAX_PLACEMENT_OUTPOSTS = 16;
899
+ const OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS = 20;
900
+ const OUTPOST_HIRE_SUMMARY_WAIT_MS = 15_000;
901
+ const OUTPOST_SINGLE_LANE_WAIT_ENV = "HQ_LANES_OUTPOST_SINGLE_LANE_WAIT_MS";
902
+ const OUTPOST_SINGLE_LANE_WAIT_MAX_MS = 600_000;
903
+ function outpostSingleLaneWaitMs(env) {
904
+ const raw = env[OUTPOST_SINGLE_LANE_WAIT_ENV]?.trim() || String(PICKUP_WAIT_MS);
905
+ const value = /^\d+$/.test(raw) ? Number(raw) : Number.NaN;
906
+ if (!Number.isSafeInteger(value) ||
907
+ value < CAPACITY_PROBE_TIMEOUT_MS ||
908
+ value > OUTPOST_SINGLE_LANE_WAIT_MAX_MS) {
909
+ throw new LanesError("outpost_wait_invalid", `${OUTPOST_SINGLE_LANE_WAIT_ENV} must be an integer from ${CAPACITY_PROBE_TIMEOUT_MS} to ${OUTPOST_SINGLE_LANE_WAIT_MAX_MS} milliseconds`, { env: OUTPOST_SINGLE_LANE_WAIT_ENV });
910
+ }
911
+ return value;
912
+ }
913
+ function outpostRouteMatchesHost(route, host) {
914
+ return typeof route.instanceName === "string" &&
915
+ route.instanceName.trim() !== "" &&
916
+ sanitizeHostComponent(route.instanceName.trim()) === sanitizeHostComponent(host.trim());
917
+ }
918
+ function outpostPickupTimeoutMessage(outpostId, waitMs, operation) {
919
+ return `Outpost ${outpostId} pickup timed out after ${waitMs}ms while ${operation}. Raise ${OUTPOST_SINGLE_LANE_WAIT_ENV} (milliseconds) and retry.`;
920
+ }
889
921
  function outpostIdentityUnknownError(message, expectedId, details = {}) {
890
922
  const expected = expectedId ?? "<expected-outpost-id>";
891
923
  const command = `hq lanes outpost-identity backfill --id ${expected}`;
@@ -1025,14 +1057,20 @@ function normalizedLaneListQuery(query) {
1025
1057
  };
1026
1058
  }
1027
1059
  async function readOutpostLaneRecordsUncached(ctx, outpostId, query) {
1060
+ const targetedLookup = query.hierarchy !== undefined || query.laneIds !== undefined;
1061
+ const waitMs = targetedLookup ? outpostSingleLaneWaitMs(ctx.env) : CAPACITY_PROBE_TIMEOUT_MS;
1028
1062
  let result;
1029
1063
  try {
1030
- result = await dispatchOutpostLaneList(ctx, outpostId, query, CAPACITY_PROBE_TIMEOUT_MS / 1000, CAPACITY_PROBE_TIMEOUT_MS);
1064
+ result = await dispatchOutpostLaneList(ctx, outpostId, query, Math.ceil(waitMs / 1000), waitMs);
1031
1065
  }
1032
1066
  catch (err) {
1067
+ const message = dispatchErrorMessage(err);
1068
+ if (/timed out waiting for outpost pickup/i.test(message)) {
1069
+ throw new LanesError("outpost_lane_list_unavailable", outpostPickupTimeoutMessage(outpostId, waitMs, "reading the requested lane"), { outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
1070
+ }
1033
1071
  if (isLanesError(err))
1034
1072
  throw err;
1035
- throw new LanesError("outpost_lane_list_unavailable", `Could not read lanes from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { outpost_id: outpostId });
1073
+ throw new LanesError("outpost_lane_list_unavailable", `Could not read lanes from Outpost ${outpostId}: ${message}`, { outpost_id: outpostId });
1036
1074
  }
1037
1075
  if (result.done !== true || result.status !== "Success" || result.exitCode !== 0) {
1038
1076
  throw new LanesError("outpost_lane_list_unavailable", `Outpost ${outpostId} did not return a successful lane list`, { outpost_id: outpostId, status: result.status, done: result.done ?? false, exit_code: result.exitCode });
@@ -1119,17 +1157,34 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
1119
1157
  if (!hostSuffix || sanitizeHostComponent(hostSuffix) !== hostSuffix) {
1120
1158
  throw commandLaneNotFound(laneId, options.outpostHint === true);
1121
1159
  }
1122
- let rawOutpostIds;
1160
+ let rawOutpostRoutes;
1123
1161
  try {
1124
- rawOutpostIds = await ctx.listOutpostIds();
1162
+ rawOutpostRoutes = await ctx.listOutpostRoutes();
1125
1163
  }
1126
1164
  catch (err) {
1127
1165
  throw new LanesError("outpost_list_unavailable", `Could not resolve registered Outposts for lane ${laneId}: ${dispatchErrorMessage(err)}`, { lane_id: laneId });
1128
1166
  }
1129
- if (!Array.isArray(rawOutpostIds)) {
1167
+ if (!Array.isArray(rawOutpostRoutes)) {
1130
1168
  throw new LanesError("outpost_list_invalid", "Registered Outpost list is not an array", { lane_id: laneId });
1131
1169
  }
1132
- const outpostIds = rawOutpostIds.map((id) => typeof id === "string" ? id.trim() : "");
1170
+ const outpostRoutes = rawOutpostRoutes.map((raw, index) => {
1171
+ if (raw === null || typeof raw !== "object" || Array.isArray(raw) ||
1172
+ typeof raw.outpostId !== "string" ||
1173
+ !raw.outpostId ||
1174
+ (raw.instanceName !== undefined &&
1175
+ typeof raw.instanceName !== "string")) {
1176
+ throw new LanesError("outpost_list_invalid", `Registered Outpost row ${index} is malformed`, {
1177
+ lane_id: laneId,
1178
+ index,
1179
+ });
1180
+ }
1181
+ const row = raw;
1182
+ return {
1183
+ outpostId: row.outpostId.trim(),
1184
+ ...(typeof row.instanceName === "string" ? { instanceName: row.instanceName.trim() } : {}),
1185
+ };
1186
+ });
1187
+ const outpostIds = outpostRoutes.map(({ outpostId }) => outpostId);
1133
1188
  if (outpostIds.length > MAX_PLACEMENT_OUTPOSTS ||
1134
1189
  outpostIds.some((id) => id === "") ||
1135
1190
  new Set(outpostIds).size !== outpostIds.length) {
@@ -1142,9 +1197,23 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
1142
1197
  if (requestedOutpostId !== undefined && !outpostIds.includes(requestedOutpostId)) {
1143
1198
  throw new LanesError("outpost_not_registered", `Outpost ${requestedOutpostId} is not registered`, { lane_id: laneId, outpost_id: requestedOutpostId });
1144
1199
  }
1200
+ const hostRoutes = outpostRoutes.filter((route) => outpostRouteMatchesHost(route, hostSuffix));
1201
+ const identity = ctx.readOutpostIdentity();
1202
+ const localHostRoutes = outpostRoutes.filter((route) => outpostRouteMatchesHost(route, ctx.hostname()));
1203
+ if (localHostRoutes.length > 1 || (identity.kind === "id" && localHostRoutes.some((route) => route.outpostId !== identity.id))) {
1204
+ throw new LanesError("outpost_host_ambiguous", "Registered Outpost host identity conflicts with this host's identity", { lane_id: laneId, local_outpost_id: identity.kind === "id" ? identity.id : undefined });
1205
+ }
1206
+ const localOutpostId = identity.kind === "id" ? identity.id : localHostRoutes[0]?.outpostId;
1207
+ const hostOutpostIds = hostRoutes.map(({ outpostId }) => outpostId);
1208
+ const hostOutpostIdSet = new Set(hostOutpostIds);
1145
1209
  const matches = [];
1146
1210
  const failedOutpostIds = [];
1147
- const lookupOutpostIds = requestedOutpostId === undefined ? outpostIds : [requestedOutpostId];
1211
+ const lookupOutpostIds = requestedOutpostId !== undefined
1212
+ ? [requestedOutpostId]
1213
+ : [
1214
+ ...hostOutpostIds.filter((id) => id !== localOutpostId),
1215
+ ...outpostIds.filter((id) => id !== localOutpostId && !hostOutpostIdSet.has(id)),
1216
+ ];
1148
1217
  for (const outpostId of lookupOutpostIds) {
1149
1218
  let ownerLanes;
1150
1219
  try {
@@ -1156,6 +1225,8 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
1156
1225
  catch (err) {
1157
1226
  if (requestedOutpostId !== undefined)
1158
1227
  throw err;
1228
+ if (isLanesError(err) && err.error === "outpost_wait_invalid")
1229
+ throw err;
1159
1230
  failedOutpostIds.push(outpostId);
1160
1231
  continue;
1161
1232
  }
@@ -3547,6 +3618,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
3547
3618
  throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch to`, { lane_id: lane.lane_id });
3548
3619
  }
3549
3620
  const outpostId = lane.host.id;
3621
+ const waitMs = outpostSingleLaneWaitMs(ctx.env);
3550
3622
  const profileRepinContext = action === "resume" &&
3551
3623
  options.worker !== undefined &&
3552
3624
  callerMatchesSenior(ctx, lane.senior)
@@ -3583,6 +3655,8 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
3583
3655
  try {
3584
3656
  const dispatchCommand = (contexts) => ctx.dispatchOutpostExec({
3585
3657
  outpostId,
3658
+ timeoutSeconds: Math.ceil(waitMs / 1000),
3659
+ waitMs,
3586
3660
  command: buildOutpostLaneCommand({
3587
3661
  action: action === "grant"
3588
3662
  ? "grant-target"
@@ -3636,7 +3710,10 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
3636
3710
  }
3637
3711
  }
3638
3712
  catch (err) {
3639
- throw new LanesError("outpost_dispatch_failed", `Could not dispatch ${action} for lane ${lane.lane_id} to Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, outpost_id: outpostId });
3713
+ const message = dispatchErrorMessage(err);
3714
+ throw new LanesError("outpost_dispatch_failed", /timed out waiting for outpost pickup/i.test(message)
3715
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `dispatching ${action} for lane ${lane.lane_id}`)
3716
+ : `Could not dispatch ${action} for lane ${lane.lane_id} to Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
3640
3717
  }
3641
3718
  if (grantContextRejected) {
3642
3719
  throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing this fix before it can resume lane ${lane.lane_id} under its primary-hosted senior`, {
@@ -3834,6 +3911,7 @@ function parseReparentDescendantIds(value, laneId, outpostId) {
3834
3911
  return ids;
3835
3912
  }
3836
3913
  async function readOutpostLaneSnapshot(ctx, outpostId, laneId) {
3914
+ const waitMs = outpostSingleLaneWaitMs(ctx.env);
3837
3915
  let dispatched;
3838
3916
  try {
3839
3917
  dispatched = await ctx.dispatchOutpostExec({
@@ -3843,12 +3921,15 @@ async function readOutpostLaneSnapshot(ctx, outpostId, laneId) {
3843
3921
  laneId,
3844
3922
  hqBinary: outpostCliOverride(ctx, outpostId),
3845
3923
  }),
3846
- timeoutSeconds: OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS,
3847
- waitMs: OUTPOST_HIRE_SUMMARY_WAIT_MS,
3924
+ timeoutSeconds: Math.ceil(waitMs / 1000),
3925
+ waitMs,
3848
3926
  });
3849
3927
  }
3850
3928
  catch (err) {
3851
- throw new LanesError("outpost_lane_sync_unavailable", `Could not refresh descendant lane ${laneId} from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: laneId, outpost_id: outpostId });
3929
+ const message = dispatchErrorMessage(err);
3930
+ throw new LanesError("outpost_lane_sync_unavailable", /timed out waiting for outpost pickup/i.test(message)
3931
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `reading lane ${laneId}`)
3932
+ : `Could not refresh descendant lane ${laneId} from Outpost ${outpostId}: ${message}`, { lane_id: laneId, outpost_id: outpostId });
3852
3933
  }
3853
3934
  if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
3854
3935
  throw new LanesError("outpost_lane_sync_unavailable", `Outpost ${outpostId} did not confirm the snapshot for descendant lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId, command_id: dispatched.commandId });
@@ -5534,12 +5615,12 @@ async function prepareAccountMove(ctx, lane, input) {
5534
5615
  const at = now.toISOString();
5535
5616
  const from = lane.account ?? null;
5536
5617
  // The capacity decision and the record write run under the target
5537
- // account's admission lock, the same serialization lane creation uses. A
5538
- // slot is taken by a running lane or by a lane another move has already
5539
- // pointed at the account whose spawn has not claimed it yet
5540
- // (`pending_resume` set by this write, cleared by the claim), so two moves
5541
- // into one free slot cannot both start.
5542
- return withLaneAdmissionLock(hqRoot, lane.provider, async () => {
5618
+ // account's admission lock, or while reconcile holds all provider
5619
+ // admission locks. A slot is taken by a running lane or a lane another move
5620
+ // has pointed at its target account but whose spawn has not claimed it yet
5621
+ // (`pending_resume` is set by this write and cleared by the claim), so two
5622
+ // moves into one free slot cannot both start.
5623
+ const writeMove = async () => {
5543
5624
  const admission = input.deferTurnStart
5544
5625
  ? undefined
5545
5626
  : await assessLaneAdmission(ctx, lane, {
@@ -5602,7 +5683,10 @@ async function prepareAccountMove(ctx, lane, input) {
5602
5683
  return next;
5603
5684
  }), input.grantContext);
5604
5685
  return { lane: moved, from, to: target, copy, queued };
5605
- }, ctx.sleep);
5686
+ };
5687
+ return input.admissionLocksHeld
5688
+ ? writeMove()
5689
+ : withLaneAdmissionLock(hqRoot, lane.provider, writeMove, ctx.sleep);
5606
5690
  }
5607
5691
  async function selectAutomaticClaudeResumeAccount(ctx, lane, workerProfile, options = {}) {
5608
5692
  if (lane.provider !== "claude")
@@ -6370,22 +6454,27 @@ function parseRemoteCancelledQuestionIds(value, laneId, outpostId, commandId) {
6370
6454
  }
6371
6455
  return value;
6372
6456
  }
6373
- async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId) {
6457
+ async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId, waitMs) {
6374
6458
  let dispatched;
6375
6459
  try {
6376
6460
  dispatched = await ctx.dispatchOutpostExec({
6377
6461
  outpostId,
6378
6462
  command: buildOutpostLaneCommand({ action: "show", laneId }),
6379
- timeoutSeconds: 120,
6380
- waitMs: 120_000,
6463
+ timeoutSeconds: Math.ceil(waitMs / 1000),
6464
+ waitMs,
6381
6465
  });
6382
6466
  }
6383
6467
  catch (err) {
6384
- throw new LanesError("outpost_delivery_unconfirmed", `Could not read terminal lane ${laneId} from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, {
6468
+ const message = dispatchErrorMessage(err);
6469
+ throw new LanesError("outpost_delivery_unconfirmed", /timed out waiting for outpost pickup/i.test(message)
6470
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `confirming terminal lane ${laneId}`)
6471
+ : `Could not read terminal lane ${laneId} from Outpost ${outpostId}: ${message}`, {
6385
6472
  lane_id: laneId,
6386
6473
  outpost_id: outpostId,
6387
6474
  command_id: stopCommandId,
6388
6475
  sync_command_id: null,
6476
+ wait_ms: waitMs,
6477
+ wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV,
6389
6478
  });
6390
6479
  }
6391
6480
  const commandId = dispatched.commandId?.trim();
@@ -6454,17 +6543,21 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
6454
6543
  throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch stop to`, { lane_id: lane.lane_id });
6455
6544
  }
6456
6545
  const outpostId = lane.host.id;
6546
+ const waitMs = outpostSingleLaneWaitMs(ctx.env);
6457
6547
  let dispatched;
6458
6548
  try {
6459
6549
  dispatched = await ctx.dispatchOutpostExec({
6460
6550
  outpostId,
6461
6551
  command: buildOutpostLaneCommand({ action: "stop", laneId: lane.lane_id, reason }),
6462
- timeoutSeconds: 120,
6463
- waitMs: 120_000,
6552
+ timeoutSeconds: Math.ceil(waitMs / 1000),
6553
+ waitMs,
6464
6554
  });
6465
6555
  }
6466
6556
  catch (err) {
6467
- throw new LanesError("outpost_dispatch_failed", `Could not dispatch stop for lane ${lane.lane_id} to Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, outpost_id: outpostId });
6557
+ const message = dispatchErrorMessage(err);
6558
+ throw new LanesError("outpost_dispatch_failed", /timed out waiting for outpost pickup/i.test(message)
6559
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `stopping lane ${lane.lane_id}`)
6560
+ : `Could not dispatch stop for lane ${lane.lane_id} to Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
6468
6561
  }
6469
6562
  if (reason !== undefined && outpostRejectedStopReasonOption(dispatched)) {
6470
6563
  throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing stop reason support before it can stop lane ${lane.lane_id}`, {
@@ -6512,7 +6605,7 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
6512
6605
  }
6513
6606
  state = remote.state;
6514
6607
  alreadyTerminal = true;
6515
- const owner = await readOutpostTerminalLane(ctx, lane.lane_id, outpostId, commandId);
6608
+ const owner = await readOutpostTerminalLane(ctx, lane.lane_id, outpostId, commandId, waitMs);
6516
6609
  if (owner.lane.state !== state) {
6517
6610
  throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} returned a different terminal state for lane ${lane.lane_id}`, {
6518
6611
  lane_id: lane.lane_id,
@@ -6993,6 +7086,93 @@ async function runReconcile(ctx, opts, recoverQueuedMessages) {
6993
7086
  recoverQueuedMessages,
6994
7087
  mesh: ctx.mesh,
6995
7088
  env: ctx.env,
7089
+ prepareKeepAliveClaudeResume: async (lane) => {
7090
+ const recoveredLane = laneWithRecoveredProviderSession(hqRoot, lane);
7091
+ const pendingModel = recoveredLane.pending_settings?.model;
7092
+ const selectionLane = pendingModel === undefined
7093
+ ? recoveredLane
7094
+ : { ...recoveredLane, model: pendingModel, model_source: undefined };
7095
+ let automatic;
7096
+ try {
7097
+ automatic = await selectAutomaticClaudeResumeAccount(ctx, selectionLane, undefined, {
7098
+ repairUnavailableAccount: true,
7099
+ });
7100
+ }
7101
+ catch (err) {
7102
+ return {
7103
+ lane: recoveredLane,
7104
+ selectionFailure: err instanceof LanesError
7105
+ ? `${err.error}: ${err.message}`
7106
+ : err instanceof Error
7107
+ ? err.message
7108
+ : String(err),
7109
+ };
7110
+ }
7111
+ if (automatic === undefined)
7112
+ return { lane: recoveredLane };
7113
+ const { selection } = automatic;
7114
+ if (opts.dryRun) {
7115
+ return {
7116
+ lane: {
7117
+ ...selectionLane,
7118
+ account: selection.account,
7119
+ account_selection: selection,
7120
+ ...(pendingModel !== undefined
7121
+ ? { model: pendingModel, model_source: "cli" }
7122
+ : automatic.preservedModel !== undefined
7123
+ ? { model: automatic.preservedModel, model_source: "preserved" }
7124
+ : {}),
7125
+ },
7126
+ };
7127
+ }
7128
+ let latest;
7129
+ try {
7130
+ latest = readLane(hqRoot, lane.lane_id);
7131
+ }
7132
+ catch (err) {
7133
+ throw new LanesError("keep_alive_account_selection_lane_unreadable", `Cannot re-read lane ${lane.lane_id} before applying Claude account selection: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id });
7134
+ }
7135
+ if (latest.version !== lane.version ||
7136
+ latest.state !== lane.state ||
7137
+ latest.keep_alive !== true ||
7138
+ latest.keep_alive_opted_in_at !== lane.keep_alive_opted_in_at ||
7139
+ latest.provider_session_id !== lane.provider_session_id ||
7140
+ latest.last_resume?.session_id !== lane.last_resume?.session_id)
7141
+ return { lane: latest };
7142
+ if (selection.account === lane.account) {
7143
+ return { lane: persistClaudeAccountSelection(hqRoot, recoveredLane, selection) };
7144
+ }
7145
+ try {
7146
+ const moved = await prepareAccountMove(ctx, recoveredLane, {
7147
+ account: selection.account,
7148
+ cliModel: pendingModel,
7149
+ accountHistoryReason: "keep_alive_resume",
7150
+ ...(pendingModel === undefined && automatic.preservedModel !== undefined
7151
+ ? { preservedModel: automatic.preservedModel }
7152
+ : {}),
7153
+ deferTurnStart: true,
7154
+ admissionLocksHeld: true,
7155
+ accountSelection: selection,
7156
+ });
7157
+ return { lane: moved.lane };
7158
+ }
7159
+ catch (err) {
7160
+ let latest;
7161
+ try {
7162
+ latest = readLane(hqRoot, lane.lane_id);
7163
+ }
7164
+ catch (readErr) {
7165
+ return {
7166
+ lane,
7167
+ selectionFailure: `account move failed: ${err instanceof Error ? err.message : String(err)}; lane re-read failed: ${readErr instanceof Error ? readErr.message : String(readErr)}`,
7168
+ };
7169
+ }
7170
+ return {
7171
+ lane: latest,
7172
+ selectionFailure: `account move failed: ${err instanceof LanesError ? `${err.error}: ${err.message}` : err instanceof Error ? err.message : String(err)}`,
7173
+ };
7174
+ }
7175
+ },
6996
7176
  });
6997
7177
  printJson(summary);
6998
7178
  if (!summary.ok) {
@@ -92,6 +92,11 @@ export interface ReconcileDeps {
92
92
  meshEmitTimeoutMs?: number;
93
93
  /** Fresh keep-alive eligibility read; injected to exercise opt-in changes before queueing. */
94
94
  readKeepAliveRecheckLane?: (hqRoot: string, laneId: string) => LaneRecord;
95
+ /** Reuse the command's Claude resume selection and session-move path for keep-alive. */
96
+ prepareKeepAliveClaudeResume?: (lane: LaneRecord) => Promise<{
97
+ lane: LaneRecord;
98
+ selectionFailure?: string;
99
+ }>;
95
100
  /** Process-table snapshot seam for the opt-in runaway-search guard. */
96
101
  runawayToolProcessLister?: () => readonly RunawayProcessInfo[];
97
102
  /** Test seam: prevent real process signals while exercising the guard. */
@@ -1001,7 +1001,7 @@ export async function runPassBody(hqRoot, deps) {
1001
1001
  await withAllLaneAdmissionLocks(hqRoot, async () => {
1002
1002
  const freshLanes = loadLaneRecords(hqRoot, summary);
1003
1003
  const freshQuestions = loadQuestionRecords(hqRoot, summary);
1004
- resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, startDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
1004
+ await resumeKeepAlive(hqRoot, freshLanes, freshQuestions, summary, startDeps, now, keepAliveFlagByCompany, supersessionLineageAwareByCompany);
1005
1005
  const drainLanes = loadLaneRecords(hqRoot, summary);
1006
1006
  await admitQueued(hqRoot, drainLanes, summary, startDeps, now);
1007
1007
  await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, startDeps);
@@ -1723,7 +1723,7 @@ function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
1723
1723
  throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
1724
1724
  }
1725
1725
  }
1726
- function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCompany, supersessionLineageAwareByCompany) {
1726
+ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCompany, supersessionLineageAwareByCompany) {
1727
1727
  const nowMs = now.getTime();
1728
1728
  const nowIso = now.toISOString();
1729
1729
  const config = loadLanesConfig(hqRoot);
@@ -2108,6 +2108,44 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
2108
2108
  });
2109
2109
  continue;
2110
2110
  }
2111
+ const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
2112
+ if (seniorId !== undefined) {
2113
+ let grantPreflight;
2114
+ try {
2115
+ grantPreflight = withHireGrantAdmission(hqRoot, seniorId, (snapshot) => {
2116
+ const latestLane = readLane(hqRoot, current.lane_id);
2117
+ if (latestLane.version !== current.version ||
2118
+ latestLane.state !== current.state ||
2119
+ latestLane.keep_alive !== true ||
2120
+ latestLane.keep_alive_opted_in_at !== current.keep_alive_opted_in_at ||
2121
+ latestLane.senior.kind !== "lane" ||
2122
+ latestLane.senior.id !== seniorId)
2123
+ return { kind: "stale" };
2124
+ const refusal = terminalChildResumeHireGrantError(latestLane, snapshot);
2125
+ return refusal === null
2126
+ ? { kind: "eligible" }
2127
+ : { kind: "refused", error: refusal.error, message: refusal.message };
2128
+ });
2129
+ }
2130
+ catch (err) {
2131
+ markDirty(summary, {
2132
+ kind: "lane",
2133
+ id: current.lane_id,
2134
+ why: `keep-alive hire admission preflight failed: ${err instanceof Error ? err.message : String(err)}`,
2135
+ });
2136
+ continue;
2137
+ }
2138
+ if (grantPreflight.kind === "stale")
2139
+ continue;
2140
+ if (grantPreflight.kind === "refused") {
2141
+ summary.skipped.push({
2142
+ kind: "lane",
2143
+ id: current.lane_id,
2144
+ why: `keep_alive_hire_admission: ${grantPreflight.error}: ${grantPreflight.message}`,
2145
+ });
2146
+ continue;
2147
+ }
2148
+ }
2111
2149
  const resumedFrom = current.state === "done" ? "done" : "interrupted";
2112
2150
  const resumeText = resumedFrom === "interrupted"
2113
2151
  ? LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
@@ -2118,6 +2156,114 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
2118
2156
  if (carriesAnsweredQuestions && !deps.dryRun) {
2119
2157
  writeAnsweredQuestions(hqRoot, current, questions);
2120
2158
  }
2159
+ let accountSelectionFailure;
2160
+ if (current.provider === "claude" && deps.prepareKeepAliveClaudeResume !== undefined) {
2161
+ const optInBeforePreparation = current.keep_alive_opted_in_at;
2162
+ let preparedLane;
2163
+ try {
2164
+ const prepared = await deps.prepareKeepAliveClaudeResume(current);
2165
+ preparedLane = prepared.lane;
2166
+ accountSelectionFailure = prepared.selectionFailure;
2167
+ }
2168
+ catch (err) {
2169
+ // Account balancing is best-effort for keep-alive. A failure to read
2170
+ // balance inputs must not prevent the existing session from resuming.
2171
+ try {
2172
+ preparedLane = readLane(hqRoot, current.lane_id);
2173
+ }
2174
+ catch (readErr) {
2175
+ markDirty(summary, {
2176
+ kind: "lane",
2177
+ id: current.lane_id,
2178
+ why: `keep-alive account selection failed and lane could not be re-read: ${readErr instanceof Error ? readErr.message : String(readErr)}`,
2179
+ });
2180
+ continue;
2181
+ }
2182
+ accountSelectionFailure = err instanceof Error ? err.message : String(err);
2183
+ }
2184
+ const preparedSessionId = preparedLane.provider_session_id ?? preparedLane.last_resume?.session_id;
2185
+ if (preparedLane.state !== initialState ||
2186
+ preparedLane.keep_alive !== true ||
2187
+ !laneRunsHere(preparedLane, deps.host) ||
2188
+ preparedSessionId !== currentSessionId)
2189
+ continue;
2190
+ let latest;
2191
+ try {
2192
+ latest = deps.dryRun
2193
+ ? (deps.readKeepAliveRecheckLane?.(hqRoot, current.lane_id) ?? readLane(hqRoot, current.lane_id))
2194
+ : readLane(hqRoot, current.lane_id);
2195
+ }
2196
+ catch (err) {
2197
+ markDirty(summary, {
2198
+ kind: "lane",
2199
+ id: current.lane_id,
2200
+ why: `keep-alive eligibility recheck failed after account selection: ${err instanceof Error ? err.message : String(err)}`,
2201
+ });
2202
+ continue;
2203
+ }
2204
+ if (latest.state !== initialState || !laneRunsHere(latest, deps.host))
2205
+ continue;
2206
+ if (latest.keep_alive !== true) {
2207
+ recordKeepAliveSkip(hqRoot, latest, "not_opted_in", summary, deps);
2208
+ continue;
2209
+ }
2210
+ if (latest.keep_alive_opted_in_at !== optInBeforePreparation) {
2211
+ summary.skipped.push({
2212
+ kind: "lane",
2213
+ id: latest.lane_id,
2214
+ why: "keep_alive_eligibility_changed_during_account_selection",
2215
+ });
2216
+ continue;
2217
+ }
2218
+ const latestSessionId = latest.provider_session_id ?? latest.last_resume?.session_id;
2219
+ if (latestSessionId !== currentSessionId)
2220
+ continue;
2221
+ let latestEligibilityAtMs;
2222
+ if (initialState === "done") {
2223
+ const envelope = readLatestEnvelopeRecord(hqRoot, latest.lane_id);
2224
+ if (!envelope || envelope.envelope.decision !== "done")
2225
+ continue;
2226
+ if (doneKeepAliveVerdict(latest, envelope.envelope, latest.keep_alive_standing === true && hasPendingKeepAliveWork(hqRoot, latest, questions)) !== "eligible")
2227
+ continue;
2228
+ try {
2229
+ latestEligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, latest, envelope.ref);
2230
+ }
2231
+ catch (err) {
2232
+ markDirty(summary, {
2233
+ kind: "lane",
2234
+ id: latest.lane_id,
2235
+ why: `keep-alive envelope timestamp unavailable after account selection: ${err instanceof Error ? err.message : String(err)}`,
2236
+ });
2237
+ continue;
2238
+ }
2239
+ }
2240
+ else {
2241
+ latestEligibilityAtMs = Date.parse(latest.interrupted_at ?? "");
2242
+ }
2243
+ const latestOptedInAtMs = Date.parse(latest.keep_alive_opted_in_at ?? "");
2244
+ if (!Number.isFinite(latestEligibilityAtMs) || !Number.isFinite(latestOptedInAtMs))
2245
+ continue;
2246
+ if (latestEligibilityAtMs <= latestOptedInAtMs) {
2247
+ recordKeepAliveSkip(hqRoot, latest, initialState === "done" ? "envelope_before_opt_in" : "interruption_before_opt_in", summary, deps);
2248
+ continue;
2249
+ }
2250
+ current = deps.dryRun ? preparedLane : latest;
2251
+ }
2252
+ if (accountSelectionFailure !== undefined) {
2253
+ summary.skipped.push({
2254
+ kind: "lane",
2255
+ id: current.lane_id,
2256
+ why: `keep_alive_account_selection_failed: ${accountSelectionFailure}`,
2257
+ });
2258
+ if (!deps.dryRun) {
2259
+ recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
2260
+ event: "keep_alive_account_selection_failed",
2261
+ at: nowIso,
2262
+ account: current.account ?? null,
2263
+ reason: accountSelectionFailure,
2264
+ });
2265
+ }
2266
+ }
2121
2267
  const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
2122
2268
  latestLane.state === current.state &&
2123
2269
  latestLane.keep_alive === true &&
@@ -2145,7 +2291,6 @@ function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, flagByCom
2145
2291
  keep_alive_exhausted_at: undefined,
2146
2292
  keep_alive_blocked_reason: undefined,
2147
2293
  }), deps.dryRun);
2148
- const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
2149
2294
  let queued;
2150
2295
  if (seniorId) {
2151
2296
  try {
@@ -500,9 +500,8 @@ export interface LaneRecord {
500
500
  /** Eligible accounts for a queue entry created without an explicit account. */
501
501
  admission_accounts?: string[];
502
502
  /**
503
- * Every operator move of the lane's session to another account of the
504
- * same provider (`hq lanes resume|message --account`), newest last, capped
505
- * like the other histories.
503
+ * Moves of the lane's session to another account of the same provider,
504
+ * newest last and capped like the other histories.
506
505
  */
507
506
  account_history?: AccountHistoryEntry[];
508
507
  /** Bounded audit history for explicitly authorized resumes from failed. */
@@ -665,7 +664,7 @@ export interface AccountHistoryEntry {
665
664
  from: string | null;
666
665
  to: string;
667
666
  at: string;
668
- reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "operator_move";
667
+ reason: "first_assignment" | "model_change" | "usage_limit" | "account_unavailable" | "recorded_account_logged_out" | "load_balance" | "keep_alive_resume" | "operator_move";
669
668
  }
670
669
  export interface FailedResumeHistoryEntry {
671
670
  at: string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@indigoai-us/hq-cli",
3
- "version": "5.278.0",
3
+ "version": "5.280.0",
4
4
  "description": "HQ by Indigo management CLI — modules and cloud sync",
5
5
  "main": "dist/index.js",
6
6
  "bin": {