@indigoai-us/hq-cli 5.279.0 → 5.281.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -36,7 +36,7 @@ import { createOutpostProfileRepinContext, parseOutpostProfileRepinContext, } fr
36
36
  import { createOutpostRegistryWorkerApprovalContext, parseOutpostRegistryWorkerApprovalContext, } from "../lib/lanes/outpost-registry-worker-approval.js";
37
37
  import { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity, resolveAdmissionHost, } from "../lib/lanes/admission-host.js";
38
38
  export { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity } from "../lib/lanes/admission-host.js";
39
- import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostQuestionListCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
39
+ import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostQuestionListCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, PICKUP_WAIT_MS, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
40
40
  import { outpostHireSummaryCacheKey, readOutpostHireSummaryCache, writeOutpostHireSummaryCache, } from "../lib/lanes/outpost-hire-cache.js";
41
41
  import { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "../lib/lanes/host-capacity.js";
42
42
  import { BALANCE_MOVABLE_STATES, selectClaudeAccount, } from "../lib/lanes/claude-balance.js";
@@ -61,6 +61,7 @@ import { assertQuestionBodySecretSafe } from "../lib/lanes/questions.js";
61
61
  import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
62
62
  import { parseCliEffort, resolveLaneEffort, } from "../lib/lanes/effort.js";
63
63
  import { LanesError } from "../lib/lanes/errors.js";
64
+ import { readLatestEnvelopeRecord } from "../lib/lanes/watcher.js";
64
65
  import { loadQuotaState, quotaBlockFor } from "../lib/lanes/quota.js";
65
66
  import { SESSION_MOVE_REASON } from "../lib/lanes/move-account.js";
66
67
  import { managerStatusGateEnabledForLane, resolveManagerStatusGateForLane, } from "../lib/lanes/manager-status-gate.js";
@@ -74,6 +75,8 @@ export { spawnLaneWorker };
74
75
  export { questionBodyAbsPath };
75
76
  export { MAX_LANE_DEPTH };
76
77
  export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
78
+ export const LANE_REMOTE_MIRROR_SYNC_FLAG = "lanes.remote-done-mirror";
79
+ const REMOTE_MIRROR_REFRESH_BATCH_SIZE = 100;
77
80
  /** The lane → mesh seam. Defined in lib/lanes/mesh-events.ts; the watcher shares it. */
78
81
  export { emitLaneDecisionEvents };
79
82
  function withoutProviderHome(value) {
@@ -159,6 +162,10 @@ async function defaultListOutpostIds() {
159
162
  const rows = await defaultOutpostsClient().list();
160
163
  return rows.map((row) => row.outpostId);
161
164
  }
165
+ async function defaultListOutpostRoutes() {
166
+ const rows = await defaultOutpostsClient().list();
167
+ return rows.map(({ outpostId, instanceName }) => ({ outpostId, instanceName }));
168
+ }
162
169
  async function defaultStageOutpostPayload(outpostId, bytes) {
163
170
  return stageOutpostPayload(defaultOutpostsClient(), outpostId, bytes);
164
171
  }
@@ -218,6 +225,9 @@ function resolveDeps(deps = {}) {
218
225
  };
219
226
  const bindProcessCwd = deps.hqRoot === undefined;
220
227
  const resolveRoot = deps.hqRoot ?? (() => resolveLanesHqRoot({ env, cwd: cwd() }));
228
+ const listOutpostRoutes = deps.listOutpostRoutes ?? (deps.listOutpostIds
229
+ ? async () => (await deps.listOutpostIds()).map((outpostId) => ({ outpostId }))
230
+ : defaultListOutpostRoutes);
221
231
  const hqRoot = () => {
222
232
  callerCwd();
223
233
  const root = resolveRoot();
@@ -236,6 +246,7 @@ function resolveDeps(deps = {}) {
236
246
  resolveCompany: deps.resolveCompany ?? defaultResolveCompany,
237
247
  resolveActiveCompany: deps.resolveActiveCompany ?? defaultResolveActiveCompany,
238
248
  listOutpostIds: deps.listOutpostIds ?? defaultListOutpostIds,
249
+ listOutpostRoutes,
239
250
  readOutpostIdentity: deps.readOutpostIdentity ?? (() => readOutpostIdentity()),
240
251
  spawnLogin: deps.spawnLogin ?? defaultSpawnLogin,
241
252
  throttleDeps: deps.throttleDeps ?? defaultThrottleDeps(env),
@@ -611,6 +622,7 @@ async function fetchOwnerLaneHireFields(ctx, lane) {
611
622
  }
612
623
  const outpostId = lane.host.id;
613
624
  const hqBinary = outpostCliOverride(ctx, outpostId);
625
+ const waitMs = outpostSingleLaneWaitMs(ctx.env);
614
626
  let dispatched;
615
627
  try {
616
628
  dispatched = await ctx.dispatchOutpostExec({
@@ -620,12 +632,15 @@ async function fetchOwnerLaneHireFields(ctx, lane) {
620
632
  laneId: lane.lane_id,
621
633
  hqBinary,
622
634
  }),
623
- timeoutSeconds: OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS,
624
- waitMs: OUTPOST_HIRE_SUMMARY_WAIT_MS,
635
+ timeoutSeconds: Math.ceil(waitMs / 1000),
636
+ waitMs,
625
637
  });
626
638
  }
627
639
  catch (err) {
628
- throw new LanesError("outpost_lane_hire_summary_unavailable", `Could not read hire summary for lane ${lane.lane_id} from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, outpost_id: outpostId });
640
+ const message = dispatchErrorMessage(err);
641
+ throw new LanesError("outpost_lane_hire_summary_unavailable", /timed out waiting for outpost pickup/i.test(message)
642
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `reading hire summary for lane ${lane.lane_id}`)
643
+ : `Could not read hire summary for lane ${lane.lane_id} from Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId });
629
644
  }
630
645
  if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
631
646
  throw new LanesError("outpost_lane_hire_summary_unavailable", `Outpost ${outpostId} did not confirm the hire summary for lane ${lane.lane_id}`, { lane_id: lane.lane_id, outpost_id: outpostId, command_id: dispatched.commandId });
@@ -702,7 +717,7 @@ async function dispatchOutpostLaneList(ctx, outpostId, query, timeoutSeconds, wa
702
717
  timeoutSeconds,
703
718
  waitMs,
704
719
  });
705
- if (!rejectedLaneListFlag(filtered, query))
720
+ if (query.strictSnapshot || !rejectedLaneListFlag(filtered, query))
706
721
  return filtered;
707
722
  return ctx.dispatchOutpostExec({
708
723
  outpostId,
@@ -808,8 +823,6 @@ async function batchOwnerLaneHireFields(ctx, lanes) {
808
823
  writeOutpostHireSummaryCache(ctx.hqRoot(), cacheUpdates, ctx.now().getTime(), cached);
809
824
  return outcomes;
810
825
  }
811
- const OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS = 20;
812
- const OUTPOST_HIRE_SUMMARY_WAIT_MS = 15_000;
813
826
  const OUTPOST_HIRE_SUMMARY_CONCURRENCY = 4;
814
827
  async function laneHirePresentation(ctx, lane, lanes) {
815
828
  const local = laneHireFields(lane, lanes);
@@ -864,6 +877,7 @@ async function withResumeHireAdmission(ctx, lane, fn, grantContext) {
864
877
  !listLaneIds(hqRoot).includes(lane.senior.id)) {
865
878
  throw new LanesError("hire_grant_context_required", `Resume of Outpost lane ${lane.lane_id} requires the primary senior's hire-grant context`, { lane_id: lane.lane_id, senior_lane_id: lane.senior.id });
866
879
  }
880
+ await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), new Set([lane.senior.id]));
867
881
  return withHireGrantAdmission(hqRoot, lane.senior.id, (snapshot) => {
868
882
  const current = readLane(hqRoot, lane.lane_id);
869
883
  if (grantContext !== undefined &&
@@ -886,6 +900,28 @@ async function withResumeHireAdmission(ctx, lane, fn, grantContext) {
886
900
  }
887
901
  const CAPACITY_PROBE_TIMEOUT_MS = 15_000;
888
902
  const MAX_PLACEMENT_OUTPOSTS = 16;
903
+ const OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS = 20;
904
+ const OUTPOST_HIRE_SUMMARY_WAIT_MS = 15_000;
905
+ const OUTPOST_SINGLE_LANE_WAIT_ENV = "HQ_LANES_OUTPOST_SINGLE_LANE_WAIT_MS";
906
+ const OUTPOST_SINGLE_LANE_WAIT_MAX_MS = 600_000;
907
+ function outpostSingleLaneWaitMs(env) {
908
+ const raw = env[OUTPOST_SINGLE_LANE_WAIT_ENV]?.trim() || String(PICKUP_WAIT_MS);
909
+ const value = /^\d+$/.test(raw) ? Number(raw) : Number.NaN;
910
+ if (!Number.isSafeInteger(value) ||
911
+ value < CAPACITY_PROBE_TIMEOUT_MS ||
912
+ value > OUTPOST_SINGLE_LANE_WAIT_MAX_MS) {
913
+ throw new LanesError("outpost_wait_invalid", `${OUTPOST_SINGLE_LANE_WAIT_ENV} must be an integer from ${CAPACITY_PROBE_TIMEOUT_MS} to ${OUTPOST_SINGLE_LANE_WAIT_MAX_MS} milliseconds`, { env: OUTPOST_SINGLE_LANE_WAIT_ENV });
914
+ }
915
+ return value;
916
+ }
917
+ function outpostRouteMatchesHost(route, host) {
918
+ return typeof route.instanceName === "string" &&
919
+ route.instanceName.trim() !== "" &&
920
+ sanitizeHostComponent(route.instanceName.trim()) === sanitizeHostComponent(host.trim());
921
+ }
922
+ function outpostPickupTimeoutMessage(outpostId, waitMs, operation) {
923
+ return `Outpost ${outpostId} pickup timed out after ${waitMs}ms while ${operation}. Raise ${OUTPOST_SINGLE_LANE_WAIT_ENV} (milliseconds) and retry.`;
924
+ }
889
925
  function outpostIdentityUnknownError(message, expectedId, details = {}) {
890
926
  const expected = expectedId ?? "<expected-outpost-id>";
891
927
  const command = `hq lanes outpost-identity backfill --id ${expected}`;
@@ -1022,17 +1058,24 @@ function normalizedLaneListQuery(query) {
1022
1058
  ...(query.active ? { active: true } : {}),
1023
1059
  ...(query.hierarchy === undefined ? {} : { hierarchy: query.hierarchy }),
1024
1060
  ...(query.snapshot ? { snapshot: true } : {}),
1061
+ ...(query.strictSnapshot ? { strictSnapshot: true } : {}),
1025
1062
  };
1026
1063
  }
1027
1064
  async function readOutpostLaneRecordsUncached(ctx, outpostId, query) {
1065
+ const targetedLookup = query.hierarchy !== undefined || query.laneIds !== undefined;
1066
+ const waitMs = targetedLookup ? outpostSingleLaneWaitMs(ctx.env) : CAPACITY_PROBE_TIMEOUT_MS;
1028
1067
  let result;
1029
1068
  try {
1030
- result = await dispatchOutpostLaneList(ctx, outpostId, query, CAPACITY_PROBE_TIMEOUT_MS / 1000, CAPACITY_PROBE_TIMEOUT_MS);
1069
+ result = await dispatchOutpostLaneList(ctx, outpostId, query, Math.ceil(waitMs / 1000), waitMs);
1031
1070
  }
1032
1071
  catch (err) {
1072
+ const message = dispatchErrorMessage(err);
1073
+ if (/timed out waiting for outpost pickup/i.test(message)) {
1074
+ throw new LanesError("outpost_lane_list_unavailable", outpostPickupTimeoutMessage(outpostId, waitMs, "reading the requested lane"), { outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
1075
+ }
1033
1076
  if (isLanesError(err))
1034
1077
  throw err;
1035
- throw new LanesError("outpost_lane_list_unavailable", `Could not read lanes from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { outpost_id: outpostId });
1078
+ throw new LanesError("outpost_lane_list_unavailable", `Could not read lanes from Outpost ${outpostId}: ${message}`, { outpost_id: outpostId });
1036
1079
  }
1037
1080
  if (result.done !== true || result.status !== "Success" || result.exitCode !== 0) {
1038
1081
  throw new LanesError("outpost_lane_list_unavailable", `Outpost ${outpostId} did not return a successful lane list`, { outpost_id: outpostId, status: result.status, done: result.done ?? false, exit_code: result.exitCode });
@@ -1106,6 +1149,156 @@ async function readOutpostLaneRecords(ctx, outpostId, requestedQuery = {}) {
1106
1149
  byOutpost.set(outpostId, { queryKey, promise });
1107
1150
  return promise;
1108
1151
  }
1152
+ /** Refresh only primary-created, non-terminal Outpost mirrors in one bounded owner query per Outpost. */
1153
+ async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
1154
+ const result = {
1155
+ updated: new Map(),
1156
+ unverifiedRemote: new Map(),
1157
+ };
1158
+ const host = reconcileHostFromCtx(ctx);
1159
+ if (host.outpostId !== null || parentLaneIds?.size === 0)
1160
+ return result;
1161
+ let primaryIdentity;
1162
+ const originMatchesThisPrimary = (lane) => {
1163
+ if (lane.origin_host_source !== "imds") {
1164
+ return lane.origin_host === host.hostname;
1165
+ }
1166
+ primaryIdentity ??= ctx.resolveMachineIdentity();
1167
+ if (primaryIdentity.status !== "ok")
1168
+ return false;
1169
+ const primaryImdsId = primaryIdentity.imds ??
1170
+ (primaryIdentity.source === "imds" ? primaryIdentity.id : null);
1171
+ return primaryImdsId !== null && primaryImdsId === lane.origin_host;
1172
+ };
1173
+ const candidates = records
1174
+ .filter((lane) => lane.host.kind === "outpost" &&
1175
+ originMatchesThisPrimary(lane) &&
1176
+ (parentLaneIds === undefined || (lane.senior.kind === "lane" &&
1177
+ lane.senior.id !== undefined &&
1178
+ parentLaneIds.has(lane.senior.id))) &&
1179
+ !isTerminalLaneState(lane.state))
1180
+ .sort((a, b) => a.lane_id.localeCompare(b.lane_id));
1181
+ if (candidates.length === 0)
1182
+ return result;
1183
+ const enabledByCompany = new Map();
1184
+ const enabledCandidates = [];
1185
+ for (const lane of candidates) {
1186
+ const companyKey = JSON.stringify([lane.company?.uid ?? null, lane.company?.slug ?? null]);
1187
+ let enabled = enabledByCompany.get(companyKey);
1188
+ if (enabled === undefined) {
1189
+ enabled = await readLanesRegistryFlag(LANE_REMOTE_MIRROR_SYNC_FLAG, {
1190
+ company: lane.company,
1191
+ env: ctx.env,
1192
+ });
1193
+ enabledByCompany.set(companyKey, enabled);
1194
+ }
1195
+ if (enabled)
1196
+ enabledCandidates.push(lane);
1197
+ }
1198
+ if (enabledCandidates.length === 0)
1199
+ return result;
1200
+ const byOutpost = new Map();
1201
+ for (const lane of enabledCandidates) {
1202
+ const group = byOutpost.get(lane.host.id) ?? [];
1203
+ group.push(lane);
1204
+ byOutpost.set(lane.host.id, group);
1205
+ }
1206
+ for (const [outpostId, group] of byOutpost) {
1207
+ const batch = group.slice(0, REMOTE_MIRROR_REFRESH_BATCH_SIZE);
1208
+ const overflow = group.slice(REMOTE_MIRROR_REFRESH_BATCH_SIZE);
1209
+ if (overflow.length > 0) {
1210
+ const ids = overflow.map((lane) => lane.lane_id);
1211
+ for (const lane of overflow) {
1212
+ result.unverifiedRemote.set(lane.lane_id, "refresh_batch_limit");
1213
+ }
1214
+ process.stderr.write(`hq lanes: unverified remote mirrors from Outpost ${outpostId} remain counted; refresh batch limit left lanes ${ids.join(", ")} unread\n`);
1215
+ }
1216
+ let ownerRecords;
1217
+ try {
1218
+ ownerRecords = await readOutpostLaneRecords(ctx, outpostId, {
1219
+ laneIds: batch.map((lane) => lane.lane_id),
1220
+ snapshot: true,
1221
+ strictSnapshot: true,
1222
+ });
1223
+ }
1224
+ catch (error) {
1225
+ const detail = isLanesError(error)
1226
+ ? `${error.error}: ${error.message}`
1227
+ : dispatchErrorMessage(error);
1228
+ for (const lane of batch)
1229
+ result.unverifiedRemote.set(lane.lane_id, detail);
1230
+ process.stderr.write(`hq lanes: unverified remote mirrors from Outpost ${outpostId} remain counted: ${detail}; lanes ${batch.map((lane) => lane.lane_id).join(", ")}\n`);
1231
+ continue;
1232
+ }
1233
+ for (const mirror of batch) {
1234
+ const matches = ownerRecords.filter((lane) => lane.lane_id === mirror.lane_id);
1235
+ const owner = matches[0];
1236
+ const ownerIsHostedByOutpost = owner !== undefined && ((owner.host.kind === "outpost" && owner.host.id === outpostId) ||
1237
+ (owner.host.kind === "local" && owner.host.id.trim() !== ""));
1238
+ if (matches.length !== 1 ||
1239
+ owner === undefined ||
1240
+ !ownerIsHostedByOutpost ||
1241
+ (owner.company?.uid ?? null) !== (mirror.company?.uid ?? null) ||
1242
+ (owner.company?.slug ?? null) !== (mirror.company?.slug ?? null)) {
1243
+ const detail = matches.length === 0
1244
+ ? "owner did not return the requested lane"
1245
+ : matches.length > 1
1246
+ ? "owner returned duplicate lane rows"
1247
+ : "owner lane identity did not match its mirror";
1248
+ result.unverifiedRemote.set(mirror.lane_id, detail);
1249
+ process.stderr.write(`hq lanes: unverified remote mirror ${mirror.lane_id} from Outpost ${outpostId} remains counted: ${detail}\n`);
1250
+ continue;
1251
+ }
1252
+ const endedAt = isTerminalLaneState(owner.state)
1253
+ ? owner.last_envelope?.at ?? owner.updated_at
1254
+ : undefined;
1255
+ const envelopeSummary = isTerminalLaneState(owner.state)
1256
+ ? owner.envelope_summary
1257
+ : undefined;
1258
+ const before = readLane(hqRoot, mirror.lane_id);
1259
+ if (before.host.kind !== "outpost" || before.host.id !== outpostId ||
1260
+ !originMatchesThisPrimary(before) || isTerminalLaneState(before.state)) {
1261
+ result.updated.set(mirror.lane_id, before);
1262
+ continue;
1263
+ }
1264
+ const needsUpdate = before.state !== owner.state ||
1265
+ before.queued_reason !== (owner.state === "queued" ? owner.queued_reason : undefined) ||
1266
+ before.ended_at !== endedAt ||
1267
+ before.envelope_summary !== envelopeSummary ||
1268
+ (isTerminalLaneState(owner.state) && (JSON.stringify(before.last_envelope) !== JSON.stringify(owner.last_envelope) ||
1269
+ before.story_complete !== owner.story_complete ||
1270
+ before.story_complete_reason !== owner.story_complete_reason ||
1271
+ before.story_status !== owner.story_status ||
1272
+ before.terminal_reason !== owner.terminal_reason));
1273
+ if (!needsUpdate) {
1274
+ result.updated.set(mirror.lane_id, before);
1275
+ continue;
1276
+ }
1277
+ const updated = retryUpdateLane(hqRoot, mirror.lane_id, "reconcile", (current) => {
1278
+ if (current.host.kind !== "outpost" || current.host.id !== outpostId ||
1279
+ !originMatchesThisPrimary(current) || isTerminalLaneState(current.state))
1280
+ return current;
1281
+ return {
1282
+ ...current,
1283
+ state: owner.state,
1284
+ queued_reason: owner.state === "queued" ? owner.queued_reason : undefined,
1285
+ ended_at: endedAt,
1286
+ envelope_summary: envelopeSummary,
1287
+ ...(isTerminalLaneState(owner.state) ? {
1288
+ last_envelope: owner.last_envelope,
1289
+ story_complete: owner.story_complete,
1290
+ story_complete_reason: owner.story_complete_reason,
1291
+ story_status: owner.story_status,
1292
+ terminal_reason: owner.terminal_reason,
1293
+ terminal_details: owner.terminal_details,
1294
+ } : {}),
1295
+ };
1296
+ });
1297
+ result.updated.set(mirror.lane_id, updated);
1298
+ }
1299
+ }
1300
+ return result;
1301
+ }
1109
1302
  async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
1110
1303
  try {
1111
1304
  return { lane: readLane(ctx.hqRoot(), laneId), remoteOnly: false };
@@ -1119,17 +1312,34 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
1119
1312
  if (!hostSuffix || sanitizeHostComponent(hostSuffix) !== hostSuffix) {
1120
1313
  throw commandLaneNotFound(laneId, options.outpostHint === true);
1121
1314
  }
1122
- let rawOutpostIds;
1315
+ let rawOutpostRoutes;
1123
1316
  try {
1124
- rawOutpostIds = await ctx.listOutpostIds();
1317
+ rawOutpostRoutes = await ctx.listOutpostRoutes();
1125
1318
  }
1126
1319
  catch (err) {
1127
1320
  throw new LanesError("outpost_list_unavailable", `Could not resolve registered Outposts for lane ${laneId}: ${dispatchErrorMessage(err)}`, { lane_id: laneId });
1128
1321
  }
1129
- if (!Array.isArray(rawOutpostIds)) {
1322
+ if (!Array.isArray(rawOutpostRoutes)) {
1130
1323
  throw new LanesError("outpost_list_invalid", "Registered Outpost list is not an array", { lane_id: laneId });
1131
1324
  }
1132
- const outpostIds = rawOutpostIds.map((id) => typeof id === "string" ? id.trim() : "");
1325
+ const outpostRoutes = rawOutpostRoutes.map((raw, index) => {
1326
+ if (raw === null || typeof raw !== "object" || Array.isArray(raw) ||
1327
+ typeof raw.outpostId !== "string" ||
1328
+ !raw.outpostId ||
1329
+ (raw.instanceName !== undefined &&
1330
+ typeof raw.instanceName !== "string")) {
1331
+ throw new LanesError("outpost_list_invalid", `Registered Outpost row ${index} is malformed`, {
1332
+ lane_id: laneId,
1333
+ index,
1334
+ });
1335
+ }
1336
+ const row = raw;
1337
+ return {
1338
+ outpostId: row.outpostId.trim(),
1339
+ ...(typeof row.instanceName === "string" ? { instanceName: row.instanceName.trim() } : {}),
1340
+ };
1341
+ });
1342
+ const outpostIds = outpostRoutes.map(({ outpostId }) => outpostId);
1133
1343
  if (outpostIds.length > MAX_PLACEMENT_OUTPOSTS ||
1134
1344
  outpostIds.some((id) => id === "") ||
1135
1345
  new Set(outpostIds).size !== outpostIds.length) {
@@ -1142,9 +1352,23 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
1142
1352
  if (requestedOutpostId !== undefined && !outpostIds.includes(requestedOutpostId)) {
1143
1353
  throw new LanesError("outpost_not_registered", `Outpost ${requestedOutpostId} is not registered`, { lane_id: laneId, outpost_id: requestedOutpostId });
1144
1354
  }
1355
+ const hostRoutes = outpostRoutes.filter((route) => outpostRouteMatchesHost(route, hostSuffix));
1356
+ const identity = ctx.readOutpostIdentity();
1357
+ const localHostRoutes = outpostRoutes.filter((route) => outpostRouteMatchesHost(route, ctx.hostname()));
1358
+ if (localHostRoutes.length > 1 || (identity.kind === "id" && localHostRoutes.some((route) => route.outpostId !== identity.id))) {
1359
+ throw new LanesError("outpost_host_ambiguous", "Registered Outpost host identity conflicts with this host's identity", { lane_id: laneId, local_outpost_id: identity.kind === "id" ? identity.id : undefined });
1360
+ }
1361
+ const localOutpostId = identity.kind === "id" ? identity.id : localHostRoutes[0]?.outpostId;
1362
+ const hostOutpostIds = hostRoutes.map(({ outpostId }) => outpostId);
1363
+ const hostOutpostIdSet = new Set(hostOutpostIds);
1145
1364
  const matches = [];
1146
1365
  const failedOutpostIds = [];
1147
- const lookupOutpostIds = requestedOutpostId === undefined ? outpostIds : [requestedOutpostId];
1366
+ const lookupOutpostIds = requestedOutpostId !== undefined
1367
+ ? [requestedOutpostId]
1368
+ : [
1369
+ ...hostOutpostIds.filter((id) => id !== localOutpostId),
1370
+ ...outpostIds.filter((id) => id !== localOutpostId && !hostOutpostIdSet.has(id)),
1371
+ ];
1148
1372
  for (const outpostId of lookupOutpostIds) {
1149
1373
  let ownerLanes;
1150
1374
  try {
@@ -1156,6 +1380,8 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
1156
1380
  catch (err) {
1157
1381
  if (requestedOutpostId !== undefined)
1158
1382
  throw err;
1383
+ if (isLanesError(err) && err.error === "outpost_wait_invalid")
1384
+ throw err;
1159
1385
  failedOutpostIds.push(outpostId);
1160
1386
  continue;
1161
1387
  }
@@ -2964,6 +3190,9 @@ async function runCreate(ctx, opts) {
2964
3190
  : {}),
2965
3191
  attribution_history: [],
2966
3192
  };
3193
+ if (senior.kind === "lane" && senior.id) {
3194
+ await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), new Set([senior.id]));
3195
+ }
2967
3196
  const lane = senior.kind === "lane"
2968
3197
  ? withHireGrantAdmission(hqRoot, senior.id, ({ parent, children, parentGrant, availableChildGrant }) => {
2969
3198
  if (isTerminalLaneState(parent.state)) {
@@ -3547,6 +3776,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
3547
3776
  throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch to`, { lane_id: lane.lane_id });
3548
3777
  }
3549
3778
  const outpostId = lane.host.id;
3779
+ const waitMs = outpostSingleLaneWaitMs(ctx.env);
3550
3780
  const profileRepinContext = action === "resume" &&
3551
3781
  options.worker !== undefined &&
3552
3782
  callerMatchesSenior(ctx, lane.senior)
@@ -3583,6 +3813,8 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
3583
3813
  try {
3584
3814
  const dispatchCommand = (contexts) => ctx.dispatchOutpostExec({
3585
3815
  outpostId,
3816
+ timeoutSeconds: Math.ceil(waitMs / 1000),
3817
+ waitMs,
3586
3818
  command: buildOutpostLaneCommand({
3587
3819
  action: action === "grant"
3588
3820
  ? "grant-target"
@@ -3636,7 +3868,10 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
3636
3868
  }
3637
3869
  }
3638
3870
  catch (err) {
3639
- throw new LanesError("outpost_dispatch_failed", `Could not dispatch ${action} for lane ${lane.lane_id} to Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, outpost_id: outpostId });
3871
+ const message = dispatchErrorMessage(err);
3872
+ throw new LanesError("outpost_dispatch_failed", /timed out waiting for outpost pickup/i.test(message)
3873
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `dispatching ${action} for lane ${lane.lane_id}`)
3874
+ : `Could not dispatch ${action} for lane ${lane.lane_id} to Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
3640
3875
  }
3641
3876
  if (grantContextRejected) {
3642
3877
  throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing this fix before it can resume lane ${lane.lane_id} under its primary-hosted senior`, {
@@ -3834,6 +4069,7 @@ function parseReparentDescendantIds(value, laneId, outpostId) {
3834
4069
  return ids;
3835
4070
  }
3836
4071
  async function readOutpostLaneSnapshot(ctx, outpostId, laneId) {
4072
+ const waitMs = outpostSingleLaneWaitMs(ctx.env);
3837
4073
  let dispatched;
3838
4074
  try {
3839
4075
  dispatched = await ctx.dispatchOutpostExec({
@@ -3843,12 +4079,15 @@ async function readOutpostLaneSnapshot(ctx, outpostId, laneId) {
3843
4079
  laneId,
3844
4080
  hqBinary: outpostCliOverride(ctx, outpostId),
3845
4081
  }),
3846
- timeoutSeconds: OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS,
3847
- waitMs: OUTPOST_HIRE_SUMMARY_WAIT_MS,
4082
+ timeoutSeconds: Math.ceil(waitMs / 1000),
4083
+ waitMs,
3848
4084
  });
3849
4085
  }
3850
4086
  catch (err) {
3851
- throw new LanesError("outpost_lane_sync_unavailable", `Could not refresh descendant lane ${laneId} from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: laneId, outpost_id: outpostId });
4087
+ const message = dispatchErrorMessage(err);
4088
+ throw new LanesError("outpost_lane_sync_unavailable", /timed out waiting for outpost pickup/i.test(message)
4089
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `reading lane ${laneId}`)
4090
+ : `Could not refresh descendant lane ${laneId} from Outpost ${outpostId}: ${message}`, { lane_id: laneId, outpost_id: outpostId });
3852
4091
  }
3853
4092
  if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
3854
4093
  throw new LanesError("outpost_lane_sync_unavailable", `Outpost ${outpostId} did not confirm the snapshot for descendant lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId, command_id: dispatched.commandId });
@@ -4147,7 +4386,7 @@ async function runList(ctx, opts) {
4147
4386
  const hierarchyIds = opts.hierarchy === undefined
4148
4387
  ? undefined
4149
4388
  : new Set(selectOutpostHireGrantHierarchy(opts.hierarchy, laneSource).map((lane) => lane.lane_id));
4150
- const selectedLanes = laneSource
4389
+ let selectedLanes = laneSource
4151
4390
  .filter((lane) => {
4152
4391
  if (requestedLaneIds.length > 0 && !requestedIds.has(lane.lane_id))
4153
4392
  return false;
@@ -4173,9 +4412,27 @@ async function runList(ctx, opts) {
4173
4412
  })
4174
4413
  .sort(compareLaneQueueOrder);
4175
4414
  if (opts.snapshot) {
4176
- printJson({ ok: true, action: "list", lanes: selectedLanes });
4415
+ printJson({
4416
+ ok: true,
4417
+ action: "list",
4418
+ lanes: selectedLanes.map((lane) => {
4419
+ if (!isTerminalLaneState(lane.state))
4420
+ return lane;
4421
+ const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
4422
+ return envelope?.summary ? { ...lane, envelope_summary: envelope.summary } : lane;
4423
+ }),
4424
+ });
4177
4425
  return;
4178
4426
  }
4427
+ const remoteRefresh = await refreshRemoteLaneMirrors(ctx, hqRoot, laneSource, new Set(selectedLanes
4428
+ .filter((lane) => !isTerminalLaneState(lane.state))
4429
+ .map((lane) => lane.lane_id)));
4430
+ for (const [laneId, updated] of remoteRefresh.updated) {
4431
+ const index = laneSource.findIndex((lane) => lane.lane_id === laneId);
4432
+ if (index >= 0)
4433
+ laneSource[index] = updated;
4434
+ }
4435
+ selectedLanes = selectedLanes.map((lane) => remoteRefresh.updated.get(lane.lane_id) ?? lane);
4179
4436
  const now = ctx.now();
4180
4437
  const host = reconcileHostFromCtx(ctx);
4181
4438
  const allLanes = laneSource;
@@ -4232,6 +4489,7 @@ async function runList(ctx, opts) {
4232
4489
  return {
4233
4490
  ...lane,
4234
4491
  ...hireByLane.get(lane.lane_id),
4492
+ ...(remoteRefresh.unverifiedRemote.has(lane.lane_id) ? { unverified_remote: true } : {}),
4235
4493
  stories: laneStories(lane),
4236
4494
  story_ids: laneStories(lane).map((story) => story.story_id),
4237
4495
  no_story_reason: lane.no_story_reason ?? null,
@@ -4268,6 +4526,9 @@ async function runList(ctx, opts) {
4268
4526
  ok: true,
4269
4527
  action: "list",
4270
4528
  lanes,
4529
+ ...(remoteRefresh.unverifiedRemote.size > 0
4530
+ ? { unverified_remote_lanes: [...remoteRefresh.unverifiedRemote.keys()] }
4531
+ : {}),
4271
4532
  ...(host_capacity !== undefined ? { host_capacity } : {}),
4272
4533
  };
4273
4534
  if (opts.human)
@@ -4687,6 +4948,13 @@ async function applyHireGrant(ctx, laneId, hireGrant, authorization, grantContex
4687
4948
  ? [initial.senior.id]
4688
4949
  : []),
4689
4950
  ];
4951
+ if (authorization === "senior") {
4952
+ const parentLaneIds = new Set([initial.lane_id]);
4953
+ if (initial.senior.kind === "lane" && initial.senior.id) {
4954
+ parentLaneIds.add(initial.senior.id);
4955
+ }
4956
+ await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), parentLaneIds);
4957
+ }
4690
4958
  const updated = ctx.withGrantHireLocks(hqRoot, lockedLaneIds, () => {
4691
4959
  const current = readLane(hqRoot, laneId);
4692
4960
  const currentCallerAuthorization = authorization === "senior"
@@ -6140,6 +6408,12 @@ async function reparentOnOwner(ctx, laneId, opts) {
6140
6408
  const nextSenior = parseSeniorRef(opts.senior);
6141
6409
  assertSeniorExists(hqRoot, nextSenior, ctx);
6142
6410
  const initial = readLane(hqRoot, laneId);
6411
+ const refreshParents = new Set();
6412
+ if (initial.senior.kind === "lane" && initial.senior.id)
6413
+ refreshParents.add(initial.senior.id);
6414
+ if (nextSenior.kind === "lane" && nextSenior.id)
6415
+ refreshParents.add(nextSenior.id);
6416
+ await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot), refreshParents);
6143
6417
  const lockIds = [
6144
6418
  laneId,
6145
6419
  ...(initial.senior.kind === "lane" && initial.senior.id ? [initial.senior.id] : []),
@@ -6299,12 +6573,58 @@ const CANONICAL_LOG_STREAMS = [
6299
6573
  ];
6300
6574
  async function runLogs(ctx, laneId, opts) {
6301
6575
  const hqRoot = ctx.hqRoot();
6302
- readLane(hqRoot, laneId);
6303
- const logsDir = path.join(runDir(hqRoot, laneId), "logs");
6576
+ const lane = readLane(hqRoot, laneId);
6304
6577
  const tail = parseOptionalInt(opts.tail, "--tail");
6305
6578
  if (tail === 0) {
6306
6579
  throw new LanesError("invalid_integer", "--tail must be a positive integer", { flag: "--tail", value: opts.tail });
6307
6580
  }
6581
+ if (lane.host.kind === "outpost" &&
6582
+ !laneRunsHere(lane, reconcileHostFromCtx(ctx)) &&
6583
+ await readLanesRegistryFlag(LANE_REMOTE_MIRROR_SYNC_FLAG, {
6584
+ company: lane.company,
6585
+ env: ctx.env,
6586
+ })) {
6587
+ const outpostId = lane.host.id;
6588
+ const waitMs = outpostSingleLaneWaitMs(ctx.env);
6589
+ let dispatched;
6590
+ try {
6591
+ dispatched = await ctx.dispatchOutpostExec({
6592
+ outpostId,
6593
+ command: buildOutpostLaneCommand({
6594
+ action: "logs",
6595
+ laneId,
6596
+ hqBinary: outpostCliOverride(ctx, outpostId),
6597
+ ...(tail === undefined ? {} : { tail: String(tail) }),
6598
+ json: true,
6599
+ }),
6600
+ timeoutSeconds: Math.ceil(waitMs / 1000),
6601
+ waitMs,
6602
+ });
6603
+ }
6604
+ catch (error) {
6605
+ throw new LanesError("outpost_lane_logs_unavailable", `Could not read logs for lane ${laneId} from Outpost ${outpostId}: ${dispatchErrorMessage(error)}`, { lane_id: laneId, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
6606
+ }
6607
+ if (dispatched.done !== true || dispatched.status !== "Success" || dispatched.exitCode !== 0) {
6608
+ throw new LanesError("outpost_lane_logs_unavailable", `Outpost ${outpostId} did not return logs for lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId, status: dispatched.status, exit_code: dispatched.exitCode });
6609
+ }
6610
+ let body;
6611
+ try {
6612
+ body = JSON.parse(dispatched.stdout);
6613
+ }
6614
+ catch (error) {
6615
+ throw new LanesError("outpost_lane_logs_invalid", `Outpost ${outpostId} returned invalid logs JSON for lane ${laneId}: ${dispatchErrorMessage(error)}`, { lane_id: laneId, outpost_id: outpostId });
6616
+ }
6617
+ if (body === null || typeof body !== "object" || Array.isArray(body) ||
6618
+ body.ok !== true ||
6619
+ body.action !== "logs" ||
6620
+ body.lane_id !== laneId ||
6621
+ !Array.isArray(body.streams)) {
6622
+ throw new LanesError("outpost_lane_logs_invalid", `Outpost ${outpostId} returned a malformed logs payload for lane ${laneId}`, { lane_id: laneId, outpost_id: outpostId });
6623
+ }
6624
+ printJson(body);
6625
+ return;
6626
+ }
6627
+ const logsDir = path.join(runDir(hqRoot, laneId), "logs");
6308
6628
  const names = new Set(CANONICAL_LOG_STREAMS);
6309
6629
  if (fs.existsSync(logsDir)) {
6310
6630
  for (const name of fs.readdirSync(logsDir)) {
@@ -6373,22 +6693,27 @@ function parseRemoteCancelledQuestionIds(value, laneId, outpostId, commandId) {
6373
6693
  }
6374
6694
  return value;
6375
6695
  }
6376
- async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId) {
6696
+ async function readOutpostTerminalLane(ctx, laneId, outpostId, stopCommandId, waitMs) {
6377
6697
  let dispatched;
6378
6698
  try {
6379
6699
  dispatched = await ctx.dispatchOutpostExec({
6380
6700
  outpostId,
6381
6701
  command: buildOutpostLaneCommand({ action: "show", laneId }),
6382
- timeoutSeconds: 120,
6383
- waitMs: 120_000,
6702
+ timeoutSeconds: Math.ceil(waitMs / 1000),
6703
+ waitMs,
6384
6704
  });
6385
6705
  }
6386
6706
  catch (err) {
6387
- throw new LanesError("outpost_delivery_unconfirmed", `Could not read terminal lane ${laneId} from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, {
6707
+ const message = dispatchErrorMessage(err);
6708
+ throw new LanesError("outpost_delivery_unconfirmed", /timed out waiting for outpost pickup/i.test(message)
6709
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `confirming terminal lane ${laneId}`)
6710
+ : `Could not read terminal lane ${laneId} from Outpost ${outpostId}: ${message}`, {
6388
6711
  lane_id: laneId,
6389
6712
  outpost_id: outpostId,
6390
6713
  command_id: stopCommandId,
6391
6714
  sync_command_id: null,
6715
+ wait_ms: waitMs,
6716
+ wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV,
6392
6717
  });
6393
6718
  }
6394
6719
  const commandId = dispatched.commandId?.trim();
@@ -6457,17 +6782,21 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
6457
6782
  throw new LanesError("outpost_dispatch_failed", `Lane ${lane.lane_id} has no Outpost host to dispatch stop to`, { lane_id: lane.lane_id });
6458
6783
  }
6459
6784
  const outpostId = lane.host.id;
6785
+ const waitMs = outpostSingleLaneWaitMs(ctx.env);
6460
6786
  let dispatched;
6461
6787
  try {
6462
6788
  dispatched = await ctx.dispatchOutpostExec({
6463
6789
  outpostId,
6464
6790
  command: buildOutpostLaneCommand({ action: "stop", laneId: lane.lane_id, reason }),
6465
- timeoutSeconds: 120,
6466
- waitMs: 120_000,
6791
+ timeoutSeconds: Math.ceil(waitMs / 1000),
6792
+ waitMs,
6467
6793
  });
6468
6794
  }
6469
6795
  catch (err) {
6470
- throw new LanesError("outpost_dispatch_failed", `Could not dispatch stop for lane ${lane.lane_id} to Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { lane_id: lane.lane_id, outpost_id: outpostId });
6796
+ const message = dispatchErrorMessage(err);
6797
+ throw new LanesError("outpost_dispatch_failed", /timed out waiting for outpost pickup/i.test(message)
6798
+ ? outpostPickupTimeoutMessage(outpostId, waitMs, `stopping lane ${lane.lane_id}`)
6799
+ : `Could not dispatch stop for lane ${lane.lane_id} to Outpost ${outpostId}: ${message}`, { lane_id: lane.lane_id, outpost_id: outpostId, wait_ms: waitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
6471
6800
  }
6472
6801
  if (reason !== undefined && outpostRejectedStopReasonOption(dispatched)) {
6473
6802
  throw new LanesError("outpost_cli_upgrade_required", `Outpost ${outpostId} needs an hq-cli release containing stop reason support before it can stop lane ${lane.lane_id}`, {
@@ -6515,7 +6844,7 @@ async function dispatchOutpostLaneStop(ctx, lane, reason, quiet = false) {
6515
6844
  }
6516
6845
  state = remote.state;
6517
6846
  alreadyTerminal = true;
6518
- const owner = await readOutpostTerminalLane(ctx, lane.lane_id, outpostId, commandId);
6847
+ const owner = await readOutpostTerminalLane(ctx, lane.lane_id, outpostId, commandId, waitMs);
6519
6848
  if (owner.lane.state !== state) {
6520
6849
  throw new LanesError("outpost_delivery_unconfirmed", `Outpost ${outpostId} returned a different terminal state for lane ${lane.lane_id}`, {
6521
6850
  lane_id: lane.lane_id,
@@ -6969,6 +7298,9 @@ function writeAnswerDeliveryFailed(hqRoot, laneId, questionId, err, at) {
6969
7298
  async function runReconcile(ctx, opts, recoverQueuedMessages) {
6970
7299
  const fromEnv = ctx.env.HQ_ROOT?.trim();
6971
7300
  const hqRoot = fromEnv && ctx.existsSync(fromEnv) ? fromEnv : ctx.hqRoot();
7301
+ const mirrorRefresh = opts.dryRun
7302
+ ? { updated: new Map(), unverifiedRemote: new Map() }
7303
+ : await refreshRemoteLaneMirrors(ctx, hqRoot, loadAllLanes(hqRoot));
6972
7304
  const summary = await runReconcilePass(hqRoot, {
6973
7305
  now: ctx.now,
6974
7306
  host: reconcileHostFromCtx(ctx),
@@ -7084,7 +7416,12 @@ async function runReconcile(ctx, opts, recoverQueuedMessages) {
7084
7416
  }
7085
7417
  },
7086
7418
  });
7087
- printJson(summary);
7419
+ printJson({
7420
+ ...summary,
7421
+ ...(mirrorRefresh.unverifiedRemote.size > 0
7422
+ ? { unverified_remote_lanes: [...mirrorRefresh.unverifiedRemote.keys()] }
7423
+ : {}),
7424
+ });
7088
7425
  if (!summary.ok) {
7089
7426
  process.exit(1);
7090
7427
  }