@indigoai-us/hq-cli 5.203.0 → 5.205.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/dist/command-catalog.generated.d.ts +33 -0
  3. package/dist/command-catalog.generated.js +43 -0
  4. package/dist/commands/lanes.d.ts +3 -0
  5. package/dist/commands/lanes.js +253 -50
  6. package/dist/lib/lanes/account-home.d.ts +8 -0
  7. package/dist/lib/lanes/account-home.js +10 -0
  8. package/dist/lib/lanes/accounts.d.ts +4 -1
  9. package/dist/lib/lanes/accounts.js +15 -10
  10. package/dist/lib/lanes/admission.js +6 -0
  11. package/dist/lib/lanes/claude-account-balance.d.ts +3 -0
  12. package/dist/lib/lanes/claude-account-balance.js +19 -4
  13. package/dist/lib/lanes/config.d.ts +2 -1
  14. package/dist/lib/lanes/config.js +55 -13
  15. package/dist/lib/lanes/hire-grants.d.ts +4 -0
  16. package/dist/lib/lanes/hire-grants.js +62 -0
  17. package/dist/lib/lanes/host-capacity.d.ts +4 -0
  18. package/dist/lib/lanes/host-capacity.js +8 -1
  19. package/dist/lib/lanes/index.d.ts +3 -1
  20. package/dist/lib/lanes/index.js +2 -1
  21. package/dist/lib/lanes/keep-alive.d.ts +8 -0
  22. package/dist/lib/lanes/keep-alive.js +14 -0
  23. package/dist/lib/lanes/liveness.d.ts +1 -0
  24. package/dist/lib/lanes/liveness.js +3 -1
  25. package/dist/lib/lanes/outpost-dispatch.d.ts +1 -0
  26. package/dist/lib/lanes/outpost-dispatch.js +2 -0
  27. package/dist/lib/lanes/profile.d.ts +1 -0
  28. package/dist/lib/lanes/profile.js +8 -0
  29. package/dist/lib/lanes/provider-rejection.d.ts +1 -1
  30. package/dist/lib/lanes/provider-rejection.js +15 -5
  31. package/dist/lib/lanes/reconcile.d.ts +3 -0
  32. package/dist/lib/lanes/reconcile.js +293 -13
  33. package/dist/lib/lanes/spawn.d.ts +1 -0
  34. package/dist/lib/lanes/spawn.js +30 -14
  35. package/dist/lib/lanes/store.js +25 -0
  36. package/dist/lib/lanes/types.d.ts +15 -0
  37. package/dist/lib/lanes/types.js +1 -0
  38. package/package.json +1 -1
@@ -28,6 +28,7 @@ const PROFILE_META_KEYS = new Set([
28
28
  "max_concurrent",
29
29
  "source",
30
30
  "accounts",
31
+ "keep_alive",
31
32
  ]);
32
33
  function isPlainObject(value) {
33
34
  return value !== null && typeof value === "object" && !Array.isArray(value);
@@ -164,6 +165,12 @@ function parseMeta(id, dir, raw) {
164
165
  value: raw.fast_mode,
165
166
  });
166
167
  }
168
+ if (raw.keep_alive !== undefined && typeof raw.keep_alive !== "boolean") {
169
+ throw profileError(id, dir, "meta.yaml keep_alive must be a boolean when present", {
170
+ field: "keep_alive",
171
+ value: raw.keep_alive,
172
+ });
173
+ }
167
174
  let version;
168
175
  if (raw.version !== undefined) {
169
176
  if (typeof raw.version !== "string" || raw.version.trim().length === 0) {
@@ -211,6 +218,7 @@ function parseMeta(id, dir, raw) {
211
218
  ...(account !== undefined ? { account } : {}),
212
219
  ...(effort !== undefined ? { effort } : {}),
213
220
  ...(raw.fast_mode !== undefined ? { fast_mode: raw.fast_mode } : {}),
221
+ ...(raw.keep_alive !== undefined ? { keep_alive: raw.keep_alive } : {}),
214
222
  ...(maxConcurrent !== undefined ? { max_concurrent: maxConcurrent } : {}),
215
223
  ...(source !== undefined ? { source } : {}),
216
224
  ...(accounts !== undefined ? { accounts } : {}),
@@ -35,7 +35,7 @@ export interface ProviderRejection {
35
35
  /** Derived from the status: a transient class (408, 425, 5xx) is retryable; a 4xx is not. */
36
36
  retryable: boolean;
37
37
  /** Which event on the stream carried the failure. */
38
- event: "codex.turn.failed" | "claude.result.is_error" | "grok.error";
38
+ event: "codex.turn.failed" | "claude.result.is_error" | "claude.result.login_failure" | "grok.error";
39
39
  }
40
40
  export declare function isProviderLoginFailure(failure: ProviderRejection | null): boolean;
41
41
  export declare function capMessage(message: string): string;
@@ -28,7 +28,7 @@ export const QUOTA_STATUS = 429;
28
28
  function isProviderLoginMessage(provider, message) {
29
29
  switch (provider) {
30
30
  case "claude":
31
- return /\bnot logged in\b/i.test(message);
31
+ return /(?:\bnot logged in\b|\bplease run\s+\/login\b)/i.test(message);
32
32
  case "codex":
33
33
  return /\bnot logged in\b/i.test(message) ||
34
34
  /\bmissing bearer or basic authentication in header\b/i.test(message);
@@ -40,6 +40,11 @@ function isProviderLoginMessage(provider, message) {
40
40
  }
41
41
  }
42
42
  }
43
+ /** Successful Claude results can contain worker prose, so only accept its short CLI login banner. */
44
+ function isClaudeSuccessLoginMessage(message) {
45
+ const trimmed = message.trim();
46
+ return trimmed.length <= 80 && /^not logged in\b.{0,40}please run \/login\.?$/i.test(trimmed);
47
+ }
43
48
  export function isProviderLoginFailure(failure) {
44
49
  return failure !== null && isProviderLoginMessage(failure.provider, failure.message);
45
50
  }
@@ -163,11 +168,14 @@ export function parseClaudeResultRejection(text) {
163
168
  const found = scanJsonLinesBackward(text, (rec) => {
164
169
  if (rec.type !== "result")
165
170
  return null;
166
- if (rec.is_error !== true)
171
+ const message = typeof rec.result === "string" ? rec.result : "";
172
+ const loginFailure = rec.is_error === true
173
+ ? isProviderLoginMessage("claude", message)
174
+ : isClaudeSuccessLoginMessage(message);
175
+ if (rec.is_error !== true && !loginFailure)
167
176
  return NOT_REJECTED;
168
177
  const status = statusOf(rec.api_error_status);
169
- const message = typeof rec.result === "string" ? rec.result : "";
170
- if (status === null && !isProviderLoginMessage("claude", message))
178
+ if (status === null && !loginFailure)
171
179
  return NOT_REJECTED;
172
180
  return {
173
181
  reason: PROVIDER_REJECTED_REASON,
@@ -176,7 +184,9 @@ export function parseClaudeResultRejection(text) {
176
184
  status,
177
185
  error_type: typeof rec.subtype === "string" ? rec.subtype : null,
178
186
  retryable: retryableFromStatus(status),
179
- event: "claude.result.is_error",
187
+ event: loginFailure && rec.is_error !== true
188
+ ? "claude.result.login_failure"
189
+ : "claude.result.is_error",
180
190
  };
181
191
  });
182
192
  return found === NOT_REJECTED ? null : found;
@@ -8,6 +8,7 @@
8
8
  import { type LivenessDeps } from "./liveness.js";
9
9
  import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
10
10
  import { type ThrottleDeps } from "./throttle.js";
11
+ import { readLanesRegistryFlag } from "./registry-flag.js";
11
12
  import type { LaneRecord, LivenessState, QueuedReason, QuestionRecord, SeniorRef } from "./types.js";
12
13
  import { type LaneMesh } from "./mesh-events.js";
13
14
  /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
@@ -71,6 +72,8 @@ export interface ReconcileDeps {
71
72
  readSessionSenior?: (sessionId: string) => SeniorRef;
72
73
  presenceConnected?: boolean;
73
74
  env?: NodeJS.ProcessEnv;
75
+ /** Flag reader seam used to assert rollout reads happen before admission locks. */
76
+ readRegistryFlag?: typeof readLanesRegistryFlag;
74
77
  /** Detached mesh seam for dead-watcher recovery. */
75
78
  mesh?: LaneMesh | null;
76
79
  /** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
@@ -10,21 +10,26 @@ import * as path from "node:path";
10
10
  import { parseUserYaml } from "../../utils/user-yaml-error.js";
11
11
  import { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
12
12
  import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
13
+ import { effectiveAccountHome } from "./account-home.js";
13
14
  import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
14
15
  import { MAX_LANE_DEPTH } from "./constants.js";
15
16
  import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./errors.js";
17
+ import { LANE_HIRE_GRANTS_FLAG, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./hire-grants.js";
16
18
  import { loadWorkerProfile } from "./profile.js";
17
19
  import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
18
20
  import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, markSpawnFailure, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
19
21
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
20
22
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
23
+ import { readLanesRegistryFlag } from "./registry-flag.js";
24
+ import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
25
+ import { providerSupportsSessionResume } from "./delivery.js";
21
26
  import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
22
27
  import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
23
- import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
28
+ import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, writeWatcherDecision, } from "./watcher.js";
24
29
  import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
25
30
  import { laneStories, storyStatusForEnvelope } from "./stories.js";
26
31
  import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
27
- import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
32
+ import { PROVIDER_REJECTED_REASON, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
28
33
  /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
29
34
  export const QUESTION_AGE_MS = 60 * 60 * 1000;
30
35
  /** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
@@ -38,6 +43,13 @@ export const PICKUP_VALIDATION_MAX_AGE_MS = 15 * 60 * 1000;
38
43
  export const PICKUP_VALIDATION_TIMEOUT_REASON = "pickup_validation_timeout";
39
44
  export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
40
45
  export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
46
+ function spawnDepsForHost(deps) {
47
+ return {
48
+ ...(deps.spawnLaneDeps ?? {}),
49
+ hostname: deps.host.hostname,
50
+ admissionHost: deps.host,
51
+ };
52
+ }
41
53
  function logSkip(skip) {
42
54
  process.stderr.write(`hq lanes reconcile: skipped ${skip.kind}${skip.id ? ` ${skip.id}` : ""}: ${skip.why}\n`);
43
55
  }
@@ -326,12 +338,16 @@ function availableBalancedAccount(hqRoot, lane, occupying, config, host, profile
326
338
  continue;
327
339
  if (config.providers[lane.provider].accounts[account]?.enabled !== true)
328
340
  continue;
329
- const status = accountStatus(lane.provider, account, config);
341
+ const status = accountStatus(lane.provider, account, config, {
342
+ hostname: host.hostname,
343
+ existsSync: (file) => fs.existsSync(file),
344
+ });
330
345
  if (!isCredentialProven(status))
331
346
  continue;
332
347
  try {
333
348
  resolveAccount(lane.provider, account, config, {
334
349
  hqRoot,
350
+ hostname: host.hostname,
335
351
  ...(pinnedModel !== undefined ? { model: pinnedModel } : {}),
336
352
  });
337
353
  }
@@ -655,7 +671,7 @@ export async function runPassBody(hqRoot, deps) {
655
671
  };
656
672
  const readSessionSenior = deps.readSessionSenior ??
657
673
  ((sessionId) => defaultReadSessionSenior(hqRoot, sessionId));
658
- const configuredClaudeHomes = Object.values(config.providers.claude.accounts).map((account) => account.home);
674
+ const configuredClaudeHomes = Object.values(config.providers.claude.accounts).map((account) => effectiveAccountHome(account, deps.host.hostname).home);
659
675
  const sessionStateCache = new Map();
660
676
  const sessionState = (sessionId) => {
661
677
  if (sessionStateCache.has(sessionId)) {
@@ -780,11 +796,51 @@ export async function runPassBody(hqRoot, deps) {
780
796
  });
781
797
  }
782
798
  }
799
+ const readRegistryFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
800
+ const keepAliveFlagCompanies = new Map();
801
+ const keepAliveHireGrantFlagCompanies = new Map();
802
+ for (const lane of lanes) {
803
+ if (lane.state !== "done" ||
804
+ lane.keep_alive !== true ||
805
+ !laneRunsHere(lane, deps.host) ||
806
+ lane.terminal_reason === PROVIDER_REJECTED_REASON ||
807
+ lane.terminal_reason === "account_not_logged_in" ||
808
+ !providerSupportsSessionResume(lane.provider) ||
809
+ !(lane.provider_session_id ?? lane.last_resume?.session_id))
810
+ continue;
811
+ const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
812
+ if (!envelope || envelope.decision !== "done" || envelope.story_complete === true)
813
+ continue;
814
+ const key = keepAliveCompanyKey(lane.company);
815
+ keepAliveFlagCompanies.set(key, lane.company);
816
+ if (lane.senior.kind === "lane" && lane.senior.id) {
817
+ keepAliveHireGrantFlagCompanies.set(key, lane.company);
818
+ }
819
+ }
820
+ const keepAliveFlagByCompany = new Map();
821
+ const keepAliveHireGrantFlagByCompany = new Map();
822
+ const env = deps.env ?? process.env;
823
+ await Promise.all([
824
+ ...[...keepAliveFlagCompanies.entries()].map(async ([key, company]) => {
825
+ keepAliveFlagByCompany.set(key, await readRegistryFlag(LEAD_LANE_KEEP_ALIVE_FLAG, {
826
+ company: company ?? undefined,
827
+ env,
828
+ }));
829
+ }),
830
+ ...[...keepAliveHireGrantFlagCompanies.entries()].map(async ([key, company]) => {
831
+ keepAliveHireGrantFlagByCompany.set(key, await readRegistryFlag(LANE_HIRE_GRANTS_FLAG, {
832
+ company: company ?? undefined,
833
+ env,
834
+ }));
835
+ }),
836
+ ]);
783
837
  await withAllLaneAdmissionLocks(hqRoot, async () => {
784
838
  const freshLanes = loadLaneRecords(hqRoot, summary);
839
+ resumeDoneKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
840
+ const drainLanes = loadLaneRecords(hqRoot, summary);
785
841
  const freshQuestions = loadQuestionRecords(hqRoot, summary);
786
- await admitQueued(hqRoot, freshLanes, summary, deps, now);
787
- await resumeAwaiting(hqRoot, freshLanes, freshQuestions, summary, deps);
842
+ await admitQueued(hqRoot, drainLanes, summary, deps, now);
843
+ await resumeAwaiting(hqRoot, drainLanes, freshQuestions, summary, deps);
788
844
  });
789
845
  if (!deps.dryRun && summary.errors.length === 0) {
790
846
  try {
@@ -887,7 +943,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
887
943
  }
888
944
  if (!watcherAlive && !deps.dryRun) {
889
945
  try {
890
- armLaneWatcher(hqRoot, lane.lane_id, deps.spawnLaneDeps ?? {});
946
+ armLaneWatcher(hqRoot, lane.lane_id, spawnDepsForHost(deps));
891
947
  summary.heartbeats.push(`${lane.lane_id}:watcher-rearmed`);
892
948
  }
893
949
  catch (err) {
@@ -941,10 +997,10 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
941
997
  repairCandidate.envelope === null &&
942
998
  (!repairCandidate.missing || envelopeRepairMode(classified.provider) === "resume")) {
943
999
  const spawnRepair = deps.spawnRepair ??
944
- ((root, repairLane, input) => spawnEnvelopeRepair(root, repairLane, input, deps.spawnLaneDeps));
1000
+ ((root, repairLane, input) => spawnEnvelopeRepair(root, repairLane, input, spawnDepsForHost(deps)));
945
1001
  const repair = decideEnvelopeRepair(hqRoot, lane.lane_id, repairCandidate, nowMs, spawnRepair, () => undefined);
946
1002
  if (repair.action === "spawned") {
947
- armLaneWatcher(hqRoot, lane.lane_id, deps.spawnLaneDeps);
1003
+ armLaneWatcher(hqRoot, lane.lane_id, spawnDepsForHost(deps));
948
1004
  summary.heartbeats.push(`${lane.lane_id}:envelope-repair`);
949
1005
  continue;
950
1006
  }
@@ -969,8 +1025,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
969
1025
  if (!deps.dryRun && accountRetryReason && classified.account) {
970
1026
  try {
971
1027
  const recovered = await retryLaneOnAlternateAccount(hqRoot, classified, accountRetryReason, "reconcile", {
972
- ...(deps.spawnLaneDeps ?? {}),
973
- admissionHost: deps.host,
1028
+ ...spawnDepsForHost(deps),
974
1029
  });
975
1030
  summary.dead_host.push({
976
1031
  id: lane.lane_id,
@@ -1197,7 +1252,7 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
1197
1252
  nowMs,
1198
1253
  });
1199
1254
  const spawnFn = deps.spawn ??
1200
- ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
1255
+ ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
1201
1256
  const running = lanes.filter((lane) => (lane.state === "running" ||
1202
1257
  hasFreshAdmissionReservation(lane, now.getTime()) ||
1203
1258
  hasActivePendingResume(lane, now.getTime())) &&
@@ -1311,6 +1366,231 @@ async function admitQueued(hqRoot, lanes, summary, deps, now) {
1311
1366
  writeThrottleState(filePath, applyThrottleDecision(base, applied, nowMs), throttleDeps);
1312
1367
  }
1313
1368
  }
1369
+ function keepAliveCompanyKey(company) {
1370
+ return JSON.stringify([company?.uid ?? null, company?.slug ?? null]);
1371
+ }
1372
+ function hasKeepAliveProgress(lane) {
1373
+ return (lane.keep_alive_resume_count ?? 0) !== 0 ||
1374
+ lane.keep_alive_last_resume_at !== undefined ||
1375
+ lane.keep_alive_next_eligible_at !== undefined ||
1376
+ lane.keep_alive_exhausted_at !== undefined;
1377
+ }
1378
+ function clearedKeepAliveProgress(lane) {
1379
+ return {
1380
+ ...lane,
1381
+ keep_alive_resume_count: 0,
1382
+ keep_alive_last_resume_at: undefined,
1383
+ keep_alive_next_eligible_at: undefined,
1384
+ keep_alive_exhausted_at: undefined,
1385
+ };
1386
+ }
1387
+ function keepAliveProgressResetByHuman(lane) {
1388
+ const count = lane.keep_alive_resume_count ?? 0;
1389
+ const lastAutoMs = Date.parse(lane.keep_alive_last_resume_at ?? "");
1390
+ if (count > 0 && !Number.isFinite(lastAutoMs))
1391
+ return true;
1392
+ const messageMs = Date.parse(lane.last_message_at ?? "");
1393
+ const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT
1394
+ ? Number.NaN
1395
+ : Date.parse(lane.last_resume?.at ?? "");
1396
+ return Number.isFinite(lastAutoMs) &&
1397
+ ((Number.isFinite(messageMs) && messageMs > lastAutoMs) ||
1398
+ (Number.isFinite(resumeMs) && resumeMs > lastAutoMs));
1399
+ }
1400
+ function recordKeepAliveEvent(hqRoot, laneId, summary, event) {
1401
+ try {
1402
+ writeWatcherDecision(watcherEventsPath(hqRoot, laneId), {
1403
+ lane_id: laneId,
1404
+ ...event,
1405
+ });
1406
+ }
1407
+ catch (err) {
1408
+ markDirty(summary, {
1409
+ kind: "lane",
1410
+ id: laneId,
1411
+ why: `keep-alive event write failed: ${err instanceof Error ? err.message : String(err)}`,
1412
+ });
1413
+ }
1414
+ }
1415
+ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
1416
+ const nowMs = now.getTime();
1417
+ const nowIso = now.toISOString();
1418
+ const config = loadLanesConfig(hqRoot);
1419
+ for (const lane of lanes) {
1420
+ if (lane.state !== "done" ||
1421
+ lane.keep_alive !== true ||
1422
+ !laneRunsHere(lane, deps.host))
1423
+ continue;
1424
+ const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
1425
+ const envelope = latest?.envelope;
1426
+ if (!envelope || envelope.decision !== "done")
1427
+ continue;
1428
+ if (envelope.story_complete === true) {
1429
+ if (hasKeepAliveProgress(lane)) {
1430
+ casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
1431
+ current.state === "done" &&
1432
+ current.keep_alive === true, (current) => clearedKeepAliveProgress(current), deps.dryRun);
1433
+ }
1434
+ continue;
1435
+ }
1436
+ if (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
1437
+ lane.terminal_reason === "account_not_logged_in" ||
1438
+ !providerSupportsSessionResume(lane.provider) ||
1439
+ !(lane.provider_session_id ?? lane.last_resume?.session_id))
1440
+ continue;
1441
+ if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
1442
+ continue;
1443
+ const current = readLane(hqRoot, lane.lane_id);
1444
+ if (current.version !== lane.version ||
1445
+ current.state !== "done" ||
1446
+ current.keep_alive !== true ||
1447
+ !laneRunsHere(current, deps.host))
1448
+ continue;
1449
+ const currentEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id)?.envelope;
1450
+ if (!currentEnvelope || currentEnvelope.decision !== "done")
1451
+ continue;
1452
+ if (currentEnvelope.story_complete === true) {
1453
+ if (hasKeepAliveProgress(current)) {
1454
+ casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1455
+ latestLane.state === "done" &&
1456
+ latestLane.keep_alive === true, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
1457
+ }
1458
+ continue;
1459
+ }
1460
+ const providerSessionId = current.provider_session_id ?? current.last_resume?.session_id;
1461
+ if (current.terminal_reason === PROVIDER_REJECTED_REASON ||
1462
+ current.terminal_reason === "account_not_logged_in" ||
1463
+ typeof providerSessionId !== "string" ||
1464
+ providerSessionId.trim().length === 0)
1465
+ continue;
1466
+ const resetByHuman = keepAliveProgressResetByHuman(current);
1467
+ const attempts = resetByHuman ? 0 : current.keep_alive_resume_count ?? 0;
1468
+ if (attempts >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES) {
1469
+ if (current.keep_alive_exhausted_at !== undefined)
1470
+ continue;
1471
+ const exhausted = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1472
+ latestLane.state === "done" &&
1473
+ latestLane.keep_alive === true &&
1474
+ (latestLane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES &&
1475
+ latestLane.keep_alive_exhausted_at === undefined, (latestLane) => ({ ...latestLane, keep_alive_exhausted_at: nowIso }), deps.dryRun);
1476
+ if (exhausted.changed && !deps.dryRun) {
1477
+ recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
1478
+ event: "keep_alive_exhausted",
1479
+ at: nowIso,
1480
+ attempt: attempts,
1481
+ });
1482
+ }
1483
+ continue;
1484
+ }
1485
+ const attempt = attempts + 1;
1486
+ const doneAtMs = Date.parse(current.updated_at);
1487
+ const attemptEligibleMs = doneAtMs + leadLaneKeepAliveBackoffMs(attempt);
1488
+ const storedNextMs = resetByHuman
1489
+ ? Number.NaN
1490
+ : Date.parse(current.keep_alive_next_eligible_at ?? "");
1491
+ const nextEligibleMs = Number.isFinite(storedNextMs)
1492
+ ? Math.max(attemptEligibleMs, storedNextMs)
1493
+ : attemptEligibleMs;
1494
+ if (nowMs < nextEligibleMs) {
1495
+ if (resetByHuman && hasKeepAliveProgress(current)) {
1496
+ casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1497
+ latestLane.state === "done" &&
1498
+ latestLane.keep_alive === true, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
1499
+ }
1500
+ continue;
1501
+ }
1502
+ const nextEligibleAt = new Date(nowMs + leadLaneKeepAliveBackoffMs(attempt + 1)).toISOString();
1503
+ const queuedCount = loadLaneRecords(hqRoot, summary).filter((queuedLane) => queuedLane.state === "queued" &&
1504
+ queuedLane.provider === current.provider &&
1505
+ laneRunsHere(queuedLane, deps.host) &&
1506
+ !hasFreshAdmissionReservation(queuedLane, nowMs)).length;
1507
+ const maxQueued = config.providers[current.provider].max_queued;
1508
+ if (queuedCount >= maxQueued) {
1509
+ summary.skipped.push({
1510
+ kind: "lane",
1511
+ id: current.lane_id,
1512
+ why: `lane_queue_full: ${current.provider} queue has ${queuedCount}/${maxQueued} lanes`,
1513
+ });
1514
+ continue;
1515
+ }
1516
+ const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1517
+ latestLane.state === "done" &&
1518
+ latestLane.keep_alive === true &&
1519
+ (expectedSeniorId === undefined ||
1520
+ (latestLane.senior.kind === "lane" && latestLane.senior.id === expectedSeniorId)), (latestLane) => ({
1521
+ ...latestLane,
1522
+ state: "queued",
1523
+ queued_reason: undefined,
1524
+ queued_at: nowIso,
1525
+ admission_reserved_at: undefined,
1526
+ terminal_reason: undefined,
1527
+ terminal_details: undefined,
1528
+ pending_resume: {
1529
+ session_id: providerSessionId,
1530
+ trigger: "resume",
1531
+ resumed_from: "done",
1532
+ text: LEAD_LANE_KEEP_ALIVE_RESUME_TEXT,
1533
+ compact: true,
1534
+ at: nowIso,
1535
+ },
1536
+ keep_alive_resume_count: attempt,
1537
+ keep_alive_last_resume_at: nowIso,
1538
+ keep_alive_next_eligible_at: nextEligibleAt,
1539
+ keep_alive_exhausted_at: undefined,
1540
+ }), deps.dryRun);
1541
+ const seniorId = current.senior.kind === "lane" ? current.senior.id : undefined;
1542
+ let queued;
1543
+ if (seniorId &&
1544
+ hireGrantFlagByCompany.get(keepAliveCompanyKey(current.company)) === true) {
1545
+ try {
1546
+ queued = withHireGrantAdmission(hqRoot, seniorId, (snapshot) => {
1547
+ const latestLane = readLane(hqRoot, current.lane_id);
1548
+ if (latestLane.version !== current.version ||
1549
+ latestLane.senior.kind !== "lane" ||
1550
+ latestLane.senior.id !== seniorId)
1551
+ return { changed: false, record: latestLane };
1552
+ const refusal = terminalChildResumeHireGrantError(latestLane, snapshot);
1553
+ if (refusal !== null) {
1554
+ summary.skipped.push({
1555
+ kind: "lane",
1556
+ id: current.lane_id,
1557
+ why: `keep_alive_hire_admission: ${refusal.error}: ${refusal.message}`,
1558
+ });
1559
+ return { changed: false, record: latestLane };
1560
+ }
1561
+ return queue(seniorId);
1562
+ });
1563
+ }
1564
+ catch (err) {
1565
+ markDirty(summary, {
1566
+ kind: "lane",
1567
+ id: current.lane_id,
1568
+ why: `keep-alive hire admission failed: ${err instanceof Error ? err.message : String(err)}`,
1569
+ });
1570
+ continue;
1571
+ }
1572
+ }
1573
+ else {
1574
+ queued = queue(seniorId);
1575
+ }
1576
+ if (!queued.changed)
1577
+ continue;
1578
+ summary.resumed.push({
1579
+ id: current.lane_id,
1580
+ from: "done",
1581
+ to: "queued",
1582
+ reason: "keep_alive",
1583
+ });
1584
+ if (!deps.dryRun) {
1585
+ recordKeepAliveEvent(hqRoot, current.lane_id, summary, {
1586
+ event: "keep_alive_resume",
1587
+ at: nowIso,
1588
+ attempt,
1589
+ next_eligible_at: nextEligibleAt,
1590
+ });
1591
+ }
1592
+ }
1593
+ }
1314
1594
  async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneIds) {
1315
1595
  const config = loadLanesConfig(hqRoot);
1316
1596
  const awaiting = lanes.filter((lane) => lane.state === "awaiting_input" &&
@@ -1336,7 +1616,7 @@ async function resumeAwaiting(hqRoot, lanes, questions, summary, deps, onlyLaneI
1336
1616
  const throttle = await decideAdmission(hqRoot, config.throttle, throttleDeps);
1337
1617
  throttleDecision = throttle.decision;
1338
1618
  }
1339
- const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, deps.spawnLaneDeps));
1619
+ const spawnFn = deps.spawn ?? ((root, lane) => spawnLaneWorker(root, lane, spawnDepsForHost(deps)));
1340
1620
  const nowMs = deps.now().getTime();
1341
1621
  const fallbackAnswerAt = new Date(nowMs).toISOString();
1342
1622
  const answerAtByLane = new Map();
@@ -68,6 +68,7 @@ export type ProveDetachedFn = (pid: number) => DetachProof;
68
68
  export type KillProcessFn = (pid: number, signal?: NodeJS.Signals) => void;
69
69
  export interface SpawnLaneDeps {
70
70
  env?: NodeJS.ProcessEnv;
71
+ hostname?: string;
71
72
  /** The caller's host scope for admission checks; reconcile supplies its resolved host. */
72
73
  admissionHost?: AdmissionHost;
73
74
  /** Provider model catalog dependencies, shared with create in tests. */