@indigoai-us/hq-cli 5.216.0 → 5.218.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,22 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [5.218.0] — 2026-09-26
6
+
7
+ ### Fixed
8
+
9
+ - The reconcile summary now lists opted-in lanes it could not resume and why.
10
+ Stderr provides reason-specific guidance and recommends reapplying opt-in only
11
+ when that can help.
12
+
13
+ ## [5.217.0] — 2026-09-26
14
+
15
+ ### Fixed
16
+
17
+ - Keep-alive resumes a lane interrupted by `exited_with_partial_output` when
18
+ the interruption followed its explicit opt-in. Quota and host-loss
19
+ interruptions, login failures, and stopped lanes remain ineligible.
20
+
5
21
  ## [5.216.0] — 2026-09-26
6
22
 
7
23
  ### Fixed
@@ -5,6 +5,7 @@ export declare const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
5
5
  export declare const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS: number;
6
6
  export declare const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS: number;
7
7
  export declare const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
8
+ export declare const LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT = "Automatic keep-alive resume: your previous turn was interrupted by a provider process exit. Check your last state before continuing. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
8
9
  export type KeepAliveBlockedReason = NonNullable<LaneRecord["keep_alive_blocked_reason"]>;
9
10
  export declare function retiredLaneRegistryPath(hqRoot: string): string;
10
11
  /** Missing registry means no retired lanes; every present invalid file fails closed. */
@@ -7,6 +7,7 @@ export const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
7
7
  export const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS = 5 * 60 * 1000;
8
8
  export const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS = 60 * 60 * 1000;
9
9
  export const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
10
+ export const LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT = "Automatic keep-alive resume: your previous turn was interrupted by a provider process exit. Check your last state before continuing. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
10
11
  export function retiredLaneRegistryPath(hqRoot) {
11
12
  return path.join(hqRoot, "workspace", "lane-briefs", "juniors.json");
12
13
  }
@@ -9,6 +9,7 @@ import { type LivenessDeps } from "./liveness.js";
9
9
  import { type EnvelopeRepairSpawnInput, type SpawnLaneDeps } from "./spawn.js";
10
10
  import { type ThrottleDeps } from "./throttle.js";
11
11
  import { readLanesRegistryFlag } from "./registry-flag.js";
12
+ import { type KeepAliveBlockedReason } from "./keep-alive.js";
12
13
  import type { LaneRecord, LivenessState, QueuedReason, QuestionRecord, SeniorRef } from "./types.js";
13
14
  import { type LaneMesh } from "./mesh-events.js";
14
15
  /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
@@ -49,6 +50,10 @@ export interface ReconcileSummary {
49
50
  dead_host: ReconcileChange[];
50
51
  heartbeats: string[];
51
52
  skipped: ReconcileSkip[];
53
+ keep_alive_blocked: Array<{
54
+ id: string;
55
+ reason: KeepAliveBlockedReason;
56
+ }>;
52
57
  errors: Array<{
53
58
  message: string;
54
59
  code?: string;
@@ -78,6 +83,8 @@ export interface ReconcileDeps {
78
83
  mesh?: LaneMesh | null;
79
84
  /** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
80
85
  meshEmitTimeoutMs?: number;
86
+ /** Fresh keep-alive eligibility read; injected to exercise opt-in changes before queueing. */
87
+ readKeepAliveRecheckLane?: (hqRoot: string, laneId: string) => LaneRecord;
81
88
  /**
82
89
  * Awaited after lane/question records and liveness are loaded, before any
83
90
  * CAS write. Test seam so two in-process passes can share a snapshot.
@@ -21,11 +21,11 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
21
21
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
22
22
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
23
23
  import { readLanesRegistryFlag } from "./registry-flag.js";
24
- import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
24
+ import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
25
25
  import { providerSupportsSessionResume } from "./delivery.js";
26
26
  import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
27
27
  import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
28
- import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, writeWatcherDecision, } from "./watcher.js";
28
+ import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
29
29
  import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
30
30
  import { laneStories, storyStatusForEnvelope } from "./stories.js";
31
31
  import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
@@ -66,6 +66,7 @@ function emptySummary(dryRun) {
66
66
  dead_host: [],
67
67
  heartbeats: [],
68
68
  skipped: [],
69
+ keep_alive_blocked: [],
69
70
  errors: [],
70
71
  heartbeat_at: null,
71
72
  };
@@ -800,7 +801,7 @@ export async function runPassBody(hqRoot, deps) {
800
801
  const keepAliveFlagCompanies = new Map();
801
802
  const keepAliveHireGrantFlagCompanies = new Map();
802
803
  for (const lane of lanes) {
803
- if (!(lane.state === "done" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
804
+ if (!(lane.state === "done" || lane.state === "interrupted" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
804
805
  (lane.keep_alive !== true && lane.worker_profile === undefined) ||
805
806
  !laneRunsHere(lane, deps.host) ||
806
807
  (lane.state === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
@@ -838,7 +839,7 @@ export async function runPassBody(hqRoot, deps) {
838
839
  ]);
839
840
  await withAllLaneAdmissionLocks(hqRoot, async () => {
840
841
  const freshLanes = loadLaneRecords(hqRoot, summary);
841
- resumeDoneKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
842
+ resumeKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
842
843
  const drainLanes = loadLaneRecords(hqRoot, summary);
843
844
  const freshQuestions = loadQuestionRecords(hqRoot, summary);
844
845
  await admitQueued(hqRoot, drainLanes, summary, deps, now);
@@ -1059,6 +1060,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1059
1060
  state: repairedClassification.state,
1060
1061
  queued_reason: undefined,
1061
1062
  terminal_reason: repairedClassification.reason ?? "host_lost",
1063
+ interrupted_at: repairedClassification.state === "interrupted" ? nowIso : undefined,
1062
1064
  ...(providerRejection
1063
1065
  ? { terminal_details: providerRejectionDetails(providerRejection, current) }
1064
1066
  : {}),
@@ -1175,6 +1177,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1175
1177
  state: "interrupted",
1176
1178
  queued_reason: undefined,
1177
1179
  terminal_reason: "host_lost",
1180
+ interrupted_at: nowIso,
1178
1181
  }), deps.dryRun);
1179
1182
  if (result.changed) {
1180
1183
  summary.dead_host.push({
@@ -1392,7 +1395,8 @@ function keepAliveProgressResetByHuman(lane) {
1392
1395
  if (count > 0 && !Number.isFinite(lastAutoMs))
1393
1396
  return true;
1394
1397
  const messageMs = Date.parse(lane.last_message_at ?? "");
1395
- const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT
1398
+ const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT ||
1399
+ lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
1396
1400
  ? Number.NaN
1397
1401
  : Date.parse(lane.last_resume?.at ?? "");
1398
1402
  return Number.isFinite(lastAutoMs) &&
@@ -1414,12 +1418,38 @@ function recordKeepAliveEvent(hqRoot, laneId, summary, event) {
1414
1418
  });
1415
1419
  }
1416
1420
  }
1421
+ function keepAliveSkipGuidance(laneId, reason) {
1422
+ switch (reason) {
1423
+ case "missing_opt_in_time":
1424
+ return `if appropriate, run hq lanes keep-alive ${laneId} --on`;
1425
+ case "not_opted_in":
1426
+ return "explicit opt-in is required before keep-alive can resume this lane";
1427
+ case "envelope_before_opt_in":
1428
+ return `the latest envelope predates opt-in; review the lane with hq lanes show ${laneId}`;
1429
+ case "interruption_before_opt_in":
1430
+ return `the interruption predates opt-in; review the lane with hq lanes show ${laneId}`;
1431
+ case "missing_interruption_time":
1432
+ return "the interruption time is unavailable; inspect the watcher classification before resuming";
1433
+ case "retired":
1434
+ return "the lane is retired; resolve its retired status before resuming";
1435
+ case "superseded":
1436
+ return "a newer lead exists for this project; continue on that lane";
1437
+ case "retired_list_invalid":
1438
+ return "the retired lane list is invalid; repair workspace/lane-briefs/juniors.json before retrying";
1439
+ case "terminal_state":
1440
+ return `the lane is in a terminal state; review it with hq lanes show ${laneId} before starting replacement work`;
1441
+ }
1442
+ }
1417
1443
  function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
1418
1444
  summary.skipped.push({
1419
1445
  kind: "lane",
1420
1446
  id: lane.lane_id,
1421
1447
  why: `keep_alive_${reason}${detail ? `: ${detail}` : ""}`,
1422
1448
  });
1449
+ if (lane.keep_alive === true) {
1450
+ summary.keep_alive_blocked.push({ id: lane.lane_id, reason });
1451
+ process.stderr.write(`hq lanes reconcile: keep-alive blocked for lane ${lane.lane_id} (${reason}); ${keepAliveSkipGuidance(lane.lane_id, reason)}\n`);
1452
+ }
1423
1453
  if (deps.dryRun || lane.keep_alive_blocked_reason === reason)
1424
1454
  return;
1425
1455
  try {
@@ -1433,6 +1463,36 @@ function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
1433
1463
  });
1434
1464
  }
1435
1465
  }
1466
+ function readLatestInterruptedClassificationAt(hqRoot, lane) {
1467
+ const logPath = watcherOutLogPath(hqRoot, lane.lane_id);
1468
+ let body;
1469
+ try {
1470
+ body = fs.readFileSync(logPath, "utf8");
1471
+ }
1472
+ catch (err) {
1473
+ if (err.code === "ENOENT")
1474
+ return undefined;
1475
+ throw err;
1476
+ }
1477
+ let latestClassification;
1478
+ for (const line of body.split(/\r?\n/).filter(Boolean)) {
1479
+ const parsed = JSON.parse(line);
1480
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
1481
+ throw new Error("watcher log contains a non-object decision record");
1482
+ }
1483
+ const record = parsed;
1484
+ if (record.event === "classification")
1485
+ latestClassification = record;
1486
+ }
1487
+ const at = latestClassification?.at;
1488
+ if (latestClassification?.state !== "interrupted" ||
1489
+ latestClassification.reason !== lane.terminal_reason ||
1490
+ typeof at !== "string" ||
1491
+ !Number.isFinite(Date.parse(at)) ||
1492
+ new Date(Date.parse(at)).toISOString() !== at)
1493
+ return undefined;
1494
+ return at;
1495
+ }
1436
1496
  function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
1437
1497
  const filePath = path.join(laneRunDir(hqRoot, lane.lane_id), ref);
1438
1498
  try {
@@ -1442,7 +1502,7 @@ function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
1442
1502
  throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
1443
1503
  }
1444
1504
  }
1445
- function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
1505
+ function resumeKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
1446
1506
  const nowMs = now.getTime();
1447
1507
  const nowIso = now.toISOString();
1448
1508
  const config = loadLanesConfig(hqRoot);
@@ -1456,34 +1516,97 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1456
1516
  markError(summary, err);
1457
1517
  }
1458
1518
  }
1459
- for (const lane of lanes) {
1460
- if (!laneRunsHere(lane, deps.host))
1519
+ for (const originalLane of lanes) {
1520
+ if (!laneRunsHere(originalLane, deps.host))
1461
1521
  continue;
1462
- if (lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") {
1463
- if (lane.keep_alive === true &&
1464
- flagByCompany.get(keepAliveCompanyKey(lane.company)) === true)
1465
- recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps);
1522
+ if (originalLane.state === "killed" ||
1523
+ originalLane.state === "cancelled" ||
1524
+ originalLane.state === "failed") {
1525
+ if (originalLane.keep_alive === true &&
1526
+ flagByCompany.get(keepAliveCompanyKey(originalLane.company)) === true)
1527
+ recordKeepAliveSkip(hqRoot, originalLane, "terminal_state", summary, deps);
1466
1528
  continue;
1467
1529
  }
1468
- if (lane.state !== "done" ||
1469
- (lane.keep_alive !== true && lane.worker_profile === undefined))
1470
- continue;
1471
- const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
1472
- const envelope = latest?.envelope;
1473
- if (!envelope || envelope.decision !== "done")
1530
+ if ((originalLane.state !== "done" && originalLane.state !== "interrupted") ||
1531
+ (originalLane.keep_alive !== true && originalLane.worker_profile === undefined))
1474
1532
  continue;
1475
- if (envelope.story_complete === true) {
1476
- if (hasKeepAliveProgress(lane)) {
1477
- casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
1478
- current.state === "done" &&
1479
- current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
1533
+ const initialState = originalLane.state;
1534
+ let lane = originalLane;
1535
+ if (initialState === "done") {
1536
+ const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
1537
+ if (!envelope || envelope.decision !== "done")
1538
+ continue;
1539
+ if (envelope.story_complete === true) {
1540
+ if (hasKeepAliveProgress(lane)) {
1541
+ casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
1542
+ current.state === "done" &&
1543
+ current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
1544
+ }
1545
+ continue;
1480
1546
  }
1481
- continue;
1482
1547
  }
1483
1548
  if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
1484
1549
  continue;
1485
- if (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
1486
- lane.terminal_reason === "account_not_logged_in") {
1550
+ // Persist a stable timestamp for legacy interrupted rows before a later
1551
+ // keep-alive status write can advance their general updated_at clock. The
1552
+ // watcher classification is the interruption time; updated_at is only an
1553
+ // upper bound and can postdate explicit opt-in.
1554
+ if (initialState === "interrupted" && lane.keep_alive === true && lane.interrupted_at === undefined) {
1555
+ let interruptedAt;
1556
+ try {
1557
+ interruptedAt = readLatestInterruptedClassificationAt(hqRoot, lane);
1558
+ }
1559
+ catch (err) {
1560
+ const detail = `watcher classification read failed: ${err instanceof Error ? err.message : String(err)}`;
1561
+ recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, detail);
1562
+ const skip = summary.skipped[summary.skipped.length - 1];
1563
+ if (skip) {
1564
+ summary.ok = false;
1565
+ logSkip(skip);
1566
+ }
1567
+ continue;
1568
+ }
1569
+ if (interruptedAt === undefined) {
1570
+ recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "no latest matching watcher classification event");
1571
+ const skip = summary.skipped[summary.skipped.length - 1];
1572
+ if (skip) {
1573
+ summary.ok = false;
1574
+ logSkip(skip);
1575
+ }
1576
+ continue;
1577
+ }
1578
+ try {
1579
+ const backfill = casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
1580
+ current.state === "interrupted" &&
1581
+ current.keep_alive === true &&
1582
+ current.terminal_reason === lane.terminal_reason &&
1583
+ current.interrupted_at === undefined, (current) => ({ ...current, interrupted_at: interruptedAt }), deps.dryRun);
1584
+ lane = backfill.record;
1585
+ if (lane.interrupted_at === undefined) {
1586
+ recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "lane changed before watcher classification timestamp could be recorded");
1587
+ const skip = summary.skipped[summary.skipped.length - 1];
1588
+ if (skip) {
1589
+ summary.ok = false;
1590
+ logSkip(skip);
1591
+ }
1592
+ continue;
1593
+ }
1594
+ }
1595
+ catch (err) {
1596
+ markDirty(summary, {
1597
+ kind: "lane",
1598
+ id: lane.lane_id,
1599
+ why: `keep-alive interruption timestamp backfill failed: ${err instanceof Error ? err.message : String(err)}`,
1600
+ });
1601
+ continue;
1602
+ }
1603
+ }
1604
+ if (initialState === "interrupted" && lane.terminal_reason !== "exited_with_partial_output") {
1605
+ recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason ?? "missing interruption reason");
1606
+ continue;
1607
+ }
1608
+ if (initialState === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
1609
+ lane.terminal_reason === "account_not_logged_in")) {
1487
1610
  recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason);
1488
1611
  continue;
1489
1612
  }
@@ -1496,20 +1619,35 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1496
1619
  recordKeepAliveSkip(hqRoot, lane, "missing_opt_in_time", summary, deps);
1497
1620
  continue;
1498
1621
  }
1499
- let envelopeAtMs;
1500
- try {
1501
- envelopeAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
1622
+ let eligibilityAtMs;
1623
+ if (initialState === "done") {
1624
+ const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
1625
+ if (!latest || latest.envelope.decision !== "done" || latest.envelope.story_complete === true)
1626
+ continue;
1627
+ try {
1628
+ eligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
1629
+ }
1630
+ catch (err) {
1631
+ markDirty(summary, {
1632
+ kind: "lane",
1633
+ id: lane.lane_id,
1634
+ why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
1635
+ });
1636
+ continue;
1637
+ }
1502
1638
  }
1503
- catch (err) {
1504
- markDirty(summary, {
1505
- kind: "lane",
1506
- id: lane.lane_id,
1507
- why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
1508
- });
1509
- continue;
1639
+ else {
1640
+ eligibilityAtMs = Date.parse(lane.interrupted_at ?? "");
1641
+ if (!Number.isFinite(eligibilityAtMs)) {
1642
+ recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps);
1643
+ continue;
1644
+ }
1510
1645
  }
1511
- if (envelopeAtMs <= optedInAtMs) {
1512
- recordKeepAliveSkip(hqRoot, lane, "envelope_before_opt_in", summary, deps);
1646
+ const initialTimeBlockReason = initialState === "done"
1647
+ ? "envelope_before_opt_in"
1648
+ : "interruption_before_opt_in";
1649
+ if (eligibilityAtMs <= optedInAtMs) {
1650
+ recordKeepAliveSkip(hqRoot, lane, initialTimeBlockReason, summary, deps);
1513
1651
  continue;
1514
1652
  }
1515
1653
  if (retiredLaneIds === null) {
@@ -1521,33 +1659,54 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1521
1659
  recordKeepAliveSkip(hqRoot, lane, exclusion, summary, deps);
1522
1660
  continue;
1523
1661
  }
1662
+ const providerSessionId = lane.provider_session_id ?? lane.last_resume?.session_id;
1524
1663
  if (!providerSupportsSessionResume(lane.provider) ||
1525
- !(lane.provider_session_id ?? lane.last_resume?.session_id))
1664
+ typeof providerSessionId !== "string" ||
1665
+ providerSessionId.trim().length === 0) {
1666
+ if (initialState === "interrupted") {
1667
+ recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, "provider session cannot be resumed");
1668
+ }
1526
1669
  continue;
1527
- const keepAliveStillEnabled = (candidate) => candidate.keep_alive === true &&
1528
- candidate.keep_alive_opted_in_at === lane.keep_alive_opted_in_at;
1529
- const current = readLane(hqRoot, lane.lane_id);
1530
- if (current.version !== lane.version ||
1531
- current.state !== "done" ||
1532
- !keepAliveStillEnabled(current) ||
1533
- !laneRunsHere(current, deps.host))
1670
+ }
1671
+ // This second read is intentionally independent of the pass snapshot. The
1672
+ // final timestamp comparison below closes the opt-in race before the CAS.
1673
+ const rechecked = deps.readKeepAliveRecheckLane?.(hqRoot, lane.lane_id) ?? readLane(hqRoot, lane.lane_id);
1674
+ // Dry-run backfills only in memory. Carry that simulated legacy timestamp
1675
+ // through the fresh read when the stored lane version has not changed.
1676
+ const current = deps.dryRun &&
1677
+ initialState === "interrupted" &&
1678
+ originalLane.interrupted_at === undefined &&
1679
+ lane.interrupted_at !== undefined &&
1680
+ rechecked.version === lane.version &&
1681
+ rechecked.interrupted_at === undefined
1682
+ ? { ...rechecked, interrupted_at: lane.interrupted_at }
1683
+ : rechecked;
1684
+ if (current.state !== initialState || !laneRunsHere(current, deps.host))
1534
1685
  continue;
1535
- const currentEnvelopeRecord = readLatestEnvelopeRecord(hqRoot, current.lane_id);
1536
- const currentEnvelope = currentEnvelopeRecord?.envelope;
1537
- if (!currentEnvelope || currentEnvelope.decision !== "done")
1686
+ if (current.keep_alive !== true) {
1687
+ recordKeepAliveSkip(hqRoot, current, "not_opted_in", summary, deps);
1538
1688
  continue;
1539
- if (currentEnvelope.story_complete === true) {
1540
- if (hasKeepAliveProgress(current)) {
1541
- casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1542
- latestLane.state === "done" &&
1543
- keepAliveStillEnabled(latestLane), (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
1544
- }
1689
+ }
1690
+ if (initialState === "interrupted" && current.terminal_reason !== "exited_with_partial_output") {
1691
+ recordKeepAliveSkip(hqRoot, current, "terminal_state", summary, deps, current.terminal_reason ?? "missing interruption reason");
1545
1692
  continue;
1546
1693
  }
1547
- if (currentEnvelopeRecord !== null) {
1548
- let currentEnvelopeAtMs;
1694
+ let currentEligibilityAtMs;
1695
+ if (initialState === "done") {
1696
+ const currentEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id);
1697
+ if (!currentEnvelope || currentEnvelope.envelope.decision !== "done")
1698
+ continue;
1699
+ if (currentEnvelope.envelope.story_complete === true) {
1700
+ if (hasKeepAliveProgress(current)) {
1701
+ casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1702
+ latestLane.state === "done" &&
1703
+ latestLane.keep_alive === true &&
1704
+ latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
1705
+ }
1706
+ continue;
1707
+ }
1549
1708
  try {
1550
- currentEnvelopeAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelopeRecord.ref);
1709
+ currentEligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelope.ref);
1551
1710
  }
1552
1711
  catch (err) {
1553
1712
  markDirty(summary, {
@@ -1557,11 +1716,26 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1557
1716
  });
1558
1717
  continue;
1559
1718
  }
1560
- if (currentEnvelopeAtMs <= Date.parse(current.keep_alive_opted_in_at ?? "")) {
1561
- recordKeepAliveSkip(hqRoot, current, "envelope_before_opt_in", summary, deps);
1719
+ }
1720
+ else {
1721
+ currentEligibilityAtMs = Date.parse(current.interrupted_at ?? "");
1722
+ if (!Number.isFinite(currentEligibilityAtMs)) {
1723
+ recordKeepAliveSkip(hqRoot, current, "missing_interruption_time", summary, deps);
1562
1724
  continue;
1563
1725
  }
1564
1726
  }
1727
+ const currentOptedInAtMs = Date.parse(current.keep_alive_opted_in_at ?? "");
1728
+ if (!Number.isFinite(currentOptedInAtMs)) {
1729
+ recordKeepAliveSkip(hqRoot, current, "missing_opt_in_time", summary, deps);
1730
+ continue;
1731
+ }
1732
+ const currentTimeBlockReason = initialState === "done"
1733
+ ? "envelope_before_opt_in"
1734
+ : "interruption_before_opt_in";
1735
+ if (currentEligibilityAtMs <= currentOptedInAtMs) {
1736
+ recordKeepAliveSkip(hqRoot, current, currentTimeBlockReason, summary, deps);
1737
+ continue;
1738
+ }
1565
1739
  if (retiredLaneIds === null) {
1566
1740
  recordKeepAliveSkip(hqRoot, current, "retired_list_invalid", summary, deps);
1567
1741
  continue;
@@ -1572,11 +1746,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1572
1746
  recordKeepAliveSkip(hqRoot, current, currentExclusion, summary, deps);
1573
1747
  continue;
1574
1748
  }
1575
- const providerSessionId = current.provider_session_id ?? current.last_resume?.session_id;
1576
- if (current.terminal_reason === PROVIDER_REJECTED_REASON ||
1577
- current.terminal_reason === "account_not_logged_in" ||
1578
- typeof providerSessionId !== "string" ||
1579
- providerSessionId.trim().length === 0)
1749
+ const currentSessionId = current.provider_session_id ?? current.last_resume?.session_id;
1750
+ if (!providerSupportsSessionResume(current.provider) ||
1751
+ typeof currentSessionId !== "string" ||
1752
+ currentSessionId.trim().length === 0)
1580
1753
  continue;
1581
1754
  const resetByHuman = keepAliveProgressResetByHuman(current);
1582
1755
  const attempts = resetByHuman ? 0 : current.keep_alive_resume_count ?? 0;
@@ -1584,8 +1757,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1584
1757
  if (current.keep_alive_exhausted_at !== undefined)
1585
1758
  continue;
1586
1759
  const exhausted = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1587
- latestLane.state === "done" &&
1588
- keepAliveStillEnabled(latestLane) &&
1760
+ latestLane.state === current.state &&
1761
+ latestLane.keep_alive === true &&
1762
+ latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
1589
1763
  (latestLane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES &&
1590
1764
  latestLane.keep_alive_exhausted_at === undefined, (latestLane) => ({ ...latestLane, keep_alive_exhausted_at: nowIso }), deps.dryRun);
1591
1765
  if (exhausted.changed && !deps.dryRun) {
@@ -1593,13 +1767,16 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1593
1767
  event: "keep_alive_exhausted",
1594
1768
  at: nowIso,
1595
1769
  attempt: attempts,
1770
+ resumed_from: current.state,
1596
1771
  });
1597
1772
  }
1598
1773
  continue;
1599
1774
  }
1600
1775
  const attempt = attempts + 1;
1601
- const doneAtMs = Date.parse(current.updated_at);
1602
- const attemptEligibleMs = doneAtMs + leadLaneKeepAliveBackoffMs(attempt);
1776
+ const terminalAtMs = initialState === "done"
1777
+ ? Date.parse(current.updated_at)
1778
+ : Date.parse(current.interrupted_at ?? "");
1779
+ const attemptEligibleMs = terminalAtMs + leadLaneKeepAliveBackoffMs(attempt);
1603
1780
  const storedNextMs = resetByHuman
1604
1781
  ? Number.NaN
1605
1782
  : Date.parse(current.keep_alive_next_eligible_at ?? "");
@@ -1609,8 +1786,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1609
1786
  if (nowMs < nextEligibleMs) {
1610
1787
  if (resetByHuman && hasKeepAliveProgress(current)) {
1611
1788
  casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1612
- latestLane.state === "done" &&
1613
- keepAliveStillEnabled(latestLane), (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
1789
+ latestLane.state === current.state &&
1790
+ latestLane.keep_alive === true &&
1791
+ latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
1614
1792
  }
1615
1793
  continue;
1616
1794
  }
@@ -1628,9 +1806,14 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1628
1806
  });
1629
1807
  continue;
1630
1808
  }
1809
+ const resumedFrom = current.state;
1810
+ const resumeText = resumedFrom === "interrupted"
1811
+ ? LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
1812
+ : LEAD_LANE_KEEP_ALIVE_RESUME_TEXT;
1631
1813
  const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1632
- latestLane.state === "done" &&
1633
- keepAliveStillEnabled(latestLane) &&
1814
+ latestLane.state === current.state &&
1815
+ latestLane.keep_alive === true &&
1816
+ latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
1634
1817
  (expectedSeniorId === undefined ||
1635
1818
  (latestLane.senior.kind === "lane" && latestLane.senior.id === expectedSeniorId)), (latestLane) => ({
1636
1819
  ...latestLane,
@@ -1641,10 +1824,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1641
1824
  terminal_reason: undefined,
1642
1825
  terminal_details: undefined,
1643
1826
  pending_resume: {
1644
- session_id: providerSessionId,
1827
+ session_id: currentSessionId,
1645
1828
  trigger: "resume",
1646
- resumed_from: "done",
1647
- text: LEAD_LANE_KEEP_ALIVE_RESUME_TEXT,
1829
+ resumed_from: resumedFrom,
1830
+ text: resumeText,
1648
1831
  compact: true,
1649
1832
  at: nowIso,
1650
1833
  },
@@ -1693,7 +1876,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1693
1876
  continue;
1694
1877
  summary.resumed.push({
1695
1878
  id: current.lane_id,
1696
- from: "done",
1879
+ from: resumedFrom,
1697
1880
  to: "queued",
1698
1881
  reason: "keep_alive",
1699
1882
  });
@@ -1703,6 +1886,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1703
1886
  at: nowIso,
1704
1887
  attempt,
1705
1888
  next_eligible_at: nextEligibleAt,
1889
+ resumed_from: resumedFrom,
1706
1890
  });
1707
1891
  }
1708
1892
  }
@@ -414,6 +414,8 @@ export function parseLaneRecord(raw, filePath) {
414
414
  "not_opted_in",
415
415
  "missing_opt_in_time",
416
416
  "envelope_before_opt_in",
417
+ "interruption_before_opt_in",
418
+ "missing_interruption_time",
417
419
  "retired",
418
420
  "superseded",
419
421
  "retired_list_invalid",
@@ -443,6 +445,7 @@ export function parseLaneRecord(raw, filePath) {
443
445
  "keep_alive_last_resume_at",
444
446
  "keep_alive_next_eligible_at",
445
447
  "keep_alive_exhausted_at",
448
+ "interrupted_at",
446
449
  ]) {
447
450
  if (raw[field] !== undefined)
448
451
  requireTimestamp(raw[field], field, filePath);
@@ -359,7 +359,7 @@ export interface LaneRecord {
359
359
  /** Time this lane most recently explicitly opted into keep-alive. */
360
360
  keep_alive_opted_in_at?: string;
361
361
  /** Why reconcile declined to resume this lane most recently. */
362
- keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
362
+ keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "interruption_before_opt_in" | "missing_interruption_time" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
363
363
  /** Consecutive automatic keep-alive resumes since the last human reset. */
364
364
  keep_alive_resume_count?: number;
365
365
  /** Time the most recent automatic keep-alive resume was queued. */
@@ -451,6 +451,8 @@ export interface LaneRecord {
451
451
  /** Set when the reconcile pass force-admits under the starvation bound. */
452
452
  admitted_reason?: string;
453
453
  terminal_reason?: string;
454
+ /** Classification time when this lane entered interrupted; stable across later metadata writes. */
455
+ interrupted_at?: string;
454
456
  /**
455
457
  * Structured detail behind a named `terminal_reason` (dispatch, pickup and
456
458
  * spawn failures), the missing-binary search path behind
@@ -230,7 +230,7 @@ export const LANE_FIELD_WRITERS = {
230
230
  last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
231
231
  last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
232
232
  overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
233
- admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
233
+ admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], interrupted_at: ["watcher", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
234
234
  quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
235
235
  provider_session_id: ["watcher", "command", "spawn"], provider_session_id_absent: ["watcher", "command"], provider_home_at_spawn: ["spawn"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], envelope_validation_errors: ["watcher"], last_progress_event_at: ["watcher"],
236
236
  };
@@ -336,6 +336,7 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
336
336
  stalled: undefined,
337
337
  stalled_at: undefined,
338
338
  stalled_reason: undefined,
339
+ interrupted_at: classification.state === "interrupted" ? at : undefined,
339
340
  envelope_validation_errors: opts.envelopeValidationErrors,
340
341
  };
341
342
  changed = true;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@indigoai-us/hq-cli",
3
- "version": "5.216.0",
3
+ "version": "5.218.0",
4
4
  "description": "HQ by Indigo management CLI — modules and cloud sync",
5
5
  "main": "dist/index.js",
6
6
  "bin": {