@indigoai-us/hq-cli 5.216.0 → 5.217.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -2,6 +2,14 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [5.217.0] — 2026-09-26
6
+
7
+ ### Fixed
8
+
9
+ - Keep-alive resumes a lane interrupted by `exited_with_partial_output` when
10
+ the interruption followed its explicit opt-in. Quota and host-loss
11
+ interruptions, login failures, and stopped lanes remain ineligible.
12
+
5
13
  ## [5.216.0] — 2026-09-26
6
14
 
7
15
  ### Fixed
@@ -5,6 +5,7 @@ export declare const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
5
5
  export declare const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS: number;
6
6
  export declare const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS: number;
7
7
  export declare const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
8
+ export declare const LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT = "Automatic keep-alive resume: your previous turn was interrupted by a provider process exit. Check your last state before continuing. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
8
9
  export type KeepAliveBlockedReason = NonNullable<LaneRecord["keep_alive_blocked_reason"]>;
9
10
  export declare function retiredLaneRegistryPath(hqRoot: string): string;
10
11
  /** Missing registry means no retired lanes; every present invalid file fails closed. */
@@ -7,6 +7,7 @@ export const LEAD_LANE_KEEP_ALIVE_MAX_RESUMES = 24;
7
7
  export const LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS = 5 * 60 * 1000;
8
8
  export const LEAD_LANE_KEEP_ALIVE_MAX_BACKOFF_MS = 60 * 60 * 1000;
9
9
  export const LEAD_LANE_KEEP_ALIVE_RESUME_TEXT = "Automatic keep-alive resume: your last turn ended with decision done and story_complete false. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
10
+ export const LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT = "Automatic keep-alive resume: your previous turn was interrupted by a provider process exit. Check your last state before continuing. Continue the open work from where you stopped. End with done only when story_complete is true; if you are waiting, use hq lanes ask or stay in the loop.";
10
11
  export function retiredLaneRegistryPath(hqRoot) {
11
12
  return path.join(hqRoot, "workspace", "lane-briefs", "juniors.json");
12
13
  }
@@ -78,6 +78,8 @@ export interface ReconcileDeps {
78
78
  mesh?: LaneMesh | null;
79
79
  /** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
80
80
  meshEmitTimeoutMs?: number;
81
+ /** Fresh keep-alive eligibility read; injected to exercise opt-in changes before queueing. */
82
+ readKeepAliveRecheckLane?: (hqRoot: string, laneId: string) => LaneRecord;
81
83
  /**
82
84
  * Awaited after lane/question records and liveness are loaded, before any
83
85
  * CAS write. Test seam so two in-process passes can share a snapshot.
@@ -21,11 +21,11 @@ import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecor
21
21
  import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
22
22
  import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
23
23
  import { readLanesRegistryFlag } from "./registry-flag.js";
24
- import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
24
+ import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, } from "./keep-alive.js";
25
25
  import { providerSupportsSessionResume } from "./delivery.js";
26
26
  import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
27
27
  import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
28
- import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, writeWatcherDecision, } from "./watcher.js";
28
+ import { SILENCE_MS, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
29
29
  import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
30
30
  import { laneStories, storyStatusForEnvelope } from "./stories.js";
31
31
  import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
@@ -800,7 +800,7 @@ export async function runPassBody(hqRoot, deps) {
800
800
  const keepAliveFlagCompanies = new Map();
801
801
  const keepAliveHireGrantFlagCompanies = new Map();
802
802
  for (const lane of lanes) {
803
- if (!(lane.state === "done" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
803
+ if (!(lane.state === "done" || lane.state === "interrupted" || lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") ||
804
804
  (lane.keep_alive !== true && lane.worker_profile === undefined) ||
805
805
  !laneRunsHere(lane, deps.host) ||
806
806
  (lane.state === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
@@ -838,7 +838,7 @@ export async function runPassBody(hqRoot, deps) {
838
838
  ]);
839
839
  await withAllLaneAdmissionLocks(hqRoot, async () => {
840
840
  const freshLanes = loadLaneRecords(hqRoot, summary);
841
- resumeDoneKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
841
+ resumeKeepAlive(hqRoot, freshLanes, summary, deps, now, keepAliveFlagByCompany, keepAliveHireGrantFlagByCompany);
842
842
  const drainLanes = loadLaneRecords(hqRoot, summary);
843
843
  const freshQuestions = loadQuestionRecords(hqRoot, summary);
844
844
  await admitQueued(hqRoot, drainLanes, summary, deps, now);
@@ -1059,6 +1059,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1059
1059
  state: repairedClassification.state,
1060
1060
  queued_reason: undefined,
1061
1061
  terminal_reason: repairedClassification.reason ?? "host_lost",
1062
+ interrupted_at: repairedClassification.state === "interrupted" ? nowIso : undefined,
1062
1063
  ...(providerRejection
1063
1064
  ? { terminal_details: providerRejectionDetails(providerRejection, current) }
1064
1065
  : {}),
@@ -1175,6 +1176,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1175
1176
  state: "interrupted",
1176
1177
  queued_reason: undefined,
1177
1178
  terminal_reason: "host_lost",
1179
+ interrupted_at: nowIso,
1178
1180
  }), deps.dryRun);
1179
1181
  if (result.changed) {
1180
1182
  summary.dead_host.push({
@@ -1392,7 +1394,8 @@ function keepAliveProgressResetByHuman(lane) {
1392
1394
  if (count > 0 && !Number.isFinite(lastAutoMs))
1393
1395
  return true;
1394
1396
  const messageMs = Date.parse(lane.last_message_at ?? "");
1395
- const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT
1397
+ const resumeMs = lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_RESUME_TEXT ||
1398
+ lane.last_resume?.text === LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
1396
1399
  ? Number.NaN
1397
1400
  : Date.parse(lane.last_resume?.at ?? "");
1398
1401
  return Number.isFinite(lastAutoMs) &&
@@ -1433,6 +1436,36 @@ function recordKeepAliveSkip(hqRoot, lane, reason, summary, deps, detail) {
1433
1436
  });
1434
1437
  }
1435
1438
  }
1439
+ function readLatestInterruptedClassificationAt(hqRoot, lane) {
1440
+ const logPath = watcherOutLogPath(hqRoot, lane.lane_id);
1441
+ let body;
1442
+ try {
1443
+ body = fs.readFileSync(logPath, "utf8");
1444
+ }
1445
+ catch (err) {
1446
+ if (err.code === "ENOENT")
1447
+ return undefined;
1448
+ throw err;
1449
+ }
1450
+ let latestClassification;
1451
+ for (const line of body.split(/\r?\n/).filter(Boolean)) {
1452
+ const parsed = JSON.parse(line);
1453
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
1454
+ throw new Error("watcher log contains a non-object decision record");
1455
+ }
1456
+ const record = parsed;
1457
+ if (record.event === "classification")
1458
+ latestClassification = record;
1459
+ }
1460
+ const at = latestClassification?.at;
1461
+ if (latestClassification?.state !== "interrupted" ||
1462
+ latestClassification.reason !== lane.terminal_reason ||
1463
+ typeof at !== "string" ||
1464
+ !Number.isFinite(Date.parse(at)) ||
1465
+ new Date(Date.parse(at)).toISOString() !== at)
1466
+ return undefined;
1467
+ return at;
1468
+ }
1436
1469
  function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
1437
1470
  const filePath = path.join(laneRunDir(hqRoot, lane.lane_id), ref);
1438
1471
  try {
@@ -1442,7 +1475,7 @@ function latestEnvelopeMtimeMs(hqRoot, lane, ref) {
1442
1475
  throw new LanesError("keep_alive_envelope_timestamp_unavailable", `Cannot stat latest envelope for lane ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: filePath });
1443
1476
  }
1444
1477
  }
1445
- function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
1478
+ function resumeKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, hireGrantFlagByCompany) {
1446
1479
  const nowMs = now.getTime();
1447
1480
  const nowIso = now.toISOString();
1448
1481
  const config = loadLanesConfig(hqRoot);
@@ -1456,34 +1489,97 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1456
1489
  markError(summary, err);
1457
1490
  }
1458
1491
  }
1459
- for (const lane of lanes) {
1460
- if (!laneRunsHere(lane, deps.host))
1492
+ for (const originalLane of lanes) {
1493
+ if (!laneRunsHere(originalLane, deps.host))
1461
1494
  continue;
1462
- if (lane.state === "killed" || lane.state === "cancelled" || lane.state === "failed") {
1463
- if (lane.keep_alive === true &&
1464
- flagByCompany.get(keepAliveCompanyKey(lane.company)) === true)
1465
- recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps);
1495
+ if (originalLane.state === "killed" ||
1496
+ originalLane.state === "cancelled" ||
1497
+ originalLane.state === "failed") {
1498
+ if (originalLane.keep_alive === true &&
1499
+ flagByCompany.get(keepAliveCompanyKey(originalLane.company)) === true)
1500
+ recordKeepAliveSkip(hqRoot, originalLane, "terminal_state", summary, deps);
1466
1501
  continue;
1467
1502
  }
1468
- if (lane.state !== "done" ||
1469
- (lane.keep_alive !== true && lane.worker_profile === undefined))
1503
+ if ((originalLane.state !== "done" && originalLane.state !== "interrupted") ||
1504
+ (originalLane.keep_alive !== true && originalLane.worker_profile === undefined))
1470
1505
  continue;
1471
- const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
1472
- const envelope = latest?.envelope;
1473
- if (!envelope || envelope.decision !== "done")
1474
- continue;
1475
- if (envelope.story_complete === true) {
1476
- if (hasKeepAliveProgress(lane)) {
1477
- casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
1478
- current.state === "done" &&
1479
- current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
1506
+ const initialState = originalLane.state;
1507
+ let lane = originalLane;
1508
+ if (initialState === "done") {
1509
+ const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
1510
+ if (!envelope || envelope.decision !== "done")
1511
+ continue;
1512
+ if (envelope.story_complete === true) {
1513
+ if (hasKeepAliveProgress(lane)) {
1514
+ casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
1515
+ current.state === "done" &&
1516
+ current.keep_alive === lane.keep_alive, (current) => ({ ...clearedKeepAliveProgress(current), keep_alive_blocked_reason: undefined }), deps.dryRun);
1517
+ }
1518
+ continue;
1480
1519
  }
1481
- continue;
1482
1520
  }
1483
1521
  if (flagByCompany.get(keepAliveCompanyKey(lane.company)) !== true)
1484
1522
  continue;
1485
- if (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
1486
- lane.terminal_reason === "account_not_logged_in") {
1523
+ // Persist a stable timestamp for legacy interrupted rows before a later
1524
+ // keep-alive status write can advance their general updated_at clock. The
1525
+ // watcher classification is the interruption time; updated_at is only an
1526
+ // upper bound and can postdate explicit opt-in.
1527
+ if (initialState === "interrupted" && lane.keep_alive === true && lane.interrupted_at === undefined) {
1528
+ let interruptedAt;
1529
+ try {
1530
+ interruptedAt = readLatestInterruptedClassificationAt(hqRoot, lane);
1531
+ }
1532
+ catch (err) {
1533
+ const detail = `watcher classification read failed: ${err instanceof Error ? err.message : String(err)}`;
1534
+ recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, detail);
1535
+ const skip = summary.skipped[summary.skipped.length - 1];
1536
+ if (skip) {
1537
+ summary.ok = false;
1538
+ logSkip(skip);
1539
+ }
1540
+ continue;
1541
+ }
1542
+ if (interruptedAt === undefined) {
1543
+ recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "no latest matching watcher classification event");
1544
+ const skip = summary.skipped[summary.skipped.length - 1];
1545
+ if (skip) {
1546
+ summary.ok = false;
1547
+ logSkip(skip);
1548
+ }
1549
+ continue;
1550
+ }
1551
+ try {
1552
+ const backfill = casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version &&
1553
+ current.state === "interrupted" &&
1554
+ current.keep_alive === true &&
1555
+ current.terminal_reason === lane.terminal_reason &&
1556
+ current.interrupted_at === undefined, (current) => ({ ...current, interrupted_at: interruptedAt }), deps.dryRun);
1557
+ lane = backfill.record;
1558
+ if (lane.interrupted_at === undefined) {
1559
+ recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps, "lane changed before watcher classification timestamp could be recorded");
1560
+ const skip = summary.skipped[summary.skipped.length - 1];
1561
+ if (skip) {
1562
+ summary.ok = false;
1563
+ logSkip(skip);
1564
+ }
1565
+ continue;
1566
+ }
1567
+ }
1568
+ catch (err) {
1569
+ markDirty(summary, {
1570
+ kind: "lane",
1571
+ id: lane.lane_id,
1572
+ why: `keep-alive interruption timestamp backfill failed: ${err instanceof Error ? err.message : String(err)}`,
1573
+ });
1574
+ continue;
1575
+ }
1576
+ }
1577
+ if (initialState === "interrupted" && lane.terminal_reason !== "exited_with_partial_output") {
1578
+ recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason ?? "missing interruption reason");
1579
+ continue;
1580
+ }
1581
+ if (initialState === "done" && (lane.terminal_reason === PROVIDER_REJECTED_REASON ||
1582
+ lane.terminal_reason === "account_not_logged_in")) {
1487
1583
  recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, lane.terminal_reason);
1488
1584
  continue;
1489
1585
  }
@@ -1496,20 +1592,35 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1496
1592
  recordKeepAliveSkip(hqRoot, lane, "missing_opt_in_time", summary, deps);
1497
1593
  continue;
1498
1594
  }
1499
- let envelopeAtMs;
1500
- try {
1501
- envelopeAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
1595
+ let eligibilityAtMs;
1596
+ if (initialState === "done") {
1597
+ const latest = readLatestEnvelopeRecord(hqRoot, lane.lane_id);
1598
+ if (!latest || latest.envelope.decision !== "done" || latest.envelope.story_complete === true)
1599
+ continue;
1600
+ try {
1601
+ eligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, lane, latest.ref);
1602
+ }
1603
+ catch (err) {
1604
+ markDirty(summary, {
1605
+ kind: "lane",
1606
+ id: lane.lane_id,
1607
+ why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
1608
+ });
1609
+ continue;
1610
+ }
1502
1611
  }
1503
- catch (err) {
1504
- markDirty(summary, {
1505
- kind: "lane",
1506
- id: lane.lane_id,
1507
- why: `keep-alive envelope timestamp unavailable: ${err instanceof Error ? err.message : String(err)}`,
1508
- });
1509
- continue;
1612
+ else {
1613
+ eligibilityAtMs = Date.parse(lane.interrupted_at ?? "");
1614
+ if (!Number.isFinite(eligibilityAtMs)) {
1615
+ recordKeepAliveSkip(hqRoot, lane, "missing_interruption_time", summary, deps);
1616
+ continue;
1617
+ }
1510
1618
  }
1511
- if (envelopeAtMs <= optedInAtMs) {
1512
- recordKeepAliveSkip(hqRoot, lane, "envelope_before_opt_in", summary, deps);
1619
+ const initialTimeBlockReason = initialState === "done"
1620
+ ? "envelope_before_opt_in"
1621
+ : "interruption_before_opt_in";
1622
+ if (eligibilityAtMs <= optedInAtMs) {
1623
+ recordKeepAliveSkip(hqRoot, lane, initialTimeBlockReason, summary, deps);
1513
1624
  continue;
1514
1625
  }
1515
1626
  if (retiredLaneIds === null) {
@@ -1521,33 +1632,54 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1521
1632
  recordKeepAliveSkip(hqRoot, lane, exclusion, summary, deps);
1522
1633
  continue;
1523
1634
  }
1635
+ const providerSessionId = lane.provider_session_id ?? lane.last_resume?.session_id;
1524
1636
  if (!providerSupportsSessionResume(lane.provider) ||
1525
- !(lane.provider_session_id ?? lane.last_resume?.session_id))
1637
+ typeof providerSessionId !== "string" ||
1638
+ providerSessionId.trim().length === 0) {
1639
+ if (initialState === "interrupted") {
1640
+ recordKeepAliveSkip(hqRoot, lane, "terminal_state", summary, deps, "provider session cannot be resumed");
1641
+ }
1526
1642
  continue;
1527
- const keepAliveStillEnabled = (candidate) => candidate.keep_alive === true &&
1528
- candidate.keep_alive_opted_in_at === lane.keep_alive_opted_in_at;
1529
- const current = readLane(hqRoot, lane.lane_id);
1530
- if (current.version !== lane.version ||
1531
- current.state !== "done" ||
1532
- !keepAliveStillEnabled(current) ||
1533
- !laneRunsHere(current, deps.host))
1643
+ }
1644
+ // This second read is intentionally independent of the pass snapshot. The
1645
+ // final timestamp comparison below closes the opt-in race before the CAS.
1646
+ const rechecked = deps.readKeepAliveRecheckLane?.(hqRoot, lane.lane_id) ?? readLane(hqRoot, lane.lane_id);
1647
+ // Dry-run backfills only in memory. Carry that simulated legacy timestamp
1648
+ // through the fresh read when the stored lane version has not changed.
1649
+ const current = deps.dryRun &&
1650
+ initialState === "interrupted" &&
1651
+ originalLane.interrupted_at === undefined &&
1652
+ lane.interrupted_at !== undefined &&
1653
+ rechecked.version === lane.version &&
1654
+ rechecked.interrupted_at === undefined
1655
+ ? { ...rechecked, interrupted_at: lane.interrupted_at }
1656
+ : rechecked;
1657
+ if (current.state !== initialState || !laneRunsHere(current, deps.host))
1534
1658
  continue;
1535
- const currentEnvelopeRecord = readLatestEnvelopeRecord(hqRoot, current.lane_id);
1536
- const currentEnvelope = currentEnvelopeRecord?.envelope;
1537
- if (!currentEnvelope || currentEnvelope.decision !== "done")
1659
+ if (current.keep_alive !== true) {
1660
+ recordKeepAliveSkip(hqRoot, current, "not_opted_in", summary, deps);
1538
1661
  continue;
1539
- if (currentEnvelope.story_complete === true) {
1540
- if (hasKeepAliveProgress(current)) {
1541
- casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1542
- latestLane.state === "done" &&
1543
- keepAliveStillEnabled(latestLane), (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
1544
- }
1662
+ }
1663
+ if (initialState === "interrupted" && current.terminal_reason !== "exited_with_partial_output") {
1664
+ recordKeepAliveSkip(hqRoot, current, "terminal_state", summary, deps, current.terminal_reason ?? "missing interruption reason");
1545
1665
  continue;
1546
1666
  }
1547
- if (currentEnvelopeRecord !== null) {
1548
- let currentEnvelopeAtMs;
1667
+ let currentEligibilityAtMs;
1668
+ if (initialState === "done") {
1669
+ const currentEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id);
1670
+ if (!currentEnvelope || currentEnvelope.envelope.decision !== "done")
1671
+ continue;
1672
+ if (currentEnvelope.envelope.story_complete === true) {
1673
+ if (hasKeepAliveProgress(current)) {
1674
+ casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1675
+ latestLane.state === "done" &&
1676
+ latestLane.keep_alive === true &&
1677
+ latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => ({ ...clearedKeepAliveProgress(latestLane), keep_alive_blocked_reason: undefined }), deps.dryRun);
1678
+ }
1679
+ continue;
1680
+ }
1549
1681
  try {
1550
- currentEnvelopeAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelopeRecord.ref);
1682
+ currentEligibilityAtMs = latestEnvelopeMtimeMs(hqRoot, current, currentEnvelope.ref);
1551
1683
  }
1552
1684
  catch (err) {
1553
1685
  markDirty(summary, {
@@ -1557,11 +1689,26 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1557
1689
  });
1558
1690
  continue;
1559
1691
  }
1560
- if (currentEnvelopeAtMs <= Date.parse(current.keep_alive_opted_in_at ?? "")) {
1561
- recordKeepAliveSkip(hqRoot, current, "envelope_before_opt_in", summary, deps);
1692
+ }
1693
+ else {
1694
+ currentEligibilityAtMs = Date.parse(current.interrupted_at ?? "");
1695
+ if (!Number.isFinite(currentEligibilityAtMs)) {
1696
+ recordKeepAliveSkip(hqRoot, current, "missing_interruption_time", summary, deps);
1562
1697
  continue;
1563
1698
  }
1564
1699
  }
1700
+ const currentOptedInAtMs = Date.parse(current.keep_alive_opted_in_at ?? "");
1701
+ if (!Number.isFinite(currentOptedInAtMs)) {
1702
+ recordKeepAliveSkip(hqRoot, current, "missing_opt_in_time", summary, deps);
1703
+ continue;
1704
+ }
1705
+ const currentTimeBlockReason = initialState === "done"
1706
+ ? "envelope_before_opt_in"
1707
+ : "interruption_before_opt_in";
1708
+ if (currentEligibilityAtMs <= currentOptedInAtMs) {
1709
+ recordKeepAliveSkip(hqRoot, current, currentTimeBlockReason, summary, deps);
1710
+ continue;
1711
+ }
1565
1712
  if (retiredLaneIds === null) {
1566
1713
  recordKeepAliveSkip(hqRoot, current, "retired_list_invalid", summary, deps);
1567
1714
  continue;
@@ -1572,11 +1719,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1572
1719
  recordKeepAliveSkip(hqRoot, current, currentExclusion, summary, deps);
1573
1720
  continue;
1574
1721
  }
1575
- const providerSessionId = current.provider_session_id ?? current.last_resume?.session_id;
1576
- if (current.terminal_reason === PROVIDER_REJECTED_REASON ||
1577
- current.terminal_reason === "account_not_logged_in" ||
1578
- typeof providerSessionId !== "string" ||
1579
- providerSessionId.trim().length === 0)
1722
+ const currentSessionId = current.provider_session_id ?? current.last_resume?.session_id;
1723
+ if (!providerSupportsSessionResume(current.provider) ||
1724
+ typeof currentSessionId !== "string" ||
1725
+ currentSessionId.trim().length === 0)
1580
1726
  continue;
1581
1727
  const resetByHuman = keepAliveProgressResetByHuman(current);
1582
1728
  const attempts = resetByHuman ? 0 : current.keep_alive_resume_count ?? 0;
@@ -1584,8 +1730,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1584
1730
  if (current.keep_alive_exhausted_at !== undefined)
1585
1731
  continue;
1586
1732
  const exhausted = casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1587
- latestLane.state === "done" &&
1588
- keepAliveStillEnabled(latestLane) &&
1733
+ latestLane.state === current.state &&
1734
+ latestLane.keep_alive === true &&
1735
+ latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
1589
1736
  (latestLane.keep_alive_resume_count ?? 0) >= LEAD_LANE_KEEP_ALIVE_MAX_RESUMES &&
1590
1737
  latestLane.keep_alive_exhausted_at === undefined, (latestLane) => ({ ...latestLane, keep_alive_exhausted_at: nowIso }), deps.dryRun);
1591
1738
  if (exhausted.changed && !deps.dryRun) {
@@ -1593,13 +1740,16 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1593
1740
  event: "keep_alive_exhausted",
1594
1741
  at: nowIso,
1595
1742
  attempt: attempts,
1743
+ resumed_from: current.state,
1596
1744
  });
1597
1745
  }
1598
1746
  continue;
1599
1747
  }
1600
1748
  const attempt = attempts + 1;
1601
- const doneAtMs = Date.parse(current.updated_at);
1602
- const attemptEligibleMs = doneAtMs + leadLaneKeepAliveBackoffMs(attempt);
1749
+ const terminalAtMs = initialState === "done"
1750
+ ? Date.parse(current.updated_at)
1751
+ : Date.parse(current.interrupted_at ?? "");
1752
+ const attemptEligibleMs = terminalAtMs + leadLaneKeepAliveBackoffMs(attempt);
1603
1753
  const storedNextMs = resetByHuman
1604
1754
  ? Number.NaN
1605
1755
  : Date.parse(current.keep_alive_next_eligible_at ?? "");
@@ -1609,8 +1759,9 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1609
1759
  if (nowMs < nextEligibleMs) {
1610
1760
  if (resetByHuman && hasKeepAliveProgress(current)) {
1611
1761
  casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1612
- latestLane.state === "done" &&
1613
- keepAliveStillEnabled(latestLane), (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
1762
+ latestLane.state === current.state &&
1763
+ latestLane.keep_alive === true &&
1764
+ latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at, (latestLane) => clearedKeepAliveProgress(latestLane), deps.dryRun);
1614
1765
  }
1615
1766
  continue;
1616
1767
  }
@@ -1628,9 +1779,14 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1628
1779
  });
1629
1780
  continue;
1630
1781
  }
1782
+ const resumedFrom = current.state;
1783
+ const resumeText = resumedFrom === "interrupted"
1784
+ ? LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT
1785
+ : LEAD_LANE_KEEP_ALIVE_RESUME_TEXT;
1631
1786
  const queue = (expectedSeniorId) => casLaneIf(hqRoot, current.lane_id, (latestLane) => latestLane.version === current.version &&
1632
- latestLane.state === "done" &&
1633
- keepAliveStillEnabled(latestLane) &&
1787
+ latestLane.state === current.state &&
1788
+ latestLane.keep_alive === true &&
1789
+ latestLane.keep_alive_opted_in_at === current.keep_alive_opted_in_at &&
1634
1790
  (expectedSeniorId === undefined ||
1635
1791
  (latestLane.senior.kind === "lane" && latestLane.senior.id === expectedSeniorId)), (latestLane) => ({
1636
1792
  ...latestLane,
@@ -1641,10 +1797,10 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1641
1797
  terminal_reason: undefined,
1642
1798
  terminal_details: undefined,
1643
1799
  pending_resume: {
1644
- session_id: providerSessionId,
1800
+ session_id: currentSessionId,
1645
1801
  trigger: "resume",
1646
- resumed_from: "done",
1647
- text: LEAD_LANE_KEEP_ALIVE_RESUME_TEXT,
1802
+ resumed_from: resumedFrom,
1803
+ text: resumeText,
1648
1804
  compact: true,
1649
1805
  at: nowIso,
1650
1806
  },
@@ -1693,7 +1849,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1693
1849
  continue;
1694
1850
  summary.resumed.push({
1695
1851
  id: current.lane_id,
1696
- from: "done",
1852
+ from: resumedFrom,
1697
1853
  to: "queued",
1698
1854
  reason: "keep_alive",
1699
1855
  });
@@ -1703,6 +1859,7 @@ function resumeDoneKeepAlive(hqRoot, lanes, summary, deps, now, flagByCompany, h
1703
1859
  at: nowIso,
1704
1860
  attempt,
1705
1861
  next_eligible_at: nextEligibleAt,
1862
+ resumed_from: resumedFrom,
1706
1863
  });
1707
1864
  }
1708
1865
  }
@@ -414,6 +414,8 @@ export function parseLaneRecord(raw, filePath) {
414
414
  "not_opted_in",
415
415
  "missing_opt_in_time",
416
416
  "envelope_before_opt_in",
417
+ "interruption_before_opt_in",
418
+ "missing_interruption_time",
417
419
  "retired",
418
420
  "superseded",
419
421
  "retired_list_invalid",
@@ -443,6 +445,7 @@ export function parseLaneRecord(raw, filePath) {
443
445
  "keep_alive_last_resume_at",
444
446
  "keep_alive_next_eligible_at",
445
447
  "keep_alive_exhausted_at",
448
+ "interrupted_at",
446
449
  ]) {
447
450
  if (raw[field] !== undefined)
448
451
  requireTimestamp(raw[field], field, filePath);
@@ -359,7 +359,7 @@ export interface LaneRecord {
359
359
  /** Time this lane most recently explicitly opted into keep-alive. */
360
360
  keep_alive_opted_in_at?: string;
361
361
  /** Why reconcile declined to resume this lane most recently. */
362
- keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
362
+ keep_alive_blocked_reason?: "not_opted_in" | "missing_opt_in_time" | "envelope_before_opt_in" | "interruption_before_opt_in" | "missing_interruption_time" | "retired" | "superseded" | "retired_list_invalid" | "terminal_state";
363
363
  /** Consecutive automatic keep-alive resumes since the last human reset. */
364
364
  keep_alive_resume_count?: number;
365
365
  /** Time the most recent automatic keep-alive resume was queued. */
@@ -451,6 +451,8 @@ export interface LaneRecord {
451
451
  /** Set when the reconcile pass force-admits under the starvation bound. */
452
452
  admitted_reason?: string;
453
453
  terminal_reason?: string;
454
+ /** Classification time when this lane entered interrupted; stable across later metadata writes. */
455
+ interrupted_at?: string;
454
456
  /**
455
457
  * Structured detail behind a named `terminal_reason` (dispatch, pickup and
456
458
  * spawn failures), the missing-binary search path behind
@@ -230,7 +230,7 @@ export const LANE_FIELD_WRITERS = {
230
230
  last_envelope: ["watcher", "reconcile"], model: ["creator", "spawn", "command"], model_source: ["creator", "spawn", "command"], effort: ["creator", "spawn", "command"], effort_source: ["creator", "spawn", "command"], fast_mode: ["spawn"],
231
231
  last_message_at: ["command"], last_compaction: ["spawn"], story_complete: ["watcher", "reconcile"], story_complete_reason: ["watcher", "reconcile"], story_status: ["watcher", "reconcile"],
232
232
  overdue: ["reconcile"], overdue_at: ["reconcile"], last_host_report_at: ["watcher", "spawn", "reconcile"], stalled: ["watcher"], stalled_at: ["watcher"], stalled_reason: ["watcher"],
233
- admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
233
+ admitted_reason: ["command", "reconcile"], terminal_reason: ["command", "watcher", "spawn", "reconcile"], interrupted_at: ["watcher", "reconcile"], terminal_details: ["command", "spawn", "watcher", "reconcile"],
234
234
  quota_reset_at: ["watcher", "reconcile"], model_quota_marker: ["watcher", "reconcile"], outpost_command_id: ["command"],
235
235
  provider_session_id: ["watcher", "command", "spawn"], provider_session_id_absent: ["watcher", "command"], provider_home_at_spawn: ["spawn"], last_resume: ["spawn"], last_run_usage: ["watcher"], envelope_repair: ["watcher", "reconcile"], envelope_validation_errors: ["watcher"], last_progress_event_at: ["watcher"],
236
236
  };
@@ -336,6 +336,7 @@ export function applyTerminalToLaneWithResult(hqRoot, laneId, classification, op
336
336
  stalled: undefined,
337
337
  stalled_at: undefined,
338
338
  stalled_reason: undefined,
339
+ interrupted_at: classification.state === "interrupted" ? at : undefined,
339
340
  envelope_validation_errors: opts.envelopeValidationErrors,
340
341
  };
341
342
  changed = true;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@indigoai-us/hq-cli",
3
- "version": "5.216.0",
3
+ "version": "5.217.0",
4
4
  "description": "HQ by Indigo management CLI — modules and cloud sync",
5
5
  "main": "dist/index.js",
6
6
  "bin": {