@indigoai-us/hq-cli 5.342.11 → 5.342.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/dist/command-catalog.generated.d.ts +1 -1
  3. package/dist/command-catalog.generated.js +1 -1
  4. package/dist/commands/agent-kit.d.ts +2 -0
  5. package/dist/commands/agent-kit.js +23 -17
  6. package/dist/commands/agent-mcp.d.ts +2 -0
  7. package/dist/commands/agent-mcp.js +21 -15
  8. package/dist/commands/lanes-keep-alive-audit.js +1 -1
  9. package/dist/commands/lanes-message.d.ts +20 -2
  10. package/dist/commands/lanes-message.js +99 -38
  11. package/dist/commands/lanes-shared.d.ts +1 -0
  12. package/dist/commands/lanes.d.ts +4 -0
  13. package/dist/commands/lanes.js +24 -30
  14. package/dist/commands/mesh.js +2 -1
  15. package/dist/commands/monitor.d.ts +14 -1
  16. package/dist/commands/monitor.js +43 -14
  17. package/dist/lib/doctor/checks/anywhere.js +11 -6
  18. package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
  19. package/dist/lib/flag-registry-worker-source.generated.js +1 -1
  20. package/dist/lib/lanes/keep-alive-audit.d.ts +5 -1
  21. package/dist/lib/lanes/keep-alive-audit.js +132 -19
  22. package/dist/lib/lanes/reconcile.js +46 -3
  23. package/dist/lib/lanes/spawn.js +79 -0
  24. package/dist/lib/lanes/types.d.ts +2 -1
  25. package/dist/utils/package-use-lease.d.ts +15 -0
  26. package/dist/utils/package-use-lease.js +94 -1
  27. package/dist/utils/resident-package-preload.d.ts +8 -1
  28. package/dist/utils/resident-package-preload.js +68 -0
  29. package/dist/utils/self-update.d.ts +5 -0
  30. package/dist/utils/self-update.js +100 -12
  31. package/docs/anywhere-runtime.md +3 -1
  32. package/package.json +4 -4
  33. package/dist/lib/db/remote-engine.d.ts +0 -58
  34. package/dist/lib/db/remote-engine.js +0 -87
  35. package/dist/lib/db/remote-sql.d.ts +0 -22
  36. package/dist/lib/db/remote-sql.js +0 -36
@@ -1,8 +1,9 @@
1
- type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "keep_alive_off" | "waiting";
1
+ type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "keep_alive_off" | "waiting";
2
2
  interface KeepAliveAuditRestart {
3
3
  at: string | null;
4
4
  trigger: string;
5
5
  caller: string | null;
6
+ workerEvent?: boolean;
6
7
  }
7
8
  interface KeepAliveAuditRow {
8
9
  lane: string;
@@ -13,6 +14,8 @@ interface KeepAliveAuditRow {
13
14
  due_at: string;
14
15
  due_reason: "next_check_at" | "backoff";
15
16
  restart: KeepAliveAuditRestart | null;
17
+ blocked_outcome?: "woken" | "skipped";
18
+ blocked_outcome_reason?: string;
16
19
  verdict: KeepAliveAuditVerdict;
17
20
  }
18
21
  interface KeepAliveAuditSummary {
@@ -41,6 +44,7 @@ export interface KeepAliveAuditResult {
41
44
  export interface StopEnvelope {
42
45
  name: string;
43
46
  mtimeMs: number;
47
+ decision: "done" | "blocked";
44
48
  nextCheckAt?: string;
45
49
  }
46
50
  export declare function readStopEnvelopes(hqRoot: string, laneId: string): {
@@ -1,11 +1,12 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
- import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS } from "./keep-alive.js";
3
+ import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS } from "./keep-alive.js";
4
4
  import { LanesError } from "./errors.js";
5
5
  import { laneRunDir } from "./spawn.js";
6
6
  import { compareEnvelopeNames } from "./envelope-sequence.js";
7
7
  import { listQuestionIds, readAllLanes, tryReadQuestion } from "./store.js";
8
- function readDoneClassifications(hqRoot, laneId) {
8
+ import { isIsoDateTime } from "./types.js";
9
+ function readTerminalClassifications(hqRoot, laneId) {
9
10
  const root = fs.realpathSync(hqRoot);
10
11
  const watcherLog = path.join(laneRunDir(root, laneId), "logs", "watcher.out");
11
12
  if (!withinRoot(root, watcherLog)) {
@@ -28,23 +29,23 @@ function readDoneClassifications(hqRoot, laneId) {
28
29
  return [];
29
30
  throw err;
30
31
  }
31
- const atTimes = [];
32
+ const classifications = [];
32
33
  for (const line of text.split(/\r?\n/)) {
33
34
  if (!line.includes('"classification"'))
34
35
  continue;
35
36
  try {
36
37
  const value = JSON.parse(line);
37
- if (!isRecord(value) || value.state !== "done")
38
+ if (!isRecord(value) || (value.state !== "done" && value.state !== "blocked"))
38
39
  continue;
39
40
  const at = parseTime(value.at);
40
41
  if (at !== undefined)
41
- atTimes.push(at);
42
+ classifications.push({ at, state: value.state });
42
43
  }
43
44
  catch (err) {
44
45
  throw new LanesError("keep_alive_audit_event_invalid", `Cannot parse lane watcher classification in ${watcherLog}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId, path: watcherLog });
45
46
  }
46
47
  }
47
- return atTimes.sort((a, b) => a - b);
48
+ return classifications.sort((a, b) => a.at - b.at);
48
49
  }
49
50
  const KEEP_ALIVE_ON_TIME_TOLERANCE_MS = 60_000;
50
51
  const KEEP_ALIVE_NOT_RESTARTED_GRACE_MS = 5 * 60_000;
@@ -55,6 +56,7 @@ const VERDICTS = [
55
56
  "late",
56
57
  "early_wake",
57
58
  "message_wake",
59
+ "work_triggered",
58
60
  "self_wake",
59
61
  "early_by_keep_alive",
60
62
  "restarted_untimed",
@@ -68,6 +70,50 @@ function parseTime(value) {
68
70
  const parsed = Date.parse(value);
69
71
  return Number.isFinite(parsed) ? parsed : undefined;
70
72
  }
73
+ function readLaneWorkerResumeEvents(hqRoot, laneId) {
74
+ const root = fs.realpathSync(hqRoot);
75
+ const eventsPath = path.join(laneRunDir(root, laneId), "watcher", "events.jsonl");
76
+ if (!withinRoot(root, eventsPath)) {
77
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher event path escapes the HQ root for ${laneId}`, { lane_id: laneId });
78
+ }
79
+ let text;
80
+ try {
81
+ const stat = fs.lstatSync(eventsPath);
82
+ if (!stat.isFile() || stat.isSymbolicLink()) {
83
+ throw new LanesError("keep_alive_audit_event_invalid", `Lane watcher events must be a regular file: ${eventsPath}`, { lane_id: laneId, path: eventsPath });
84
+ }
85
+ const realPath = fs.realpathSync(eventsPath);
86
+ if (!withinRoot(root, realPath)) {
87
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher event file escapes the HQ root for ${laneId}`, { lane_id: laneId, path: eventsPath });
88
+ }
89
+ text = fs.readFileSync(realPath, "utf8");
90
+ }
91
+ catch (err) {
92
+ if (err.code === "ENOENT")
93
+ return [];
94
+ throw err;
95
+ }
96
+ const events = [];
97
+ for (const [index, line] of text.split(/\r?\n/).entries()) {
98
+ if (!line.trim())
99
+ continue;
100
+ let value;
101
+ try {
102
+ value = JSON.parse(line);
103
+ }
104
+ catch {
105
+ continue;
106
+ }
107
+ if (!isRecord(value) || value.event !== "lane_worker_resumed")
108
+ continue;
109
+ const at = parseTime(value.resumed_at);
110
+ if (at === undefined || typeof value.reason !== "string" || !value.reason.trim()) {
111
+ throw new LanesError("keep_alive_audit_event_invalid", `Lane worker resume event has invalid timing or reason: ${eventsPath}:${index + 1}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
112
+ }
113
+ events.push({ at, reason: value.reason, caller: typeof value.caller === "string" ? value.caller : null });
114
+ }
115
+ return events.sort((a, b) => a.at - b.at);
116
+ }
71
117
  function iso(ms) {
72
118
  return new Date(ms).toISOString();
73
119
  }
@@ -116,11 +162,15 @@ export function readStopEnvelopes(hqRoot, laneId) {
116
162
  if (!isRecord(raw) || raw.schema_version !== "hq-lane-envelope.v1") {
117
163
  throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope is not hq-lane-envelope.v1: ${file}`, { lane_id: laneId, path: file });
118
164
  }
119
- if (raw.decision !== "done" || raw.story_complete !== false)
165
+ const doneStop = raw.decision === "done" && raw.story_complete === false;
166
+ const blockedStop = raw.decision === "blocked" && typeof raw.next_check_at === "string" &&
167
+ isIsoDateTime(raw.next_check_at) && Date.parse(raw.next_check_at) > stat.mtimeMs;
168
+ if (!doneStop && !blockedStop)
120
169
  continue;
121
170
  stops.push({
122
171
  name,
123
172
  mtimeMs: stat.mtimeMs,
173
+ decision: blockedStop ? "blocked" : "done",
124
174
  ...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
125
175
  });
126
176
  }
@@ -176,19 +226,20 @@ function readKeepAliveResumeEvents(hqRoot, laneId) {
176
226
  if (nextEligibleAt === undefined) {
177
227
  throw new LanesError("keep_alive_audit_event_invalid", `Keep-alive resume event has invalid timing fields: ${eventsPath}:${index + 1}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
178
228
  }
179
- const reason = value.reason === "pending_work" || value.reason === "next_check_at_due" || value.reason === "backoff"
180
- ? value.reason
181
- : undefined;
229
+ const reason = typeof value.reason === "string" ? value.reason : undefined;
182
230
  events.push({ at, nextEligibleAt, ...(reason === undefined ? {} : { reason }) });
183
231
  }
184
232
  return events.sort((a, b) => a.at - b.at);
185
233
  }
186
234
  function dueTime(stop, resumeEvents) {
235
+ const nextCheck = parseTime(stop.nextCheckAt);
236
+ if (stop.decision === "blocked" && nextCheck !== undefined && nextCheck > stop.mtimeMs) {
237
+ return { at: Math.min(nextCheck, stop.mtimeMs + LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS), reason: "next_check_at" };
238
+ }
187
239
  const previousResume = resumeEvents.filter((event) => event.at <= stop.mtimeMs).at(-1);
188
240
  const terminalPlusBackoff = stop.mtimeMs + LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS;
189
241
  let due = Math.max(terminalPlusBackoff, previousResume?.nextEligibleAt ?? 0);
190
242
  let reason = "backoff";
191
- const nextCheck = parseTime(stop.nextCheckAt);
192
243
  if (nextCheck !== undefined && nextCheck > stop.mtimeMs) {
193
244
  due = Math.max(due, Math.min(nextCheck, stop.mtimeMs + MAX_NEXT_CHECK_DELAY_MS));
194
245
  reason = "next_check_at";
@@ -286,7 +337,50 @@ function laneWasKeepAliveOffAtStop(lane, stoppedAt) {
286
337
  const optedInAt = parseTime(lane.keep_alive_opted_in_at);
287
338
  return optedInAt === undefined || optedInAt > stoppedAt;
288
339
  }
289
- function firstRestartAfter(lane, stop, nextEnvelopeMs) {
340
+ function blockedKeepAliveOutcome(lane, stop, nextEnvelopeMs) {
341
+ const terminalAt = lane.last_envelope?.ref === `envelopes/${stop.name}`
342
+ ? parseTime(lane.last_envelope.at)
343
+ : undefined;
344
+ const candidates = (lane.status_gate_history ?? []).flatMap((entry) => {
345
+ if (entry.event !== "keep_alive_blocked_resumed" && entry.event !== "keep_alive_blocked_skipped")
346
+ return [];
347
+ const at = parseTime(entry.at);
348
+ const terminal = parseTime(entry.terminal_at);
349
+ if (at === undefined || terminal === undefined)
350
+ return [];
351
+ if (nextEnvelopeMs !== undefined && at >= nextEnvelopeMs)
352
+ return [];
353
+ const sameStop = terminalAt !== undefined
354
+ ? terminal === terminalAt
355
+ : Math.abs(terminal - stop.mtimeMs) <= 10 * 60_000;
356
+ if (!sameStop)
357
+ return [];
358
+ if (entry.event === "keep_alive_blocked_skipped" && entry.reason !== "flag_off")
359
+ return [];
360
+ return [{
361
+ status: entry.event === "keep_alive_blocked_resumed" ? "woken" : "skipped",
362
+ ...(entry.reason === undefined ? {} : { reason: entry.reason }),
363
+ at: entry.at,
364
+ atMs: at,
365
+ }];
366
+ });
367
+ const latest = candidates.sort((left, right) => right.atMs - left.atMs)[0];
368
+ if (!latest)
369
+ return undefined;
370
+ return { status: latest.status, ...(latest.reason === undefined ? {} : { reason: latest.reason }), at: latest.at };
371
+ }
372
+ function firstRestartAfter(lane, stop, nextEnvelopeMs, workerResumeEvents) {
373
+ const recordedWorkerResume = workerResumeEvents
374
+ .filter((event) => event.at > stop.mtimeMs && (nextEnvelopeMs === undefined || event.at < nextEnvelopeMs))
375
+ .sort((a, b) => a.at - b.at)[0];
376
+ if (recordedWorkerResume !== undefined) {
377
+ return {
378
+ at: iso(recordedWorkerResume.at),
379
+ trigger: recordedWorkerResume.reason,
380
+ caller: recordedWorkerResume.caller,
381
+ workerEvent: true,
382
+ };
383
+ }
290
384
  const restart = (lane.resume_history ?? [])
291
385
  .map((entry) => ({ entry, at: parseTime(entry.at) }))
292
386
  .filter(({ entry, at }) => at !== undefined && at > stop.mtimeMs && (nextEnvelopeMs === undefined || at < nextEnvelopeMs) && typeof entry.trigger === "string")
@@ -329,6 +423,14 @@ function classifyVerdict(input) {
329
423
  if (restart?.trigger === "unknown" && laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
330
424
  return "keep_alive_off";
331
425
  if (restart) {
426
+ if (restart.trigger === "pending_work" || restart.trigger === "watcher" || restart.trigger === "monitor_event")
427
+ return "work_triggered";
428
+ const workEvidenceEnd = input.nextEnvelopeAt ?? Number.POSITIVE_INFINITY;
429
+ const triggeredWork = restart.workerEvent !== true && input.resumeEvents
430
+ .filter((event) => event.at > input.stoppedAt && event.at < workEvidenceEnd && event.at <= (restart.at === null ? Number.POSITIVE_INFINITY : Date.parse(restart.at)))
431
+ .some((event) => event.reason === "pending_work" || event.reason === "watcher" || event.reason === "monitor_event");
432
+ if (triggeredWork)
433
+ return "work_triggered";
332
434
  if (restart.at === null)
333
435
  return "restarted_untimed";
334
436
  const restartAt = Date.parse(restart.at);
@@ -336,12 +438,14 @@ function classifyVerdict(input) {
336
438
  return "self_wake";
337
439
  if (restart.trigger === "message" || restart.trigger === "answered_questions")
338
440
  return "message_wake";
339
- if (restart.trigger === "keep_alive") {
441
+ if (restart.trigger === "keep_alive" || restart.trigger === "next_check_at_due" || restart.trigger === "backoff") {
340
442
  const event = input.resumeEvents
341
443
  .filter((candidate) => candidate.at > input.stoppedAt && candidate.at <= restartAt + 2 * 60_000)
342
444
  .sort((left, right) => Math.abs(left.at - restartAt) - Math.abs(right.at - restartAt))[0];
343
445
  if (event?.reason === "pending_work")
344
- return "message_wake";
446
+ return "work_triggered";
447
+ if (event?.reason === "watcher" || event?.reason === "monitor_event")
448
+ return "work_triggered";
345
449
  if (event?.reason === undefined &&
346
450
  (hasDeliveredDropboxEvidence(input.hqRoot, lane.lane_id, input.stoppedAt, restartAt) ||
347
451
  hasMessageResumeEvidence(lane, input.stoppedAt, restartAt) ||
@@ -384,15 +488,19 @@ export function buildKeepAliveAudit(input) {
384
488
  try {
385
489
  const { allMtimes, stops } = readStopEnvelopes(input.hqRoot, lane.lane_id);
386
490
  const keepAliveResumeEvents = readKeepAliveResumeEvents(input.hqRoot, lane.lane_id);
387
- const doneClassifications = readDoneClassifications(input.hqRoot, lane.lane_id);
491
+ const laneWorkerResumeEvents = readLaneWorkerResumeEvents(input.hqRoot, lane.lane_id);
492
+ const terminalClassifications = readTerminalClassifications(input.hqRoot, lane.lane_id);
388
493
  for (const stop of stops) {
389
494
  if (stop.mtimeMs < sinceMs || stop.mtimeMs > nowMs)
390
495
  continue;
391
- if (!doneClassifications.some((at) => at >= stop.mtimeMs && at - stop.mtimeMs <= 10 * 60_000))
496
+ if (!terminalClassifications.some((classification) => classification.state === stop.decision && classification.at >= stop.mtimeMs && classification.at - stop.mtimeMs <= 10 * 60_000))
392
497
  continue;
393
498
  const nextEnvelopeMs = allMtimes.find((mtime) => mtime > stop.mtimeMs);
394
- const restart = firstRestartAfter(lane, stop, nextEnvelopeMs);
499
+ const blockedOutcome = stop.decision === "blocked" ? blockedKeepAliveOutcome(lane, stop, nextEnvelopeMs) : undefined;
500
+ const restart = firstRestartAfter(lane, stop, nextEnvelopeMs, laneWorkerResumeEvents);
395
501
  const due = dueTime(stop, keepAliveResumeEvents);
502
+ const reportBlockedOutcome = blockedOutcome?.status === "woken" ||
503
+ (blockedOutcome?.status === "skipped" && due.at <= nowMs);
396
504
  rows.push({
397
505
  lane: lane.lane_id,
398
506
  project: lane.project_id,
@@ -402,7 +510,11 @@ export function buildKeepAliveAudit(input) {
402
510
  due_at: iso(due.at),
403
511
  due_reason: due.reason,
404
512
  restart,
405
- verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, lane, nowMs, resumeEvents: keepAliveResumeEvents, hqRoot: input.hqRoot }),
513
+ ...(reportBlockedOutcome && blockedOutcome ? {
514
+ blocked_outcome: blockedOutcome.status,
515
+ ...(blockedOutcome.reason === undefined ? {} : { blocked_outcome_reason: blockedOutcome.reason }),
516
+ } : {}),
517
+ verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
406
518
  });
407
519
  }
408
520
  }
@@ -449,7 +561,7 @@ export function buildKeepAliveAudit(input) {
449
561
  };
450
562
  }
451
563
  export function formatKeepAliveAuditText(result) {
452
- const headings = ["LANE", "PROJECT", "PROFILE", "STOPPED", "NEXT_CHECK", "DUE", "RESTART", "VERDICT"];
564
+ const headings = ["LANE", "PROJECT", "PROFILE", "STOPPED", "NEXT_CHECK", "DUE", "RESTART", "OUTCOME", "VERDICT"];
453
565
  const values = result.rows.map((row) => [
454
566
  row.lane,
455
567
  row.project,
@@ -458,6 +570,7 @@ export function formatKeepAliveAuditText(result) {
458
570
  row.next_check_at ?? "—",
459
571
  row.due_at,
460
572
  row.restart ? `${row.restart.at ?? "untimed"} ${row.restart.trigger}${row.restart.caller ? ` (${row.restart.caller})` : ""}` : "—",
573
+ row.blocked_outcome === "woken" ? "woken" : row.blocked_outcome === "skipped" ? `due, not woken: ${(row.blocked_outcome_reason ?? "unknown").replaceAll("_", " ")}` : "—",
461
574
  row.verdict,
462
575
  ]);
463
576
  const widths = headings.map((heading, index) => Math.max(heading.length, ...values.map((row) => row[index].length)));
@@ -27,8 +27,9 @@ import { checkLaneDiskSpace, configuredCodexRolloutHomes, lowDiskGuardEnabledFor
27
27
  import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
28
28
  import { appendDropboxMessage } from "./dropbox.js";
29
29
  import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
30
- import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
30
+ import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveNextCheckAtMs, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
31
31
  import { providerSupportsSessionResume } from "./delivery.js";
32
+ import { isIsoDateTime } from "./types.js";
32
33
  import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, readQuotaSignal, recordAccountQuota, } from "./quota.js";
33
34
  import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
34
35
  import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
@@ -2252,6 +2253,9 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
2252
2253
  }
2253
2254
  const key = keepAliveCompanyKey(company);
2254
2255
  if (keepAliveFlags.get(key) !== true || blockedFlags.get(key) !== true) {
2256
+ if (keepAliveFlags.get(key) === true && blockedFlags.get(key) !== true) {
2257
+ recordBlockedKeepAliveFlagOff(hqRoot, lane, summary, deps);
2258
+ }
2255
2259
  summary.skipped.push({ kind: "lane", id: lane.lane_id, why: "keep_alive_blocked" });
2256
2260
  return;
2257
2261
  }
@@ -2323,10 +2327,22 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
2323
2327
  summary.skipped.push({ kind: "lane", id: lane.lane_id, why: "keep_alive_exhausted" });
2324
2328
  return;
2325
2329
  }
2326
- if (now.getTime() - terminalMs < LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS) {
2330
+ const blockedEnvelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
2331
+ const nextCheckValue = blockedEnvelope?.decision === "blocked" ? blockedEnvelope.next_check_at : undefined;
2332
+ const nextCheckValid = typeof nextCheckValue === "string" && isIsoDateTime(nextCheckValue) &&
2333
+ Date.parse(nextCheckValue) > terminalMs;
2334
+ const nextCheckMs = nextCheckValid
2335
+ ? keepAliveNextCheckAtMs(nextCheckValue, now.getTime(), terminalMs)
2336
+ : undefined;
2337
+ if (nextCheckMs !== undefined && nextCheckMs > now.getTime()) {
2338
+ appendBlockedKeepAliveHistory(hqRoot, lane, terminalAt, now.toISOString(), "blocked_grace", deps);
2339
+ return;
2340
+ }
2341
+ if (!nextCheckValid && now.getTime() - terminalMs < LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS) {
2327
2342
  appendBlockedKeepAliveHistory(hqRoot, lane, terminalAt, now.toISOString(), "blocked_grace", deps);
2328
2343
  return;
2329
2344
  }
2345
+ const resumeReason = nextCheckValid ? "next_check_at_due" : "backoff";
2330
2346
  const latest = deps.readKeepAliveRecheckLane?.(hqRoot, lane.lane_id) ?? readLane(hqRoot, lane.lane_id);
2331
2347
  if (latest.version !== lane.version || latest.state !== "blocked" || latest.last_envelope?.decision !== "blocked" || latest.last_envelope.at !== terminalAt)
2332
2348
  return;
@@ -2389,6 +2405,7 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
2389
2405
  at: now.toISOString(),
2390
2406
  terminal_at: terminalAt,
2391
2407
  resume_queued: true,
2408
+ reason: resumeReason,
2392
2409
  }),
2393
2410
  }), deps.dryRun);
2394
2411
  if (queued.changed) {
@@ -2399,7 +2416,7 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
2399
2416
  at: now.toISOString(),
2400
2417
  attempt: (lane.keep_alive_resume_count ?? 0) + 1,
2401
2418
  resumed_from: "blocked",
2402
- reason: "backoff",
2419
+ reason: resumeReason,
2403
2420
  ...(accountSelection !== undefined
2404
2421
  ? { account_selection: keepAliveAccountSelectionEvidence(accountSelection) }
2405
2422
  : {}),
@@ -2407,6 +2424,32 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
2407
2424
  }
2408
2425
  }
2409
2426
  }
2427
+ function recordBlockedKeepAliveFlagOff(hqRoot, lane, summary, deps) {
2428
+ const terminalAt = lane.last_envelope?.at;
2429
+ if (deps.dryRun || lane.last_envelope?.decision !== "blocked" || terminalAt === undefined)
2430
+ return;
2431
+ const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
2432
+ const nextCheckAt = envelope?.decision === "blocked" ? envelope.next_check_at : undefined;
2433
+ const terminalMs = Date.parse(lane.last_envelope.at);
2434
+ if (typeof nextCheckAt !== "string" || !isIsoDateTime(nextCheckAt) || Date.parse(nextCheckAt) <= terminalMs)
2435
+ return;
2436
+ if (lane.status_gate_history?.some((entry) => entry.event === "keep_alive_blocked_skipped" && entry.terminal_at === terminalAt && entry.reason === "flag_off"))
2437
+ return;
2438
+ try {
2439
+ casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version && current.state === "blocked" && current.last_envelope?.at === terminalAt, (current) => ({
2440
+ ...current,
2441
+ status_gate_history: appendCappedHistory(current.status_gate_history ?? [], {
2442
+ event: "keep_alive_blocked_skipped",
2443
+ at: deps.now().toISOString(),
2444
+ terminal_at: terminalAt,
2445
+ reason: "flag_off",
2446
+ }),
2447
+ }), false);
2448
+ }
2449
+ catch (err) {
2450
+ markDirty(summary, { kind: "lane", id: lane.lane_id, why: `blocked keep-alive skip record failed: ${err instanceof Error ? err.message : String(err)}` });
2451
+ }
2452
+ }
2410
2453
  function appendBlockedKeepAliveHistory(hqRoot, lane, terminalAt, at, reason, deps) {
2411
2454
  if (deps.dryRun || lane.status_gate_history?.some((entry) => entry.event === "keep_alive_blocked_grace" && entry.terminal_at === terminalAt && entry.resume_skipped_reason === reason))
2412
2455
  return;
@@ -56,6 +56,70 @@ const PROVIDER_COMMAND = {
56
56
  codex: "codex",
57
57
  grok: "grok",
58
58
  };
59
+ function keepAliveResumeReason(runDir, pendingResume) {
60
+ if (pendingResume?.keep_alive !== true || typeof pendingResume.at !== "string")
61
+ return undefined;
62
+ const eventsPath = path.join(runDir, "watcher", "events.jsonl");
63
+ let content;
64
+ try {
65
+ content = fs.readFileSync(eventsPath, "utf8");
66
+ }
67
+ catch (err) {
68
+ if (err.code === "ENOENT")
69
+ return "keep_alive";
70
+ throw err;
71
+ }
72
+ for (const line of content.split(/\r?\n/).reverse()) {
73
+ if (!line.trim())
74
+ continue;
75
+ let value;
76
+ try {
77
+ value = JSON.parse(line);
78
+ }
79
+ catch {
80
+ continue;
81
+ }
82
+ if (typeof value === "object" && value !== null && !Array.isArray(value)) {
83
+ const event = value;
84
+ if (event.event === "keep_alive_resume" && event.at === pendingResume.at && typeof event.reason === "string") {
85
+ return event.reason;
86
+ }
87
+ }
88
+ }
89
+ return "keep_alive";
90
+ }
91
+ function appendLaneWorkerResumeEvent(input) {
92
+ const eventsPath = path.join(laneRunDir(input.hqRoot, input.laneId), "watcher", "events.jsonl");
93
+ fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
94
+ let separator = "";
95
+ try {
96
+ const fd = fs.openSync(eventsPath, "r");
97
+ try {
98
+ const size = fs.fstatSync(fd).size;
99
+ if (size > 0) {
100
+ const lastByte = Buffer.alloc(1);
101
+ fs.readSync(fd, lastByte, 0, 1, size - 1);
102
+ if (lastByte[0] !== 0x0a)
103
+ separator = "\n";
104
+ }
105
+ }
106
+ finally {
107
+ fs.closeSync(fd);
108
+ }
109
+ }
110
+ catch (err) {
111
+ if (err.code !== "ENOENT")
112
+ throw err;
113
+ }
114
+ fs.appendFileSync(eventsPath, `${separator}${JSON.stringify({
115
+ event: "lane_worker_resumed",
116
+ lane_id: input.laneId,
117
+ resumed_at: input.resumedAt,
118
+ reason: input.reason,
119
+ ...(input.caller === undefined ? {} : { caller: input.caller }),
120
+ ...(input.turnId === undefined ? {} : { turn_id: input.turnId }),
121
+ })}\n`, "utf8");
122
+ }
59
123
  export function providerCommandOnPath(command, pathEnv) {
60
124
  // Test and embedded callers can intentionally omit PATH and delegate lookup
61
125
  // to their spawn implementation. A present PATH is authoritative.
@@ -2081,6 +2145,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
2081
2145
  trigger.kind === "resume" || trigger.kind === "message"
2082
2146
  ? "user"
2083
2147
  : undefined);
2148
+ const resumeReason = keepAliveResumeReason(runDir, pendingResume) ?? resumeHistoryTrigger;
2084
2149
  let statusGateHeartbeat;
2085
2150
  try {
2086
2151
  statusGateHeartbeat = managerStatusGateHeartbeat(hqRoot, existing, laneTurnId, nowIso);
@@ -2356,6 +2421,14 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
2356
2421
  ...(resumeHistoryCaller ? { caller: resumeHistoryCaller } : {}),
2357
2422
  }),
2358
2423
  }));
2424
+ appendLaneWorkerResumeEvent({
2425
+ hqRoot,
2426
+ laneId: lane.lane_id,
2427
+ resumedAt: nowIso,
2428
+ reason: resumeReason,
2429
+ ...(resumeHistoryCaller === undefined ? {} : { caller: resumeHistoryCaller }),
2430
+ turnId: laneTurnId,
2431
+ });
2359
2432
  }
2360
2433
  return laneLaunchResult(readLane(hqRoot, lane.lane_id));
2361
2434
  }
@@ -2670,6 +2743,12 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
2670
2743
  ...(callerLaneId ? { caller: callerLaneId } : {}),
2671
2744
  }),
2672
2745
  }));
2746
+ appendLaneWorkerResumeEvent({
2747
+ hqRoot,
2748
+ laneId: lane.lane_id,
2749
+ resumedAt: startedAt,
2750
+ reason: "envelope_repair",
2751
+ });
2673
2752
  return workerPid;
2674
2753
  }
2675
2754
  export { PROVIDER_HOME_ENV, PROVIDER_COMMAND };
@@ -445,10 +445,11 @@ export interface KrFreshnessState {
445
445
  stalled_fingerprint?: string;
446
446
  }
447
447
  export interface KeepAliveBlockedHistoryEntry {
448
- event: "keep_alive_blocked_grace" | "keep_alive_blocked_resumed" | "keep_alive_blocked_escalated";
448
+ event: "keep_alive_blocked_grace" | "keep_alive_blocked_resumed" | "keep_alive_blocked_escalated" | "keep_alive_blocked_skipped";
449
449
  at: string;
450
450
  terminal_at: string;
451
451
  resume_skipped_reason?: "open_question" | "blocked_grace";
452
+ reason?: "flag_off" | "backoff" | "next_check_at_due";
452
453
  resume_queued?: boolean;
453
454
  }
454
455
  export interface KeepAliveResumeHistoryEntry {
@@ -1,3 +1,4 @@
1
+ export declare const PACKAGE_USE_LEASE_HANDOFF_ENV = "HQ_CLI_PACKAGE_USE_LEASE_HANDOFF";
1
2
  export declare class PackageUseUpdatePendingError extends Error {
2
3
  constructor(message: string);
3
4
  }
@@ -10,11 +11,24 @@ export interface PackageUseLeaseRecord {
10
11
  start_time_ms: number;
11
12
  hq_version: string;
12
13
  }
14
+ export interface PackageUseLeaseHandoff {
15
+ readonly envValue: string;
16
+ waitForChild(pid: number, childAlive: () => boolean): Promise<boolean>;
17
+ cleanup(): void;
18
+ }
13
19
  export declare function npmPrefix(root: string, platform?: NodeJS.Platform): string | null;
14
20
  export declare function packageUseLeaseDirectory(prefix: string, stateDirectory?: string): string;
15
21
  export declare function packageUseUpdateRequestPath(prefix: string, stateDirectory?: string): string;
16
22
  declare function waitForUpdateRequest(requestPath: string, timeoutMs: number): void;
17
23
  export declare function acquirePackageUseLease(prefix: string, hqVersion: string, stateDirectory?: string): PackageUseLease;
24
+ /**
25
+ * Create a bounded, per-reexec rendezvous in the current lease directory.
26
+ * The child confirms readiness only after publishing its own atomic lease.
27
+ */
28
+ export declare function createPackageUseLeaseHandoff(lease: PackageUseLease, timeoutMs?: number): PackageUseLeaseHandoff;
29
+ export declare function createCurrentPackageUseLeaseHandoff(timeoutMs?: number): PackageUseLeaseHandoff | null;
30
+ /** Publish this process's lease identity to a parent waiting in the same prefix. */
31
+ export declare function publishPackageUseLeaseHandoff(lease: PackageUseLease, env?: NodeJS.ProcessEnv): boolean;
18
32
  export declare function acquireCurrentPackageUseLease(): PackageUseLease | null;
19
33
  /** Release this process's package-use lease after a long-running command has loaded its package modules. */
20
34
  export declare function releaseCurrentPackageUseLease(): void;
@@ -30,6 +44,7 @@ export declare function currentPackageUseUpdatePending(): boolean;
30
44
  export declare function currentPackageUseUpdateRequested(): boolean;
31
45
  export declare const __test__: {
32
46
  waitForUpdateRequest: typeof waitForUpdateRequest;
47
+ setCurrentPackageUseLease(lease: PackageUseLease | null): void;
33
48
  };
34
49
  export {};
35
50
  //# sourceMappingURL=package-use-lease.d.ts.map