@indigoai-us/hq-cli 5.313.0 → 5.315.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,370 @@
1
+ import * as fs from "node:fs";
2
+ import * as path from "node:path";
3
+ import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS } from "./keep-alive.js";
4
+ import { LanesError } from "./errors.js";
5
+ import { laneRunDir } from "./spawn.js";
6
+ import { readAllLanes } from "./store.js";
7
+ function readDoneClassifications(hqRoot, laneId) {
8
+ const root = fs.realpathSync(hqRoot);
9
+ const watcherLog = path.join(laneRunDir(root, laneId), "logs", "watcher.out");
10
+ if (!withinRoot(root, watcherLog)) {
11
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher log path escapes the HQ root for ${laneId}`, { lane_id: laneId });
12
+ }
13
+ let text;
14
+ try {
15
+ const stat = fs.lstatSync(watcherLog);
16
+ if (!stat.isFile() || stat.isSymbolicLink()) {
17
+ throw new LanesError("keep_alive_audit_event_invalid", `Lane watcher log must be a regular file: ${watcherLog}`, { lane_id: laneId, path: watcherLog });
18
+ }
19
+ const realPath = fs.realpathSync(watcherLog);
20
+ if (!withinRoot(root, realPath)) {
21
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher log escapes the HQ root for ${laneId}`, { lane_id: laneId, path: watcherLog });
22
+ }
23
+ text = fs.readFileSync(realPath, "utf8");
24
+ }
25
+ catch (err) {
26
+ if (err.code === "ENOENT")
27
+ return [];
28
+ throw err;
29
+ }
30
+ const atTimes = [];
31
+ for (const line of text.split(/\r?\n/)) {
32
+ if (!line.includes('"classification"'))
33
+ continue;
34
+ try {
35
+ const value = JSON.parse(line);
36
+ if (!isRecord(value) || value.state !== "done")
37
+ continue;
38
+ const at = parseTime(value.at);
39
+ if (at !== undefined)
40
+ atTimes.push(at);
41
+ }
42
+ catch (err) {
43
+ throw new LanesError("keep_alive_audit_event_invalid", `Cannot parse lane watcher classification in ${watcherLog}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId, path: watcherLog });
44
+ }
45
+ }
46
+ return atTimes.sort((a, b) => a - b);
47
+ }
48
+ const KEEP_ALIVE_ON_TIME_TOLERANCE_MS = 60_000;
49
+ const KEEP_ALIVE_NOT_RESTARTED_GRACE_MS = 5 * 60_000;
50
+ const LATE_KPI_THRESHOLD_MS = 10 * 60_000;
51
+ const MAX_NEXT_CHECK_DELAY_MS = 24 * 60 * 60_000;
52
+ const VERDICTS = [
53
+ "on_time",
54
+ "late",
55
+ "early_wake",
56
+ "message_wake",
57
+ "self_wake",
58
+ "early_by_keep_alive",
59
+ "not_restarted",
60
+ "waiting",
61
+ ];
62
+ function parseTime(value) {
63
+ if (typeof value !== "string" || !value.trim())
64
+ return undefined;
65
+ const parsed = Date.parse(value);
66
+ return Number.isFinite(parsed) ? parsed : undefined;
67
+ }
68
+ function iso(ms) {
69
+ return new Date(ms).toISOString();
70
+ }
71
+ function isRecord(value) {
72
+ return typeof value === "object" && value !== null && !Array.isArray(value);
73
+ }
74
+ function withinRoot(root, candidate) {
75
+ const relative = path.relative(root, candidate);
76
+ return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
77
+ }
78
+ function readStopEnvelopes(hqRoot, laneId) {
79
+ const root = fs.realpathSync(hqRoot);
80
+ const envelopeDir = path.join(laneRunDir(root, laneId), "envelopes");
81
+ if (!withinRoot(root, envelopeDir)) {
82
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane envelope path escapes the HQ root for ${laneId}`, { lane_id: laneId });
83
+ }
84
+ let names;
85
+ try {
86
+ const realDir = fs.realpathSync(envelopeDir);
87
+ if (!withinRoot(root, realDir)) {
88
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane envelope directory escapes the HQ root for ${laneId}`, { lane_id: laneId });
89
+ }
90
+ names = fs.readdirSync(realDir).filter((name) => /^\d+\.json$/.test(name)).sort();
91
+ }
92
+ catch (err) {
93
+ if (err.code === "ENOENT")
94
+ return { allMtimes: [], stops: [] };
95
+ throw err;
96
+ }
97
+ const allMtimes = [];
98
+ const stops = [];
99
+ for (const name of names) {
100
+ const file = path.join(envelopeDir, name);
101
+ const stat = fs.lstatSync(file);
102
+ if (!stat.isFile() || stat.isSymbolicLink()) {
103
+ throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope must be a regular file: ${file}`, { lane_id: laneId, path: file });
104
+ }
105
+ allMtimes.push(stat.mtimeMs);
106
+ let raw;
107
+ try {
108
+ raw = JSON.parse(fs.readFileSync(file, "utf8"));
109
+ }
110
+ catch (err) {
111
+ throw new LanesError("keep_alive_audit_envelope_invalid", `Cannot parse lane envelope ${file}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId, path: file });
112
+ }
113
+ if (!isRecord(raw) || raw.schema_version !== "hq-lane-envelope.v1") {
114
+ throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope is not hq-lane-envelope.v1: ${file}`, { lane_id: laneId, path: file });
115
+ }
116
+ if (raw.decision !== "done" || raw.story_complete !== false)
117
+ continue;
118
+ stops.push({
119
+ mtimeMs: stat.mtimeMs,
120
+ ...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
121
+ });
122
+ }
123
+ return { allMtimes: allMtimes.sort((a, b) => a - b), stops: stops.sort((a, b) => a.mtimeMs - b.mtimeMs) };
124
+ }
125
+ function readKeepAliveResumeEvents(hqRoot, laneId) {
126
+ const root = fs.realpathSync(hqRoot);
127
+ const eventsPath = path.join(laneRunDir(root, laneId), "watcher", "events.jsonl");
128
+ if (!withinRoot(root, eventsPath)) {
129
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher event path escapes the HQ root for ${laneId}`, { lane_id: laneId });
130
+ }
131
+ let text;
132
+ try {
133
+ const stat = fs.lstatSync(eventsPath);
134
+ if (!stat.isFile() || stat.isSymbolicLink()) {
135
+ throw new LanesError("keep_alive_audit_event_invalid", `Lane watcher events must be a regular file: ${eventsPath}`, { lane_id: laneId, path: eventsPath });
136
+ }
137
+ const realPath = fs.realpathSync(eventsPath);
138
+ if (!withinRoot(root, realPath)) {
139
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher event file escapes the HQ root for ${laneId}`, { lane_id: laneId, path: eventsPath });
140
+ }
141
+ text = fs.readFileSync(realPath, "utf8");
142
+ }
143
+ catch (err) {
144
+ if (err.code === "ENOENT")
145
+ return [];
146
+ throw err;
147
+ }
148
+ const events = [];
149
+ const lines = text.split(/\r?\n/);
150
+ for (const [index, line] of lines.entries()) {
151
+ if (!line.trim())
152
+ continue;
153
+ let value;
154
+ try {
155
+ value = JSON.parse(line);
156
+ }
157
+ catch (err) {
158
+ throw new LanesError("keep_alive_audit_event_invalid", `Cannot parse lane watcher event ${eventsPath}:${index + 1}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
159
+ }
160
+ if (!isRecord(value) || value.event !== "keep_alive_resume")
161
+ continue;
162
+ const at = parseTime(value.at);
163
+ if (at === undefined) {
164
+ throw new LanesError("keep_alive_audit_event_invalid", `Keep-alive resume event has invalid timing fields: ${eventsPath}:${index + 1}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
165
+ }
166
+ const nextEligibleAt = parseTime(value.next_eligible_at);
167
+ if (nextEligibleAt === undefined && value.resumed_from === "blocked")
168
+ continue;
169
+ if (nextEligibleAt === undefined) {
170
+ throw new LanesError("keep_alive_audit_event_invalid", `Keep-alive resume event has invalid timing fields: ${eventsPath}:${index + 1}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
171
+ }
172
+ const reason = value.reason === "pending_work" || value.reason === "next_check_at_due" || value.reason === "backoff"
173
+ ? value.reason
174
+ : undefined;
175
+ events.push({ at, nextEligibleAt, ...(reason === undefined ? {} : { reason }) });
176
+ }
177
+ return events.sort((a, b) => a.at - b.at);
178
+ }
179
+ function dueTime(stop, resumeEvents) {
180
+ const previousResume = resumeEvents.filter((event) => event.at <= stop.mtimeMs).at(-1);
181
+ const terminalPlusBackoff = stop.mtimeMs + LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS;
182
+ let due = Math.max(terminalPlusBackoff, previousResume?.nextEligibleAt ?? 0);
183
+ let reason = "backoff";
184
+ const nextCheck = parseTime(stop.nextCheckAt);
185
+ if (nextCheck !== undefined && nextCheck > stop.mtimeMs) {
186
+ due = Math.max(due, Math.min(nextCheck, stop.mtimeMs + MAX_NEXT_CHECK_DELAY_MS));
187
+ reason = "next_check_at";
188
+ }
189
+ return { at: due, reason };
190
+ }
191
+ function hasDeliveredDropboxEvidence(hqRoot, laneId, stoppedAt, restartAt) {
192
+ const root = fs.realpathSync(hqRoot);
193
+ const delivered = path.join(laneRunDir(root, laneId), "dropbox", "delivered");
194
+ if (!withinRoot(root, delivered)) {
195
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane delivered dropbox path escapes the HQ root for ${laneId}`, { lane_id: laneId, path: delivered });
196
+ }
197
+ let names;
198
+ try {
199
+ const stat = fs.lstatSync(delivered);
200
+ if (!stat.isDirectory() || stat.isSymbolicLink()) {
201
+ throw new LanesError("keep_alive_audit_event_invalid", `Lane delivered dropbox path must be a directory: ${delivered}`, { lane_id: laneId, path: delivered });
202
+ }
203
+ const realDir = fs.realpathSync(delivered);
204
+ if (!withinRoot(root, realDir)) {
205
+ throw new LanesError("keep_alive_audit_path_invalid", `Lane delivered dropbox path escapes the HQ root for ${laneId}`, { lane_id: laneId, path: delivered });
206
+ }
207
+ names = fs.readdirSync(realDir);
208
+ }
209
+ catch (err) {
210
+ if (err.code === "ENOENT")
211
+ return false;
212
+ throw err;
213
+ }
214
+ return names.some((name) => {
215
+ const file = path.join(delivered, name);
216
+ const stat = fs.lstatSync(file);
217
+ if (!stat.isFile() || stat.isSymbolicLink())
218
+ return false;
219
+ const realFile = fs.realpathSync(file);
220
+ if (!withinRoot(root, realFile))
221
+ return false;
222
+ return stat.mtimeMs >= stoppedAt && stat.mtimeMs <= restartAt + 2 * 60_000;
223
+ });
224
+ }
225
+ function firstRestartAfter(lane, stop, nextEnvelopeMs) {
226
+ const restart = (lane.resume_history ?? [])
227
+ .map((entry) => ({ entry, at: parseTime(entry.at) }))
228
+ .filter(({ entry, at }) => at !== undefined && at > stop.mtimeMs && (nextEnvelopeMs === undefined || at < nextEnvelopeMs) && typeof entry.trigger === "string")
229
+ .sort((a, b) => a.at - b.at)[0];
230
+ if (!restart || restart.at === undefined)
231
+ return null;
232
+ return {
233
+ at: iso(restart.at),
234
+ trigger: restart.entry.trigger,
235
+ caller: typeof restart.entry.caller === "string" ? restart.entry.caller : null,
236
+ };
237
+ }
238
+ function classifyVerdict(input) {
239
+ const { dueAt, restart, lane, nowMs } = input;
240
+ if (restart) {
241
+ const restartAt = Date.parse(restart.at);
242
+ if (restart.caller === lane.lane_id)
243
+ return "self_wake";
244
+ if (restart.trigger === "keep_alive") {
245
+ const event = input.resumeEvents
246
+ .filter((candidate) => candidate.at > input.stoppedAt && Math.abs(candidate.at - restartAt) <= 2 * 60_000)
247
+ .sort((left, right) => Math.abs(left.at - restartAt) - Math.abs(right.at - restartAt))[0];
248
+ if (event?.reason === "pending_work")
249
+ return "message_wake";
250
+ if (event?.reason === undefined && hasDeliveredDropboxEvidence(input.hqRoot, lane.lane_id, input.stoppedAt, restartAt)) {
251
+ return "message_wake";
252
+ }
253
+ const nextCheckAt = parseTime(input.nextCheckAt);
254
+ if (nextCheckAt !== undefined && nextCheckAt > input.stoppedAt && restartAt < Math.min(nextCheckAt, input.stoppedAt + MAX_NEXT_CHECK_DELAY_MS)) {
255
+ return "early_by_keep_alive";
256
+ }
257
+ if (restartAt < dueAt - KEEP_ALIVE_ON_TIME_TOLERANCE_MS)
258
+ return "early_by_keep_alive";
259
+ if (restartAt <= dueAt + KEEP_ALIVE_ON_TIME_TOLERANCE_MS)
260
+ return "on_time";
261
+ return "late";
262
+ }
263
+ if (restartAt < dueAt)
264
+ return "early_wake";
265
+ return "late";
266
+ }
267
+ if (lane.state === "done" && nowMs > dueAt + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS)
268
+ return "not_restarted";
269
+ return "waiting";
270
+ }
271
+ export function buildKeepAliveAudit(input) {
272
+ const nowMs = input.now.getTime();
273
+ const sinceMs = input.since === undefined
274
+ ? nowMs - 24 * 60 * 60_000
275
+ : parseTime(input.since);
276
+ if (sinceMs === undefined || !Number.isFinite(sinceMs)) {
277
+ throw new LanesError("invalid_since", "--since must be a valid ISO date-time", { since: input.since });
278
+ }
279
+ const lanes = readAllLanes(input.hqRoot).filter((lane) => (input.laneId === undefined || lane.lane_id === input.laneId) &&
280
+ typeof lane.hire_grant === "number" && Number.isFinite(lane.hire_grant) && lane.hire_grant > 0);
281
+ const rows = [];
282
+ const errors = [];
283
+ for (const lane of lanes) {
284
+ try {
285
+ const { allMtimes, stops } = readStopEnvelopes(input.hqRoot, lane.lane_id);
286
+ const keepAliveResumeEvents = readKeepAliveResumeEvents(input.hqRoot, lane.lane_id);
287
+ const doneClassifications = readDoneClassifications(input.hqRoot, lane.lane_id);
288
+ for (const stop of stops) {
289
+ if (stop.mtimeMs < sinceMs || stop.mtimeMs > nowMs)
290
+ continue;
291
+ if (!doneClassifications.some((at) => at >= stop.mtimeMs && at - stop.mtimeMs <= 10 * 60_000))
292
+ continue;
293
+ const nextEnvelopeMs = allMtimes.find((mtime) => mtime > stop.mtimeMs);
294
+ const restart = firstRestartAfter(lane, stop, nextEnvelopeMs);
295
+ const due = dueTime(stop, keepAliveResumeEvents);
296
+ rows.push({
297
+ lane: lane.lane_id,
298
+ project: lane.project_id,
299
+ profile: lane.worker_profile ?? lane.worker ?? null,
300
+ stopped_at: iso(stop.mtimeMs),
301
+ next_check_at: stop.nextCheckAt ?? null,
302
+ due_at: iso(due.at),
303
+ due_reason: due.reason,
304
+ restart,
305
+ verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, lane, nowMs, resumeEvents: keepAliveResumeEvents, hqRoot: input.hqRoot }),
306
+ });
307
+ }
308
+ }
309
+ catch (err) {
310
+ const auditError = err instanceof LanesError
311
+ ? err
312
+ : new LanesError("keep_alive_audit_read_failed", `Unable to read keep-alive audit inputs for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: laneRunDir(input.hqRoot, lane.lane_id) });
313
+ errors.push({
314
+ lane: lane.lane_id,
315
+ error: auditError.error,
316
+ path: typeof auditError.details.path === "string" ? auditError.details.path : laneRunDir(input.hqRoot, lane.lane_id),
317
+ message: auditError.message,
318
+ });
319
+ }
320
+ }
321
+ rows.sort((a, b) => a.stopped_at.localeCompare(b.stopped_at) || a.lane.localeCompare(b.lane));
322
+ const byVerdict = Object.fromEntries(VERDICTS.map((verdict) => [verdict, 0]));
323
+ let lateOverTenMinutes = 0;
324
+ for (const row of rows) {
325
+ byVerdict[row.verdict] += 1;
326
+ if (row.verdict === "late" && row.restart && Date.parse(row.restart.at) - Date.parse(row.due_at) > LATE_KPI_THRESHOLD_MS) {
327
+ lateOverTenMinutes += 1;
328
+ }
329
+ }
330
+ return {
331
+ ok: true,
332
+ since: iso(sinceMs),
333
+ rows,
334
+ summary: {
335
+ by_verdict: byVerdict,
336
+ not_restarted: byVerdict.not_restarted,
337
+ late_over_10m: lateOverTenMinutes,
338
+ early_wake_by_others: byVerdict.early_wake,
339
+ self_wake: byVerdict.self_wake,
340
+ stopped_and_not_restarted_on_their_own: byVerdict.not_restarted + lateOverTenMinutes,
341
+ },
342
+ errors,
343
+ };
344
+ }
345
+ export function formatKeepAliveAuditText(result) {
346
+ const headings = ["LANE", "PROJECT", "PROFILE", "STOPPED", "NEXT_CHECK", "DUE", "RESTART", "VERDICT"];
347
+ const values = result.rows.map((row) => [
348
+ row.lane,
349
+ row.project,
350
+ row.profile ?? "—",
351
+ row.stopped_at,
352
+ row.next_check_at ?? "—",
353
+ row.due_at,
354
+ row.restart ? `${row.restart.at} ${row.restart.trigger}${row.restart.caller ? ` (${row.restart.caller})` : ""}` : "—",
355
+ row.verdict,
356
+ ]);
357
+ const widths = headings.map((heading, index) => Math.max(heading.length, ...values.map((row) => row[index].length)));
358
+ const tableLine = (row) => row.map((value, index) => value.padEnd(widths[index])).join(" ").trimEnd();
359
+ const counts = VERDICTS.map((verdict) => `${verdict}=${result.summary.by_verdict[verdict]}`).join(", ");
360
+ return [
361
+ tableLine(headings),
362
+ ...values.map(tableLine),
363
+ "",
364
+ `Since ${result.since}; ${counts}`,
365
+ `not_restarted=${result.summary.not_restarted}, late_over_10m=${result.summary.late_over_10m}, early_wake_by_others=${result.summary.early_wake_by_others}, self_wake=${result.summary.self_wake}`,
366
+ `Stopped and not restarted on their own: ${result.summary.stopped_and_not_restarted_on_their_own}`,
367
+ ...result.errors.map((error) => `Error ${error.lane} (${error.error}) at ${error.path}: ${error.message}`),
368
+ ].join("\n");
369
+ }
370
+ //# sourceMappingURL=keep-alive-audit.js.map
@@ -27,7 +27,7 @@ import { checkLaneDiskSpace, configuredCodexRolloutHomes, lowDiskGuardEnabledFor
27
27
  import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
28
28
  import { appendDropboxMessage } from "./dropbox.js";
29
29
  import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
30
- import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
30
+ import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
31
31
  import { providerSupportsSessionResume } from "./delivery.js";
32
32
  import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, readQuotaSignal, recordAccountQuota, } from "./quota.js";
33
33
  import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
@@ -2303,6 +2303,7 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
2303
2303
  at: now.toISOString(),
2304
2304
  attempt: (lane.keep_alive_resume_count ?? 0) + 1,
2305
2305
  resumed_from: "blocked",
2306
+ reason: "backoff",
2306
2307
  ...(accountSelection !== undefined
2307
2308
  ? { account_selection: keepAliveAccountSelectionEvidence(accountSelection) }
2308
2309
  : {}),
@@ -2733,6 +2734,23 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
2733
2734
  continue;
2734
2735
  }
2735
2736
  const nextEligibleAt = new Date(nowMs + leadLaneKeepAliveBackoffMs(immediateFirstResumeEnabled ? attempt : attempt + 1)).toISOString();
2737
+ let resumeReason = "backoff";
2738
+ if (initialState === "done") {
2739
+ if (hasPendingKeepAliveWork(hqRoot, current, questions)) {
2740
+ resumeReason = "pending_work";
2741
+ }
2742
+ else {
2743
+ const finalEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id)?.envelope;
2744
+ const nextCheckMs = Date.parse(finalEnvelope?.next_check_at ?? "");
2745
+ const anchorMs = Date.parse(current.last_envelope?.at ?? current.updated_at);
2746
+ if (Number.isFinite(nextCheckMs) && Number.isFinite(anchorMs)) {
2747
+ const cappedNextCheckMs = Math.min(nextCheckMs, anchorMs + LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS);
2748
+ if (cappedNextCheckMs > nextEligibleMs && cappedNextCheckMs <= nowMs) {
2749
+ resumeReason = "next_check_at_due";
2750
+ }
2751
+ }
2752
+ }
2753
+ }
2736
2754
  const queuedCount = loadLaneRecords(hqRoot, summary).filter((queuedLane) => queuedLane.state === "queued" &&
2737
2755
  queuedLane.provider === current.provider &&
2738
2756
  laneRunsHere(queuedLane, deps.host) &&
@@ -3007,6 +3025,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
3007
3025
  attempt,
3008
3026
  next_eligible_at: nextEligibleAt,
3009
3027
  resumed_from: resumedFrom,
3028
+ reason: resumeReason,
3010
3029
  ...(accountSelection !== undefined
3011
3030
  ? { account_selection: keepAliveAccountSelectionEvidence(accountSelection) }
3012
3031
  : {}),
package/dist/sentry.d.ts CHANGED
@@ -1,5 +1,11 @@
1
- import * as Sentry from "@sentry/node";
2
1
  import type { ErrorEvent, EventHint, NodeOptions } from "@sentry/node";
2
+ type SentryModule = typeof import("@sentry/node");
3
+ /**
4
+ * The command graph touches the Sentry facade on normal startup for a
5
+ * breadcrumb and a final flush. Keep those operations inert until the SDK
6
+ * module is loaded; the first capture that needs it loads the SDK synchronously.
7
+ */
8
+ export declare const Sentry: SentryModule;
3
9
  /**
4
10
  * Drop broken-pipe (EPIPE) crashes before scrubbing/send. A closed downstream
5
11
  * reader (`hq … | head`, `source <(hq …)`, a parent that exited) is normal
@@ -18,5 +24,5 @@ export declare function epipeAwareBeforeSend(event: ErrorEvent, hint: EventHint)
18
24
  export declare function initSentry(overrides?: Partial<Pick<NodeOptions, "transport">>): void;
19
25
  /** Finish HQ's one per-invocation release-health session. */
20
26
  export declare function finishSentrySession(): void;
21
- export { Sentry };
27
+ export {};
22
28
  //# sourceMappingURL=sentry.d.ts.map
package/dist/sentry.js CHANGED
@@ -1,4 +1,4 @@
1
- import * as Sentry from "@sentry/node";
1
+ import { createRequire } from "node:module";
2
2
  import { BUNDLED_DSN } from "./sentry-dsn.generated.js";
3
3
  import { beforeSend } from "./sentry-before-send.js";
4
4
  import { beforeBreadcrumb } from "./utils/breadcrumb-buffer.js";
@@ -9,6 +9,60 @@ import { companySkillWrapperSentryTagsFor } from "./lib/company-skill-wrapper.js
9
9
  import { environmentalFsErrorMessage } from "./utils/environmental-error.js";
10
10
  import { incompleteInstallCaptureContext, incompleteInstallMessage, } from "./utils/incomplete-install-error.js";
11
11
  import { installTreeTornSentryTagsFor, sentryFingerprintFor, sentryMcpCollisionContextFor, } from "./utils/sentry-fingerprint.js";
12
+ const inertSentryScope = {
13
+ setTag: () => inertSentryScope,
14
+ setUser: () => inertSentryScope,
15
+ addBreadcrumb: () => inertSentryScope,
16
+ clearBreadcrumbs: () => inertSentryScope,
17
+ setLevel: () => inertSentryScope,
18
+ setFingerprint: () => inertSentryScope,
19
+ };
20
+ const requireSentry = createRequire(import.meta.url);
21
+ let sentryModule;
22
+ let pendingSentryBreadcrumbs = [];
23
+ function loadSentryModule() {
24
+ sentryModule ??= requireSentry("@sentry/node");
25
+ return sentryModule;
26
+ }
27
+ /**
28
+ * The command graph touches the Sentry facade on normal startup for a
29
+ * breadcrumb and a final flush. Keep those operations inert until the SDK
30
+ * module is loaded; the first capture that needs it loads the SDK synchronously.
31
+ */
32
+ export const Sentry = new Proxy({}, {
33
+ get(_target, property) {
34
+ if (property === Symbol.toStringTag)
35
+ return "Module";
36
+ if (property === "addBreadcrumb") {
37
+ return (breadcrumb) => {
38
+ if (sentryModule) {
39
+ sentryModule.addBreadcrumb(breadcrumb);
40
+ return;
41
+ }
42
+ if (pendingSentryBreadcrumbs.length >= 100)
43
+ pendingSentryBreadcrumbs.shift();
44
+ pendingSentryBreadcrumbs.push(breadcrumb);
45
+ };
46
+ }
47
+ if (property === "withScope") {
48
+ return (callback) => {
49
+ if (sentryModule)
50
+ return sentryModule.withScope(callback);
51
+ return callback(inertSentryScope);
52
+ };
53
+ }
54
+ if (property === "flush") {
55
+ return (timeout) => {
56
+ if (!sentryModule)
57
+ return Promise.resolve(true);
58
+ return sentryModule.flush(timeout);
59
+ };
60
+ }
61
+ const sdk = loadSentryModule();
62
+ const value = Reflect.get(sdk, property);
63
+ return typeof value === "function" ? value.bind(sdk) : value;
64
+ },
65
+ });
12
66
  /**
13
67
  * Drop broken-pipe (EPIPE) crashes before scrubbing/send. A closed downstream
14
68
  * reader (`hq … | head`, `source <(hq …)`, a parent that exited) is normal
@@ -172,7 +226,8 @@ export function initSentry(overrides = {}) {
172
226
  const dsn = BUNDLED_DSN || process.env.SENTRY_DSN;
173
227
  if (!dsn)
174
228
  return;
175
- Sentry.init({
229
+ const sdk = loadSentryModule();
230
+ sdk.init({
176
231
  dsn,
177
232
  // import-in-the-middle's async ESM loader makes every later CLI import
178
233
  // substantially slower. Keep HQ's explicit breadcrumbs, but give up
@@ -197,15 +252,17 @@ export function initSentry(overrides = {}) {
197
252
  // logged in). A CLI process is one user, so global setUser is correct here.
198
253
  const user = getCachedSentryUser();
199
254
  if (user)
200
- Sentry.setUser(user);
255
+ sdk.setUser(user);
256
+ for (const breadcrumb of pendingSentryBreadcrumbs)
257
+ sdk.addBreadcrumb(breadcrumb);
258
+ pendingSentryBreadcrumbs = [];
201
259
  // Release health: one session per CLI run. Ended + flushed in index.ts's
202
260
  // finally (covers every exit path). A captured error auto-marks the session
203
261
  // "errored", so crash-free numbers per release stay accurate.
204
- Sentry.startSession();
262
+ sdk.startSession();
205
263
  }
206
264
  /** Finish HQ's one per-invocation release-health session. */
207
265
  export function finishSentrySession() {
208
- Sentry.endSession();
266
+ sentryModule?.endSession();
209
267
  }
210
- export { Sentry };
211
268
  //# sourceMappingURL=sentry.js.map
@@ -1,13 +1,14 @@
1
1
  import { type FirstSyncStepOutcome } from "../lib/first-sync-next-step.js";
2
- export { CLI_FIRST_SYNC_NEXT_STEP_FLAG, FIRST_SYNC_CONFIRMATION_COPY, FIRST_SYNC_NEXT_STEP_COPY, claimFirstSyncStepMarker, firstSyncStepMarkerPaths, firstSyncStepTelemetryProperties, isFirstSyncNextStepEnabled, isHumanInteractiveCliSession, isWorkBearingSyncResult, maybeShowFirstSyncNextStep, recordFirstWorkBearingSync, shouldConfirmFirstWorkBearingSync, shouldShowFirstSyncNextStep, takeFirstSyncStepOutcomes, type FirstSyncStepOutcome, } from "../lib/first-sync-next-step.js";
2
+ export { CLI_FIRST_SYNC_NEXT_STEP_FLAG, FIRST_SYNC_CONFIRMATION_COPY, FIRST_SYNC_NEXT_STEP_COPY, claimFirstSyncStepMarker, firstSyncStepMarkerPaths, firstSyncStepTelemetryProperties, isHumanInteractiveCliSession, isWorkBearingSyncResult, maybeShowFirstSyncNextStep, recordFirstWorkBearingSync, shouldConfirmFirstWorkBearingSync, shouldShowFirstSyncNextStep, takeFirstSyncStepOutcomes, type FirstSyncStepOutcome, } from "../lib/first-sync-next-step.js";
3
3
  export declare const TELEMETRY_TIMEOUT_MS = 1200;
4
4
  /**
5
5
  * Exact stderr line printed once on the first signed-in TTY run that is not
6
6
  * `--json`. Keep this notice and its marker stable for session telemetry;
7
- * first-sync and feature-used fields have separate opt-in disclosures below.
7
+ * first-sync and feature-used fields have separate disclosures below; only
8
+ * feature-used telemetry remains opt-in.
8
9
  */
9
10
  export declare const TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp, never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
10
- export declare const FIRST_SYNC_TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp; when enabled, first-sync guidance shown or followed and the flag key cli.first-sync-next-step; never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
11
+ export declare const FIRST_SYNC_TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp; first-sync guidance shown or followed, with the identifier cli.first-sync-next-step; never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
11
12
  export declare const CLI_FEATURE_USED_TELEMETRY_FLAG = "cli.feature-used-telemetry-v1";
12
13
  export declare const CLI_FEATURE_USED_TELEMETRY_DISCLOSURE_LINE = "HQ sends a feature-use event when you run a supported CLI feature (feature kind and outcome, plus your company UID when available; never command names, file contents, or secrets). The hq-flags key cli.feature-used-telemetry-v1 controls these events. Set HQ_TELEMETRY=0 to turn this off.";
13
14
  export declare const CLI_LIMIT_AT_ACTION_PROMPT_FLAG = "billing.limit-at-action-prompt";
@@ -6,16 +6,17 @@ import { CLI_VERSION } from "../cli-version.js";
6
6
  import { isExpiring, isMachineIdentity, loadCachedTokens, loadMachineCachedTokens, } from "./cognito-session.js";
7
7
  import { resolveProcessFlagCompanyContext, resolveProcessFlagGate, } from "../lib/flag-registry.js";
8
8
  import { vaultApiFetch } from "./vault-api.js";
9
- import { firstSyncStepTelemetryProperties, isFirstSyncNextStepEnabled, isHumanInteractiveCliSession, } from "../lib/first-sync-next-step.js";
10
- export { CLI_FIRST_SYNC_NEXT_STEP_FLAG, FIRST_SYNC_CONFIRMATION_COPY, FIRST_SYNC_NEXT_STEP_COPY, claimFirstSyncStepMarker, firstSyncStepMarkerPaths, firstSyncStepTelemetryProperties, isFirstSyncNextStepEnabled, isHumanInteractiveCliSession, isWorkBearingSyncResult, maybeShowFirstSyncNextStep, recordFirstWorkBearingSync, shouldConfirmFirstWorkBearingSync, shouldShowFirstSyncNextStep, takeFirstSyncStepOutcomes, } from "../lib/first-sync-next-step.js";
9
+ import { firstSyncStepTelemetryProperties, isHumanInteractiveCliSession, } from "../lib/first-sync-next-step.js";
10
+ export { CLI_FIRST_SYNC_NEXT_STEP_FLAG, FIRST_SYNC_CONFIRMATION_COPY, FIRST_SYNC_NEXT_STEP_COPY, claimFirstSyncStepMarker, firstSyncStepMarkerPaths, firstSyncStepTelemetryProperties, isHumanInteractiveCliSession, isWorkBearingSyncResult, maybeShowFirstSyncNextStep, recordFirstWorkBearingSync, shouldConfirmFirstWorkBearingSync, shouldShowFirstSyncNextStep, takeFirstSyncStepOutcomes, } from "../lib/first-sync-next-step.js";
11
11
  export const TELEMETRY_TIMEOUT_MS = 1_200;
12
12
  /**
13
13
  * Exact stderr line printed once on the first signed-in TTY run that is not
14
14
  * `--json`. Keep this notice and its marker stable for session telemetry;
15
- * first-sync and feature-used fields have separate opt-in disclosures below.
15
+ * first-sync and feature-used fields have separate disclosures below; only
16
+ * feature-used telemetry remains opt-in.
16
17
  */
17
18
  export const TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp, never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
18
- export const FIRST_SYNC_TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp; when enabled, first-sync guidance shown or followed and the flag key cli.first-sync-next-step; never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
19
+ export const FIRST_SYNC_TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp; first-sync guidance shown or followed, with the identifier cli.first-sync-next-step; never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
19
20
  export const CLI_FEATURE_USED_TELEMETRY_FLAG = "cli.feature-used-telemetry-v1";
20
21
  export const CLI_FEATURE_USED_TELEMETRY_DISCLOSURE_LINE = "HQ sends a feature-use event when you run a supported CLI feature (feature kind and outcome, plus your company UID when available; never command names, file contents, or secrets). The hq-flags key cli.feature-used-telemetry-v1 controls these events. Set HQ_TELEMETRY=0 to turn this off.";
21
22
  const TELEMETRY_DISABLED_VALUES = new Set(["0", "false", "off"]);
@@ -289,8 +290,6 @@ export async function emitCliFirstSyncNextStepOutcome(outcome, timeoutMs = TELEM
289
290
  try {
290
291
  if (isCliTelemetryDisabled())
291
292
  return;
292
- if (!isFirstSyncNextStepEnabled())
293
- return;
294
293
  const cached = loadCachedTokens();
295
294
  const machineIdentity = isMachineIdentity();
296
295
  if (!cached ||
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@indigoai-us/hq-cli",
3
- "version": "5.313.0",
3
+ "version": "5.315.0",
4
4
  "description": "HQ by Indigo management CLI — modules and cloud sync",
5
5
  "main": "dist/index.js",
6
6
  "bin": {