@indigoai-us/hq-cli 5.313.0 → 5.315.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/assets/scaffold/core/scripts/migrate-policy-triggers.sh +12 -7
- package/dist/command-catalog.generated.d.ts +18 -0
- package/dist/command-catalog.generated.js +23 -0
- package/dist/commands/core-migrate-policy-triggers-native.js +4 -8
- package/dist/commands/lanes-keep-alive-audit.d.ts +10 -0
- package/dist/commands/lanes-keep-alive-audit.js +19 -0
- package/dist/commands/lanes-message.js +37 -28
- package/dist/commands/lanes-shared.d.ts +1 -0
- package/dist/commands/lanes.js +33 -23
- package/dist/lib/core-utils/sentry-report.d.ts +3 -3
- package/dist/lib/core-utils/sentry-report.js +23 -2
- package/dist/lib/first-sync-next-step.d.ts +0 -3
- package/dist/lib/first-sync-next-step.js +6 -16
- package/dist/lib/lanes/autopilot.d.ts +13 -0
- package/dist/lib/lanes/autopilot.js +198 -37
- package/dist/lib/lanes/flag-keys.d.ts +7 -0
- package/dist/lib/lanes/flag-keys.js +7 -0
- package/dist/lib/lanes/keep-alive-audit.d.ts +46 -0
- package/dist/lib/lanes/keep-alive-audit.js +370 -0
- package/dist/lib/lanes/reconcile.js +20 -1
- package/dist/sentry.d.ts +8 -2
- package/dist/sentry.js +63 -6
- package/dist/utils/cli-telemetry.d.ts +4 -3
- package/dist/utils/cli-telemetry.js +5 -6
- package/package.json +1 -1
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS } from "./keep-alive.js";
|
|
4
|
+
import { LanesError } from "./errors.js";
|
|
5
|
+
import { laneRunDir } from "./spawn.js";
|
|
6
|
+
import { readAllLanes } from "./store.js";
|
|
7
|
+
function readDoneClassifications(hqRoot, laneId) {
|
|
8
|
+
const root = fs.realpathSync(hqRoot);
|
|
9
|
+
const watcherLog = path.join(laneRunDir(root, laneId), "logs", "watcher.out");
|
|
10
|
+
if (!withinRoot(root, watcherLog)) {
|
|
11
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher log path escapes the HQ root for ${laneId}`, { lane_id: laneId });
|
|
12
|
+
}
|
|
13
|
+
let text;
|
|
14
|
+
try {
|
|
15
|
+
const stat = fs.lstatSync(watcherLog);
|
|
16
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
17
|
+
throw new LanesError("keep_alive_audit_event_invalid", `Lane watcher log must be a regular file: ${watcherLog}`, { lane_id: laneId, path: watcherLog });
|
|
18
|
+
}
|
|
19
|
+
const realPath = fs.realpathSync(watcherLog);
|
|
20
|
+
if (!withinRoot(root, realPath)) {
|
|
21
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher log escapes the HQ root for ${laneId}`, { lane_id: laneId, path: watcherLog });
|
|
22
|
+
}
|
|
23
|
+
text = fs.readFileSync(realPath, "utf8");
|
|
24
|
+
}
|
|
25
|
+
catch (err) {
|
|
26
|
+
if (err.code === "ENOENT")
|
|
27
|
+
return [];
|
|
28
|
+
throw err;
|
|
29
|
+
}
|
|
30
|
+
const atTimes = [];
|
|
31
|
+
for (const line of text.split(/\r?\n/)) {
|
|
32
|
+
if (!line.includes('"classification"'))
|
|
33
|
+
continue;
|
|
34
|
+
try {
|
|
35
|
+
const value = JSON.parse(line);
|
|
36
|
+
if (!isRecord(value) || value.state !== "done")
|
|
37
|
+
continue;
|
|
38
|
+
const at = parseTime(value.at);
|
|
39
|
+
if (at !== undefined)
|
|
40
|
+
atTimes.push(at);
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
throw new LanesError("keep_alive_audit_event_invalid", `Cannot parse lane watcher classification in ${watcherLog}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId, path: watcherLog });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return atTimes.sort((a, b) => a - b);
|
|
47
|
+
}
|
|
48
|
+
const KEEP_ALIVE_ON_TIME_TOLERANCE_MS = 60_000;
|
|
49
|
+
const KEEP_ALIVE_NOT_RESTARTED_GRACE_MS = 5 * 60_000;
|
|
50
|
+
const LATE_KPI_THRESHOLD_MS = 10 * 60_000;
|
|
51
|
+
const MAX_NEXT_CHECK_DELAY_MS = 24 * 60 * 60_000;
|
|
52
|
+
const VERDICTS = [
|
|
53
|
+
"on_time",
|
|
54
|
+
"late",
|
|
55
|
+
"early_wake",
|
|
56
|
+
"message_wake",
|
|
57
|
+
"self_wake",
|
|
58
|
+
"early_by_keep_alive",
|
|
59
|
+
"not_restarted",
|
|
60
|
+
"waiting",
|
|
61
|
+
];
|
|
62
|
+
function parseTime(value) {
|
|
63
|
+
if (typeof value !== "string" || !value.trim())
|
|
64
|
+
return undefined;
|
|
65
|
+
const parsed = Date.parse(value);
|
|
66
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
67
|
+
}
|
|
68
|
+
function iso(ms) {
|
|
69
|
+
return new Date(ms).toISOString();
|
|
70
|
+
}
|
|
71
|
+
function isRecord(value) {
|
|
72
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
73
|
+
}
|
|
74
|
+
function withinRoot(root, candidate) {
|
|
75
|
+
const relative = path.relative(root, candidate);
|
|
76
|
+
return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
|
|
77
|
+
}
|
|
78
|
+
function readStopEnvelopes(hqRoot, laneId) {
|
|
79
|
+
const root = fs.realpathSync(hqRoot);
|
|
80
|
+
const envelopeDir = path.join(laneRunDir(root, laneId), "envelopes");
|
|
81
|
+
if (!withinRoot(root, envelopeDir)) {
|
|
82
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane envelope path escapes the HQ root for ${laneId}`, { lane_id: laneId });
|
|
83
|
+
}
|
|
84
|
+
let names;
|
|
85
|
+
try {
|
|
86
|
+
const realDir = fs.realpathSync(envelopeDir);
|
|
87
|
+
if (!withinRoot(root, realDir)) {
|
|
88
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane envelope directory escapes the HQ root for ${laneId}`, { lane_id: laneId });
|
|
89
|
+
}
|
|
90
|
+
names = fs.readdirSync(realDir).filter((name) => /^\d+\.json$/.test(name)).sort();
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
if (err.code === "ENOENT")
|
|
94
|
+
return { allMtimes: [], stops: [] };
|
|
95
|
+
throw err;
|
|
96
|
+
}
|
|
97
|
+
const allMtimes = [];
|
|
98
|
+
const stops = [];
|
|
99
|
+
for (const name of names) {
|
|
100
|
+
const file = path.join(envelopeDir, name);
|
|
101
|
+
const stat = fs.lstatSync(file);
|
|
102
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
103
|
+
throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope must be a regular file: ${file}`, { lane_id: laneId, path: file });
|
|
104
|
+
}
|
|
105
|
+
allMtimes.push(stat.mtimeMs);
|
|
106
|
+
let raw;
|
|
107
|
+
try {
|
|
108
|
+
raw = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
throw new LanesError("keep_alive_audit_envelope_invalid", `Cannot parse lane envelope ${file}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId, path: file });
|
|
112
|
+
}
|
|
113
|
+
if (!isRecord(raw) || raw.schema_version !== "hq-lane-envelope.v1") {
|
|
114
|
+
throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope is not hq-lane-envelope.v1: ${file}`, { lane_id: laneId, path: file });
|
|
115
|
+
}
|
|
116
|
+
if (raw.decision !== "done" || raw.story_complete !== false)
|
|
117
|
+
continue;
|
|
118
|
+
stops.push({
|
|
119
|
+
mtimeMs: stat.mtimeMs,
|
|
120
|
+
...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
return { allMtimes: allMtimes.sort((a, b) => a - b), stops: stops.sort((a, b) => a.mtimeMs - b.mtimeMs) };
|
|
124
|
+
}
|
|
125
|
+
function readKeepAliveResumeEvents(hqRoot, laneId) {
|
|
126
|
+
const root = fs.realpathSync(hqRoot);
|
|
127
|
+
const eventsPath = path.join(laneRunDir(root, laneId), "watcher", "events.jsonl");
|
|
128
|
+
if (!withinRoot(root, eventsPath)) {
|
|
129
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher event path escapes the HQ root for ${laneId}`, { lane_id: laneId });
|
|
130
|
+
}
|
|
131
|
+
let text;
|
|
132
|
+
try {
|
|
133
|
+
const stat = fs.lstatSync(eventsPath);
|
|
134
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
135
|
+
throw new LanesError("keep_alive_audit_event_invalid", `Lane watcher events must be a regular file: ${eventsPath}`, { lane_id: laneId, path: eventsPath });
|
|
136
|
+
}
|
|
137
|
+
const realPath = fs.realpathSync(eventsPath);
|
|
138
|
+
if (!withinRoot(root, realPath)) {
|
|
139
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher event file escapes the HQ root for ${laneId}`, { lane_id: laneId, path: eventsPath });
|
|
140
|
+
}
|
|
141
|
+
text = fs.readFileSync(realPath, "utf8");
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
if (err.code === "ENOENT")
|
|
145
|
+
return [];
|
|
146
|
+
throw err;
|
|
147
|
+
}
|
|
148
|
+
const events = [];
|
|
149
|
+
const lines = text.split(/\r?\n/);
|
|
150
|
+
for (const [index, line] of lines.entries()) {
|
|
151
|
+
if (!line.trim())
|
|
152
|
+
continue;
|
|
153
|
+
let value;
|
|
154
|
+
try {
|
|
155
|
+
value = JSON.parse(line);
|
|
156
|
+
}
|
|
157
|
+
catch (err) {
|
|
158
|
+
throw new LanesError("keep_alive_audit_event_invalid", `Cannot parse lane watcher event ${eventsPath}:${index + 1}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
|
|
159
|
+
}
|
|
160
|
+
if (!isRecord(value) || value.event !== "keep_alive_resume")
|
|
161
|
+
continue;
|
|
162
|
+
const at = parseTime(value.at);
|
|
163
|
+
if (at === undefined) {
|
|
164
|
+
throw new LanesError("keep_alive_audit_event_invalid", `Keep-alive resume event has invalid timing fields: ${eventsPath}:${index + 1}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
|
|
165
|
+
}
|
|
166
|
+
const nextEligibleAt = parseTime(value.next_eligible_at);
|
|
167
|
+
if (nextEligibleAt === undefined && value.resumed_from === "blocked")
|
|
168
|
+
continue;
|
|
169
|
+
if (nextEligibleAt === undefined) {
|
|
170
|
+
throw new LanesError("keep_alive_audit_event_invalid", `Keep-alive resume event has invalid timing fields: ${eventsPath}:${index + 1}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
|
|
171
|
+
}
|
|
172
|
+
const reason = value.reason === "pending_work" || value.reason === "next_check_at_due" || value.reason === "backoff"
|
|
173
|
+
? value.reason
|
|
174
|
+
: undefined;
|
|
175
|
+
events.push({ at, nextEligibleAt, ...(reason === undefined ? {} : { reason }) });
|
|
176
|
+
}
|
|
177
|
+
return events.sort((a, b) => a.at - b.at);
|
|
178
|
+
}
|
|
179
|
+
function dueTime(stop, resumeEvents) {
|
|
180
|
+
const previousResume = resumeEvents.filter((event) => event.at <= stop.mtimeMs).at(-1);
|
|
181
|
+
const terminalPlusBackoff = stop.mtimeMs + LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS;
|
|
182
|
+
let due = Math.max(terminalPlusBackoff, previousResume?.nextEligibleAt ?? 0);
|
|
183
|
+
let reason = "backoff";
|
|
184
|
+
const nextCheck = parseTime(stop.nextCheckAt);
|
|
185
|
+
if (nextCheck !== undefined && nextCheck > stop.mtimeMs) {
|
|
186
|
+
due = Math.max(due, Math.min(nextCheck, stop.mtimeMs + MAX_NEXT_CHECK_DELAY_MS));
|
|
187
|
+
reason = "next_check_at";
|
|
188
|
+
}
|
|
189
|
+
return { at: due, reason };
|
|
190
|
+
}
|
|
191
|
+
function hasDeliveredDropboxEvidence(hqRoot, laneId, stoppedAt, restartAt) {
|
|
192
|
+
const root = fs.realpathSync(hqRoot);
|
|
193
|
+
const delivered = path.join(laneRunDir(root, laneId), "dropbox", "delivered");
|
|
194
|
+
if (!withinRoot(root, delivered)) {
|
|
195
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane delivered dropbox path escapes the HQ root for ${laneId}`, { lane_id: laneId, path: delivered });
|
|
196
|
+
}
|
|
197
|
+
let names;
|
|
198
|
+
try {
|
|
199
|
+
const stat = fs.lstatSync(delivered);
|
|
200
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
201
|
+
throw new LanesError("keep_alive_audit_event_invalid", `Lane delivered dropbox path must be a directory: ${delivered}`, { lane_id: laneId, path: delivered });
|
|
202
|
+
}
|
|
203
|
+
const realDir = fs.realpathSync(delivered);
|
|
204
|
+
if (!withinRoot(root, realDir)) {
|
|
205
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane delivered dropbox path escapes the HQ root for ${laneId}`, { lane_id: laneId, path: delivered });
|
|
206
|
+
}
|
|
207
|
+
names = fs.readdirSync(realDir);
|
|
208
|
+
}
|
|
209
|
+
catch (err) {
|
|
210
|
+
if (err.code === "ENOENT")
|
|
211
|
+
return false;
|
|
212
|
+
throw err;
|
|
213
|
+
}
|
|
214
|
+
return names.some((name) => {
|
|
215
|
+
const file = path.join(delivered, name);
|
|
216
|
+
const stat = fs.lstatSync(file);
|
|
217
|
+
if (!stat.isFile() || stat.isSymbolicLink())
|
|
218
|
+
return false;
|
|
219
|
+
const realFile = fs.realpathSync(file);
|
|
220
|
+
if (!withinRoot(root, realFile))
|
|
221
|
+
return false;
|
|
222
|
+
return stat.mtimeMs >= stoppedAt && stat.mtimeMs <= restartAt + 2 * 60_000;
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
function firstRestartAfter(lane, stop, nextEnvelopeMs) {
|
|
226
|
+
const restart = (lane.resume_history ?? [])
|
|
227
|
+
.map((entry) => ({ entry, at: parseTime(entry.at) }))
|
|
228
|
+
.filter(({ entry, at }) => at !== undefined && at > stop.mtimeMs && (nextEnvelopeMs === undefined || at < nextEnvelopeMs) && typeof entry.trigger === "string")
|
|
229
|
+
.sort((a, b) => a.at - b.at)[0];
|
|
230
|
+
if (!restart || restart.at === undefined)
|
|
231
|
+
return null;
|
|
232
|
+
return {
|
|
233
|
+
at: iso(restart.at),
|
|
234
|
+
trigger: restart.entry.trigger,
|
|
235
|
+
caller: typeof restart.entry.caller === "string" ? restart.entry.caller : null,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
function classifyVerdict(input) {
|
|
239
|
+
const { dueAt, restart, lane, nowMs } = input;
|
|
240
|
+
if (restart) {
|
|
241
|
+
const restartAt = Date.parse(restart.at);
|
|
242
|
+
if (restart.caller === lane.lane_id)
|
|
243
|
+
return "self_wake";
|
|
244
|
+
if (restart.trigger === "keep_alive") {
|
|
245
|
+
const event = input.resumeEvents
|
|
246
|
+
.filter((candidate) => candidate.at > input.stoppedAt && Math.abs(candidate.at - restartAt) <= 2 * 60_000)
|
|
247
|
+
.sort((left, right) => Math.abs(left.at - restartAt) - Math.abs(right.at - restartAt))[0];
|
|
248
|
+
if (event?.reason === "pending_work")
|
|
249
|
+
return "message_wake";
|
|
250
|
+
if (event?.reason === undefined && hasDeliveredDropboxEvidence(input.hqRoot, lane.lane_id, input.stoppedAt, restartAt)) {
|
|
251
|
+
return "message_wake";
|
|
252
|
+
}
|
|
253
|
+
const nextCheckAt = parseTime(input.nextCheckAt);
|
|
254
|
+
if (nextCheckAt !== undefined && nextCheckAt > input.stoppedAt && restartAt < Math.min(nextCheckAt, input.stoppedAt + MAX_NEXT_CHECK_DELAY_MS)) {
|
|
255
|
+
return "early_by_keep_alive";
|
|
256
|
+
}
|
|
257
|
+
if (restartAt < dueAt - KEEP_ALIVE_ON_TIME_TOLERANCE_MS)
|
|
258
|
+
return "early_by_keep_alive";
|
|
259
|
+
if (restartAt <= dueAt + KEEP_ALIVE_ON_TIME_TOLERANCE_MS)
|
|
260
|
+
return "on_time";
|
|
261
|
+
return "late";
|
|
262
|
+
}
|
|
263
|
+
if (restartAt < dueAt)
|
|
264
|
+
return "early_wake";
|
|
265
|
+
return "late";
|
|
266
|
+
}
|
|
267
|
+
if (lane.state === "done" && nowMs > dueAt + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS)
|
|
268
|
+
return "not_restarted";
|
|
269
|
+
return "waiting";
|
|
270
|
+
}
|
|
271
|
+
export function buildKeepAliveAudit(input) {
|
|
272
|
+
const nowMs = input.now.getTime();
|
|
273
|
+
const sinceMs = input.since === undefined
|
|
274
|
+
? nowMs - 24 * 60 * 60_000
|
|
275
|
+
: parseTime(input.since);
|
|
276
|
+
if (sinceMs === undefined || !Number.isFinite(sinceMs)) {
|
|
277
|
+
throw new LanesError("invalid_since", "--since must be a valid ISO date-time", { since: input.since });
|
|
278
|
+
}
|
|
279
|
+
const lanes = readAllLanes(input.hqRoot).filter((lane) => (input.laneId === undefined || lane.lane_id === input.laneId) &&
|
|
280
|
+
typeof lane.hire_grant === "number" && Number.isFinite(lane.hire_grant) && lane.hire_grant > 0);
|
|
281
|
+
const rows = [];
|
|
282
|
+
const errors = [];
|
|
283
|
+
for (const lane of lanes) {
|
|
284
|
+
try {
|
|
285
|
+
const { allMtimes, stops } = readStopEnvelopes(input.hqRoot, lane.lane_id);
|
|
286
|
+
const keepAliveResumeEvents = readKeepAliveResumeEvents(input.hqRoot, lane.lane_id);
|
|
287
|
+
const doneClassifications = readDoneClassifications(input.hqRoot, lane.lane_id);
|
|
288
|
+
for (const stop of stops) {
|
|
289
|
+
if (stop.mtimeMs < sinceMs || stop.mtimeMs > nowMs)
|
|
290
|
+
continue;
|
|
291
|
+
if (!doneClassifications.some((at) => at >= stop.mtimeMs && at - stop.mtimeMs <= 10 * 60_000))
|
|
292
|
+
continue;
|
|
293
|
+
const nextEnvelopeMs = allMtimes.find((mtime) => mtime > stop.mtimeMs);
|
|
294
|
+
const restart = firstRestartAfter(lane, stop, nextEnvelopeMs);
|
|
295
|
+
const due = dueTime(stop, keepAliveResumeEvents);
|
|
296
|
+
rows.push({
|
|
297
|
+
lane: lane.lane_id,
|
|
298
|
+
project: lane.project_id,
|
|
299
|
+
profile: lane.worker_profile ?? lane.worker ?? null,
|
|
300
|
+
stopped_at: iso(stop.mtimeMs),
|
|
301
|
+
next_check_at: stop.nextCheckAt ?? null,
|
|
302
|
+
due_at: iso(due.at),
|
|
303
|
+
due_reason: due.reason,
|
|
304
|
+
restart,
|
|
305
|
+
verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, lane, nowMs, resumeEvents: keepAliveResumeEvents, hqRoot: input.hqRoot }),
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
catch (err) {
|
|
310
|
+
const auditError = err instanceof LanesError
|
|
311
|
+
? err
|
|
312
|
+
: new LanesError("keep_alive_audit_read_failed", `Unable to read keep-alive audit inputs for ${lane.lane_id}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: lane.lane_id, path: laneRunDir(input.hqRoot, lane.lane_id) });
|
|
313
|
+
errors.push({
|
|
314
|
+
lane: lane.lane_id,
|
|
315
|
+
error: auditError.error,
|
|
316
|
+
path: typeof auditError.details.path === "string" ? auditError.details.path : laneRunDir(input.hqRoot, lane.lane_id),
|
|
317
|
+
message: auditError.message,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
rows.sort((a, b) => a.stopped_at.localeCompare(b.stopped_at) || a.lane.localeCompare(b.lane));
|
|
322
|
+
const byVerdict = Object.fromEntries(VERDICTS.map((verdict) => [verdict, 0]));
|
|
323
|
+
let lateOverTenMinutes = 0;
|
|
324
|
+
for (const row of rows) {
|
|
325
|
+
byVerdict[row.verdict] += 1;
|
|
326
|
+
if (row.verdict === "late" && row.restart && Date.parse(row.restart.at) - Date.parse(row.due_at) > LATE_KPI_THRESHOLD_MS) {
|
|
327
|
+
lateOverTenMinutes += 1;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return {
|
|
331
|
+
ok: true,
|
|
332
|
+
since: iso(sinceMs),
|
|
333
|
+
rows,
|
|
334
|
+
summary: {
|
|
335
|
+
by_verdict: byVerdict,
|
|
336
|
+
not_restarted: byVerdict.not_restarted,
|
|
337
|
+
late_over_10m: lateOverTenMinutes,
|
|
338
|
+
early_wake_by_others: byVerdict.early_wake,
|
|
339
|
+
self_wake: byVerdict.self_wake,
|
|
340
|
+
stopped_and_not_restarted_on_their_own: byVerdict.not_restarted + lateOverTenMinutes,
|
|
341
|
+
},
|
|
342
|
+
errors,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
export function formatKeepAliveAuditText(result) {
|
|
346
|
+
const headings = ["LANE", "PROJECT", "PROFILE", "STOPPED", "NEXT_CHECK", "DUE", "RESTART", "VERDICT"];
|
|
347
|
+
const values = result.rows.map((row) => [
|
|
348
|
+
row.lane,
|
|
349
|
+
row.project,
|
|
350
|
+
row.profile ?? "—",
|
|
351
|
+
row.stopped_at,
|
|
352
|
+
row.next_check_at ?? "—",
|
|
353
|
+
row.due_at,
|
|
354
|
+
row.restart ? `${row.restart.at} ${row.restart.trigger}${row.restart.caller ? ` (${row.restart.caller})` : ""}` : "—",
|
|
355
|
+
row.verdict,
|
|
356
|
+
]);
|
|
357
|
+
const widths = headings.map((heading, index) => Math.max(heading.length, ...values.map((row) => row[index].length)));
|
|
358
|
+
const tableLine = (row) => row.map((value, index) => value.padEnd(widths[index])).join(" ").trimEnd();
|
|
359
|
+
const counts = VERDICTS.map((verdict) => `${verdict}=${result.summary.by_verdict[verdict]}`).join(", ");
|
|
360
|
+
return [
|
|
361
|
+
tableLine(headings),
|
|
362
|
+
...values.map(tableLine),
|
|
363
|
+
"",
|
|
364
|
+
`Since ${result.since}; ${counts}`,
|
|
365
|
+
`not_restarted=${result.summary.not_restarted}, late_over_10m=${result.summary.late_over_10m}, early_wake_by_others=${result.summary.early_wake_by_others}, self_wake=${result.summary.self_wake}`,
|
|
366
|
+
`Stopped and not restarted on their own: ${result.summary.stopped_and_not_restarted_on_their_own}`,
|
|
367
|
+
...result.errors.map((error) => `Error ${error.lane} (${error.error}) at ${error.path}: ${error.message}`),
|
|
368
|
+
].join("\n");
|
|
369
|
+
}
|
|
370
|
+
//# sourceMappingURL=keep-alive-audit.js.map
|
|
@@ -27,7 +27,7 @@ import { checkLaneDiskSpace, configuredCodexRolloutHomes, lowDiskGuardEnabledFor
|
|
|
27
27
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
28
28
|
import { appendDropboxMessage } from "./dropbox.js";
|
|
29
29
|
import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
|
|
30
|
-
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
30
|
+
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
31
31
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
32
32
|
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
33
33
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
@@ -2303,6 +2303,7 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
|
|
|
2303
2303
|
at: now.toISOString(),
|
|
2304
2304
|
attempt: (lane.keep_alive_resume_count ?? 0) + 1,
|
|
2305
2305
|
resumed_from: "blocked",
|
|
2306
|
+
reason: "backoff",
|
|
2306
2307
|
...(accountSelection !== undefined
|
|
2307
2308
|
? { account_selection: keepAliveAccountSelectionEvidence(accountSelection) }
|
|
2308
2309
|
: {}),
|
|
@@ -2733,6 +2734,23 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
|
|
|
2733
2734
|
continue;
|
|
2734
2735
|
}
|
|
2735
2736
|
const nextEligibleAt = new Date(nowMs + leadLaneKeepAliveBackoffMs(immediateFirstResumeEnabled ? attempt : attempt + 1)).toISOString();
|
|
2737
|
+
let resumeReason = "backoff";
|
|
2738
|
+
if (initialState === "done") {
|
|
2739
|
+
if (hasPendingKeepAliveWork(hqRoot, current, questions)) {
|
|
2740
|
+
resumeReason = "pending_work";
|
|
2741
|
+
}
|
|
2742
|
+
else {
|
|
2743
|
+
const finalEnvelope = readLatestEnvelopeRecord(hqRoot, current.lane_id)?.envelope;
|
|
2744
|
+
const nextCheckMs = Date.parse(finalEnvelope?.next_check_at ?? "");
|
|
2745
|
+
const anchorMs = Date.parse(current.last_envelope?.at ?? current.updated_at);
|
|
2746
|
+
if (Number.isFinite(nextCheckMs) && Number.isFinite(anchorMs)) {
|
|
2747
|
+
const cappedNextCheckMs = Math.min(nextCheckMs, anchorMs + LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS);
|
|
2748
|
+
if (cappedNextCheckMs > nextEligibleMs && cappedNextCheckMs <= nowMs) {
|
|
2749
|
+
resumeReason = "next_check_at_due";
|
|
2750
|
+
}
|
|
2751
|
+
}
|
|
2752
|
+
}
|
|
2753
|
+
}
|
|
2736
2754
|
const queuedCount = loadLaneRecords(hqRoot, summary).filter((queuedLane) => queuedLane.state === "queued" &&
|
|
2737
2755
|
queuedLane.provider === current.provider &&
|
|
2738
2756
|
laneRunsHere(queuedLane, deps.host) &&
|
|
@@ -3007,6 +3025,7 @@ async function resumeKeepAlive(hqRoot, lanes, questions, summary, deps, now, fla
|
|
|
3007
3025
|
attempt,
|
|
3008
3026
|
next_eligible_at: nextEligibleAt,
|
|
3009
3027
|
resumed_from: resumedFrom,
|
|
3028
|
+
reason: resumeReason,
|
|
3010
3029
|
...(accountSelection !== undefined
|
|
3011
3030
|
? { account_selection: keepAliveAccountSelectionEvidence(accountSelection) }
|
|
3012
3031
|
: {}),
|
package/dist/sentry.d.ts
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
|
-
import * as Sentry from "@sentry/node";
|
|
2
1
|
import type { ErrorEvent, EventHint, NodeOptions } from "@sentry/node";
|
|
2
|
+
type SentryModule = typeof import("@sentry/node");
|
|
3
|
+
/**
|
|
4
|
+
* The command graph touches the Sentry facade on normal startup for a
|
|
5
|
+
* breadcrumb and a final flush. Keep those operations inert until the SDK
|
|
6
|
+
* module is loaded; the first capture that needs it loads the SDK synchronously.
|
|
7
|
+
*/
|
|
8
|
+
export declare const Sentry: SentryModule;
|
|
3
9
|
/**
|
|
4
10
|
* Drop broken-pipe (EPIPE) crashes before scrubbing/send. A closed downstream
|
|
5
11
|
* reader (`hq … | head`, `source <(hq …)`, a parent that exited) is normal
|
|
@@ -18,5 +24,5 @@ export declare function epipeAwareBeforeSend(event: ErrorEvent, hint: EventHint)
|
|
|
18
24
|
export declare function initSentry(overrides?: Partial<Pick<NodeOptions, "transport">>): void;
|
|
19
25
|
/** Finish HQ's one per-invocation release-health session. */
|
|
20
26
|
export declare function finishSentrySession(): void;
|
|
21
|
-
export {
|
|
27
|
+
export {};
|
|
22
28
|
//# sourceMappingURL=sentry.d.ts.map
|
package/dist/sentry.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
2
|
import { BUNDLED_DSN } from "./sentry-dsn.generated.js";
|
|
3
3
|
import { beforeSend } from "./sentry-before-send.js";
|
|
4
4
|
import { beforeBreadcrumb } from "./utils/breadcrumb-buffer.js";
|
|
@@ -9,6 +9,60 @@ import { companySkillWrapperSentryTagsFor } from "./lib/company-skill-wrapper.js
|
|
|
9
9
|
import { environmentalFsErrorMessage } from "./utils/environmental-error.js";
|
|
10
10
|
import { incompleteInstallCaptureContext, incompleteInstallMessage, } from "./utils/incomplete-install-error.js";
|
|
11
11
|
import { installTreeTornSentryTagsFor, sentryFingerprintFor, sentryMcpCollisionContextFor, } from "./utils/sentry-fingerprint.js";
|
|
12
|
+
const inertSentryScope = {
|
|
13
|
+
setTag: () => inertSentryScope,
|
|
14
|
+
setUser: () => inertSentryScope,
|
|
15
|
+
addBreadcrumb: () => inertSentryScope,
|
|
16
|
+
clearBreadcrumbs: () => inertSentryScope,
|
|
17
|
+
setLevel: () => inertSentryScope,
|
|
18
|
+
setFingerprint: () => inertSentryScope,
|
|
19
|
+
};
|
|
20
|
+
const requireSentry = createRequire(import.meta.url);
|
|
21
|
+
let sentryModule;
|
|
22
|
+
let pendingSentryBreadcrumbs = [];
|
|
23
|
+
function loadSentryModule() {
|
|
24
|
+
sentryModule ??= requireSentry("@sentry/node");
|
|
25
|
+
return sentryModule;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* The command graph touches the Sentry facade on normal startup for a
|
|
29
|
+
* breadcrumb and a final flush. Keep those operations inert until the SDK
|
|
30
|
+
* module is loaded; the first capture that needs it loads the SDK synchronously.
|
|
31
|
+
*/
|
|
32
|
+
export const Sentry = new Proxy({}, {
|
|
33
|
+
get(_target, property) {
|
|
34
|
+
if (property === Symbol.toStringTag)
|
|
35
|
+
return "Module";
|
|
36
|
+
if (property === "addBreadcrumb") {
|
|
37
|
+
return (breadcrumb) => {
|
|
38
|
+
if (sentryModule) {
|
|
39
|
+
sentryModule.addBreadcrumb(breadcrumb);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
if (pendingSentryBreadcrumbs.length >= 100)
|
|
43
|
+
pendingSentryBreadcrumbs.shift();
|
|
44
|
+
pendingSentryBreadcrumbs.push(breadcrumb);
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
if (property === "withScope") {
|
|
48
|
+
return (callback) => {
|
|
49
|
+
if (sentryModule)
|
|
50
|
+
return sentryModule.withScope(callback);
|
|
51
|
+
return callback(inertSentryScope);
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
if (property === "flush") {
|
|
55
|
+
return (timeout) => {
|
|
56
|
+
if (!sentryModule)
|
|
57
|
+
return Promise.resolve(true);
|
|
58
|
+
return sentryModule.flush(timeout);
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
const sdk = loadSentryModule();
|
|
62
|
+
const value = Reflect.get(sdk, property);
|
|
63
|
+
return typeof value === "function" ? value.bind(sdk) : value;
|
|
64
|
+
},
|
|
65
|
+
});
|
|
12
66
|
/**
|
|
13
67
|
* Drop broken-pipe (EPIPE) crashes before scrubbing/send. A closed downstream
|
|
14
68
|
* reader (`hq … | head`, `source <(hq …)`, a parent that exited) is normal
|
|
@@ -172,7 +226,8 @@ export function initSentry(overrides = {}) {
|
|
|
172
226
|
const dsn = BUNDLED_DSN || process.env.SENTRY_DSN;
|
|
173
227
|
if (!dsn)
|
|
174
228
|
return;
|
|
175
|
-
|
|
229
|
+
const sdk = loadSentryModule();
|
|
230
|
+
sdk.init({
|
|
176
231
|
dsn,
|
|
177
232
|
// import-in-the-middle's async ESM loader makes every later CLI import
|
|
178
233
|
// substantially slower. Keep HQ's explicit breadcrumbs, but give up
|
|
@@ -197,15 +252,17 @@ export function initSentry(overrides = {}) {
|
|
|
197
252
|
// logged in). A CLI process is one user, so global setUser is correct here.
|
|
198
253
|
const user = getCachedSentryUser();
|
|
199
254
|
if (user)
|
|
200
|
-
|
|
255
|
+
sdk.setUser(user);
|
|
256
|
+
for (const breadcrumb of pendingSentryBreadcrumbs)
|
|
257
|
+
sdk.addBreadcrumb(breadcrumb);
|
|
258
|
+
pendingSentryBreadcrumbs = [];
|
|
201
259
|
// Release health: one session per CLI run. Ended + flushed in index.ts's
|
|
202
260
|
// finally (covers every exit path). A captured error auto-marks the session
|
|
203
261
|
// "errored", so crash-free numbers per release stay accurate.
|
|
204
|
-
|
|
262
|
+
sdk.startSession();
|
|
205
263
|
}
|
|
206
264
|
/** Finish HQ's one per-invocation release-health session. */
|
|
207
265
|
export function finishSentrySession() {
|
|
208
|
-
|
|
266
|
+
sentryModule?.endSession();
|
|
209
267
|
}
|
|
210
|
-
export { Sentry };
|
|
211
268
|
//# sourceMappingURL=sentry.js.map
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { type FirstSyncStepOutcome } from "../lib/first-sync-next-step.js";
|
|
2
|
-
export { CLI_FIRST_SYNC_NEXT_STEP_FLAG, FIRST_SYNC_CONFIRMATION_COPY, FIRST_SYNC_NEXT_STEP_COPY, claimFirstSyncStepMarker, firstSyncStepMarkerPaths, firstSyncStepTelemetryProperties,
|
|
2
|
+
export { CLI_FIRST_SYNC_NEXT_STEP_FLAG, FIRST_SYNC_CONFIRMATION_COPY, FIRST_SYNC_NEXT_STEP_COPY, claimFirstSyncStepMarker, firstSyncStepMarkerPaths, firstSyncStepTelemetryProperties, isHumanInteractiveCliSession, isWorkBearingSyncResult, maybeShowFirstSyncNextStep, recordFirstWorkBearingSync, shouldConfirmFirstWorkBearingSync, shouldShowFirstSyncNextStep, takeFirstSyncStepOutcomes, type FirstSyncStepOutcome, } from "../lib/first-sync-next-step.js";
|
|
3
3
|
export declare const TELEMETRY_TIMEOUT_MS = 1200;
|
|
4
4
|
/**
|
|
5
5
|
* Exact stderr line printed once on the first signed-in TTY run that is not
|
|
6
6
|
* `--json`. Keep this notice and its marker stable for session telemetry;
|
|
7
|
-
* first-sync and feature-used fields have separate
|
|
7
|
+
* first-sync and feature-used fields have separate disclosures below; only
|
|
8
|
+
* feature-used telemetry remains opt-in.
|
|
8
9
|
*/
|
|
9
10
|
export declare const TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp, never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
|
|
10
|
-
export declare const FIRST_SYNC_TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp;
|
|
11
|
+
export declare const FIRST_SYNC_TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp; first-sync guidance shown or followed, with the identifier cli.first-sync-next-step; never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
|
|
11
12
|
export declare const CLI_FEATURE_USED_TELEMETRY_FLAG = "cli.feature-used-telemetry-v1";
|
|
12
13
|
export declare const CLI_FEATURE_USED_TELEMETRY_DISCLOSURE_LINE = "HQ sends a feature-use event when you run a supported CLI feature (feature kind and outcome, plus your company UID when available; never command names, file contents, or secrets). The hq-flags key cli.feature-used-telemetry-v1 controls these events. Set HQ_TELEMETRY=0 to turn this off.";
|
|
13
14
|
export declare const CLI_LIMIT_AT_ACTION_PROMPT_FLAG = "billing.limit-at-action-prompt";
|
|
@@ -6,16 +6,17 @@ import { CLI_VERSION } from "../cli-version.js";
|
|
|
6
6
|
import { isExpiring, isMachineIdentity, loadCachedTokens, loadMachineCachedTokens, } from "./cognito-session.js";
|
|
7
7
|
import { resolveProcessFlagCompanyContext, resolveProcessFlagGate, } from "../lib/flag-registry.js";
|
|
8
8
|
import { vaultApiFetch } from "./vault-api.js";
|
|
9
|
-
import { firstSyncStepTelemetryProperties,
|
|
10
|
-
export { CLI_FIRST_SYNC_NEXT_STEP_FLAG, FIRST_SYNC_CONFIRMATION_COPY, FIRST_SYNC_NEXT_STEP_COPY, claimFirstSyncStepMarker, firstSyncStepMarkerPaths, firstSyncStepTelemetryProperties,
|
|
9
|
+
import { firstSyncStepTelemetryProperties, isHumanInteractiveCliSession, } from "../lib/first-sync-next-step.js";
|
|
10
|
+
export { CLI_FIRST_SYNC_NEXT_STEP_FLAG, FIRST_SYNC_CONFIRMATION_COPY, FIRST_SYNC_NEXT_STEP_COPY, claimFirstSyncStepMarker, firstSyncStepMarkerPaths, firstSyncStepTelemetryProperties, isHumanInteractiveCliSession, isWorkBearingSyncResult, maybeShowFirstSyncNextStep, recordFirstWorkBearingSync, shouldConfirmFirstWorkBearingSync, shouldShowFirstSyncNextStep, takeFirstSyncStepOutcomes, } from "../lib/first-sync-next-step.js";
|
|
11
11
|
export const TELEMETRY_TIMEOUT_MS = 1_200;
|
|
12
12
|
/**
|
|
13
13
|
* Exact stderr line printed once on the first signed-in TTY run that is not
|
|
14
14
|
* `--json`. Keep this notice and its marker stable for session telemetry;
|
|
15
|
-
* first-sync and feature-used fields have separate
|
|
15
|
+
* first-sync and feature-used fields have separate disclosures below; only
|
|
16
|
+
* feature-used telemetry remains opt-in.
|
|
16
17
|
*/
|
|
17
18
|
export const TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp, never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
|
|
18
|
-
export const FIRST_SYNC_TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp;
|
|
19
|
+
export const FIRST_SYNC_TELEMETRY_DISCLOSURE_LINE = "HQ sends a usage event when you run a signed-in command (CLI version and a timestamp; first-sync guidance shown or followed, with the identifier cli.first-sync-next-step; never command names, file contents, or secrets) to improve the CLI. Set HQ_TELEMETRY=0 to turn this off.";
|
|
19
20
|
export const CLI_FEATURE_USED_TELEMETRY_FLAG = "cli.feature-used-telemetry-v1";
|
|
20
21
|
export const CLI_FEATURE_USED_TELEMETRY_DISCLOSURE_LINE = "HQ sends a feature-use event when you run a supported CLI feature (feature kind and outcome, plus your company UID when available; never command names, file contents, or secrets). The hq-flags key cli.feature-used-telemetry-v1 controls these events. Set HQ_TELEMETRY=0 to turn this off.";
|
|
21
22
|
const TELEMETRY_DISABLED_VALUES = new Set(["0", "false", "off"]);
|
|
@@ -289,8 +290,6 @@ export async function emitCliFirstSyncNextStepOutcome(outcome, timeoutMs = TELEM
|
|
|
289
290
|
try {
|
|
290
291
|
if (isCliTelemetryDisabled())
|
|
291
292
|
return;
|
|
292
|
-
if (!isFirstSyncNextStepEnabled())
|
|
293
|
-
return;
|
|
294
293
|
const cached = loadCachedTokens();
|
|
295
294
|
const machineIdentity = isMachineIdentity();
|
|
296
295
|
if (!cached ||
|