@indigoai-us/hq-cli 5.342.11 → 5.342.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/command-catalog.generated.d.ts +1 -1
- package/dist/command-catalog.generated.js +1 -1
- package/dist/commands/agent-kit.d.ts +2 -0
- package/dist/commands/agent-kit.js +23 -17
- package/dist/commands/agent-mcp.d.ts +2 -0
- package/dist/commands/agent-mcp.js +21 -15
- package/dist/commands/lanes-keep-alive-audit.js +1 -1
- package/dist/commands/lanes-message.d.ts +20 -2
- package/dist/commands/lanes-message.js +99 -38
- package/dist/commands/lanes-shared.d.ts +1 -0
- package/dist/commands/lanes.d.ts +4 -0
- package/dist/commands/lanes.js +24 -30
- package/dist/commands/mesh.js +2 -1
- package/dist/commands/monitor.d.ts +14 -1
- package/dist/commands/monitor.js +43 -14
- package/dist/lib/doctor/checks/anywhere.js +11 -6
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/lanes/keep-alive-audit.d.ts +5 -1
- package/dist/lib/lanes/keep-alive-audit.js +132 -19
- package/dist/lib/lanes/reconcile.js +46 -3
- package/dist/lib/lanes/spawn.js +79 -0
- package/dist/lib/lanes/types.d.ts +2 -1
- package/dist/utils/package-use-lease.d.ts +15 -0
- package/dist/utils/package-use-lease.js +94 -1
- package/dist/utils/resident-package-preload.d.ts +8 -1
- package/dist/utils/resident-package-preload.js +68 -0
- package/dist/utils/self-update.d.ts +5 -0
- package/dist/utils/self-update.js +100 -12
- package/docs/anywhere-runtime.md +3 -1
- package/package.json +4 -4
- package/dist/lib/db/remote-engine.d.ts +0 -58
- package/dist/lib/db/remote-engine.js +0 -87
- package/dist/lib/db/remote-sql.d.ts +0 -22
- package/dist/lib/db/remote-sql.js +0 -36
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "keep_alive_off" | "waiting";
|
|
1
|
+
type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "keep_alive_off" | "waiting";
|
|
2
2
|
interface KeepAliveAuditRestart {
|
|
3
3
|
at: string | null;
|
|
4
4
|
trigger: string;
|
|
5
5
|
caller: string | null;
|
|
6
|
+
workerEvent?: boolean;
|
|
6
7
|
}
|
|
7
8
|
interface KeepAliveAuditRow {
|
|
8
9
|
lane: string;
|
|
@@ -13,6 +14,8 @@ interface KeepAliveAuditRow {
|
|
|
13
14
|
due_at: string;
|
|
14
15
|
due_reason: "next_check_at" | "backoff";
|
|
15
16
|
restart: KeepAliveAuditRestart | null;
|
|
17
|
+
blocked_outcome?: "woken" | "skipped";
|
|
18
|
+
blocked_outcome_reason?: string;
|
|
16
19
|
verdict: KeepAliveAuditVerdict;
|
|
17
20
|
}
|
|
18
21
|
interface KeepAliveAuditSummary {
|
|
@@ -41,6 +44,7 @@ export interface KeepAliveAuditResult {
|
|
|
41
44
|
export interface StopEnvelope {
|
|
42
45
|
name: string;
|
|
43
46
|
mtimeMs: number;
|
|
47
|
+
decision: "done" | "blocked";
|
|
44
48
|
nextCheckAt?: string;
|
|
45
49
|
}
|
|
46
50
|
export declare function readStopEnvelopes(hqRoot: string, laneId: string): {
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
|
-
import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS } from "./keep-alive.js";
|
|
3
|
+
import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS } from "./keep-alive.js";
|
|
4
4
|
import { LanesError } from "./errors.js";
|
|
5
5
|
import { laneRunDir } from "./spawn.js";
|
|
6
6
|
import { compareEnvelopeNames } from "./envelope-sequence.js";
|
|
7
7
|
import { listQuestionIds, readAllLanes, tryReadQuestion } from "./store.js";
|
|
8
|
-
|
|
8
|
+
import { isIsoDateTime } from "./types.js";
|
|
9
|
+
function readTerminalClassifications(hqRoot, laneId) {
|
|
9
10
|
const root = fs.realpathSync(hqRoot);
|
|
10
11
|
const watcherLog = path.join(laneRunDir(root, laneId), "logs", "watcher.out");
|
|
11
12
|
if (!withinRoot(root, watcherLog)) {
|
|
@@ -28,23 +29,23 @@ function readDoneClassifications(hqRoot, laneId) {
|
|
|
28
29
|
return [];
|
|
29
30
|
throw err;
|
|
30
31
|
}
|
|
31
|
-
const
|
|
32
|
+
const classifications = [];
|
|
32
33
|
for (const line of text.split(/\r?\n/)) {
|
|
33
34
|
if (!line.includes('"classification"'))
|
|
34
35
|
continue;
|
|
35
36
|
try {
|
|
36
37
|
const value = JSON.parse(line);
|
|
37
|
-
if (!isRecord(value) || value.state !== "done")
|
|
38
|
+
if (!isRecord(value) || (value.state !== "done" && value.state !== "blocked"))
|
|
38
39
|
continue;
|
|
39
40
|
const at = parseTime(value.at);
|
|
40
41
|
if (at !== undefined)
|
|
41
|
-
|
|
42
|
+
classifications.push({ at, state: value.state });
|
|
42
43
|
}
|
|
43
44
|
catch (err) {
|
|
44
45
|
throw new LanesError("keep_alive_audit_event_invalid", `Cannot parse lane watcher classification in ${watcherLog}: ${err instanceof Error ? err.message : String(err)}`, { lane_id: laneId, path: watcherLog });
|
|
45
46
|
}
|
|
46
47
|
}
|
|
47
|
-
return
|
|
48
|
+
return classifications.sort((a, b) => a.at - b.at);
|
|
48
49
|
}
|
|
49
50
|
const KEEP_ALIVE_ON_TIME_TOLERANCE_MS = 60_000;
|
|
50
51
|
const KEEP_ALIVE_NOT_RESTARTED_GRACE_MS = 5 * 60_000;
|
|
@@ -55,6 +56,7 @@ const VERDICTS = [
|
|
|
55
56
|
"late",
|
|
56
57
|
"early_wake",
|
|
57
58
|
"message_wake",
|
|
59
|
+
"work_triggered",
|
|
58
60
|
"self_wake",
|
|
59
61
|
"early_by_keep_alive",
|
|
60
62
|
"restarted_untimed",
|
|
@@ -68,6 +70,50 @@ function parseTime(value) {
|
|
|
68
70
|
const parsed = Date.parse(value);
|
|
69
71
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
70
72
|
}
|
|
73
|
+
function readLaneWorkerResumeEvents(hqRoot, laneId) {
|
|
74
|
+
const root = fs.realpathSync(hqRoot);
|
|
75
|
+
const eventsPath = path.join(laneRunDir(root, laneId), "watcher", "events.jsonl");
|
|
76
|
+
if (!withinRoot(root, eventsPath)) {
|
|
77
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher event path escapes the HQ root for ${laneId}`, { lane_id: laneId });
|
|
78
|
+
}
|
|
79
|
+
let text;
|
|
80
|
+
try {
|
|
81
|
+
const stat = fs.lstatSync(eventsPath);
|
|
82
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
83
|
+
throw new LanesError("keep_alive_audit_event_invalid", `Lane watcher events must be a regular file: ${eventsPath}`, { lane_id: laneId, path: eventsPath });
|
|
84
|
+
}
|
|
85
|
+
const realPath = fs.realpathSync(eventsPath);
|
|
86
|
+
if (!withinRoot(root, realPath)) {
|
|
87
|
+
throw new LanesError("keep_alive_audit_path_invalid", `Lane watcher event file escapes the HQ root for ${laneId}`, { lane_id: laneId, path: eventsPath });
|
|
88
|
+
}
|
|
89
|
+
text = fs.readFileSync(realPath, "utf8");
|
|
90
|
+
}
|
|
91
|
+
catch (err) {
|
|
92
|
+
if (err.code === "ENOENT")
|
|
93
|
+
return [];
|
|
94
|
+
throw err;
|
|
95
|
+
}
|
|
96
|
+
const events = [];
|
|
97
|
+
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
98
|
+
if (!line.trim())
|
|
99
|
+
continue;
|
|
100
|
+
let value;
|
|
101
|
+
try {
|
|
102
|
+
value = JSON.parse(line);
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
if (!isRecord(value) || value.event !== "lane_worker_resumed")
|
|
108
|
+
continue;
|
|
109
|
+
const at = parseTime(value.resumed_at);
|
|
110
|
+
if (at === undefined || typeof value.reason !== "string" || !value.reason.trim()) {
|
|
111
|
+
throw new LanesError("keep_alive_audit_event_invalid", `Lane worker resume event has invalid timing or reason: ${eventsPath}:${index + 1}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
|
|
112
|
+
}
|
|
113
|
+
events.push({ at, reason: value.reason, caller: typeof value.caller === "string" ? value.caller : null });
|
|
114
|
+
}
|
|
115
|
+
return events.sort((a, b) => a.at - b.at);
|
|
116
|
+
}
|
|
71
117
|
function iso(ms) {
|
|
72
118
|
return new Date(ms).toISOString();
|
|
73
119
|
}
|
|
@@ -116,11 +162,15 @@ export function readStopEnvelopes(hqRoot, laneId) {
|
|
|
116
162
|
if (!isRecord(raw) || raw.schema_version !== "hq-lane-envelope.v1") {
|
|
117
163
|
throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope is not hq-lane-envelope.v1: ${file}`, { lane_id: laneId, path: file });
|
|
118
164
|
}
|
|
119
|
-
|
|
165
|
+
const doneStop = raw.decision === "done" && raw.story_complete === false;
|
|
166
|
+
const blockedStop = raw.decision === "blocked" && typeof raw.next_check_at === "string" &&
|
|
167
|
+
isIsoDateTime(raw.next_check_at) && Date.parse(raw.next_check_at) > stat.mtimeMs;
|
|
168
|
+
if (!doneStop && !blockedStop)
|
|
120
169
|
continue;
|
|
121
170
|
stops.push({
|
|
122
171
|
name,
|
|
123
172
|
mtimeMs: stat.mtimeMs,
|
|
173
|
+
decision: blockedStop ? "blocked" : "done",
|
|
124
174
|
...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
|
|
125
175
|
});
|
|
126
176
|
}
|
|
@@ -176,19 +226,20 @@ function readKeepAliveResumeEvents(hqRoot, laneId) {
|
|
|
176
226
|
if (nextEligibleAt === undefined) {
|
|
177
227
|
throw new LanesError("keep_alive_audit_event_invalid", `Keep-alive resume event has invalid timing fields: ${eventsPath}:${index + 1}`, { lane_id: laneId, path: eventsPath, line: index + 1 });
|
|
178
228
|
}
|
|
179
|
-
const reason =
|
|
180
|
-
? value.reason
|
|
181
|
-
: undefined;
|
|
229
|
+
const reason = typeof value.reason === "string" ? value.reason : undefined;
|
|
182
230
|
events.push({ at, nextEligibleAt, ...(reason === undefined ? {} : { reason }) });
|
|
183
231
|
}
|
|
184
232
|
return events.sort((a, b) => a.at - b.at);
|
|
185
233
|
}
|
|
186
234
|
function dueTime(stop, resumeEvents) {
|
|
235
|
+
const nextCheck = parseTime(stop.nextCheckAt);
|
|
236
|
+
if (stop.decision === "blocked" && nextCheck !== undefined && nextCheck > stop.mtimeMs) {
|
|
237
|
+
return { at: Math.min(nextCheck, stop.mtimeMs + LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS), reason: "next_check_at" };
|
|
238
|
+
}
|
|
187
239
|
const previousResume = resumeEvents.filter((event) => event.at <= stop.mtimeMs).at(-1);
|
|
188
240
|
const terminalPlusBackoff = stop.mtimeMs + LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS;
|
|
189
241
|
let due = Math.max(terminalPlusBackoff, previousResume?.nextEligibleAt ?? 0);
|
|
190
242
|
let reason = "backoff";
|
|
191
|
-
const nextCheck = parseTime(stop.nextCheckAt);
|
|
192
243
|
if (nextCheck !== undefined && nextCheck > stop.mtimeMs) {
|
|
193
244
|
due = Math.max(due, Math.min(nextCheck, stop.mtimeMs + MAX_NEXT_CHECK_DELAY_MS));
|
|
194
245
|
reason = "next_check_at";
|
|
@@ -286,7 +337,50 @@ function laneWasKeepAliveOffAtStop(lane, stoppedAt) {
|
|
|
286
337
|
const optedInAt = parseTime(lane.keep_alive_opted_in_at);
|
|
287
338
|
return optedInAt === undefined || optedInAt > stoppedAt;
|
|
288
339
|
}
|
|
289
|
-
function
|
|
340
|
+
function blockedKeepAliveOutcome(lane, stop, nextEnvelopeMs) {
|
|
341
|
+
const terminalAt = lane.last_envelope?.ref === `envelopes/${stop.name}`
|
|
342
|
+
? parseTime(lane.last_envelope.at)
|
|
343
|
+
: undefined;
|
|
344
|
+
const candidates = (lane.status_gate_history ?? []).flatMap((entry) => {
|
|
345
|
+
if (entry.event !== "keep_alive_blocked_resumed" && entry.event !== "keep_alive_blocked_skipped")
|
|
346
|
+
return [];
|
|
347
|
+
const at = parseTime(entry.at);
|
|
348
|
+
const terminal = parseTime(entry.terminal_at);
|
|
349
|
+
if (at === undefined || terminal === undefined)
|
|
350
|
+
return [];
|
|
351
|
+
if (nextEnvelopeMs !== undefined && at >= nextEnvelopeMs)
|
|
352
|
+
return [];
|
|
353
|
+
const sameStop = terminalAt !== undefined
|
|
354
|
+
? terminal === terminalAt
|
|
355
|
+
: Math.abs(terminal - stop.mtimeMs) <= 10 * 60_000;
|
|
356
|
+
if (!sameStop)
|
|
357
|
+
return [];
|
|
358
|
+
if (entry.event === "keep_alive_blocked_skipped" && entry.reason !== "flag_off")
|
|
359
|
+
return [];
|
|
360
|
+
return [{
|
|
361
|
+
status: entry.event === "keep_alive_blocked_resumed" ? "woken" : "skipped",
|
|
362
|
+
...(entry.reason === undefined ? {} : { reason: entry.reason }),
|
|
363
|
+
at: entry.at,
|
|
364
|
+
atMs: at,
|
|
365
|
+
}];
|
|
366
|
+
});
|
|
367
|
+
const latest = candidates.sort((left, right) => right.atMs - left.atMs)[0];
|
|
368
|
+
if (!latest)
|
|
369
|
+
return undefined;
|
|
370
|
+
return { status: latest.status, ...(latest.reason === undefined ? {} : { reason: latest.reason }), at: latest.at };
|
|
371
|
+
}
|
|
372
|
+
function firstRestartAfter(lane, stop, nextEnvelopeMs, workerResumeEvents) {
|
|
373
|
+
const recordedWorkerResume = workerResumeEvents
|
|
374
|
+
.filter((event) => event.at > stop.mtimeMs && (nextEnvelopeMs === undefined || event.at < nextEnvelopeMs))
|
|
375
|
+
.sort((a, b) => a.at - b.at)[0];
|
|
376
|
+
if (recordedWorkerResume !== undefined) {
|
|
377
|
+
return {
|
|
378
|
+
at: iso(recordedWorkerResume.at),
|
|
379
|
+
trigger: recordedWorkerResume.reason,
|
|
380
|
+
caller: recordedWorkerResume.caller,
|
|
381
|
+
workerEvent: true,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
290
384
|
const restart = (lane.resume_history ?? [])
|
|
291
385
|
.map((entry) => ({ entry, at: parseTime(entry.at) }))
|
|
292
386
|
.filter(({ entry, at }) => at !== undefined && at > stop.mtimeMs && (nextEnvelopeMs === undefined || at < nextEnvelopeMs) && typeof entry.trigger === "string")
|
|
@@ -329,6 +423,14 @@ function classifyVerdict(input) {
|
|
|
329
423
|
if (restart?.trigger === "unknown" && laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
|
|
330
424
|
return "keep_alive_off";
|
|
331
425
|
if (restart) {
|
|
426
|
+
if (restart.trigger === "pending_work" || restart.trigger === "watcher" || restart.trigger === "monitor_event")
|
|
427
|
+
return "work_triggered";
|
|
428
|
+
const workEvidenceEnd = input.nextEnvelopeAt ?? Number.POSITIVE_INFINITY;
|
|
429
|
+
const triggeredWork = restart.workerEvent !== true && input.resumeEvents
|
|
430
|
+
.filter((event) => event.at > input.stoppedAt && event.at < workEvidenceEnd && event.at <= (restart.at === null ? Number.POSITIVE_INFINITY : Date.parse(restart.at)))
|
|
431
|
+
.some((event) => event.reason === "pending_work" || event.reason === "watcher" || event.reason === "monitor_event");
|
|
432
|
+
if (triggeredWork)
|
|
433
|
+
return "work_triggered";
|
|
332
434
|
if (restart.at === null)
|
|
333
435
|
return "restarted_untimed";
|
|
334
436
|
const restartAt = Date.parse(restart.at);
|
|
@@ -336,12 +438,14 @@ function classifyVerdict(input) {
|
|
|
336
438
|
return "self_wake";
|
|
337
439
|
if (restart.trigger === "message" || restart.trigger === "answered_questions")
|
|
338
440
|
return "message_wake";
|
|
339
|
-
if (restart.trigger === "keep_alive") {
|
|
441
|
+
if (restart.trigger === "keep_alive" || restart.trigger === "next_check_at_due" || restart.trigger === "backoff") {
|
|
340
442
|
const event = input.resumeEvents
|
|
341
443
|
.filter((candidate) => candidate.at > input.stoppedAt && candidate.at <= restartAt + 2 * 60_000)
|
|
342
444
|
.sort((left, right) => Math.abs(left.at - restartAt) - Math.abs(right.at - restartAt))[0];
|
|
343
445
|
if (event?.reason === "pending_work")
|
|
344
|
-
return "
|
|
446
|
+
return "work_triggered";
|
|
447
|
+
if (event?.reason === "watcher" || event?.reason === "monitor_event")
|
|
448
|
+
return "work_triggered";
|
|
345
449
|
if (event?.reason === undefined &&
|
|
346
450
|
(hasDeliveredDropboxEvidence(input.hqRoot, lane.lane_id, input.stoppedAt, restartAt) ||
|
|
347
451
|
hasMessageResumeEvidence(lane, input.stoppedAt, restartAt) ||
|
|
@@ -384,15 +488,19 @@ export function buildKeepAliveAudit(input) {
|
|
|
384
488
|
try {
|
|
385
489
|
const { allMtimes, stops } = readStopEnvelopes(input.hqRoot, lane.lane_id);
|
|
386
490
|
const keepAliveResumeEvents = readKeepAliveResumeEvents(input.hqRoot, lane.lane_id);
|
|
387
|
-
const
|
|
491
|
+
const laneWorkerResumeEvents = readLaneWorkerResumeEvents(input.hqRoot, lane.lane_id);
|
|
492
|
+
const terminalClassifications = readTerminalClassifications(input.hqRoot, lane.lane_id);
|
|
388
493
|
for (const stop of stops) {
|
|
389
494
|
if (stop.mtimeMs < sinceMs || stop.mtimeMs > nowMs)
|
|
390
495
|
continue;
|
|
391
|
-
if (!
|
|
496
|
+
if (!terminalClassifications.some((classification) => classification.state === stop.decision && classification.at >= stop.mtimeMs && classification.at - stop.mtimeMs <= 10 * 60_000))
|
|
392
497
|
continue;
|
|
393
498
|
const nextEnvelopeMs = allMtimes.find((mtime) => mtime > stop.mtimeMs);
|
|
394
|
-
const
|
|
499
|
+
const blockedOutcome = stop.decision === "blocked" ? blockedKeepAliveOutcome(lane, stop, nextEnvelopeMs) : undefined;
|
|
500
|
+
const restart = firstRestartAfter(lane, stop, nextEnvelopeMs, laneWorkerResumeEvents);
|
|
395
501
|
const due = dueTime(stop, keepAliveResumeEvents);
|
|
502
|
+
const reportBlockedOutcome = blockedOutcome?.status === "woken" ||
|
|
503
|
+
(blockedOutcome?.status === "skipped" && due.at <= nowMs);
|
|
396
504
|
rows.push({
|
|
397
505
|
lane: lane.lane_id,
|
|
398
506
|
project: lane.project_id,
|
|
@@ -402,7 +510,11 @@ export function buildKeepAliveAudit(input) {
|
|
|
402
510
|
due_at: iso(due.at),
|
|
403
511
|
due_reason: due.reason,
|
|
404
512
|
restart,
|
|
405
|
-
|
|
513
|
+
...(reportBlockedOutcome && blockedOutcome ? {
|
|
514
|
+
blocked_outcome: blockedOutcome.status,
|
|
515
|
+
...(blockedOutcome.reason === undefined ? {} : { blocked_outcome_reason: blockedOutcome.reason }),
|
|
516
|
+
} : {}),
|
|
517
|
+
verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
|
|
406
518
|
});
|
|
407
519
|
}
|
|
408
520
|
}
|
|
@@ -449,7 +561,7 @@ export function buildKeepAliveAudit(input) {
|
|
|
449
561
|
};
|
|
450
562
|
}
|
|
451
563
|
export function formatKeepAliveAuditText(result) {
|
|
452
|
-
const headings = ["LANE", "PROJECT", "PROFILE", "STOPPED", "NEXT_CHECK", "DUE", "RESTART", "VERDICT"];
|
|
564
|
+
const headings = ["LANE", "PROJECT", "PROFILE", "STOPPED", "NEXT_CHECK", "DUE", "RESTART", "OUTCOME", "VERDICT"];
|
|
453
565
|
const values = result.rows.map((row) => [
|
|
454
566
|
row.lane,
|
|
455
567
|
row.project,
|
|
@@ -458,6 +570,7 @@ export function formatKeepAliveAuditText(result) {
|
|
|
458
570
|
row.next_check_at ?? "—",
|
|
459
571
|
row.due_at,
|
|
460
572
|
row.restart ? `${row.restart.at ?? "untimed"} ${row.restart.trigger}${row.restart.caller ? ` (${row.restart.caller})` : ""}` : "—",
|
|
573
|
+
row.blocked_outcome === "woken" ? "woken" : row.blocked_outcome === "skipped" ? `due, not woken: ${(row.blocked_outcome_reason ?? "unknown").replaceAll("_", " ")}` : "—",
|
|
461
574
|
row.verdict,
|
|
462
575
|
]);
|
|
463
576
|
const widths = headings.map((heading, index) => Math.max(heading.length, ...values.map((row) => row[index].length)));
|
|
@@ -27,8 +27,9 @@ import { checkLaneDiskSpace, configuredCodexRolloutHomes, lowDiskGuardEnabledFor
|
|
|
27
27
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
28
28
|
import { appendDropboxMessage } from "./dropbox.js";
|
|
29
29
|
import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolGuard, RUNAWAY_TOOL_GUARD_FLAG, } from "./runaway-tool-guard.js";
|
|
30
|
-
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
30
|
+
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, hasPendingKeepAliveWork, keepAliveNextCheckAtMs, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
31
31
|
import { providerSupportsSessionResume } from "./delivery.js";
|
|
32
|
+
import { isIsoDateTime } from "./types.js";
|
|
32
33
|
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
33
34
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
34
35
|
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
@@ -2252,6 +2253,9 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
|
|
|
2252
2253
|
}
|
|
2253
2254
|
const key = keepAliveCompanyKey(company);
|
|
2254
2255
|
if (keepAliveFlags.get(key) !== true || blockedFlags.get(key) !== true) {
|
|
2256
|
+
if (keepAliveFlags.get(key) === true && blockedFlags.get(key) !== true) {
|
|
2257
|
+
recordBlockedKeepAliveFlagOff(hqRoot, lane, summary, deps);
|
|
2258
|
+
}
|
|
2255
2259
|
summary.skipped.push({ kind: "lane", id: lane.lane_id, why: "keep_alive_blocked" });
|
|
2256
2260
|
return;
|
|
2257
2261
|
}
|
|
@@ -2323,10 +2327,22 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
|
|
|
2323
2327
|
summary.skipped.push({ kind: "lane", id: lane.lane_id, why: "keep_alive_exhausted" });
|
|
2324
2328
|
return;
|
|
2325
2329
|
}
|
|
2326
|
-
|
|
2330
|
+
const blockedEnvelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
2331
|
+
const nextCheckValue = blockedEnvelope?.decision === "blocked" ? blockedEnvelope.next_check_at : undefined;
|
|
2332
|
+
const nextCheckValid = typeof nextCheckValue === "string" && isIsoDateTime(nextCheckValue) &&
|
|
2333
|
+
Date.parse(nextCheckValue) > terminalMs;
|
|
2334
|
+
const nextCheckMs = nextCheckValid
|
|
2335
|
+
? keepAliveNextCheckAtMs(nextCheckValue, now.getTime(), terminalMs)
|
|
2336
|
+
: undefined;
|
|
2337
|
+
if (nextCheckMs !== undefined && nextCheckMs > now.getTime()) {
|
|
2338
|
+
appendBlockedKeepAliveHistory(hqRoot, lane, terminalAt, now.toISOString(), "blocked_grace", deps);
|
|
2339
|
+
return;
|
|
2340
|
+
}
|
|
2341
|
+
if (!nextCheckValid && now.getTime() - terminalMs < LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS) {
|
|
2327
2342
|
appendBlockedKeepAliveHistory(hqRoot, lane, terminalAt, now.toISOString(), "blocked_grace", deps);
|
|
2328
2343
|
return;
|
|
2329
2344
|
}
|
|
2345
|
+
const resumeReason = nextCheckValid ? "next_check_at_due" : "backoff";
|
|
2330
2346
|
const latest = deps.readKeepAliveRecheckLane?.(hqRoot, lane.lane_id) ?? readLane(hqRoot, lane.lane_id);
|
|
2331
2347
|
if (latest.version !== lane.version || latest.state !== "blocked" || latest.last_envelope?.decision !== "blocked" || latest.last_envelope.at !== terminalAt)
|
|
2332
2348
|
return;
|
|
@@ -2389,6 +2405,7 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
|
|
|
2389
2405
|
at: now.toISOString(),
|
|
2390
2406
|
terminal_at: terminalAt,
|
|
2391
2407
|
resume_queued: true,
|
|
2408
|
+
reason: resumeReason,
|
|
2392
2409
|
}),
|
|
2393
2410
|
}), deps.dryRun);
|
|
2394
2411
|
if (queued.changed) {
|
|
@@ -2399,7 +2416,7 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
|
|
|
2399
2416
|
at: now.toISOString(),
|
|
2400
2417
|
attempt: (lane.keep_alive_resume_count ?? 0) + 1,
|
|
2401
2418
|
resumed_from: "blocked",
|
|
2402
|
-
reason:
|
|
2419
|
+
reason: resumeReason,
|
|
2403
2420
|
...(accountSelection !== undefined
|
|
2404
2421
|
? { account_selection: keepAliveAccountSelectionEvidence(accountSelection) }
|
|
2405
2422
|
: {}),
|
|
@@ -2407,6 +2424,32 @@ async function resumeBlockedKeepAlive(hqRoot, lane, questions, summary, deps, no
|
|
|
2407
2424
|
}
|
|
2408
2425
|
}
|
|
2409
2426
|
}
|
|
2427
|
+
function recordBlockedKeepAliveFlagOff(hqRoot, lane, summary, deps) {
|
|
2428
|
+
const terminalAt = lane.last_envelope?.at;
|
|
2429
|
+
if (deps.dryRun || lane.last_envelope?.decision !== "blocked" || terminalAt === undefined)
|
|
2430
|
+
return;
|
|
2431
|
+
const envelope = readLatestEnvelopeRecord(hqRoot, lane.lane_id)?.envelope;
|
|
2432
|
+
const nextCheckAt = envelope?.decision === "blocked" ? envelope.next_check_at : undefined;
|
|
2433
|
+
const terminalMs = Date.parse(lane.last_envelope.at);
|
|
2434
|
+
if (typeof nextCheckAt !== "string" || !isIsoDateTime(nextCheckAt) || Date.parse(nextCheckAt) <= terminalMs)
|
|
2435
|
+
return;
|
|
2436
|
+
if (lane.status_gate_history?.some((entry) => entry.event === "keep_alive_blocked_skipped" && entry.terminal_at === terminalAt && entry.reason === "flag_off"))
|
|
2437
|
+
return;
|
|
2438
|
+
try {
|
|
2439
|
+
casLaneIf(hqRoot, lane.lane_id, (current) => current.version === lane.version && current.state === "blocked" && current.last_envelope?.at === terminalAt, (current) => ({
|
|
2440
|
+
...current,
|
|
2441
|
+
status_gate_history: appendCappedHistory(current.status_gate_history ?? [], {
|
|
2442
|
+
event: "keep_alive_blocked_skipped",
|
|
2443
|
+
at: deps.now().toISOString(),
|
|
2444
|
+
terminal_at: terminalAt,
|
|
2445
|
+
reason: "flag_off",
|
|
2446
|
+
}),
|
|
2447
|
+
}), false);
|
|
2448
|
+
}
|
|
2449
|
+
catch (err) {
|
|
2450
|
+
markDirty(summary, { kind: "lane", id: lane.lane_id, why: `blocked keep-alive skip record failed: ${err instanceof Error ? err.message : String(err)}` });
|
|
2451
|
+
}
|
|
2452
|
+
}
|
|
2410
2453
|
function appendBlockedKeepAliveHistory(hqRoot, lane, terminalAt, at, reason, deps) {
|
|
2411
2454
|
if (deps.dryRun || lane.status_gate_history?.some((entry) => entry.event === "keep_alive_blocked_grace" && entry.terminal_at === terminalAt && entry.resume_skipped_reason === reason))
|
|
2412
2455
|
return;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -56,6 +56,70 @@ const PROVIDER_COMMAND = {
|
|
|
56
56
|
codex: "codex",
|
|
57
57
|
grok: "grok",
|
|
58
58
|
};
|
|
59
|
+
function keepAliveResumeReason(runDir, pendingResume) {
|
|
60
|
+
if (pendingResume?.keep_alive !== true || typeof pendingResume.at !== "string")
|
|
61
|
+
return undefined;
|
|
62
|
+
const eventsPath = path.join(runDir, "watcher", "events.jsonl");
|
|
63
|
+
let content;
|
|
64
|
+
try {
|
|
65
|
+
content = fs.readFileSync(eventsPath, "utf8");
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
if (err.code === "ENOENT")
|
|
69
|
+
return "keep_alive";
|
|
70
|
+
throw err;
|
|
71
|
+
}
|
|
72
|
+
for (const line of content.split(/\r?\n/).reverse()) {
|
|
73
|
+
if (!line.trim())
|
|
74
|
+
continue;
|
|
75
|
+
let value;
|
|
76
|
+
try {
|
|
77
|
+
value = JSON.parse(line);
|
|
78
|
+
}
|
|
79
|
+
catch {
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
83
|
+
const event = value;
|
|
84
|
+
if (event.event === "keep_alive_resume" && event.at === pendingResume.at && typeof event.reason === "string") {
|
|
85
|
+
return event.reason;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return "keep_alive";
|
|
90
|
+
}
|
|
91
|
+
function appendLaneWorkerResumeEvent(input) {
|
|
92
|
+
const eventsPath = path.join(laneRunDir(input.hqRoot, input.laneId), "watcher", "events.jsonl");
|
|
93
|
+
fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
|
|
94
|
+
let separator = "";
|
|
95
|
+
try {
|
|
96
|
+
const fd = fs.openSync(eventsPath, "r");
|
|
97
|
+
try {
|
|
98
|
+
const size = fs.fstatSync(fd).size;
|
|
99
|
+
if (size > 0) {
|
|
100
|
+
const lastByte = Buffer.alloc(1);
|
|
101
|
+
fs.readSync(fd, lastByte, 0, 1, size - 1);
|
|
102
|
+
if (lastByte[0] !== 0x0a)
|
|
103
|
+
separator = "\n";
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
fs.closeSync(fd);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
if (err.code !== "ENOENT")
|
|
112
|
+
throw err;
|
|
113
|
+
}
|
|
114
|
+
fs.appendFileSync(eventsPath, `${separator}${JSON.stringify({
|
|
115
|
+
event: "lane_worker_resumed",
|
|
116
|
+
lane_id: input.laneId,
|
|
117
|
+
resumed_at: input.resumedAt,
|
|
118
|
+
reason: input.reason,
|
|
119
|
+
...(input.caller === undefined ? {} : { caller: input.caller }),
|
|
120
|
+
...(input.turnId === undefined ? {} : { turn_id: input.turnId }),
|
|
121
|
+
})}\n`, "utf8");
|
|
122
|
+
}
|
|
59
123
|
export function providerCommandOnPath(command, pathEnv) {
|
|
60
124
|
// Test and embedded callers can intentionally omit PATH and delegate lookup
|
|
61
125
|
// to their spawn implementation. A present PATH is authoritative.
|
|
@@ -2081,6 +2145,7 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2081
2145
|
trigger.kind === "resume" || trigger.kind === "message"
|
|
2082
2146
|
? "user"
|
|
2083
2147
|
: undefined);
|
|
2148
|
+
const resumeReason = keepAliveResumeReason(runDir, pendingResume) ?? resumeHistoryTrigger;
|
|
2084
2149
|
let statusGateHeartbeat;
|
|
2085
2150
|
try {
|
|
2086
2151
|
statusGateHeartbeat = managerStatusGateHeartbeat(hqRoot, existing, laneTurnId, nowIso);
|
|
@@ -2356,6 +2421,14 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
2356
2421
|
...(resumeHistoryCaller ? { caller: resumeHistoryCaller } : {}),
|
|
2357
2422
|
}),
|
|
2358
2423
|
}));
|
|
2424
|
+
appendLaneWorkerResumeEvent({
|
|
2425
|
+
hqRoot,
|
|
2426
|
+
laneId: lane.lane_id,
|
|
2427
|
+
resumedAt: nowIso,
|
|
2428
|
+
reason: resumeReason,
|
|
2429
|
+
...(resumeHistoryCaller === undefined ? {} : { caller: resumeHistoryCaller }),
|
|
2430
|
+
turnId: laneTurnId,
|
|
2431
|
+
});
|
|
2359
2432
|
}
|
|
2360
2433
|
return laneLaunchResult(readLane(hqRoot, lane.lane_id));
|
|
2361
2434
|
}
|
|
@@ -2670,6 +2743,12 @@ export function spawnEnvelopeRepair(hqRoot, lane, input, deps = {}) {
|
|
|
2670
2743
|
...(callerLaneId ? { caller: callerLaneId } : {}),
|
|
2671
2744
|
}),
|
|
2672
2745
|
}));
|
|
2746
|
+
appendLaneWorkerResumeEvent({
|
|
2747
|
+
hqRoot,
|
|
2748
|
+
laneId: lane.lane_id,
|
|
2749
|
+
resumedAt: startedAt,
|
|
2750
|
+
reason: "envelope_repair",
|
|
2751
|
+
});
|
|
2673
2752
|
return workerPid;
|
|
2674
2753
|
}
|
|
2675
2754
|
export { PROVIDER_HOME_ENV, PROVIDER_COMMAND };
|
|
@@ -445,10 +445,11 @@ export interface KrFreshnessState {
|
|
|
445
445
|
stalled_fingerprint?: string;
|
|
446
446
|
}
|
|
447
447
|
export interface KeepAliveBlockedHistoryEntry {
|
|
448
|
-
event: "keep_alive_blocked_grace" | "keep_alive_blocked_resumed" | "keep_alive_blocked_escalated";
|
|
448
|
+
event: "keep_alive_blocked_grace" | "keep_alive_blocked_resumed" | "keep_alive_blocked_escalated" | "keep_alive_blocked_skipped";
|
|
449
449
|
at: string;
|
|
450
450
|
terminal_at: string;
|
|
451
451
|
resume_skipped_reason?: "open_question" | "blocked_grace";
|
|
452
|
+
reason?: "flag_off" | "backoff" | "next_check_at_due";
|
|
452
453
|
resume_queued?: boolean;
|
|
453
454
|
}
|
|
454
455
|
export interface KeepAliveResumeHistoryEntry {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export declare const PACKAGE_USE_LEASE_HANDOFF_ENV = "HQ_CLI_PACKAGE_USE_LEASE_HANDOFF";
|
|
1
2
|
export declare class PackageUseUpdatePendingError extends Error {
|
|
2
3
|
constructor(message: string);
|
|
3
4
|
}
|
|
@@ -10,11 +11,24 @@ export interface PackageUseLeaseRecord {
|
|
|
10
11
|
start_time_ms: number;
|
|
11
12
|
hq_version: string;
|
|
12
13
|
}
|
|
14
|
+
export interface PackageUseLeaseHandoff {
|
|
15
|
+
readonly envValue: string;
|
|
16
|
+
waitForChild(pid: number, childAlive: () => boolean): Promise<boolean>;
|
|
17
|
+
cleanup(): void;
|
|
18
|
+
}
|
|
13
19
|
export declare function npmPrefix(root: string, platform?: NodeJS.Platform): string | null;
|
|
14
20
|
export declare function packageUseLeaseDirectory(prefix: string, stateDirectory?: string): string;
|
|
15
21
|
export declare function packageUseUpdateRequestPath(prefix: string, stateDirectory?: string): string;
|
|
16
22
|
declare function waitForUpdateRequest(requestPath: string, timeoutMs: number): void;
|
|
17
23
|
export declare function acquirePackageUseLease(prefix: string, hqVersion: string, stateDirectory?: string): PackageUseLease;
|
|
24
|
+
/**
|
|
25
|
+
* Create a bounded, per-reexec rendezvous in the current lease directory.
|
|
26
|
+
* The child confirms readiness only after publishing its own atomic lease.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createPackageUseLeaseHandoff(lease: PackageUseLease, timeoutMs?: number): PackageUseLeaseHandoff;
|
|
29
|
+
export declare function createCurrentPackageUseLeaseHandoff(timeoutMs?: number): PackageUseLeaseHandoff | null;
|
|
30
|
+
/** Publish this process's lease identity to a parent waiting in the same prefix. */
|
|
31
|
+
export declare function publishPackageUseLeaseHandoff(lease: PackageUseLease, env?: NodeJS.ProcessEnv): boolean;
|
|
18
32
|
export declare function acquireCurrentPackageUseLease(): PackageUseLease | null;
|
|
19
33
|
/** Release this process's package-use lease after a long-running command has loaded its package modules. */
|
|
20
34
|
export declare function releaseCurrentPackageUseLease(): void;
|
|
@@ -30,6 +44,7 @@ export declare function currentPackageUseUpdatePending(): boolean;
|
|
|
30
44
|
export declare function currentPackageUseUpdateRequested(): boolean;
|
|
31
45
|
export declare const __test__: {
|
|
32
46
|
waitForUpdateRequest: typeof waitForUpdateRequest;
|
|
47
|
+
setCurrentPackageUseLease(lease: PackageUseLease | null): void;
|
|
33
48
|
};
|
|
34
49
|
export {};
|
|
35
50
|
//# sourceMappingURL=package-use-lease.d.ts.map
|