@indigoai-us/hq-cli 5.202.1 → 5.204.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +24 -0
- package/assets/lanes-hooks/hq-lanes-drain.mjs +36 -15
- package/dist/command-catalog.generated.d.ts +195 -0
- package/dist/command-catalog.generated.js +249 -0
- package/dist/command-registration-plan.d.ts +6 -0
- package/dist/command-registration-plan.js +1 -0
- package/dist/commands/lanes.d.ts +4 -0
- package/dist/commands/lanes.js +350 -108
- package/dist/commands/monitor.d.ts +58 -0
- package/dist/commands/monitor.js +965 -0
- package/dist/lib/lanes/account-home.d.ts +8 -0
- package/dist/lib/lanes/account-home.js +10 -0
- package/dist/lib/lanes/accounts.d.ts +4 -1
- package/dist/lib/lanes/accounts.js +15 -10
- package/dist/lib/lanes/claude-account-balance.d.ts +3 -0
- package/dist/lib/lanes/claude-account-balance.js +19 -4
- package/dist/lib/lanes/config.d.ts +2 -1
- package/dist/lib/lanes/config.js +55 -13
- package/dist/lib/lanes/delivery-core.d.ts +15 -0
- package/dist/lib/lanes/delivery-core.js +21 -0
- package/dist/lib/lanes/delivery.d.ts +21 -5
- package/dist/lib/lanes/delivery.js +71 -7
- package/dist/lib/lanes/dropbox.d.ts +13 -1
- package/dist/lib/lanes/dropbox.js +61 -5
- package/dist/lib/lanes/host-capacity.d.ts +4 -0
- package/dist/lib/lanes/host-capacity.js +8 -1
- package/dist/lib/lanes/index.d.ts +3 -1
- package/dist/lib/lanes/index.js +2 -1
- package/dist/lib/lanes/liveness.d.ts +1 -0
- package/dist/lib/lanes/liveness.js +3 -1
- package/dist/lib/lanes/move-account.js +1 -1
- package/dist/lib/lanes/reconcile.js +20 -9
- package/dist/lib/lanes/spawn.d.ts +7 -3
- package/dist/lib/lanes/spawn.js +71 -22
- package/dist/lib/lanes/types.d.ts +8 -0
- package/dist/lib/monitor.d.ts +153 -0
- package/dist/lib/monitor.js +519 -0
- package/dist/main.js +40 -28
- package/dist/utils/cli-network-lifecycle.d.ts +4 -0
- package/dist/utils/cli-network-lifecycle.js +11 -0
- package/package.json +1 -1
|
@@ -0,0 +1,519 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { formatMonitorEvent as formatDropboxMonitorEvent } from "./lanes/dropbox.js";
|
|
5
|
+
import { LanesError } from "./lanes/errors.js";
|
|
6
|
+
import { readProcStarttime } from "./lanes/spawn.js";
|
|
7
|
+
export const MONITOR_SCHEMA_VERSION = "hq-monitor.v1";
|
|
8
|
+
export const MONITOR_FLAG = "monitor.enabled";
|
|
9
|
+
export const MONITOR_FLAG_DEFAULT = false;
|
|
10
|
+
export const MONITOR_DEFAULT_TIMEOUT_MS = 5 * 60_000;
|
|
11
|
+
export const MONITOR_DEFAULT_CHECKIN_MS = 55 * 60_000;
|
|
12
|
+
export const MONITOR_PROMPT_CACHE_TTL_MS = 60 * 60_000;
|
|
13
|
+
export const MONITOR_BATCH_WINDOW_MS = 200;
|
|
14
|
+
export const MONITOR_POLL_INTERVAL_MS = 1_000;
|
|
15
|
+
export const MONITOR_RATE_WINDOW_MS = 10_000;
|
|
16
|
+
/** Sustained stdout floods stop after 3 windows; the per-window cap can be overridden with HQ_MONITOR_MAX_EVENTS_PER_WINDOW. */
|
|
17
|
+
export const MONITOR_MAX_EVENTS_PER_WINDOW = 60;
|
|
18
|
+
export const MONITOR_MAX_FLOOD_WINDOWS = 3;
|
|
19
|
+
export const MONITOR_LOCK_STALE_MS = 30_000;
|
|
20
|
+
export const MONITOR_MAX_TIMER_DELAY_MS = 2_147_000_000;
|
|
21
|
+
const SESSION_ID_RE = /^[A-Za-z0-9_-]{1,128}$/;
|
|
22
|
+
const SAFE_LANE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
23
|
+
const RUNTIME_ENV = {
|
|
24
|
+
claude: "CLAUDE_CODE_SESSION_ID",
|
|
25
|
+
codex: "CODEX_THREAD_ID",
|
|
26
|
+
grok: "GROK_SESSION_ID",
|
|
27
|
+
};
|
|
28
|
+
const RUNTIME_COMMANDS = new Set(["claude", "codex", "grok"]);
|
|
29
|
+
function monitorError(error, message, details = {}) {
|
|
30
|
+
return new LanesError(error, message, details);
|
|
31
|
+
}
|
|
32
|
+
/** This guard is first in monitor start so a conflicting invocation has no side effects. */
|
|
33
|
+
export function assertMonitorTimeoutCompatibility(persistent, timeoutProvided) {
|
|
34
|
+
if (persistent && timeoutProvided) {
|
|
35
|
+
throw monitorError("persistent_timeout_conflict", "--persistent cannot be used with --timeout.", {});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export function parseMonitorDuration(value, kind) {
|
|
39
|
+
const errorName = kind === "timeout" ? "monitor_timeout_invalid" : "monitor_checkin_invalid";
|
|
40
|
+
const match = /^(\d+(?:\.\d+)?)(ms|s|m|h)?$/i.exec(value.trim());
|
|
41
|
+
if (!match) {
|
|
42
|
+
throw monitorError(errorName, `Invalid --${kind} duration ${JSON.stringify(value)}. Use a number with optional ms, s, m, or h suffix.`, { value });
|
|
43
|
+
}
|
|
44
|
+
const amount = Number(match[1]);
|
|
45
|
+
const unit = (match[2] ?? "ms").toLowerCase();
|
|
46
|
+
const multiplier = unit === "h" ? 3_600_000 : unit === "m" ? 60_000 : unit === "s" ? 1_000 : 1;
|
|
47
|
+
const milliseconds = amount * multiplier;
|
|
48
|
+
if (!Number.isFinite(milliseconds) || milliseconds <= 0 || milliseconds > Number.MAX_SAFE_INTEGER) {
|
|
49
|
+
throw monitorError(errorName, `Invalid --${kind} duration ${JSON.stringify(value)}. Duration must be positive and finite.`, { value });
|
|
50
|
+
}
|
|
51
|
+
const rounded = Math.ceil(milliseconds);
|
|
52
|
+
if (kind === "checkin" && rounded >= MONITOR_PROMPT_CACHE_TTL_MS) {
|
|
53
|
+
throw monitorError("checkin_exceeds_cache_ttl", "--checkin must be less than 60 minutes so the prompt cache can be reused.", { checkin_ms: rounded, cache_ttl_ms: MONITOR_PROMPT_CACHE_TTL_MS });
|
|
54
|
+
}
|
|
55
|
+
return rounded;
|
|
56
|
+
}
|
|
57
|
+
function assertLaneTargetId(laneId) {
|
|
58
|
+
if (!SAFE_LANE_ID_RE.test(laneId) || laneId === "." || laneId === "..") {
|
|
59
|
+
throw monitorError("monitor_target_invalid", `Invalid lane target ${JSON.stringify(laneId)}.`, { lane_id: laneId });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
export function assertMonitorSessionId(sessionId) {
|
|
63
|
+
if (!SESSION_ID_RE.test(sessionId)) {
|
|
64
|
+
throw monitorError("monitor_session_id_invalid", "Session ids must contain only letters, numbers, underscores, or hyphens and be 1–128 characters long.", { session_id: sessionId });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
export function parseMonitorTarget(value) {
|
|
68
|
+
const laneMatch = /^lane:(.+)$/.exec(value);
|
|
69
|
+
if (laneMatch?.[1]) {
|
|
70
|
+
assertLaneTargetId(laneMatch[1]);
|
|
71
|
+
return { kind: "lane", laneId: laneMatch[1] };
|
|
72
|
+
}
|
|
73
|
+
const sessionMatch = /^session:(claude|codex|grok):(.+)$/.exec(value);
|
|
74
|
+
if (sessionMatch?.[1] && sessionMatch[2]) {
|
|
75
|
+
const provider = sessionMatch[1];
|
|
76
|
+
assertMonitorSessionId(sessionMatch[2]);
|
|
77
|
+
return { kind: "session", provider, sessionId: sessionMatch[2] };
|
|
78
|
+
}
|
|
79
|
+
throw monitorError("monitor_target_invalid", "--target must be lane:<lane-id> or session:<claude|codex|grok>:<session-id>.", { target: value });
|
|
80
|
+
}
|
|
81
|
+
export function monitorTargetKey(target) {
|
|
82
|
+
return target.kind === "lane"
|
|
83
|
+
? `lane:${target.laneId}`
|
|
84
|
+
: `session:${target.provider}:${target.sessionId}`;
|
|
85
|
+
}
|
|
86
|
+
export function detectMonitorTarget(options) {
|
|
87
|
+
const laneId = options.env.HQ_LANE_ID?.trim();
|
|
88
|
+
if (laneId) {
|
|
89
|
+
assertLaneTargetId(laneId);
|
|
90
|
+
return { kind: "lane", laneId };
|
|
91
|
+
}
|
|
92
|
+
const runtime = findNearestMonitorRuntime({
|
|
93
|
+
parentPid: options.parentPid,
|
|
94
|
+
readProcess: options.readProcess,
|
|
95
|
+
});
|
|
96
|
+
if (runtime) {
|
|
97
|
+
const provider = runtime.provider;
|
|
98
|
+
const sessionId = options.env[RUNTIME_ENV[provider]]?.trim() ?? "";
|
|
99
|
+
if (!sessionId) {
|
|
100
|
+
throw monitorError("monitor_target_unresolved", `Found ${provider} in the parent process chain but ${RUNTIME_ENV[provider]} is unset; pass --target explicitly.`, { provider, env_var: RUNTIME_ENV[provider] });
|
|
101
|
+
}
|
|
102
|
+
assertMonitorSessionId(sessionId);
|
|
103
|
+
return { kind: "session", provider, sessionId };
|
|
104
|
+
}
|
|
105
|
+
throw monitorError("monitor_target_unresolved", "Could not find a Claude, Codex, or Grok session in the parent process chain; pass --target lane:<id> or --target session:<provider>:<id>.", {});
|
|
106
|
+
}
|
|
107
|
+
/** The nearest runtime process in the caller chain, shared by target detection and hook mapping. */
|
|
108
|
+
export function findNearestMonitorRuntime(options) {
|
|
109
|
+
let pid = options.parentPid;
|
|
110
|
+
const visited = new Set();
|
|
111
|
+
while (Number.isInteger(pid) && pid > 1 && !visited.has(pid)) {
|
|
112
|
+
visited.add(pid);
|
|
113
|
+
const entry = options.readProcess(pid);
|
|
114
|
+
if (!entry)
|
|
115
|
+
return null;
|
|
116
|
+
const command = path.basename(entry.command).replace(/\.exe$/i, "").toLowerCase();
|
|
117
|
+
if (RUNTIME_COMMANDS.has(command))
|
|
118
|
+
return { provider: command, pid };
|
|
119
|
+
pid = entry.ppid;
|
|
120
|
+
}
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
export function monitorSessionDir(hqRoot, provider, sessionId) {
|
|
124
|
+
assertMonitorSessionId(sessionId);
|
|
125
|
+
return path.join(hqRoot, "workspace", "monitors", "sessions", `${provider}-${sessionId}`);
|
|
126
|
+
}
|
|
127
|
+
export function monitorSessionHasActiveMarkers(hqRoot, provider, sessionId) {
|
|
128
|
+
const activeDir = path.join(monitorSessionDir(hqRoot, provider, sessionId), "active");
|
|
129
|
+
try {
|
|
130
|
+
return fs.readdirSync(activeDir).some((name) => !name.startsWith("."));
|
|
131
|
+
}
|
|
132
|
+
catch (err) {
|
|
133
|
+
if (err.code === "ENOENT")
|
|
134
|
+
return false;
|
|
135
|
+
throw err;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
export function monitorSessionHasActiveMarkerForOwner(hqRoot, provider, sessionId, owner) {
|
|
139
|
+
const activeDir = path.join(monitorSessionDir(hqRoot, provider, sessionId), "active");
|
|
140
|
+
let names;
|
|
141
|
+
try {
|
|
142
|
+
names = fs.readdirSync(activeDir).filter((name) => !name.startsWith("."));
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
if (err.code === "ENOENT")
|
|
146
|
+
return false;
|
|
147
|
+
throw err;
|
|
148
|
+
}
|
|
149
|
+
return names.some((name) => {
|
|
150
|
+
const marker = JSON.parse(fs.readFileSync(path.join(activeDir, name), "utf8"));
|
|
151
|
+
const markerOwner = marker.owner;
|
|
152
|
+
return markerOwner !== undefined && markerOwner !== null
|
|
153
|
+
&& markerOwner.pid === owner.pid
|
|
154
|
+
&& markerOwner.starttime === owner.starttime;
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
export function monitorSessionHasInboxFile(hqRoot, provider, sessionId) {
|
|
158
|
+
return fs.existsSync(path.join(monitorSessionDir(hqRoot, provider, sessionId), "dropbox", "inbox.jsonl"));
|
|
159
|
+
}
|
|
160
|
+
export function monitorSessionInboxHasEvents(hqRoot, provider, sessionId) {
|
|
161
|
+
const inbox = path.join(monitorSessionDir(hqRoot, provider, sessionId), "dropbox", "inbox.jsonl");
|
|
162
|
+
try {
|
|
163
|
+
return fs.readFileSync(inbox, "utf8").split(/\r?\n/).some((line) => line.trim() !== "");
|
|
164
|
+
}
|
|
165
|
+
catch (err) {
|
|
166
|
+
if (err.code === "ENOENT")
|
|
167
|
+
return false;
|
|
168
|
+
throw err;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
export function monitorSessionHasWork(hqRoot, provider, sessionId) {
|
|
172
|
+
return monitorSessionHasActiveMarkers(hqRoot, provider, sessionId)
|
|
173
|
+
|| monitorSessionHasInboxFile(hqRoot, provider, sessionId);
|
|
174
|
+
}
|
|
175
|
+
export function monitorDir(hqRoot, monitorId) {
|
|
176
|
+
if (!/^[A-Za-z0-9_-]{1,128}$/.test(monitorId)) {
|
|
177
|
+
throw monitorError("monitor_id_invalid", `Invalid monitor id ${JSON.stringify(monitorId)}.`, { monitor_id: monitorId });
|
|
178
|
+
}
|
|
179
|
+
return path.join(hqRoot, "workspace", "monitors", monitorId);
|
|
180
|
+
}
|
|
181
|
+
export function makeMonitorId() {
|
|
182
|
+
return `mon_${Date.now().toString(36)}_${randomUUID().replaceAll("-", "")}`;
|
|
183
|
+
}
|
|
184
|
+
export function formatMonitorEvent(input) {
|
|
185
|
+
return formatDropboxMonitorEvent({
|
|
186
|
+
text: input.text,
|
|
187
|
+
kind: "monitor_event",
|
|
188
|
+
monitor_id: input.monitorId,
|
|
189
|
+
description: input.description,
|
|
190
|
+
sequence: input.sequence,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
export function nextMonitorEventSequence(state) {
|
|
194
|
+
return (state.event_sequence ?? 0) + 1;
|
|
195
|
+
}
|
|
196
|
+
export function nextMonitorTimeoutDelay(timeoutMs, elapsedMs) {
|
|
197
|
+
return Math.max(0, Math.min(timeoutMs - elapsedMs, MONITOR_MAX_TIMER_DELAY_MS));
|
|
198
|
+
}
|
|
199
|
+
export function recordMonitorEventState(state, event) {
|
|
200
|
+
return {
|
|
201
|
+
...state,
|
|
202
|
+
event_sequence: event.seq,
|
|
203
|
+
event_count: state.event_count + (event.delivery === "delivered" ? 1 : 0),
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
/** Keep the supervisor alive until its command has closed so TERM escalation can run. */
|
|
207
|
+
export function requestMonitorStop(childPid, terminate, finishWithoutChild) {
|
|
208
|
+
if (childPid === null) {
|
|
209
|
+
finishWithoutChild();
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
terminate();
|
|
213
|
+
}
|
|
214
|
+
export function parseHookSessionId(payloadText) {
|
|
215
|
+
let payload;
|
|
216
|
+
try {
|
|
217
|
+
payload = JSON.parse(payloadText);
|
|
218
|
+
}
|
|
219
|
+
catch (err) {
|
|
220
|
+
throw monitorError("monitor_hook_payload_invalid", `Hook payload is not valid JSON: ${err instanceof Error ? err.message : String(err)}.`, {});
|
|
221
|
+
}
|
|
222
|
+
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
|
|
223
|
+
throw monitorError("monitor_hook_payload_invalid", "Hook payload must be a JSON object with session_id.", {});
|
|
224
|
+
}
|
|
225
|
+
const sessionId = payload.session_id;
|
|
226
|
+
if (typeof sessionId !== "string") {
|
|
227
|
+
throw monitorError("monitor_hook_session_id_missing", "Hook payload has no string session_id.", {});
|
|
228
|
+
}
|
|
229
|
+
assertMonitorSessionId(sessionId);
|
|
230
|
+
return sessionId;
|
|
231
|
+
}
|
|
232
|
+
export function recordHookSessionMapping(hqRoot, provider, environmentSessionId, payloadSessionId, runtime) {
|
|
233
|
+
assertMonitorSessionId(payloadSessionId);
|
|
234
|
+
const environmentId = environmentSessionId?.trim();
|
|
235
|
+
if (!environmentId || environmentId === payloadSessionId || !runtime || runtime.provider !== provider)
|
|
236
|
+
return;
|
|
237
|
+
assertMonitorSessionId(environmentId);
|
|
238
|
+
const mappingPath = sessionMappingPath(hqRoot, runtime);
|
|
239
|
+
writeJsonAtomic(mappingPath, {
|
|
240
|
+
provider,
|
|
241
|
+
runtime_pid: runtime.pid,
|
|
242
|
+
runtime_starttime: runtime.starttime,
|
|
243
|
+
environment_session_id: environmentId,
|
|
244
|
+
payload_session_id: payloadSessionId,
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
export function resolveMappedSessionId(hqRoot, provider, environmentSessionId, runtime) {
|
|
248
|
+
assertMonitorSessionId(environmentSessionId);
|
|
249
|
+
if (!runtime || runtime.provider !== provider)
|
|
250
|
+
return environmentSessionId;
|
|
251
|
+
const mappingPath = sessionMappingPath(hqRoot, runtime);
|
|
252
|
+
try {
|
|
253
|
+
const parsed = JSON.parse(fs.readFileSync(mappingPath, "utf8"));
|
|
254
|
+
const mapped = parsed.payload_session_id;
|
|
255
|
+
if (parsed.provider === provider &&
|
|
256
|
+
parsed.runtime_pid === runtime.pid &&
|
|
257
|
+
parsed.runtime_starttime === runtime.starttime &&
|
|
258
|
+
parsed.environment_session_id === environmentSessionId &&
|
|
259
|
+
typeof mapped === "string") {
|
|
260
|
+
assertMonitorSessionId(mapped);
|
|
261
|
+
return mapped;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
catch (err) {
|
|
265
|
+
if (err.code !== "ENOENT") {
|
|
266
|
+
throw monitorError("monitor_session_map_unreadable", `Could not read session map: ${err instanceof Error ? err.message : String(err)}.`, { path: mappingPath });
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
return environmentSessionId;
|
|
270
|
+
}
|
|
271
|
+
function sessionMappingPath(hqRoot, runtime) {
|
|
272
|
+
return path.join(hqRoot, "workspace", "monitors", "session-map", `${runtime.provider}-${runtime.pid}-${runtime.starttime}.json`);
|
|
273
|
+
}
|
|
274
|
+
function writeJsonAtomic(filePath, value) {
|
|
275
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
276
|
+
const tmp = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
|
|
277
|
+
fs.writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
|
|
278
|
+
try {
|
|
279
|
+
fs.renameSync(tmp, filePath);
|
|
280
|
+
}
|
|
281
|
+
catch (err) {
|
|
282
|
+
fs.rmSync(tmp, { force: true });
|
|
283
|
+
throw err;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
function readJson(filePath) {
|
|
287
|
+
try {
|
|
288
|
+
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
289
|
+
}
|
|
290
|
+
catch (err) {
|
|
291
|
+
if (err.code === "ENOENT")
|
|
292
|
+
return null;
|
|
293
|
+
throw monitorError("monitor_state_unreadable", `Could not read ${filePath}: ${err instanceof Error ? err.message : String(err)}.`, { path: filePath });
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
function targetStorageDir(hqRoot, targetKey) {
|
|
297
|
+
const key = createHash("sha256").update(targetKey).digest("hex");
|
|
298
|
+
return path.join(hqRoot, "workspace", "monitors", "targets", key);
|
|
299
|
+
}
|
|
300
|
+
function defaultPidIsAlive(pid, starttime, readStarttime) {
|
|
301
|
+
try {
|
|
302
|
+
process.kill(pid, 0);
|
|
303
|
+
}
|
|
304
|
+
catch (err) {
|
|
305
|
+
const code = err.code;
|
|
306
|
+
if (code === "ESRCH")
|
|
307
|
+
return false;
|
|
308
|
+
if (code !== "EPERM")
|
|
309
|
+
throw err;
|
|
310
|
+
}
|
|
311
|
+
if (starttime === null)
|
|
312
|
+
return true;
|
|
313
|
+
const current = readStarttime(pid);
|
|
314
|
+
return current === null || current === starttime;
|
|
315
|
+
}
|
|
316
|
+
async function acquireTargetLock(lockFile, deps) {
|
|
317
|
+
fs.mkdirSync(path.dirname(lockFile), { recursive: true });
|
|
318
|
+
const token = randomUUID();
|
|
319
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
320
|
+
const owner = { token, pid: process.pid, starttime: readProcStarttime(process.pid), at: deps.now().getTime() };
|
|
321
|
+
try {
|
|
322
|
+
fs.writeFileSync(lockFile, `${JSON.stringify(owner)}\n`, { flag: "wx", mode: 0o600 });
|
|
323
|
+
return { token, pid: owner.pid, starttime: owner.starttime };
|
|
324
|
+
}
|
|
325
|
+
catch (err) {
|
|
326
|
+
if (err.code !== "EEXIST")
|
|
327
|
+
throw err;
|
|
328
|
+
}
|
|
329
|
+
const existing = readJson(lockFile);
|
|
330
|
+
if (existing && Number.isInteger(existing.pid)) {
|
|
331
|
+
const start = typeof existing.starttime === "number" ? existing.starttime : null;
|
|
332
|
+
if (deps.pidIsAlive(existing.pid, start))
|
|
333
|
+
return null;
|
|
334
|
+
}
|
|
335
|
+
else {
|
|
336
|
+
try {
|
|
337
|
+
const age = deps.now().getTime() - fs.statSync(lockFile).mtimeMs;
|
|
338
|
+
if (age < MONITOR_LOCK_STALE_MS)
|
|
339
|
+
return null;
|
|
340
|
+
}
|
|
341
|
+
catch (err) {
|
|
342
|
+
if (err.code === "ENOENT")
|
|
343
|
+
continue;
|
|
344
|
+
throw err;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
try {
|
|
348
|
+
fs.unlinkSync(lockFile);
|
|
349
|
+
}
|
|
350
|
+
catch (err) {
|
|
351
|
+
if (err.code !== "ENOENT")
|
|
352
|
+
throw err;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
return null;
|
|
356
|
+
}
|
|
357
|
+
function releaseTargetLock(lockFile, token) {
|
|
358
|
+
const current = readJson(lockFile);
|
|
359
|
+
if (current?.token !== token)
|
|
360
|
+
return;
|
|
361
|
+
try {
|
|
362
|
+
fs.unlinkSync(lockFile);
|
|
363
|
+
}
|
|
364
|
+
catch (err) {
|
|
365
|
+
if (err.code !== "ENOENT")
|
|
366
|
+
throw err;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
export async function withMonitorTargetLock(hqRoot, targetKey, operation, options = {}) {
|
|
370
|
+
const dir = targetStorageDir(hqRoot, targetKey);
|
|
371
|
+
const activityPath = path.join(dir, "activity.json");
|
|
372
|
+
const lockFile = path.join(dir, "activity.lock");
|
|
373
|
+
const deps = {
|
|
374
|
+
now: options.now ?? (() => new Date()),
|
|
375
|
+
pidIsAlive: options.pidIsAlive ?? ((pid, starttime) => defaultPidIsAlive(pid, starttime, readProcStarttime)),
|
|
376
|
+
sleep: options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
|
|
377
|
+
};
|
|
378
|
+
let owner = null;
|
|
379
|
+
for (let attempt = 0; attempt < 100 && owner === null; attempt += 1) {
|
|
380
|
+
owner = await acquireTargetLock(lockFile, deps);
|
|
381
|
+
if (owner === null)
|
|
382
|
+
await deps.sleep(10);
|
|
383
|
+
}
|
|
384
|
+
if (!owner)
|
|
385
|
+
throw monitorError("monitor_target_lock_busy", `Could not acquire target delivery lock for ${targetKey}.`, { target: targetKey });
|
|
386
|
+
try {
|
|
387
|
+
return await operation(activityPath);
|
|
388
|
+
}
|
|
389
|
+
finally {
|
|
390
|
+
releaseTargetLock(lockFile, owner.token);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
export async function recordMonitorTargetDelivery(hqRoot, targetKey, now = () => new Date(), kind = "monitor_event") {
|
|
394
|
+
await withMonitorTargetLock(hqRoot, targetKey, (activityPath) => {
|
|
395
|
+
recordMonitorTargetDeliveryAtPath(activityPath, now, kind);
|
|
396
|
+
}, { now });
|
|
397
|
+
}
|
|
398
|
+
export async function recordMonitorTargetActivity(hqRoot, targetKey, now = () => new Date()) {
|
|
399
|
+
const target = parseMonitorTarget(targetKey);
|
|
400
|
+
if (target.kind === "session" && !monitorSessionHasWork(hqRoot, target.provider, target.sessionId))
|
|
401
|
+
return;
|
|
402
|
+
await withMonitorTargetLock(hqRoot, targetKey, (activityPath) => {
|
|
403
|
+
const current = readJson(activityPath);
|
|
404
|
+
const activityAt = now().getTime();
|
|
405
|
+
writeJsonAtomic(activityPath, current
|
|
406
|
+
? { ...current, last_activity_ms: activityAt }
|
|
407
|
+
: {
|
|
408
|
+
last_delivery_ms: activityAt,
|
|
409
|
+
last_delivery_kind: "start",
|
|
410
|
+
checkin_count: 0,
|
|
411
|
+
last_activity_ms: activityAt,
|
|
412
|
+
});
|
|
413
|
+
}, { now });
|
|
414
|
+
}
|
|
415
|
+
export function recordMonitorTargetDeliveryAtPath(activityPath, now = () => new Date(), kind = "monitor_event") {
|
|
416
|
+
const current = readJson(activityPath);
|
|
417
|
+
writeJsonAtomic(activityPath, {
|
|
418
|
+
last_delivery_ms: now().getTime(),
|
|
419
|
+
last_delivery_kind: kind,
|
|
420
|
+
checkin_count: current?.checkin_count ?? 0,
|
|
421
|
+
...(current?.last_activity_ms === undefined ? {} : { last_activity_ms: current.last_activity_ms }),
|
|
422
|
+
...(current?.last_checkin_attempt_ms === undefined ? {} : { last_checkin_attempt_ms: current.last_checkin_attempt_ms }),
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
function monitorIsLive(state, deps) {
|
|
426
|
+
return state?.status === "running" && state.pid !== null &&
|
|
427
|
+
deps.pidIsAlive(state.pid, state.starttime);
|
|
428
|
+
}
|
|
429
|
+
function activeTargetMonitors(hqRoot, targetKey, deps) {
|
|
430
|
+
const root = path.join(hqRoot, "workspace", "monitors");
|
|
431
|
+
let names;
|
|
432
|
+
try {
|
|
433
|
+
names = fs.readdirSync(root, { withFileTypes: true })
|
|
434
|
+
.filter((entry) => entry.isDirectory() && entry.name !== "sessions" && entry.name !== "targets")
|
|
435
|
+
.map((entry) => entry.name);
|
|
436
|
+
}
|
|
437
|
+
catch (err) {
|
|
438
|
+
if (err.code === "ENOENT")
|
|
439
|
+
return [];
|
|
440
|
+
throw err;
|
|
441
|
+
}
|
|
442
|
+
const monitors = [];
|
|
443
|
+
for (const name of names) {
|
|
444
|
+
const dir = path.join(root, name);
|
|
445
|
+
const meta = readJson(path.join(dir, "meta.json"));
|
|
446
|
+
const state = readJson(path.join(dir, "state.json"));
|
|
447
|
+
if (meta?.target_key !== targetKey || !monitorIsLive(state, deps))
|
|
448
|
+
continue;
|
|
449
|
+
monitors.push({ meta, state });
|
|
450
|
+
}
|
|
451
|
+
return monitors;
|
|
452
|
+
}
|
|
453
|
+
/** Elapsed idle time in whole units; live timers never land on exact multiples. */
|
|
454
|
+
export function formatElapsed(ms) {
|
|
455
|
+
const minutes = Math.floor(ms / 60_000);
|
|
456
|
+
if (minutes >= 60) {
|
|
457
|
+
const hours = Math.floor(minutes / 60);
|
|
458
|
+
const rest = minutes % 60;
|
|
459
|
+
return rest === 0 ? `${hours}h` : `${hours}h ${rest}m`;
|
|
460
|
+
}
|
|
461
|
+
if (minutes >= 1)
|
|
462
|
+
return `${minutes}m`;
|
|
463
|
+
return `${Math.max(0, Math.floor(ms / 1_000))}s`;
|
|
464
|
+
}
|
|
465
|
+
function formatCheckinText(elapsedMs, monitors) {
|
|
466
|
+
const descriptions = monitors.map(({ meta }) => meta.description).join(", ");
|
|
467
|
+
return `hq monitor check-in — idle for ${formatElapsed(elapsedMs)}; ${monitors.length} monitor(s) still armed: ${descriptions}. If you are idle, confirm nothing needs action and end your turn. If you are mid-task, ignore this and continue.`;
|
|
468
|
+
}
|
|
469
|
+
export async function runMonitorCheckinTick(hqRoot, targetKey, deps) {
|
|
470
|
+
if (!(await deps.isWakeable(targetKey)))
|
|
471
|
+
return { sent: false, sequence: null, text: null, outcome: null };
|
|
472
|
+
return withMonitorTargetLock(hqRoot, targetKey, async (activityPath) => {
|
|
473
|
+
const monitors = activeTargetMonitors(hqRoot, targetKey, deps);
|
|
474
|
+
const eligible = monitors.filter(({ meta }) => meta.checkin_ms !== null && meta.checkin_ms > 0);
|
|
475
|
+
if (monitors.length === 0 || eligible.length === 0) {
|
|
476
|
+
return { sent: false, sequence: null, text: null, outcome: null };
|
|
477
|
+
}
|
|
478
|
+
const shortestInterval = Math.min(...eligible.map(({ meta }) => meta.checkin_ms));
|
|
479
|
+
const nowMs = deps.now().getTime();
|
|
480
|
+
const activity = readJson(activityPath);
|
|
481
|
+
const targetActivityAt = await deps.readTargetActivityAt?.(targetKey) ?? null;
|
|
482
|
+
if (!activity) {
|
|
483
|
+
writeJsonAtomic(activityPath, {
|
|
484
|
+
last_delivery_ms: targetActivityAt ?? nowMs,
|
|
485
|
+
last_delivery_kind: "start",
|
|
486
|
+
checkin_count: 0,
|
|
487
|
+
...(targetActivityAt === null ? {} : { last_activity_ms: targetActivityAt }),
|
|
488
|
+
});
|
|
489
|
+
return { sent: false, sequence: null, text: null, outcome: null };
|
|
490
|
+
}
|
|
491
|
+
const lastActivityMs = Math.max(activity.last_delivery_ms, activity.last_activity_ms ?? Number.NEGATIVE_INFINITY, targetActivityAt ?? Number.NEGATIVE_INFINITY);
|
|
492
|
+
const elapsed = nowMs - lastActivityMs;
|
|
493
|
+
if (elapsed < shortestInterval)
|
|
494
|
+
return { sent: false, sequence: null, text: null, outcome: null };
|
|
495
|
+
if (activity.last_checkin_attempt_ms !== undefined && nowMs - activity.last_checkin_attempt_ms < shortestInterval) {
|
|
496
|
+
return { sent: false, sequence: null, text: null, outcome: null };
|
|
497
|
+
}
|
|
498
|
+
const text = formatCheckinText(elapsed, monitors);
|
|
499
|
+
const sequence = activity.checkin_count + 1;
|
|
500
|
+
const delivery = await deps.deliver(text, sequence);
|
|
501
|
+
if (!delivery.delivered) {
|
|
502
|
+
writeJsonAtomic(activityPath, { ...activity, last_checkin_attempt_ms: nowMs });
|
|
503
|
+
return { sent: false, sequence: null, text, outcome: delivery.outcome };
|
|
504
|
+
}
|
|
505
|
+
writeJsonAtomic(activityPath, {
|
|
506
|
+
last_delivery_ms: nowMs,
|
|
507
|
+
last_delivery_kind: "checkin",
|
|
508
|
+
checkin_count: sequence,
|
|
509
|
+
last_checkin_attempt_ms: nowMs,
|
|
510
|
+
});
|
|
511
|
+
return { sent: true, sequence, text, outcome: delivery.outcome };
|
|
512
|
+
}, { now: deps.now, pidIsAlive: deps.pidIsAlive, sleep: deps.sleep });
|
|
513
|
+
}
|
|
514
|
+
export function readMonitorPidLiveness(pid, starttime, readStarttime = readProcStarttime) {
|
|
515
|
+
if (pid === null || !Number.isInteger(pid) || pid <= 0)
|
|
516
|
+
return false;
|
|
517
|
+
return defaultPidIsAlive(pid, starttime, readStarttime);
|
|
518
|
+
}
|
|
519
|
+
//# sourceMappingURL=monitor.js.map
|
package/dist/main.js
CHANGED
|
@@ -53,6 +53,7 @@ import { registerCommandsWithRecovery } from "./startup-registration.js";
|
|
|
53
53
|
import { installTreeTornCaptureContext, installTreeTornStderrLine, isInstallTreeTornError, } from "./utils/install-tree-torn.js";
|
|
54
54
|
import { isVaultAccessDeniedError, vaultAccessDeniedMessage, } from "./utils/vault-access-denied-error.js";
|
|
55
55
|
import { fallbackOperatorMessage, unexpectedCliErrorMessage } from "./utils/unexpected-cli-error.js";
|
|
56
|
+
import { runGlobalNetworkLifecycle } from "./utils/cli-network-lifecycle.js";
|
|
56
57
|
/**
|
|
57
58
|
* Total time one CLI invocation may await foreground network work. This spans
|
|
58
59
|
* the hard version decision, pre-action observability, and release-health
|
|
@@ -152,7 +153,7 @@ export function handleStreamError(err, deps = defaultStreamErrorDependencies) {
|
|
|
152
153
|
}
|
|
153
154
|
process.stdout.on("error", (err) => handleStreamError(err));
|
|
154
155
|
process.stderr.on("error", (err) => handleStreamError(err));
|
|
155
|
-
initSentry();
|
|
156
|
+
runGlobalNetworkLifecycle(process.argv, () => initSentry());
|
|
156
157
|
const program = new Command();
|
|
157
158
|
program
|
|
158
159
|
.name("hq")
|
|
@@ -166,13 +167,15 @@ program.hook("preAction", async () => {
|
|
|
166
167
|
// regardless, because the timer cannot be serviced while the event loop is
|
|
167
168
|
// blocked. Keep anything added here asynchronous, or separately cheap: this
|
|
168
169
|
// hook is on the path of EVERY hq command.
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
170
|
+
await runGlobalNetworkLifecycle(process.argv, async () => {
|
|
171
|
+
const budget = activeForegroundNetworkBudget;
|
|
172
|
+
if (!budget)
|
|
173
|
+
return;
|
|
174
|
+
await budget.waitFor(OBSERVABILITY_FOREGROUND_WAIT_MS, FINALIZATION_FOREGROUND_WAIT_MS, (timeoutMs) => Promise.all([
|
|
175
|
+
emitCliSessionStarted(timeoutMs),
|
|
176
|
+
reportCliClientHealthInvocation({ timeoutMs }),
|
|
177
|
+
]));
|
|
178
|
+
});
|
|
176
179
|
});
|
|
177
180
|
export async function runCli() {
|
|
178
181
|
const foregroundNetworkBudget = new ForegroundNetworkBudget();
|
|
@@ -180,17 +183,19 @@ export async function runCli() {
|
|
|
180
183
|
// Begin one registry refresh without awaiting it. Every command gate reads
|
|
181
184
|
// only the client's held snapshot and falls back locally, so an offline or
|
|
182
185
|
// slow registry cannot delay command parsing or alter a command failure.
|
|
183
|
-
|
|
186
|
+
runGlobalNetworkLifecycle(process.argv, () => {
|
|
187
|
+
void kickFlagRegistryReadiness();
|
|
188
|
+
});
|
|
184
189
|
// Set when a self-update re-exec'd this command on a newer CLI: the child
|
|
185
190
|
// already did the work, so this process only has to carry its exit status
|
|
186
191
|
// out (after the finally block's telemetry, hence not process.exit here).
|
|
187
192
|
let reexecStatus = null;
|
|
188
193
|
try {
|
|
189
|
-
Sentry.addBreadcrumb({
|
|
194
|
+
runGlobalNetworkLifecycle(process.argv, () => Sentry.addBreadcrumb({
|
|
190
195
|
category: "command",
|
|
191
196
|
message: sanitizeArgv(process.argv.slice(2)).join(" "),
|
|
192
197
|
level: "info",
|
|
193
|
-
});
|
|
198
|
+
}));
|
|
194
199
|
// Version handling, both halves. Skipped for inspection flags
|
|
195
200
|
// (`--version`, `--help`) so users debugging a broken install can still
|
|
196
201
|
// introspect what they have, and silent on any failure — neither half may
|
|
@@ -204,7 +209,9 @@ export async function runCli() {
|
|
|
204
209
|
// it (hq-pro's `updateRecommended` and the cached npm latest from
|
|
205
210
|
// `version-check.ts`); whichever fires first re-execs, and the child
|
|
206
211
|
// carries a guard env so it can never update again.
|
|
207
|
-
|
|
212
|
+
const versionAction = await runGlobalNetworkLifecycle(process.argv, async () => {
|
|
213
|
+
if (shouldSkipGate(process.argv))
|
|
214
|
+
return "continue";
|
|
208
215
|
const gate = (await foregroundNetworkBudget.waitFor(VERSION_GATE_FOREGROUND_WAIT_MS, FINALIZATION_FOREGROUND_WAIT_MS, (timeoutMs) => enforceVersionGate(async (decision) => {
|
|
209
216
|
const outcome = await autoUpdateAndReexec(process.argv, decision.latestVersion);
|
|
210
217
|
if (outcome.action === "reexec")
|
|
@@ -212,16 +219,19 @@ export async function runCli() {
|
|
|
212
219
|
return outcome.action === "reexec";
|
|
213
220
|
}, { timeoutMs }))) ?? "continue";
|
|
214
221
|
if (gate === "reexec")
|
|
215
|
-
return;
|
|
222
|
+
return "reexec";
|
|
216
223
|
const cachedLatest = staleAgainstCachedLatest();
|
|
217
224
|
if (cachedLatest) {
|
|
218
225
|
const outcome = await autoUpdateAndReexec(process.argv, cachedLatest);
|
|
219
226
|
if (outcome.action === "reexec") {
|
|
220
227
|
reexecStatus = outcome.reexecStatus ?? 0;
|
|
221
|
-
return;
|
|
228
|
+
return "reexec";
|
|
222
229
|
}
|
|
223
230
|
}
|
|
224
|
-
|
|
231
|
+
return "continue";
|
|
232
|
+
});
|
|
233
|
+
if (versionAction === "reexec")
|
|
234
|
+
return;
|
|
225
235
|
// Register every contributor for the selected root only. Root help and
|
|
226
236
|
// unknown-command recovery use generated metadata, so they keep the full
|
|
227
237
|
// Commander surface without evaluating every command implementation.
|
|
@@ -256,19 +266,21 @@ export async function runCli() {
|
|
|
256
266
|
finally {
|
|
257
267
|
// Plan-limit nag (US-016): stderr-only, never throws, never touches
|
|
258
268
|
// process.exitCode — safe to run after exit codes have been set.
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
269
|
+
await runGlobalNetworkLifecycle(process.argv, async () => {
|
|
270
|
+
emitPlanLimitNag();
|
|
271
|
+
// Release health: finalize the per-run session before the flush.
|
|
272
|
+
finishSentrySession();
|
|
273
|
+
// Neither task may turn a successful command into Node's
|
|
274
|
+
// `unsettled top-level await` exit. They are observability-only after the
|
|
275
|
+
// command has completed, so a bounded best-effort wait is the terminal
|
|
276
|
+
// lifecycle boundary for this invocation. The budget reserves this 500 ms
|
|
277
|
+
// slice even when the command itself was long-running.
|
|
278
|
+
const releaseHealthTimeoutMs = foregroundNetworkBudget.timeoutFor(FINALIZATION_FOREGROUND_WAIT_MS);
|
|
279
|
+
await settleWithin([
|
|
280
|
+
refreshVersionCache({ timeoutMs: releaseHealthTimeoutMs }),
|
|
281
|
+
Sentry.flush(releaseHealthTimeoutMs),
|
|
282
|
+
], releaseHealthTimeoutMs);
|
|
283
|
+
});
|
|
272
284
|
// Last, so it wins over anything the (skipped) command path would have
|
|
273
285
|
// set: the re-exec'd child's status IS this invocation's result.
|
|
274
286
|
if (reexecStatus !== null)
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare function isMonitorFastPathCommand(argv: readonly string[]): boolean;
|
|
2
|
+
/** Skip process-wide network lifecycle work for hooks and detached monitor supervisors. */
|
|
3
|
+
export declare function runGlobalNetworkLifecycle<T>(argv: readonly string[], operation: () => T): T | undefined;
|
|
4
|
+
//# sourceMappingURL=cli-network-lifecycle.d.ts.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
const MONITOR_HIDDEN_NETWORK_BYPASS = new Set(["drain", "wait", "enabled", "_run"]);
|
|
2
|
+
export function isMonitorFastPathCommand(argv) {
|
|
3
|
+
return argv[2] === "monitor" && MONITOR_HIDDEN_NETWORK_BYPASS.has(argv[3] ?? "");
|
|
4
|
+
}
|
|
5
|
+
/** Skip process-wide network lifecycle work for hooks and detached monitor supervisors. */
|
|
6
|
+
export function runGlobalNetworkLifecycle(argv, operation) {
|
|
7
|
+
if (isMonitorFastPathCommand(argv))
|
|
8
|
+
return undefined;
|
|
9
|
+
return operation();
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=cli-network-lifecycle.js.map
|