@indigoai-us/hq-cli 5.202.1 → 5.203.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +24 -0
- package/assets/lanes-hooks/hq-lanes-drain.mjs +36 -15
- package/dist/command-catalog.generated.d.ts +168 -0
- package/dist/command-catalog.generated.js +214 -0
- package/dist/command-registration-plan.d.ts +6 -0
- package/dist/command-registration-plan.js +1 -0
- package/dist/commands/lanes.d.ts +1 -0
- package/dist/commands/lanes.js +150 -83
- package/dist/commands/monitor.d.ts +58 -0
- package/dist/commands/monitor.js +965 -0
- package/dist/lib/lanes/delivery-core.d.ts +15 -0
- package/dist/lib/lanes/delivery-core.js +21 -0
- package/dist/lib/lanes/delivery.d.ts +21 -5
- package/dist/lib/lanes/delivery.js +71 -7
- package/dist/lib/lanes/dropbox.d.ts +13 -1
- package/dist/lib/lanes/dropbox.js +61 -5
- package/dist/lib/lanes/index.d.ts +1 -1
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/move-account.js +1 -1
- package/dist/lib/lanes/spawn.d.ts +6 -3
- package/dist/lib/lanes/spawn.js +41 -8
- package/dist/lib/lanes/types.d.ts +3 -0
- package/dist/lib/monitor.d.ts +153 -0
- package/dist/lib/monitor.js +519 -0
- package/dist/main.js +40 -28
- package/dist/utils/cli-network-lifecycle.d.ts +4 -0
- package/dist/utils/cli-network-lifecycle.js +11 -0
- package/package.json +1 -1
|
@@ -0,0 +1,965 @@
|
|
|
1
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { readLanesRegistryFlag } from "../lib/lanes/registry-flag.js";
|
|
5
|
+
import { resolveProcessFlagCompanyContext } from "../lib/flag-registry.js";
|
|
6
|
+
import { resolveLanesHqRoot } from "../lib/lanes/hq-root.js";
|
|
7
|
+
import { LanesError } from "../lib/lanes/errors.js";
|
|
8
|
+
import { readToolActivity, tryReadLane } from "../lib/lanes/index.js";
|
|
9
|
+
import { appendDropboxMessageAtPath, drainDropboxAtPath, formatDropboxMessages } from "../lib/lanes/dropbox.js";
|
|
10
|
+
import { dispatchLaneDelivery } from "../lib/lanes/delivery-core.js";
|
|
11
|
+
import { MONITOR_FLAG, MONITOR_FLAG_DEFAULT, MONITOR_DEFAULT_CHECKIN_MS, MONITOR_DEFAULT_TIMEOUT_MS, MONITOR_POLL_INTERVAL_MS, MONITOR_BATCH_WINDOW_MS, MONITOR_MAX_EVENTS_PER_WINDOW, MONITOR_MAX_FLOOD_WINDOWS, MONITOR_RATE_WINDOW_MS, MONITOR_SCHEMA_VERSION, monitorDir, monitorSessionDir, monitorSessionHasActiveMarkerForOwner, monitorSessionHasActiveMarkers, monitorSessionHasWork, monitorSessionInboxHasEvents, monitorTargetKey, assertMonitorTimeoutCompatibility, parseMonitorDuration, parseMonitorTarget, detectMonitorTarget, findNearestMonitorRuntime, makeMonitorId, recordMonitorTargetDelivery, recordMonitorTargetActivity, recordMonitorTargetDeliveryAtPath, runMonitorCheckinTick, readMonitorPidLiveness, recordHookSessionMapping, resolveMappedSessionId, parseHookSessionId, withMonitorTargetLock, nextMonitorEventSequence, recordMonitorEventState, requestMonitorStop, nextMonitorTimeoutDelay } from "../lib/monitor.js";
|
|
12
|
+
import { readProcStarttime } from "../lib/lanes/spawn.js";
|
|
13
|
+
const RUNTIME_ENV = {
|
|
14
|
+
claude: "CLAUDE_CODE_SESSION_ID",
|
|
15
|
+
codex: "CODEX_THREAD_ID",
|
|
16
|
+
grok: "GROK_SESSION_ID",
|
|
17
|
+
};
|
|
18
|
+
function fail(error) {
|
|
19
|
+
const value = error instanceof LanesError
|
|
20
|
+
? error.toJSON()
|
|
21
|
+
: { error: "internal", message: error instanceof Error ? error.message : String(error) };
|
|
22
|
+
process.stderr.write(`${JSON.stringify(value)}\n`);
|
|
23
|
+
process.exit(1);
|
|
24
|
+
}
|
|
25
|
+
function printJson(value) {
|
|
26
|
+
process.stdout.write(`${JSON.stringify(value)}\n`);
|
|
27
|
+
}
|
|
28
|
+
function readJson(filePath) {
|
|
29
|
+
try {
|
|
30
|
+
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
31
|
+
}
|
|
32
|
+
catch (err) {
|
|
33
|
+
if (err.code === "ENOENT")
|
|
34
|
+
return null;
|
|
35
|
+
throw new LanesError("monitor_state_unreadable", `Could not read ${filePath}: ${err instanceof Error ? err.message : String(err)}.`, { path: filePath });
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function writeJsonAtomic(filePath, value) {
|
|
39
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
40
|
+
const tmp = `${filePath}.${process.pid}.${Date.now()}.tmp`;
|
|
41
|
+
fs.writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
|
|
42
|
+
try {
|
|
43
|
+
fs.renameSync(tmp, filePath);
|
|
44
|
+
}
|
|
45
|
+
catch (err) {
|
|
46
|
+
fs.rmSync(tmp, { force: true });
|
|
47
|
+
throw err;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
function statePath(hqRoot, id) {
|
|
51
|
+
return path.join(monitorDir(hqRoot, id), "state.json");
|
|
52
|
+
}
|
|
53
|
+
function errorPath(error) {
|
|
54
|
+
return error instanceof Error ? error.message : String(error);
|
|
55
|
+
}
|
|
56
|
+
function writeStderr(text) {
|
|
57
|
+
process.stderr.write(text);
|
|
58
|
+
}
|
|
59
|
+
function waitSleep(milliseconds) {
|
|
60
|
+
return new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
61
|
+
}
|
|
62
|
+
function procEntry(pid) {
|
|
63
|
+
try {
|
|
64
|
+
const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
65
|
+
const close = stat.lastIndexOf(")");
|
|
66
|
+
const fields = stat.slice(close + 2).split(/\s+/);
|
|
67
|
+
const ppid = Number(fields[1]);
|
|
68
|
+
const commandLine = fs.readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0").filter(Boolean);
|
|
69
|
+
const command = commandLine[0] ?? fs.readFileSync(`/proc/${pid}/comm`, "utf8").trim();
|
|
70
|
+
if (!Number.isInteger(ppid) || !command)
|
|
71
|
+
return null;
|
|
72
|
+
return { pid, ppid, command };
|
|
73
|
+
}
|
|
74
|
+
catch (err) {
|
|
75
|
+
if (err.code === "ENOENT" || err.code === "ESRCH")
|
|
76
|
+
return null;
|
|
77
|
+
throw err;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
export function findSessionOwner(target, deps = {}) {
|
|
81
|
+
if (target.kind !== "session")
|
|
82
|
+
return null;
|
|
83
|
+
const env = deps.env ?? process.env;
|
|
84
|
+
let pid = deps.parentPid ?? process.ppid;
|
|
85
|
+
const readProcess = deps.readProcess ?? procEntry;
|
|
86
|
+
const readStarttime = deps.readStarttime ?? readProcStarttime;
|
|
87
|
+
const seen = new Set();
|
|
88
|
+
while (pid > 1 && !seen.has(pid)) {
|
|
89
|
+
seen.add(pid);
|
|
90
|
+
const entry = readProcess(pid);
|
|
91
|
+
if (!entry)
|
|
92
|
+
return null;
|
|
93
|
+
const command = path.basename(entry.command).replace(/\.exe$/i, "").toLowerCase();
|
|
94
|
+
if (command === target.provider && env[RUNTIME_ENV[target.provider]]?.trim() === target.sessionId) {
|
|
95
|
+
return { pid, starttime: readStarttime(pid) };
|
|
96
|
+
}
|
|
97
|
+
pid = entry.ppid;
|
|
98
|
+
}
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
export function sessionOwnerTarget(discovered, resolved) {
|
|
102
|
+
return discovered.kind === "session" ? discovered : resolved;
|
|
103
|
+
}
|
|
104
|
+
function processCompanyRef() {
|
|
105
|
+
const context = resolveProcessFlagCompanyContext();
|
|
106
|
+
if (!context.companyUid && !context.companySlug)
|
|
107
|
+
return null;
|
|
108
|
+
return { uid: context.companyUid ?? "", slug: context.companySlug ?? "" };
|
|
109
|
+
}
|
|
110
|
+
async function monitorEnabled(company = processCompanyRef()) {
|
|
111
|
+
return readLanesRegistryFlag(MONITOR_FLAG, {
|
|
112
|
+
company,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
export async function runMonitorEnabledCommand(deps = {}) {
|
|
116
|
+
let enabled = false;
|
|
117
|
+
try {
|
|
118
|
+
enabled = await (deps.readFlag ?? ((company) => monitorEnabled(company)))(processCompanyRef());
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
const name = error instanceof Error ? error.name : "UnknownError";
|
|
122
|
+
(deps.writeError ?? ((text) => process.stderr.write(text)))(`${JSON.stringify({
|
|
123
|
+
event: "monitor.enabled.flag_read_failed",
|
|
124
|
+
flag: MONITOR_FLAG,
|
|
125
|
+
error: name,
|
|
126
|
+
fallback: false,
|
|
127
|
+
})}\n`);
|
|
128
|
+
}
|
|
129
|
+
(deps.writeOut ?? ((text) => process.stdout.write(text)))(`${enabled ? "true" : "false"}\n`);
|
|
130
|
+
return enabled ? 0 : 1;
|
|
131
|
+
}
|
|
132
|
+
function ensureEnabled(company) {
|
|
133
|
+
return monitorEnabled(company).then((enabled) => {
|
|
134
|
+
if (!enabled)
|
|
135
|
+
throw new LanesError("monitor_disabled", "hq monitor is disabled by the monitor.enabled feature flag.", { flag: MONITOR_FLAG, default: MONITOR_FLAG_DEFAULT });
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
function targetWakeDescription(target) {
|
|
139
|
+
if (target.kind === "lane")
|
|
140
|
+
return "lane dropbox while running; lane resume when done or blocked with a recorded provider session";
|
|
141
|
+
if (target.provider === "claude")
|
|
142
|
+
return "Claude session inbox on the next tool call or user message, and the async Stop waiter while idle";
|
|
143
|
+
// Grok discards UserPromptSubmit hook output, so its events arrive only on a tool call.
|
|
144
|
+
if (target.provider === "grok")
|
|
145
|
+
return "grok session inbox on the next tool call; idle wake is not available";
|
|
146
|
+
return `${target.provider} session inbox on the next tool call or user message; idle wake is not available`;
|
|
147
|
+
}
|
|
148
|
+
export function armText(meta) {
|
|
149
|
+
const lifetime = meta.persistent
|
|
150
|
+
? `persistent: runs until \`hq monitor stop ${meta.monitor_id}\`, the script exits, or the target ends`
|
|
151
|
+
: `expires in ${formatDuration(meta.timeout_ms ?? MONITOR_DEFAULT_TIMEOUT_MS)} unless the command exits first`;
|
|
152
|
+
const cache = meta.persistent || (meta.timeout_ms ?? 0) > 60 * 60_000
|
|
153
|
+
? meta.checkin_ms === null
|
|
154
|
+
? "\nPrompt caching only works within an hour. With check-ins off, the next turn may read uncached."
|
|
155
|
+
: `\nPrompt caching only works within an hour. hq monitor checks in every ${formatDuration(meta.checkin_ms)} while you are idle so your next turn reads from cache; keep check-in turns short.`
|
|
156
|
+
: "";
|
|
157
|
+
return `Monitor started (task ${meta.monitor_id}, ${lifetime}). You will be notified on each event. Target wake path: ${targetWakeDescription(meta.target)}.${cache}\nKeep working — do not poll or sleep. Events may arrive while you are waiting for the user — an event is not their reply.`;
|
|
158
|
+
}
|
|
159
|
+
function formatDuration(milliseconds) {
|
|
160
|
+
if (milliseconds % 3_600_000 === 0)
|
|
161
|
+
return `${milliseconds / 3_600_000}h`;
|
|
162
|
+
if (milliseconds % 60_000 === 0)
|
|
163
|
+
return `${milliseconds / 60_000}m`;
|
|
164
|
+
if (milliseconds % 1_000 === 0)
|
|
165
|
+
return `${milliseconds / 1_000}s`;
|
|
166
|
+
return `${milliseconds}ms`;
|
|
167
|
+
}
|
|
168
|
+
function monitorTimeout(timeoutMs, onTimeout) {
|
|
169
|
+
const startedAt = performance.now();
|
|
170
|
+
let timer = null;
|
|
171
|
+
let cancelled = false;
|
|
172
|
+
const schedule = () => {
|
|
173
|
+
if (cancelled)
|
|
174
|
+
return;
|
|
175
|
+
const elapsedMs = performance.now() - startedAt;
|
|
176
|
+
const delayMs = nextMonitorTimeoutDelay(timeoutMs, elapsedMs);
|
|
177
|
+
if (delayMs <= 0) {
|
|
178
|
+
onTimeout();
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
timer = setTimeout(schedule, delayMs);
|
|
182
|
+
};
|
|
183
|
+
schedule();
|
|
184
|
+
return {
|
|
185
|
+
clear: () => {
|
|
186
|
+
cancelled = true;
|
|
187
|
+
if (timer)
|
|
188
|
+
clearTimeout(timer);
|
|
189
|
+
},
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function sessionEnvironmentId(provider) {
|
|
193
|
+
return process.env[RUNTIME_ENV[provider]]?.trim() || undefined;
|
|
194
|
+
}
|
|
195
|
+
function nearestRuntimeIdentityInProcessChain() {
|
|
196
|
+
const nearest = findNearestMonitorRuntime({ parentPid: process.ppid, readProcess: procEntry });
|
|
197
|
+
if (!nearest)
|
|
198
|
+
return undefined;
|
|
199
|
+
const starttime = readProcStarttime(nearest.pid);
|
|
200
|
+
return starttime === null ? undefined : { ...nearest, starttime };
|
|
201
|
+
}
|
|
202
|
+
export function resolveMonitorStartTarget(hqRoot, discovered, explicitlyProvided, runtime) {
|
|
203
|
+
if (explicitlyProvided || discovered.kind !== "session" || discovered.provider !== "codex")
|
|
204
|
+
return discovered;
|
|
205
|
+
return { ...discovered, sessionId: resolveMappedSessionId(hqRoot, discovered.provider, discovered.sessionId, runtime) };
|
|
206
|
+
}
|
|
207
|
+
async function startMonitor(options, argv = []) {
|
|
208
|
+
// Keep this ahead of flag reads, target detection and every filesystem write.
|
|
209
|
+
assertMonitorTimeoutCompatibility(options.persistent === true, options.timeout !== undefined);
|
|
210
|
+
if (!options.description?.trim())
|
|
211
|
+
throw new LanesError("monitor_description_required", "--description must be non-empty.", {});
|
|
212
|
+
const timeoutMs = options.persistent
|
|
213
|
+
? null
|
|
214
|
+
: options.timeout === undefined
|
|
215
|
+
? MONITOR_DEFAULT_TIMEOUT_MS
|
|
216
|
+
: parseMonitorDuration(options.timeout, "timeout");
|
|
217
|
+
const checkinMs = options.checkin?.trim().toLowerCase() === "off"
|
|
218
|
+
? null
|
|
219
|
+
: options.checkin === undefined
|
|
220
|
+
? MONITOR_DEFAULT_CHECKIN_MS
|
|
221
|
+
: parseMonitorDuration(options.checkin, "checkin");
|
|
222
|
+
if (Boolean(options.command?.trim()) === (argv.length > 0)) {
|
|
223
|
+
throw new LanesError("monitor_command_required", "Provide exactly one of --command or argv after --.", {});
|
|
224
|
+
}
|
|
225
|
+
const env = process.env;
|
|
226
|
+
const hqRoot = resolveLanesHqRoot({ env, cwd: process.cwd() });
|
|
227
|
+
const discovered = options.target
|
|
228
|
+
? parseMonitorTarget(options.target)
|
|
229
|
+
: detectMonitorTarget({ env, parentPid: process.ppid, readProcess: procEntry });
|
|
230
|
+
const runtime = options.target === undefined ? nearestRuntimeIdentityInProcessChain() : undefined;
|
|
231
|
+
const target = resolveMonitorStartTarget(hqRoot, discovered, options.target !== undefined, runtime);
|
|
232
|
+
const targetLane = target.kind === "lane" ? tryReadLane(hqRoot, target.laneId) : null;
|
|
233
|
+
if (target.kind === "lane" && !targetLane) {
|
|
234
|
+
throw new LanesError("monitor_lane_not_found", `Lane ${target.laneId} was not found.`, { lane_id: target.laneId });
|
|
235
|
+
}
|
|
236
|
+
const company = target.kind === "lane" ? targetLane?.company ?? null : processCompanyRef();
|
|
237
|
+
await ensureEnabled(company);
|
|
238
|
+
const cliEntrypoint = process.argv[1];
|
|
239
|
+
if (!cliEntrypoint)
|
|
240
|
+
throw new LanesError("monitor_cli_path_missing", "Could not locate the hq CLI entry point for the detached supervisor.", {});
|
|
241
|
+
const id = makeMonitorId();
|
|
242
|
+
const dir = monitorDir(hqRoot, id);
|
|
243
|
+
const meta = {
|
|
244
|
+
schema_version: MONITOR_SCHEMA_VERSION,
|
|
245
|
+
monitor_id: id,
|
|
246
|
+
target,
|
|
247
|
+
target_key: monitorTargetKey(target),
|
|
248
|
+
company,
|
|
249
|
+
description: options.description.trim(),
|
|
250
|
+
command: options.command?.trim() || null,
|
|
251
|
+
argv: argv.length ? argv : null,
|
|
252
|
+
cwd: process.cwd(),
|
|
253
|
+
created_at: new Date().toISOString(),
|
|
254
|
+
timeout_ms: timeoutMs,
|
|
255
|
+
persistent: options.persistent === true,
|
|
256
|
+
checkin_ms: checkinMs,
|
|
257
|
+
session_owner: findSessionOwner(sessionOwnerTarget(discovered, target)),
|
|
258
|
+
};
|
|
259
|
+
fs.mkdirSync(path.dirname(dir), { recursive: true, mode: 0o700 });
|
|
260
|
+
fs.mkdirSync(dir, { recursive: false, mode: 0o700 });
|
|
261
|
+
fs.writeFileSync(path.join(dir, "meta.json"), `${JSON.stringify(meta, null, 2)}\n`, { mode: 0o600 });
|
|
262
|
+
fs.writeFileSync(path.join(dir, "events.jsonl"), "", { mode: 0o600 });
|
|
263
|
+
fs.writeFileSync(path.join(dir, "stderr.log"), "", { mode: 0o600 });
|
|
264
|
+
writeJsonAtomic(statePath(hqRoot, id), {
|
|
265
|
+
status: "starting",
|
|
266
|
+
pid: null,
|
|
267
|
+
starttime: null,
|
|
268
|
+
event_count: 0,
|
|
269
|
+
event_sequence: 0,
|
|
270
|
+
});
|
|
271
|
+
if (target.kind === "session") {
|
|
272
|
+
const activeDir = path.join(monitorSessionDir(hqRoot, target.provider, target.sessionId), "active");
|
|
273
|
+
fs.mkdirSync(activeDir, { recursive: true });
|
|
274
|
+
fs.writeFileSync(path.join(activeDir, id), `${JSON.stringify({ monitor_id: id, owner: meta.session_owner })}\n`, { mode: 0o600 });
|
|
275
|
+
}
|
|
276
|
+
await recordMonitorTargetDelivery(hqRoot, meta.target_key, () => new Date(), "start");
|
|
277
|
+
const child = spawn(process.execPath, [cliEntrypoint, "monitor", "_run", id], {
|
|
278
|
+
cwd: meta.cwd,
|
|
279
|
+
env: { ...env, HQ_ROOT: hqRoot },
|
|
280
|
+
detached: true,
|
|
281
|
+
stdio: "ignore",
|
|
282
|
+
});
|
|
283
|
+
child.once("error", (err) => {
|
|
284
|
+
const current = readJson(statePath(hqRoot, id));
|
|
285
|
+
if (current)
|
|
286
|
+
writeJsonAtomic(statePath(hqRoot, id), { ...current, status: "terminal", finished_at: new Date().toISOString(), terminal_reason: "supervisor_spawn_failed" });
|
|
287
|
+
if (meta.target.kind === "session")
|
|
288
|
+
fs.rmSync(path.join(monitorSessionDir(hqRoot, meta.target.provider, meta.target.sessionId), "active", id), { force: true });
|
|
289
|
+
process.stderr.write(`hq monitor: supervisor spawn failed: ${err.message}\n`);
|
|
290
|
+
});
|
|
291
|
+
child.unref();
|
|
292
|
+
if (!child.pid)
|
|
293
|
+
throw new LanesError("monitor_supervisor_spawn_failed", "The detached monitor supervisor did not start.", {});
|
|
294
|
+
const running = {
|
|
295
|
+
status: "running",
|
|
296
|
+
pid: child.pid,
|
|
297
|
+
starttime: readProcStarttime(child.pid),
|
|
298
|
+
started_at: new Date().toISOString(),
|
|
299
|
+
event_count: 0,
|
|
300
|
+
};
|
|
301
|
+
const latest = readJson(statePath(hqRoot, id));
|
|
302
|
+
writeJsonAtomic(statePath(hqRoot, id), latest?.pid === child.pid ? latest : running);
|
|
303
|
+
if (options.json)
|
|
304
|
+
printJson({ ok: true, monitor_id: id, target, state: running, message: armText(meta) });
|
|
305
|
+
else
|
|
306
|
+
process.stdout.write(`${armText(meta)}\n`);
|
|
307
|
+
}
|
|
308
|
+
function monitorIds(hqRoot) {
|
|
309
|
+
const root = path.join(hqRoot, "workspace", "monitors");
|
|
310
|
+
try {
|
|
311
|
+
return fs.readdirSync(root, { withFileTypes: true })
|
|
312
|
+
.filter((entry) => entry.isDirectory() && !["sessions", "targets", "session-map"].includes(entry.name))
|
|
313
|
+
.map((entry) => entry.name);
|
|
314
|
+
}
|
|
315
|
+
catch (err) {
|
|
316
|
+
if (err.code === "ENOENT")
|
|
317
|
+
return [];
|
|
318
|
+
throw err;
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
function stateForDisplay(hqRoot, id, raw) {
|
|
322
|
+
if (!raw || raw.status === "terminal" || raw.pid === null)
|
|
323
|
+
return raw;
|
|
324
|
+
if (readMonitorPidLiveness(raw.pid, raw.starttime))
|
|
325
|
+
return raw;
|
|
326
|
+
return { ...raw, status: "terminal", terminal_reason: "supervisor_died", finished_at: raw.finished_at ?? new Date().toISOString() };
|
|
327
|
+
}
|
|
328
|
+
async function listMonitors(options) {
|
|
329
|
+
const hqRoot = resolveLanesHqRoot();
|
|
330
|
+
const rows = monitorIds(hqRoot).flatMap((id) => {
|
|
331
|
+
const meta = readJson(path.join(monitorDir(hqRoot, id), "meta.json"));
|
|
332
|
+
const state = stateForDisplay(hqRoot, id, readJson(statePath(hqRoot, id)));
|
|
333
|
+
if (!meta || !state || (!options.all && state.status !== "running"))
|
|
334
|
+
return [];
|
|
335
|
+
return [{ monitor_id: id, description: meta.description, target: meta.target, state }];
|
|
336
|
+
});
|
|
337
|
+
if (options.json)
|
|
338
|
+
printJson({ monitors: rows });
|
|
339
|
+
else
|
|
340
|
+
for (const row of rows)
|
|
341
|
+
process.stdout.write(`${row.monitor_id}\t${row.state.status}\t${row.description}\t${monitorTargetKey(row.target)}${row.state.terminal_reason ? `\t${row.state.terminal_reason}` : ""}\n`);
|
|
342
|
+
}
|
|
343
|
+
async function showMonitor(id, json) {
|
|
344
|
+
const hqRoot = resolveLanesHqRoot();
|
|
345
|
+
const dir = monitorDir(hqRoot, id);
|
|
346
|
+
const meta = readJson(path.join(dir, "meta.json"));
|
|
347
|
+
const state = stateForDisplay(hqRoot, id, readJson(statePath(hqRoot, id)));
|
|
348
|
+
if (!meta || !state)
|
|
349
|
+
throw new LanesError("monitor_not_found", `Monitor ${id} was not found.`, { monitor_id: id });
|
|
350
|
+
const events = fs.readFileSync(path.join(dir, "events.jsonl"), "utf8").split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
351
|
+
if (json)
|
|
352
|
+
printJson({ meta, state, events });
|
|
353
|
+
else
|
|
354
|
+
process.stdout.write(`${JSON.stringify({ meta, state, events }, null, 2)}\n`);
|
|
355
|
+
}
|
|
356
|
+
async function stopMonitor(id) {
|
|
357
|
+
const hqRoot = resolveLanesHqRoot();
|
|
358
|
+
const dir = monitorDir(hqRoot, id);
|
|
359
|
+
const meta = readJson(path.join(dir, "meta.json"));
|
|
360
|
+
const state = readJson(statePath(hqRoot, id));
|
|
361
|
+
if (!meta || !state)
|
|
362
|
+
throw new LanesError("monitor_not_found", `Monitor ${id} was not found.`, { monitor_id: id });
|
|
363
|
+
if (state.status !== "terminal" && state.pid !== null && readMonitorPidLiveness(state.pid, state.starttime)) {
|
|
364
|
+
try {
|
|
365
|
+
process.kill(state.pid, "SIGTERM");
|
|
366
|
+
}
|
|
367
|
+
catch (err) {
|
|
368
|
+
if (err.code !== "ESRCH")
|
|
369
|
+
throw err;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
else if (state.status !== "terminal") {
|
|
373
|
+
writeJsonAtomic(statePath(hqRoot, id), { ...state, status: "terminal", terminal_reason: "supervisor_died", finished_at: new Date().toISOString() });
|
|
374
|
+
removeActiveMarker(hqRoot, meta);
|
|
375
|
+
}
|
|
376
|
+
const deadline = Date.now() + 5_000;
|
|
377
|
+
let current = readJson(statePath(hqRoot, id));
|
|
378
|
+
while (current && current.status !== "terminal" && Date.now() < deadline) {
|
|
379
|
+
await waitSleep(50);
|
|
380
|
+
current = readJson(statePath(hqRoot, id));
|
|
381
|
+
}
|
|
382
|
+
if (current?.status !== "terminal") {
|
|
383
|
+
throw new LanesError("monitor_stop_timeout", `Monitor ${id} did not record its stopped state within five seconds.`, { monitor_id: id });
|
|
384
|
+
}
|
|
385
|
+
process.stdout.write(`Monitor ${id} stopped.\n`);
|
|
386
|
+
}
|
|
387
|
+
async function monitorLogs(id, stderr) {
|
|
388
|
+
const hqRoot = resolveLanesHqRoot();
|
|
389
|
+
const filePath = path.join(monitorDir(hqRoot, id), stderr ? "stderr.log" : "events.jsonl");
|
|
390
|
+
if (!fs.existsSync(filePath))
|
|
391
|
+
throw new LanesError("monitor_not_found", `Monitor ${id} was not found.`, { monitor_id: id });
|
|
392
|
+
process.stdout.write(fs.readFileSync(filePath, "utf8"));
|
|
393
|
+
}
|
|
394
|
+
function appendEventRecord(hqRoot, id, state, event) {
|
|
395
|
+
const dir = monitorDir(hqRoot, id);
|
|
396
|
+
fs.appendFileSync(path.join(dir, "events.jsonl"), `${JSON.stringify(event)}\n`, { mode: 0o600 });
|
|
397
|
+
const next = recordMonitorEventState(state, event);
|
|
398
|
+
writeJsonAtomic(statePath(hqRoot, id), next);
|
|
399
|
+
return next;
|
|
400
|
+
}
|
|
401
|
+
function removeActiveMarker(hqRoot, meta) {
|
|
402
|
+
if (meta.target.kind !== "session")
|
|
403
|
+
return;
|
|
404
|
+
fs.rmSync(path.join(monitorSessionDir(hqRoot, meta.target.provider, meta.target.sessionId), "active", meta.monitor_id), { force: true });
|
|
405
|
+
}
|
|
406
|
+
async function deliverLaneEvent(hqRoot, meta, sequence, text) {
|
|
407
|
+
if (meta.target.kind !== "lane")
|
|
408
|
+
throw new LanesError("monitor_lane_target_required", "Lane delivery requires a lane target.");
|
|
409
|
+
const lane = tryReadLane(hqRoot, meta.target.laneId);
|
|
410
|
+
if (!lane)
|
|
411
|
+
return { delivered: false, outcome: "lane_not_found" };
|
|
412
|
+
let dispatched;
|
|
413
|
+
try {
|
|
414
|
+
dispatched = await dispatchLaneDelivery(lane, {
|
|
415
|
+
dropbox: () => invokeLaneMonitorMessage(hqRoot, meta, sequence, text),
|
|
416
|
+
resume: () => invokeLaneMonitorMessage(hqRoot, meta, sequence, text),
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
catch (err) {
|
|
420
|
+
if (err instanceof LanesError && err.error === "lane_not_accepting_messages")
|
|
421
|
+
return { delivered: false, outcome: `undeliverable:${err.message}` };
|
|
422
|
+
throw err;
|
|
423
|
+
}
|
|
424
|
+
return { ...dispatched.value, outcome: `${dispatched.mechanism}:${dispatched.value.outcome}` };
|
|
425
|
+
}
|
|
426
|
+
function invokeLaneMonitorMessage(hqRoot, meta, sequence, text) {
|
|
427
|
+
if (meta.target.kind !== "lane")
|
|
428
|
+
throw new LanesError("monitor_lane_target_required", "Lane delivery requires a lane target.");
|
|
429
|
+
const cliEntrypoint = process.argv[1];
|
|
430
|
+
if (!cliEntrypoint)
|
|
431
|
+
throw new LanesError("monitor_cli_path_missing", "Could not locate the hq CLI entry point for lane delivery.");
|
|
432
|
+
const result = spawnSync(process.execPath, [cliEntrypoint, "lanes", "_monitor-message", meta.target.laneId], {
|
|
433
|
+
cwd: meta.cwd,
|
|
434
|
+
env: { ...process.env, HQ_ROOT: hqRoot },
|
|
435
|
+
input: JSON.stringify({ text, monitor_id: meta.monitor_id, description: meta.description, sequence }),
|
|
436
|
+
encoding: "utf8",
|
|
437
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
438
|
+
});
|
|
439
|
+
if (result.error)
|
|
440
|
+
throw result.error;
|
|
441
|
+
if (result.status !== 0) {
|
|
442
|
+
let error = `lane_delivery_exit_${result.status ?? "unknown"}`;
|
|
443
|
+
try {
|
|
444
|
+
const parsed = JSON.parse(result.stdout.trim());
|
|
445
|
+
if (typeof parsed.error === "string")
|
|
446
|
+
error = parsed.error;
|
|
447
|
+
}
|
|
448
|
+
catch (err) {
|
|
449
|
+
if (!(err instanceof SyntaxError))
|
|
450
|
+
throw err;
|
|
451
|
+
}
|
|
452
|
+
return { delivered: false, outcome: error };
|
|
453
|
+
}
|
|
454
|
+
try {
|
|
455
|
+
const output = JSON.parse(result.stdout.trim());
|
|
456
|
+
return { delivered: output.accepted === true, outcome: String(output.delivery_status ?? output.delivery ?? "accepted") };
|
|
457
|
+
}
|
|
458
|
+
catch (err) {
|
|
459
|
+
throw new LanesError("monitor_lane_delivery_invalid_response", `The lane delivery command returned invalid JSON: ${errorPath(err)}.`, {});
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
function deliverSessionEvent(hqRoot, meta, sequence, text) {
|
|
463
|
+
if (meta.target.kind !== "session")
|
|
464
|
+
throw new LanesError("monitor_session_target_required", "Session delivery requires a session target.");
|
|
465
|
+
const box = path.join(monitorSessionDir(hqRoot, meta.target.provider, meta.target.sessionId), "dropbox");
|
|
466
|
+
appendDropboxMessageAtPath(box, {
|
|
467
|
+
at: new Date().toISOString(),
|
|
468
|
+
kind: "monitor_event",
|
|
469
|
+
monitor_id: meta.monitor_id,
|
|
470
|
+
description: meta.description,
|
|
471
|
+
sequence,
|
|
472
|
+
text,
|
|
473
|
+
});
|
|
474
|
+
return { delivered: true, outcome: "session_inbox" };
|
|
475
|
+
}
|
|
476
|
+
async function deliverTargetEvent(hqRoot, meta, sequence, text) {
|
|
477
|
+
return meta.target.kind === "lane"
|
|
478
|
+
? deliverLaneEvent(hqRoot, meta, sequence, text)
|
|
479
|
+
: deliverSessionEvent(hqRoot, meta, sequence, text);
|
|
480
|
+
}
|
|
481
|
+
export async function deliverNormalEvent(hqRoot, meta, state, rawText) {
|
|
482
|
+
return withMonitorTargetLock(hqRoot, meta.target_key, async (activityPath) => {
|
|
483
|
+
const current = readJson(statePath(hqRoot, meta.monitor_id)) ?? state;
|
|
484
|
+
const sequence = nextMonitorEventSequence(current);
|
|
485
|
+
const outcome = await deliverTargetEvent(hqRoot, meta, sequence, rawText);
|
|
486
|
+
const event = { seq: sequence, at: new Date().toISOString(), text: rawText, delivery: outcome.delivered ? "delivered" : `undeliverable:${outcome.outcome}` };
|
|
487
|
+
const next = appendEventRecord(hqRoot, meta.monitor_id, current, event);
|
|
488
|
+
if (outcome.delivered)
|
|
489
|
+
recordMonitorTargetDeliveryAtPath(activityPath);
|
|
490
|
+
return next;
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
function targetEnded(hqRoot, meta) {
|
|
494
|
+
if (meta.target.kind === "session") {
|
|
495
|
+
return meta.session_owner !== null && !readMonitorPidLiveness(meta.session_owner.pid, meta.session_owner.starttime);
|
|
496
|
+
}
|
|
497
|
+
const lane = tryReadLane(hqRoot, meta.target.laneId);
|
|
498
|
+
return !lane || ["killed", "cancelled", "failed"].includes(lane.state);
|
|
499
|
+
}
|
|
500
|
+
async function monitorCheckin(hqRoot, meta, state) {
|
|
501
|
+
await runMonitorCheckinTick(hqRoot, meta.target_key, {
|
|
502
|
+
now: () => new Date(),
|
|
503
|
+
isWakeable: async (targetKey) => {
|
|
504
|
+
const monitor = activeTargetMeta(hqRoot, targetKey)[0];
|
|
505
|
+
if (!monitor)
|
|
506
|
+
return false;
|
|
507
|
+
if (monitor.target.kind === "session")
|
|
508
|
+
return monitor.target.provider === "claude";
|
|
509
|
+
const lane = tryReadLane(hqRoot, monitor.target.laneId);
|
|
510
|
+
return lane !== null && (lane.state === "running" ||
|
|
511
|
+
((lane.state === "done" || lane.state === "blocked") && Boolean(lane.provider_session_id)));
|
|
512
|
+
},
|
|
513
|
+
readTargetActivityAt: (targetKey) => {
|
|
514
|
+
if (meta.target.kind !== "lane" || targetKey !== meta.target_key)
|
|
515
|
+
return null;
|
|
516
|
+
const activityAt = readToolActivity(hqRoot, meta.target.laneId)?.at;
|
|
517
|
+
if (!activityAt)
|
|
518
|
+
return null;
|
|
519
|
+
const parsed = Date.parse(activityAt);
|
|
520
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
521
|
+
},
|
|
522
|
+
deliver: async (text) => {
|
|
523
|
+
const monitor = activeTargetMeta(hqRoot, meta.target_key)[0];
|
|
524
|
+
if (!monitor)
|
|
525
|
+
return { delivered: false, outcome: "no_live_monitor" };
|
|
526
|
+
const monitorState = readJson(statePath(hqRoot, monitor.monitor_id)) ?? state;
|
|
527
|
+
const sequence = nextMonitorEventSequence(monitorState);
|
|
528
|
+
const outcome = await deliverTargetEvent(hqRoot, monitor, sequence, text);
|
|
529
|
+
const event = { seq: sequence, at: new Date().toISOString(), text, delivery: outcome.delivered ? "delivered" : `undeliverable:${outcome.outcome}` };
|
|
530
|
+
appendEventRecord(hqRoot, monitor.monitor_id, monitorState, event);
|
|
531
|
+
return outcome;
|
|
532
|
+
},
|
|
533
|
+
pidIsAlive: readMonitorPidLiveness,
|
|
534
|
+
readStarttime: readProcStarttime,
|
|
535
|
+
});
|
|
536
|
+
return readJson(statePath(hqRoot, meta.monitor_id)) ?? state;
|
|
537
|
+
}
|
|
538
|
+
function activeTargetMeta(hqRoot, targetKey) {
|
|
539
|
+
return monitorIds(hqRoot).flatMap((id) => {
|
|
540
|
+
const meta = readJson(path.join(monitorDir(hqRoot, id), "meta.json"));
|
|
541
|
+
const state = readJson(statePath(hqRoot, id));
|
|
542
|
+
return meta?.target_key === targetKey && state?.status === "running" && state.pid !== null && readMonitorPidLiveness(state.pid, state.starttime) ? [meta] : [];
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
export async function runSupervisor(id, deps = {}) {
|
|
546
|
+
const hqRoot = (deps.resolveRoot ?? resolveLanesHqRoot)();
|
|
547
|
+
const dir = monitorDir(hqRoot, id);
|
|
548
|
+
const meta = readJson(path.join(dir, "meta.json"));
|
|
549
|
+
const stateAtStart = readJson(statePath(hqRoot, id));
|
|
550
|
+
if (!meta || !stateAtStart)
|
|
551
|
+
throw new LanesError("monitor_not_found", `Monitor ${id} was not found.`, { monitor_id: id });
|
|
552
|
+
if (!(await (deps.monitorEnabled ?? monitorEnabled)(meta.company))) {
|
|
553
|
+
writeJsonAtomic(statePath(hqRoot, id), { ...stateAtStart, status: "terminal", finished_at: new Date().toISOString(), terminal_reason: "monitor_disabled" });
|
|
554
|
+
removeActiveMarker(hqRoot, meta);
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
557
|
+
let state = { ...stateAtStart, status: "running", pid: process.pid, starttime: readProcStarttime(process.pid), started_at: new Date().toISOString() };
|
|
558
|
+
writeJsonAtomic(statePath(hqRoot, id), state);
|
|
559
|
+
let commandChild = null;
|
|
560
|
+
let finished = false;
|
|
561
|
+
let finishing = false;
|
|
562
|
+
let terminalReason = null;
|
|
563
|
+
let timeoutTimer = null;
|
|
564
|
+
let pollTimer = null;
|
|
565
|
+
let batchTimer = null;
|
|
566
|
+
let killEscalationTimer = null;
|
|
567
|
+
let buffered = "";
|
|
568
|
+
let batchLines = [];
|
|
569
|
+
let deliveryQueue = Promise.resolve();
|
|
570
|
+
let windowStartedAt = Date.now();
|
|
571
|
+
let windowEvents = 0;
|
|
572
|
+
let suppressed = 0;
|
|
573
|
+
let floodWindows = 0;
|
|
574
|
+
let checkinRunning = false;
|
|
575
|
+
const deliver = deps.deliver ?? deliverNormalEvent;
|
|
576
|
+
const perWindowCap = (() => {
|
|
577
|
+
const candidate = Number.parseInt(process.env.HQ_MONITOR_MAX_EVENTS_PER_WINDOW ?? "", 10);
|
|
578
|
+
return Number.isInteger(candidate) && candidate > 0 ? candidate : MONITOR_MAX_EVENTS_PER_WINDOW;
|
|
579
|
+
})();
|
|
580
|
+
const writeStderr = (chunk) => fs.appendFileSync(path.join(dir, "stderr.log"), chunk);
|
|
581
|
+
const killCommand = () => {
|
|
582
|
+
if (!commandChild?.pid)
|
|
583
|
+
return;
|
|
584
|
+
try {
|
|
585
|
+
process.kill(-commandChild.pid, "SIGTERM");
|
|
586
|
+
}
|
|
587
|
+
catch (err) {
|
|
588
|
+
if (err.code !== "ESRCH")
|
|
589
|
+
throw err;
|
|
590
|
+
}
|
|
591
|
+
if (!killEscalationTimer) {
|
|
592
|
+
killEscalationTimer = setTimeout(() => {
|
|
593
|
+
if (!commandChild?.pid)
|
|
594
|
+
return;
|
|
595
|
+
try {
|
|
596
|
+
process.kill(-commandChild.pid, "SIGKILL");
|
|
597
|
+
}
|
|
598
|
+
catch (err) {
|
|
599
|
+
if (err.code !== "ESRCH")
|
|
600
|
+
writeStderr(`command kill escalation failed: ${errorPath(err)}\n`);
|
|
601
|
+
}
|
|
602
|
+
}, 1_500);
|
|
603
|
+
}
|
|
604
|
+
};
|
|
605
|
+
const flushBatch = () => {
|
|
606
|
+
if (batchTimer)
|
|
607
|
+
clearTimeout(batchTimer);
|
|
608
|
+
batchTimer = null;
|
|
609
|
+
const batch = batchLines.splice(0, batchLines.length);
|
|
610
|
+
batchLines = [];
|
|
611
|
+
if (batch.length === 0 || finished)
|
|
612
|
+
return deliveryQueue;
|
|
613
|
+
deliveryQueue = deliveryQueue.then(async () => {
|
|
614
|
+
const now = Date.now();
|
|
615
|
+
if (now - windowStartedAt >= MONITOR_RATE_WINDOW_MS) {
|
|
616
|
+
const windowsPassed = Math.floor((now - windowStartedAt) / MONITOR_RATE_WINDOW_MS);
|
|
617
|
+
if (suppressed > 0) {
|
|
618
|
+
state = await deliver(hqRoot, meta, state, `[monitor ${id} suppressed ${suppressed} events — output rate exceeded]`);
|
|
619
|
+
floodWindows += 1;
|
|
620
|
+
suppressed = 0;
|
|
621
|
+
}
|
|
622
|
+
else {
|
|
623
|
+
floodWindows = 0;
|
|
624
|
+
}
|
|
625
|
+
windowStartedAt += windowsPassed * MONITOR_RATE_WINDOW_MS;
|
|
626
|
+
windowEvents = 0;
|
|
627
|
+
if (floodWindows >= MONITOR_MAX_FLOOD_WINDOWS) {
|
|
628
|
+
terminalReason = "rate_exceeded";
|
|
629
|
+
killCommand();
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
const remaining = Math.max(0, perWindowCap - windowEvents);
|
|
633
|
+
const permitted = batch.slice(0, remaining);
|
|
634
|
+
suppressed += batch.length - permitted.length;
|
|
635
|
+
windowEvents += permitted.length;
|
|
636
|
+
if (permitted.length > 0)
|
|
637
|
+
state = await deliver(hqRoot, meta, state, permitted.join("\n"));
|
|
638
|
+
if (floodWindows >= MONITOR_MAX_FLOOD_WINDOWS) {
|
|
639
|
+
terminalReason = "rate_exceeded";
|
|
640
|
+
killCommand();
|
|
641
|
+
}
|
|
642
|
+
}).catch((err) => {
|
|
643
|
+
writeStderr(`event delivery failed: ${errorPath(err)}\n`);
|
|
644
|
+
});
|
|
645
|
+
return deliveryQueue;
|
|
646
|
+
};
|
|
647
|
+
const finish = async (exitCode) => {
|
|
648
|
+
if (finished || finishing)
|
|
649
|
+
return;
|
|
650
|
+
finishing = true;
|
|
651
|
+
timeoutTimer?.clear();
|
|
652
|
+
if (pollTimer)
|
|
653
|
+
clearInterval(pollTimer);
|
|
654
|
+
if (batchTimer)
|
|
655
|
+
clearTimeout(batchTimer);
|
|
656
|
+
if (killEscalationTimer)
|
|
657
|
+
clearTimeout(killEscalationTimer);
|
|
658
|
+
if (buffered.length > 0)
|
|
659
|
+
batchLines.push(buffered);
|
|
660
|
+
buffered = "";
|
|
661
|
+
if (batchLines.length > 0)
|
|
662
|
+
flushBatch();
|
|
663
|
+
await deliveryQueue;
|
|
664
|
+
finished = true;
|
|
665
|
+
const reason = terminalReason ?? (exitCode === null ? "supervisor_died" : "script_exit");
|
|
666
|
+
const priorEvents = state.event_count;
|
|
667
|
+
if (reason !== "stopped" && reason !== "target_ended" && reason !== "supervisor_died") {
|
|
668
|
+
const terminalText = reason === "expired"
|
|
669
|
+
? `[hq monitor ${id} expired after ${formatDuration(meta.timeout_ms ?? MONITOR_DEFAULT_TIMEOUT_MS)} with ${priorEvents} events delivered. Re-arm with \`hq monitor start ...\` if you still need the watch — and widen the filter if silence was unexpected.]`
|
|
670
|
+
: reason === "rate_exceeded"
|
|
671
|
+
? `[hq monitor ${id} stopped: rate_exceeded after suppressing sustained output.]`
|
|
672
|
+
: `[hq monitor ${id} ${JSON.stringify(meta.description)} exited with code ${exitCode ?? 1} after ${priorEvents} events]`;
|
|
673
|
+
try {
|
|
674
|
+
state = await deliver(hqRoot, meta, state, terminalText);
|
|
675
|
+
}
|
|
676
|
+
catch (err) {
|
|
677
|
+
writeStderr(`terminal event delivery failed: ${errorPath(err)}\n`);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
state = { ...state, status: "terminal", finished_at: new Date().toISOString(), exit_code: exitCode, terminal_reason: reason };
|
|
681
|
+
writeJsonAtomic(statePath(hqRoot, id), state);
|
|
682
|
+
removeActiveMarker(hqRoot, meta);
|
|
683
|
+
};
|
|
684
|
+
const stop = () => {
|
|
685
|
+
terminalReason = "stopped";
|
|
686
|
+
requestMonitorStop(commandChild?.pid ?? null, killCommand, () => {
|
|
687
|
+
void finish(null).catch((err) => { writeStderr(`supervisor shutdown failed: ${errorPath(err)}\n`); process.exitCode = 1; });
|
|
688
|
+
});
|
|
689
|
+
};
|
|
690
|
+
process.on("SIGTERM", stop);
|
|
691
|
+
process.on("SIGINT", stop);
|
|
692
|
+
pollTimer = setInterval(() => {
|
|
693
|
+
if (targetEnded(hqRoot, meta)) {
|
|
694
|
+
terminalReason = "target_ended";
|
|
695
|
+
killCommand();
|
|
696
|
+
}
|
|
697
|
+
if (!checkinRunning) {
|
|
698
|
+
checkinRunning = true;
|
|
699
|
+
void monitorCheckin(hqRoot, meta, state)
|
|
700
|
+
.then((next) => { state = next; })
|
|
701
|
+
.catch((err) => writeStderr(`check-in failed: ${errorPath(err)}\n`))
|
|
702
|
+
.finally(() => { checkinRunning = false; });
|
|
703
|
+
}
|
|
704
|
+
}, MONITOR_POLL_INTERVAL_MS);
|
|
705
|
+
if (meta.timeout_ms !== null)
|
|
706
|
+
timeoutTimer = monitorTimeout(meta.timeout_ms, () => {
|
|
707
|
+
terminalReason = "expired";
|
|
708
|
+
killCommand();
|
|
709
|
+
});
|
|
710
|
+
const childArgs = meta.command !== null ? ["-c", meta.command] : (meta.argv ?? []);
|
|
711
|
+
const executable = meta.command !== null ? "bash" : childArgs[0];
|
|
712
|
+
const args = meta.command !== null ? childArgs : childArgs.slice(1);
|
|
713
|
+
if (!executable)
|
|
714
|
+
throw new LanesError("monitor_command_required", "No command was recorded for this monitor.");
|
|
715
|
+
commandChild = (deps.spawn ?? spawn)(executable, args, { cwd: meta.cwd, env: process.env, detached: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
716
|
+
if (commandChild.pid) {
|
|
717
|
+
state = { ...state, child_pid: commandChild.pid, child_starttime: readProcStarttime(commandChild.pid) };
|
|
718
|
+
writeJsonAtomic(statePath(hqRoot, id), state);
|
|
719
|
+
}
|
|
720
|
+
commandChild.stdout?.on("data", (chunk) => {
|
|
721
|
+
buffered += chunk.toString("utf8");
|
|
722
|
+
const lines = buffered.split(/\r?\n/);
|
|
723
|
+
buffered = lines.pop() ?? "";
|
|
724
|
+
batchLines.push(...lines.filter((line) => line.length > 0));
|
|
725
|
+
if (batchLines.length > 0 && !batchTimer)
|
|
726
|
+
batchTimer = setTimeout(() => { void flushBatch().catch((err) => writeStderr(`event delivery failed: ${errorPath(err)}\n`)); }, MONITOR_BATCH_WINDOW_MS);
|
|
727
|
+
});
|
|
728
|
+
commandChild.stderr?.on("data", writeStderr);
|
|
729
|
+
commandChild.once("error", (err) => {
|
|
730
|
+
writeStderr(`command start failed: ${err.message}\n`);
|
|
731
|
+
void finish(127).catch((cause) => { writeStderr(`supervisor finish failed: ${errorPath(cause)}\n`); process.exitCode = 1; });
|
|
732
|
+
});
|
|
733
|
+
commandChild.once("close", (code) => {
|
|
734
|
+
void finish(code).catch((err) => { writeStderr(`supervisor finish failed: ${errorPath(err)}\n`); process.exitCode = 1; });
|
|
735
|
+
});
|
|
736
|
+
await new Promise((resolve) => {
|
|
737
|
+
const done = setInterval(() => {
|
|
738
|
+
if (finished) {
|
|
739
|
+
clearInterval(done);
|
|
740
|
+
resolve();
|
|
741
|
+
}
|
|
742
|
+
}, 50);
|
|
743
|
+
done.unref();
|
|
744
|
+
});
|
|
745
|
+
process.off("SIGTERM", stop);
|
|
746
|
+
process.off("SIGINT", stop);
|
|
747
|
+
}
|
|
748
|
+
async function readStdin() {
|
|
749
|
+
const chunks = [];
|
|
750
|
+
for await (const chunk of process.stdin)
|
|
751
|
+
chunks.push(Buffer.from(chunk));
|
|
752
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
753
|
+
}
|
|
754
|
+
export async function runMonitorHookSafely(kind, operation, writeError = writeStderr) {
|
|
755
|
+
try {
|
|
756
|
+
return await operation();
|
|
757
|
+
}
|
|
758
|
+
catch (err) {
|
|
759
|
+
writeError(`hq monitor ${kind}: ${errorPath(err)}\n`);
|
|
760
|
+
return { stdout: "", exitCode: 0 };
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
export async function drainSession(provider, event, deps = {}) {
|
|
764
|
+
if (!(await (deps.monitorEnabled ?? (() => monitorEnabled()))()))
|
|
765
|
+
return { stdout: "", exitCode: 0 };
|
|
766
|
+
const hqRoot = (deps.resolveRoot ?? resolveLanesHqRoot)();
|
|
767
|
+
const payloadId = parseHookSessionId(await (deps.readStdin ?? readStdin)());
|
|
768
|
+
const envId = (deps.sessionEnvironmentId ?? sessionEnvironmentId)(provider);
|
|
769
|
+
const runtime = (deps.nearestRuntimeIdentity ?? nearestRuntimeIdentityInProcessChain)();
|
|
770
|
+
const codexRuntimeOwnsMapping = provider === "codex" && envId !== undefined && runtime?.provider === "codex";
|
|
771
|
+
const payloadHasWork = monitorSessionHasWork(hqRoot, provider, payloadId);
|
|
772
|
+
const sessionIds = new Set();
|
|
773
|
+
if (payloadHasWork)
|
|
774
|
+
sessionIds.add(payloadId);
|
|
775
|
+
if (codexRuntimeOwnsMapping && envId !== payloadId && runtime) {
|
|
776
|
+
const mappedSessionId = resolveMappedSessionId(hqRoot, provider, envId, runtime);
|
|
777
|
+
const ownsRuntimeTarget = monitorSessionHasActiveMarkerForOwner(hqRoot, provider, envId, runtime);
|
|
778
|
+
if (ownsRuntimeTarget || mappedSessionId === payloadId) {
|
|
779
|
+
if (monitorSessionHasWork(hqRoot, provider, envId))
|
|
780
|
+
sessionIds.add(envId);
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
if (sessionIds.size === 0)
|
|
784
|
+
return { stdout: "", exitCode: 0 };
|
|
785
|
+
if (codexRuntimeOwnsMapping && envId && runtime?.provider === "codex" && payloadId !== envId
|
|
786
|
+
&& (payloadHasWork || sessionIds.has(envId))) {
|
|
787
|
+
recordHookSessionMapping(hqRoot, provider, envId, payloadId, runtime);
|
|
788
|
+
}
|
|
789
|
+
const messages = [];
|
|
790
|
+
for (const sessionId of sessionIds) {
|
|
791
|
+
const targetKey = monitorTargetKey({ kind: "session", provider, sessionId });
|
|
792
|
+
await (deps.recordActivity ?? recordMonitorTargetActivity)(hqRoot, targetKey);
|
|
793
|
+
const box = path.join(monitorSessionDir(hqRoot, provider, sessionId), "dropbox");
|
|
794
|
+
messages.push(...(deps.drainDropbox ?? drainDropboxAtPath)(box));
|
|
795
|
+
}
|
|
796
|
+
messages.sort((left, right) => (left.at ?? "").localeCompare(right.at ?? ""));
|
|
797
|
+
const stdout = messages.length > 0
|
|
798
|
+
? `${JSON.stringify({ hookSpecificOutput: { hookEventName: event, additionalContext: formatDropboxMessages(messages) } })}\n`
|
|
799
|
+
: "";
|
|
800
|
+
return { stdout, exitCode: 0 };
|
|
801
|
+
}
|
|
802
|
+
function sessionMonitorIds(hqRoot, provider, sessionId) {
|
|
803
|
+
const activeDir = path.join(monitorSessionDir(hqRoot, provider, sessionId), "active");
|
|
804
|
+
try {
|
|
805
|
+
return fs.readdirSync(activeDir);
|
|
806
|
+
}
|
|
807
|
+
catch (err) {
|
|
808
|
+
if (err.code === "ENOENT")
|
|
809
|
+
return [];
|
|
810
|
+
throw err;
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
function waiterIsCurrent(waiterPath, token) {
|
|
814
|
+
const current = readJson(waiterPath);
|
|
815
|
+
return current?.token === token;
|
|
816
|
+
}
|
|
817
|
+
function waitMonitorPoll(sleep = waitSleep) {
|
|
818
|
+
return sleep(MONITOR_POLL_INTERVAL_MS);
|
|
819
|
+
}
|
|
820
|
+
export async function waitForSession(provider, deps = {}) {
|
|
821
|
+
if (!(await (deps.monitorEnabled ?? (() => monitorEnabled()))()))
|
|
822
|
+
return { stdout: "", exitCode: 0 };
|
|
823
|
+
const hqRoot = (deps.resolveRoot ?? resolveLanesHqRoot)();
|
|
824
|
+
const payloadId = parseHookSessionId(await (deps.readStdin ?? readStdin)());
|
|
825
|
+
if (!monitorSessionHasActiveMarkers(hqRoot, provider, payloadId)
|
|
826
|
+
&& !monitorSessionInboxHasEvents(hqRoot, provider, payloadId)) {
|
|
827
|
+
return { stdout: "", exitCode: 0 };
|
|
828
|
+
}
|
|
829
|
+
const sessionId = payloadId;
|
|
830
|
+
await (deps.recordActivity ?? recordMonitorTargetActivity)(hqRoot, monitorTargetKey({ kind: "session", provider, sessionId }));
|
|
831
|
+
const sessionDir = monitorSessionDir(hqRoot, provider, payloadId);
|
|
832
|
+
const box = path.join(sessionDir, "dropbox");
|
|
833
|
+
const waiterDir = path.join(sessionDir, "waiters");
|
|
834
|
+
const waiterPath = path.join(waiterDir, "claude-stop.json");
|
|
835
|
+
const token = `${process.pid}-${Date.now()}`;
|
|
836
|
+
writeJsonAtomic(waiterPath, { token, pid: process.pid, starttime: readProcStarttime(process.pid), at: new Date().toISOString() });
|
|
837
|
+
while (waiterIsCurrent(waiterPath, token)) {
|
|
838
|
+
const messages = (deps.drainDropbox ?? drainDropboxAtPath)(box, { canDrain: () => waiterIsCurrent(waiterPath, token) });
|
|
839
|
+
if (messages.length > 0) {
|
|
840
|
+
return { stdout: `${formatDropboxMessages(messages)}\n`, exitCode: 2 };
|
|
841
|
+
}
|
|
842
|
+
const ids = reapStaleSessionMarkers(hqRoot, provider, payloadId);
|
|
843
|
+
if (ids.length === 0 && !fs.existsSync(path.join(box, "inbox.jsonl")))
|
|
844
|
+
return { stdout: "", exitCode: 0 };
|
|
845
|
+
if (ids.length === 0 || sessionOwnersGone(hqRoot, ids) || !waiterIsCurrent(waiterPath, token))
|
|
846
|
+
return { stdout: "", exitCode: 0 };
|
|
847
|
+
await waitMonitorPoll(deps.sleep);
|
|
848
|
+
}
|
|
849
|
+
return { stdout: "", exitCode: 0 };
|
|
850
|
+
}
|
|
851
|
+
function reapStaleSessionMarkers(hqRoot, provider, sessionId) {
|
|
852
|
+
const ids = sessionMonitorIds(hqRoot, provider, sessionId);
|
|
853
|
+
const live = [];
|
|
854
|
+
for (const id of ids) {
|
|
855
|
+
const state = readJson(statePath(hqRoot, id));
|
|
856
|
+
const meta = readJson(path.join(monitorDir(hqRoot, id), "meta.json"));
|
|
857
|
+
if (state?.status === "running" && state.pid !== null && !readMonitorPidLiveness(state.pid, state.starttime)) {
|
|
858
|
+
writeJsonAtomic(statePath(hqRoot, id), { ...state, status: "terminal", terminal_reason: "supervisor_died", finished_at: new Date().toISOString() });
|
|
859
|
+
if (meta)
|
|
860
|
+
removeActiveMarker(hqRoot, meta);
|
|
861
|
+
continue;
|
|
862
|
+
}
|
|
863
|
+
if (state?.status === "terminal") {
|
|
864
|
+
if (meta)
|
|
865
|
+
removeActiveMarker(hqRoot, meta);
|
|
866
|
+
continue;
|
|
867
|
+
}
|
|
868
|
+
live.push(id);
|
|
869
|
+
}
|
|
870
|
+
return live;
|
|
871
|
+
}
|
|
872
|
+
function sessionOwnersGone(hqRoot, ids) {
|
|
873
|
+
const owners = ids.flatMap((id) => {
|
|
874
|
+
const meta = readJson(path.join(monitorDir(hqRoot, id), "meta.json"));
|
|
875
|
+
return meta?.session_owner ? [meta.session_owner] : [];
|
|
876
|
+
});
|
|
877
|
+
return owners.length > 0 && owners.every((owner) => !readMonitorPidLiveness(owner.pid, owner.starttime));
|
|
878
|
+
}
|
|
879
|
+
export function registerMonitorCommand(parent) {
|
|
880
|
+
const monitor = parent.command("monitor").description("Run detached monitors and deliver their output into agent context");
|
|
881
|
+
monitor.command("enabled", { hidden: true }).action(async () => {
|
|
882
|
+
process.exitCode = await runMonitorEnabledCommand();
|
|
883
|
+
});
|
|
884
|
+
monitor.command("start")
|
|
885
|
+
.description("Start a detached monitor script")
|
|
886
|
+
.requiredOption("--description <text>", "Why this monitor exists")
|
|
887
|
+
.option("--command <sh>", "Run one shell command under bash -c")
|
|
888
|
+
.option("--timeout <duration>", "Deadline; bare numbers are milliseconds")
|
|
889
|
+
.option("--persistent", "Run until the script exits, the target ends, or stopped")
|
|
890
|
+
.option("--target <target>", "lane:<id> or session:<provider>:<session-id>")
|
|
891
|
+
.option("--checkin <duration|off>", "Per-target check-in interval; defaults to 55m")
|
|
892
|
+
.option("--json", "Emit JSON")
|
|
893
|
+
.argument("[argv...]", "Executable and arguments after --")
|
|
894
|
+
.action(async (argv, options) => {
|
|
895
|
+
try {
|
|
896
|
+
await startMonitor(options, argv ?? []);
|
|
897
|
+
}
|
|
898
|
+
catch (err) {
|
|
899
|
+
fail(err);
|
|
900
|
+
}
|
|
901
|
+
});
|
|
902
|
+
monitor.command("list").description("List active monitors").option("--all", "Include terminal monitors").option("--json", "Emit JSON")
|
|
903
|
+
.action(async (options) => { try {
|
|
904
|
+
await listMonitors(options);
|
|
905
|
+
}
|
|
906
|
+
catch (err) {
|
|
907
|
+
fail(err);
|
|
908
|
+
} });
|
|
909
|
+
monitor.command("show").description("Show monitor metadata, state and events").argument("<monitor-id>")
|
|
910
|
+
.option("--json", "Emit JSON").action(async (id, options) => { try {
|
|
911
|
+
await showMonitor(id, options.json);
|
|
912
|
+
}
|
|
913
|
+
catch (err) {
|
|
914
|
+
fail(err);
|
|
915
|
+
} });
|
|
916
|
+
monitor.command("stop").description("Stop a running monitor").argument("<monitor-id>")
|
|
917
|
+
.action(async (id) => { try {
|
|
918
|
+
await stopMonitor(id);
|
|
919
|
+
}
|
|
920
|
+
catch (err) {
|
|
921
|
+
fail(err);
|
|
922
|
+
} });
|
|
923
|
+
monitor.command("logs").description("Read monitor events or stderr").argument("<monitor-id>")
|
|
924
|
+
.option("--stderr", "Read stderr.log instead of events").action(async (id, options) => { try {
|
|
925
|
+
await monitorLogs(id, options.stderr === true);
|
|
926
|
+
}
|
|
927
|
+
catch (err) {
|
|
928
|
+
fail(err);
|
|
929
|
+
} });
|
|
930
|
+
monitor.command("_run", { hidden: true }).argument("<monitor-id>").action(async (id) => {
|
|
931
|
+
try {
|
|
932
|
+
await runSupervisor(id);
|
|
933
|
+
}
|
|
934
|
+
catch (err) {
|
|
935
|
+
process.stderr.write(`hq monitor supervisor: ${errorPath(err)}\n`);
|
|
936
|
+
process.exitCode = 1;
|
|
937
|
+
}
|
|
938
|
+
});
|
|
939
|
+
monitor.command("drain", { hidden: true }).requiredOption("--provider <provider>").requiredOption("--event <event>")
|
|
940
|
+
.action(async (options) => {
|
|
941
|
+
const result = await runMonitorHookSafely("drain", async () => {
|
|
942
|
+
if (!["claude", "codex", "grok"].includes(options.provider))
|
|
943
|
+
throw new LanesError("monitor_provider_invalid", "--provider must be claude, codex, or grok.");
|
|
944
|
+
if (!["PreToolUse", "UserPromptSubmit"].includes(options.event))
|
|
945
|
+
throw new LanesError("monitor_hook_event_invalid", "--event must be PreToolUse or UserPromptSubmit.");
|
|
946
|
+
return drainSession(options.provider, options.event);
|
|
947
|
+
});
|
|
948
|
+
if (result.stdout)
|
|
949
|
+
process.stdout.write(result.stdout);
|
|
950
|
+
process.exitCode = result.exitCode;
|
|
951
|
+
});
|
|
952
|
+
monitor.command("wait", { hidden: true }).requiredOption("--provider <provider>")
|
|
953
|
+
.action(async (options) => {
|
|
954
|
+
const result = await runMonitorHookSafely("wait", async () => {
|
|
955
|
+
if (options.provider !== "claude")
|
|
956
|
+
throw new LanesError("monitor_wait_provider_unsupported", "Only Claude Stop hooks can wait for session monitor events.", { provider: options.provider });
|
|
957
|
+
return waitForSession(options.provider);
|
|
958
|
+
});
|
|
959
|
+
if (result.stdout)
|
|
960
|
+
process.stdout.write(result.stdout);
|
|
961
|
+
process.exitCode = result.exitCode;
|
|
962
|
+
});
|
|
963
|
+
return monitor;
|
|
964
|
+
}
|
|
965
|
+
//# sourceMappingURL=monitor.js.map
|