@ours.network/fleet 0.10.4 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/fleet-query-service.d.ts +11 -0
- package/dist/application/fleet-query-service.js +9 -1
- package/dist/cli.js +200 -3
- package/dist/config.d.ts +2 -0
- package/dist/config.js +3 -1
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +22 -0
- package/dist/duration.d.ts +5 -0
- package/dist/duration.js +20 -0
- package/dist/ops.d.ts +16 -0
- package/dist/ops.js +112 -3
- package/dist/paths.d.ts +1 -0
- package/dist/paths.js +1 -0
- package/dist/resolved-plan.js +7 -0
- package/dist/watchdog/alerts.d.ts +34 -0
- package/dist/watchdog/alerts.js +78 -0
- package/dist/watchdog/briefing.d.ts +65 -0
- package/dist/watchdog/briefing.js +181 -0
- package/dist/watchdog/config.d.ts +49 -0
- package/dist/watchdog/config.js +114 -0
- package/dist/watchdog/query.d.ts +78 -0
- package/dist/watchdog/query.js +124 -0
- package/dist/watchdog/report.d.ts +53 -0
- package/dist/watchdog/report.js +126 -0
- package/dist/watchdog/run.d.ts +61 -0
- package/dist/watchdog/run.js +318 -0
- package/dist/watchdog/scheduler.d.ts +105 -0
- package/dist/watchdog/scheduler.js +244 -0
- package/dist/watchdog/service.d.ts +46 -0
- package/dist/watchdog/service.js +179 -0
- package/dist/watchdog/store.d.ts +85 -0
- package/dist/watchdog/store.js +226 -0
- package/dist/web/runtime.js +46 -2
- package/dist/web/server.d.ts +2 -0
- package/dist/web/server.js +18 -0
- package/dist/web-app/assets/{TerminalView-DcImdrI1.js → TerminalView-BvcIkuIF.js} +1 -1
- package/dist/web-app/assets/index-B-jtLAkp.css +1 -0
- package/dist/web-app/assets/index-CUN7ksTw.js +9 -0
- package/dist/web-app/index.html +2 -2
- package/package.json +1 -1
- package/dist/web-app/assets/index-BokQN1Ao.js +0 -9
- package/dist/web-app/assets/index-lAXzaOZM.css +0 -1
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { type Exec } from '../exec.js';
|
|
2
|
+
export declare const WATCHDOG_SYSTEMD_UNIT = "ours-fleet-watchdogs.service";
|
|
3
|
+
export declare const WATCHDOG_LAUNCHD_LABEL = "network.ours.fleet.watchdogs";
|
|
4
|
+
/**
|
|
5
|
+
* Supervises the single long-running watchdog-scheduler process (the hidden
|
|
6
|
+
* `_run-watchdogs` command, Task 9's `runScheduler`) the same way
|
|
7
|
+
* `WebServiceManager` (src/web/service.ts) supervises the web console: a
|
|
8
|
+
* private systemd --user unit on Linux, a launchd LaunchAgent on macOS.
|
|
9
|
+
*/
|
|
10
|
+
export declare class WatchdogServiceManager {
|
|
11
|
+
private readonly exec;
|
|
12
|
+
private readonly platform;
|
|
13
|
+
constructor(exec?: Exec, platform?: NodeJS.Platform);
|
|
14
|
+
get definitionPath(): string;
|
|
15
|
+
/** false when explicitly disabled (OURS_FLEET_SUPERVISOR=none) or on an unsupported platform. */
|
|
16
|
+
supervised(): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Writes the unit/plist and returns whether its content actually changed
|
|
19
|
+
* (finding #4): `binPath`/`configPath` are the only inputs that ever
|
|
20
|
+
* change this content, and neither reflects a watchdog's `interval:` or
|
|
21
|
+
* any other config value inside `watchdogs:` — so `changed` here can never
|
|
22
|
+
* by itself justify a restart on every config edit. It's one of the two
|
|
23
|
+
* signals reconcileWatchdogScheduler (ops.ts) combines with a config
|
|
24
|
+
* fingerprint before deciding restart vs. the idempotent start.
|
|
25
|
+
*/
|
|
26
|
+
install(binPath: string, configPath?: string): Promise<{
|
|
27
|
+
changed: boolean;
|
|
28
|
+
}>;
|
|
29
|
+
start(): Promise<void>;
|
|
30
|
+
stop(): Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* `start()` is a no-op on an already-active unit — systemctl start against
|
|
33
|
+
* a running service just returns 0 without reloading anything, and
|
|
34
|
+
* launchctl kickstart (without -k) behaves the same way. That means a
|
|
35
|
+
* config change (new/changed watchdogs) never reaches a live scheduler
|
|
36
|
+
* process via reconcileWatchdogScheduler's `install` + `start` pair (final
|
|
37
|
+
* review #4). `restart()` mirrors WebServiceManager.restart: an
|
|
38
|
+
* unconditional restart on Linux, and `kickstart -k` (force-restart) on
|
|
39
|
+
* macOS, falling back to stop+start if the kickstart itself fails.
|
|
40
|
+
*/
|
|
41
|
+
restart(): Promise<void>;
|
|
42
|
+
status(): Promise<string>;
|
|
43
|
+
uninstall(): Promise<void>;
|
|
44
|
+
private requireInstalled;
|
|
45
|
+
private must;
|
|
46
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync } from 'node:fs';
|
|
2
|
+
import { dirname, isAbsolute, join, resolve } from 'node:path';
|
|
3
|
+
import { replaceFileAtomically } from '../atomic-file.js';
|
|
4
|
+
import { FleetError } from '../application/errors.js';
|
|
5
|
+
import { realExec } from '../exec.js';
|
|
6
|
+
import { home } from '../paths.js';
|
|
7
|
+
export const WATCHDOG_SYSTEMD_UNIT = 'ours-fleet-watchdogs.service';
|
|
8
|
+
export const WATCHDOG_LAUNCHD_LABEL = 'network.ours.fleet.watchdogs';
|
|
9
|
+
/**
|
|
10
|
+
* Supervises the single long-running watchdog-scheduler process (the hidden
|
|
11
|
+
* `_run-watchdogs` command, Task 9's `runScheduler`) the same way
|
|
12
|
+
* `WebServiceManager` (src/web/service.ts) supervises the web console: a
|
|
13
|
+
* private systemd --user unit on Linux, a launchd LaunchAgent on macOS.
|
|
14
|
+
*/
|
|
15
|
+
export class WatchdogServiceManager {
|
|
16
|
+
exec;
|
|
17
|
+
platform;
|
|
18
|
+
constructor(exec = realExec, platform = process.platform) {
|
|
19
|
+
this.exec = exec;
|
|
20
|
+
this.platform = platform;
|
|
21
|
+
}
|
|
22
|
+
get definitionPath() {
|
|
23
|
+
return this.platform === 'linux'
|
|
24
|
+
? join(home(), '.config', 'systemd', 'user', WATCHDOG_SYSTEMD_UNIT)
|
|
25
|
+
: join(home(), 'Library', 'LaunchAgents', `${WATCHDOG_LAUNCHD_LABEL}.plist`);
|
|
26
|
+
}
|
|
27
|
+
/** false when explicitly disabled (OURS_FLEET_SUPERVISOR=none) or on an unsupported platform. */
|
|
28
|
+
supervised() {
|
|
29
|
+
if (process.env.OURS_FLEET_SUPERVISOR === 'none')
|
|
30
|
+
return false;
|
|
31
|
+
return this.platform === 'linux' || this.platform === 'darwin';
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Writes the unit/plist and returns whether its content actually changed
|
|
35
|
+
* (finding #4): `binPath`/`configPath` are the only inputs that ever
|
|
36
|
+
* change this content, and neither reflects a watchdog's `interval:` or
|
|
37
|
+
* any other config value inside `watchdogs:` — so `changed` here can never
|
|
38
|
+
* by itself justify a restart on every config edit. It's one of the two
|
|
39
|
+
* signals reconcileWatchdogScheduler (ops.ts) combines with a config
|
|
40
|
+
* fingerprint before deciding restart vs. the idempotent start.
|
|
41
|
+
*/
|
|
42
|
+
async install(binPath, configPath) {
|
|
43
|
+
const resolvedScript = resolveExecutable(binPath, 'ours-fleet CLI script');
|
|
44
|
+
const runtime = resolveExecutable(process.execPath, 'Node runtime');
|
|
45
|
+
const config = configPath ? resolve(configPath) : undefined;
|
|
46
|
+
const content = this.platform === 'linux'
|
|
47
|
+
? watchdogSystemdUnit(runtime, resolvedScript, config)
|
|
48
|
+
: watchdogLaunchdPlist(runtime, resolvedScript, config);
|
|
49
|
+
let previous;
|
|
50
|
+
try {
|
|
51
|
+
previous = readFileSync(this.definitionPath, 'utf8');
|
|
52
|
+
}
|
|
53
|
+
catch { /* absent: definitely changed */ }
|
|
54
|
+
const changed = previous !== content;
|
|
55
|
+
mkdirSync(dirname(this.definitionPath), { recursive: true, mode: 0o700 });
|
|
56
|
+
replaceFileAtomically(this.definitionPath, content, 0o600);
|
|
57
|
+
if (this.platform === 'linux') {
|
|
58
|
+
await this.must('systemctl', ['--user', 'daemon-reload']);
|
|
59
|
+
await this.must('systemctl', ['--user', 'enable', WATCHDOG_SYSTEMD_UNIT]);
|
|
60
|
+
}
|
|
61
|
+
return { changed };
|
|
62
|
+
}
|
|
63
|
+
async start() {
|
|
64
|
+
this.requireInstalled();
|
|
65
|
+
if (this.platform === 'linux') {
|
|
66
|
+
await this.must('systemctl', ['--user', 'start', WATCHDOG_SYSTEMD_UNIT]);
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const domain = `gui/${uid()}`;
|
|
70
|
+
const loaded = await this.exec('launchctl', ['print', `${domain}/${WATCHDOG_LAUNCHD_LABEL}`]);
|
|
71
|
+
if (loaded.code === 0)
|
|
72
|
+
await this.must('launchctl', ['kickstart', `${domain}/${WATCHDOG_LAUNCHD_LABEL}`]);
|
|
73
|
+
else
|
|
74
|
+
await this.must('launchctl', ['bootstrap', domain, this.definitionPath]);
|
|
75
|
+
}
|
|
76
|
+
async stop() {
|
|
77
|
+
if (this.platform === 'linux') {
|
|
78
|
+
// Tolerate "not loaded" (systemctl exit 5) the same way the launchd
|
|
79
|
+
// branch below tolerates "could not find service": a watchdog-less
|
|
80
|
+
// fleet's `up`/`down` calls stop() defensively even when the unit was
|
|
81
|
+
// never installed, and that must not surface as an error (final
|
|
82
|
+
// review #3).
|
|
83
|
+
const result = await this.exec('systemctl', ['--user', 'stop', WATCHDOG_SYSTEMD_UNIT]);
|
|
84
|
+
if (result.code !== 0 && result.code !== 5 && !/not loaded/i.test(`${result.stdout}\n${result.stderr}`))
|
|
85
|
+
throw new FleetError('control_unavailable', `systemctl --user stop ${WATCHDOG_SYSTEMD_UNIT} failed: ${result.stderr.trim() || `exit ${result.code}`}`);
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
const result = await this.exec('launchctl', ['bootout', `gui/${uid()}/${WATCHDOG_LAUNCHD_LABEL}`]);
|
|
89
|
+
if (result.code !== 0 && !/could not find service|no such process/i.test(`${result.stdout}\n${result.stderr}`))
|
|
90
|
+
throw new FleetError('control_unavailable', `launchctl bootout failed: ${result.stderr.trim()}`);
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* `start()` is a no-op on an already-active unit — systemctl start against
|
|
94
|
+
* a running service just returns 0 without reloading anything, and
|
|
95
|
+
* launchctl kickstart (without -k) behaves the same way. That means a
|
|
96
|
+
* config change (new/changed watchdogs) never reaches a live scheduler
|
|
97
|
+
* process via reconcileWatchdogScheduler's `install` + `start` pair (final
|
|
98
|
+
* review #4). `restart()` mirrors WebServiceManager.restart: an
|
|
99
|
+
* unconditional restart on Linux, and `kickstart -k` (force-restart) on
|
|
100
|
+
* macOS, falling back to stop+start if the kickstart itself fails.
|
|
101
|
+
*/
|
|
102
|
+
async restart() {
|
|
103
|
+
this.requireInstalled();
|
|
104
|
+
if (this.platform === 'linux') {
|
|
105
|
+
await this.must('systemctl', ['--user', 'restart', WATCHDOG_SYSTEMD_UNIT]);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const result = await this.exec('launchctl', ['kickstart', '-k', `gui/${uid()}/${WATCHDOG_LAUNCHD_LABEL}`]);
|
|
109
|
+
if (result.code !== 0) {
|
|
110
|
+
await this.stop();
|
|
111
|
+
await this.start();
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
async status() {
|
|
115
|
+
if (this.platform === 'linux') {
|
|
116
|
+
const result = await this.exec('systemctl', [
|
|
117
|
+
'--user', 'show', WATCHDOG_SYSTEMD_UNIT, '-p', 'LoadState', '-p', 'ActiveState',
|
|
118
|
+
'-p', 'SubState', '-p', 'ExecMainPID', '--no-pager',
|
|
119
|
+
]);
|
|
120
|
+
return result.stdout.trim() || result.stderr.trim() || `exit ${result.code}`;
|
|
121
|
+
}
|
|
122
|
+
const result = await this.exec('launchctl', ['print', `gui/${uid()}/${WATCHDOG_LAUNCHD_LABEL}`]);
|
|
123
|
+
return result.code === 0 ? result.stdout.trim() : `not loaded (${WATCHDOG_LAUNCHD_LABEL})`;
|
|
124
|
+
}
|
|
125
|
+
async uninstall() {
|
|
126
|
+
if (this.platform === 'linux') {
|
|
127
|
+
await this.exec('systemctl', ['--user', 'disable', '--now', WATCHDOG_SYSTEMD_UNIT]);
|
|
128
|
+
rmSync(this.definitionPath, { force: true });
|
|
129
|
+
await this.exec('systemctl', ['--user', 'daemon-reload']);
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
await this.stop();
|
|
133
|
+
rmSync(this.definitionPath, { force: true });
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
requireInstalled() {
|
|
137
|
+
if (!existsSync(this.definitionPath))
|
|
138
|
+
throw new FleetError('prerequisite_unavailable', 'watchdog scheduler service is not installed');
|
|
139
|
+
}
|
|
140
|
+
async must(command, args) {
|
|
141
|
+
const result = await this.exec(command, args);
|
|
142
|
+
if (result.code !== 0)
|
|
143
|
+
throw new FleetError('control_unavailable', `${command} ${args.join(' ')} failed: ${result.stderr.trim() || `exit ${result.code}`}`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
function uid() { return process.getuid?.() ?? 501; }
|
|
147
|
+
function resolveExecutable(executable, label) {
|
|
148
|
+
const absolute = isAbsolute(executable) ? executable : resolve(executable);
|
|
149
|
+
try {
|
|
150
|
+
return realpathSync(absolute);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
throw new FleetError('prerequisite_unavailable', `${label} does not exist: ${absolute}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
function systemdQuote(value) {
|
|
157
|
+
return `"${value.replace(/[%\\"]/g, char => char === '%' ? '%%' : `\\${char}`)}"`;
|
|
158
|
+
}
|
|
159
|
+
function watchdogSystemdUnit(runtime, script, configuration) {
|
|
160
|
+
const config = configuration ? ` -c ${systemdQuote(configuration)}` : '';
|
|
161
|
+
return `[Unit]\nDescription=ours-fleet watchdog scheduler\nAfter=default.target\n\n`
|
|
162
|
+
+ `[Service]\nType=simple\nExecStart=${systemdQuote(runtime)} ${systemdQuote(script)} _run-watchdogs${config}\n`
|
|
163
|
+
+ `Restart=on-failure\nRestartSec=5\nTimeoutStopSec=15\n\n`
|
|
164
|
+
+ `[Install]\nWantedBy=default.target\n`;
|
|
165
|
+
}
|
|
166
|
+
const xml = (value) => value.replace(/[&<>"']/g, char => ({
|
|
167
|
+
'&': '&', '<': '<', '>': '>', '"': '"', "'": ''',
|
|
168
|
+
}[char]));
|
|
169
|
+
function watchdogLaunchdPlist(runtime, script, configuration) {
|
|
170
|
+
const config = configuration
|
|
171
|
+
? `<string>-c</string><string>${xml(configuration)}</string>` : '';
|
|
172
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n`
|
|
173
|
+
+ `<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n`
|
|
174
|
+
+ `<plist version="1.0"><dict>\n<key>Label</key><string>${WATCHDOG_LAUNCHD_LABEL}</string>\n`
|
|
175
|
+
+ `<key>ProgramArguments</key><array><string>${xml(runtime)}</string><string>${xml(script)}</string>`
|
|
176
|
+
+ `<string>_run-watchdogs</string>${config}</array>\n`
|
|
177
|
+
+ `<key>RunAtLoad</key><true/><key>KeepAlive</key><dict><key>SuccessfulExit</key><false/></dict>\n`
|
|
178
|
+
+ `<key>ProcessType</key><string>Background</string>\n</dict></plist>\n`;
|
|
179
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import type { WatchdogReport, WatchdogReportStatus } from './report.js';
|
|
2
|
+
/**
|
|
3
|
+
* A lock directory briefly exists before acquireRunLock can rename its
|
|
4
|
+
* prewritten owner.json into it. Missing/corrupt owner metadata is therefore
|
|
5
|
+
* reclaimable only after this grace period, never while that acquisition
|
|
6
|
+
* window may still be in progress.
|
|
7
|
+
*/
|
|
8
|
+
export declare const RUN_LOCK_OWNER_GRACE_MS = 10000;
|
|
9
|
+
/**
|
|
10
|
+
* Choke point every other helper in this module goes through to reach a
|
|
11
|
+
* watchdog's on-disk state. Validates `name` BEFORE any join/mkdir (defense
|
|
12
|
+
* in depth, finding #1): a caller that forgets its own ROLE_NAME_RE guard
|
|
13
|
+
* (as the CLI's `watchdogKnown` once did) must not be able to turn an
|
|
14
|
+
* unvalidated `name` into `join(watchdogsRoot(), '../../victim')` and mkdir
|
|
15
|
+
* an arbitrary path on disk.
|
|
16
|
+
*/
|
|
17
|
+
export declare function watchdogDir(name: string): string;
|
|
18
|
+
export declare function reportsDir(name: string): string;
|
|
19
|
+
export interface RunLockOwner {
|
|
20
|
+
pid: number;
|
|
21
|
+
at: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* mkdir-as-mutex: atomic across processes, unlike a lock file (open+O_EXCL
|
|
25
|
+
* would work too, but a directory needs no cleanup of file contents and
|
|
26
|
+
* can't be partially written). EEXIST means another run holds it.
|
|
27
|
+
*
|
|
28
|
+
* Stamps `owner.json` with our pid (finding #2): ownership metadata is what
|
|
29
|
+
* lets a later `reclaimStaleRunLock` tell a lock abandoned by a dead process
|
|
30
|
+
* apart from one genuinely held by a live run. A naive "mkdir, then
|
|
31
|
+
* writeFileSync(owner.json)" leaves an unnecessarily wide window —
|
|
32
|
+
* between the mkdir succeeding and the write landing — where the lock dir
|
|
33
|
+
* exists but owner.json doesn't yet. A `reclaimStaleRunLock` call from
|
|
34
|
+
* another process landing in exactly that window would see a lock with no
|
|
35
|
+
* (or corrupt) owner metadata and treat a genuinely live lock as a legacy
|
|
36
|
+
* one, reclaiming it out from under us. Do all slow work (building the JSON,
|
|
37
|
+
* writing it, chmod) to a per-pid temp file BEFORE mkdir, then publish it with
|
|
38
|
+
* one rename. There is still an unavoidable interval between those two
|
|
39
|
+
* syscalls; reclaimStaleRunLock protects it by treating a fresh ownerless lock
|
|
40
|
+
* as held for RUN_LOCK_OWNER_GRACE_MS. The temp file is unique per-pid (this
|
|
41
|
+
* function is synchronous, so there's no same-process concurrent-call hazard
|
|
42
|
+
* either) and is cleaned up if mkdir loses the race.
|
|
43
|
+
*/
|
|
44
|
+
export declare function acquireRunLock(name: string): boolean;
|
|
45
|
+
/**
|
|
46
|
+
* Release-tolerant of absence: a lock already gone (or never acquired) is not
|
|
47
|
+
* an error. Recursive because the lock dir now holds `owner.json` alongside
|
|
48
|
+
* the mkdir mutex itself (finding #2) — a plain rmdir would fail ENOTEMPTY.
|
|
49
|
+
*/
|
|
50
|
+
export declare function releaseRunLock(name: string): void;
|
|
51
|
+
/** Reads a run lock's owner metadata; missing or corrupt yields undefined. */
|
|
52
|
+
export declare function readRunLockOwner(name: string): RunLockOwner | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* Owner-mandated (finding #2): only a DEMONSTRABLY stale run lock may be
|
|
55
|
+
* reclaimed — a lock is stale iff it isn't held at all, its owner metadata
|
|
56
|
+
* names a dead pid, or its owner metadata is missing/corrupt AND the lock dir
|
|
57
|
+
* is older than RUN_LOCK_OWNER_GRACE_MS. The age gate closes the interprocess
|
|
58
|
+
* interval between acquireRunLock's mkdir and owner.json rename: a concurrent
|
|
59
|
+
* scheduler sees a fresh ownerless lock as held, not stale. A live owner (a
|
|
60
|
+
* foreground `watchdog-run`, another scheduler instance, an in-progress run)
|
|
61
|
+
* is left strictly alone: reclaiming it would let two runs share the same temp
|
|
62
|
+
* dir.
|
|
63
|
+
* Returns true when the lock is (now) not held — whether because it was
|
|
64
|
+
* already absent or because a stale lock was just removed; false when a live
|
|
65
|
+
* lock was found and deliberately left in place.
|
|
66
|
+
*/
|
|
67
|
+
export declare function reclaimStaleRunLock(name: string): boolean;
|
|
68
|
+
/** Lexical-chronological UTC run id, e.g. '20260731T115000Z'. */
|
|
69
|
+
export declare function formatRunId(d: Date): string;
|
|
70
|
+
export interface RunListEntry {
|
|
71
|
+
runId: string;
|
|
72
|
+
status: WatchdogReportStatus;
|
|
73
|
+
startedAt: string;
|
|
74
|
+
finishedAt: string;
|
|
75
|
+
summary: WatchdogReport['summary'];
|
|
76
|
+
error: string | null;
|
|
77
|
+
}
|
|
78
|
+
export declare function writeReport(name: string, report: WatchdogReport): string;
|
|
79
|
+
/** Newest-first run listing; a corrupt report file yields a synthetic 'error' entry rather than throwing. */
|
|
80
|
+
export declare function listRuns(name: string): RunListEntry[];
|
|
81
|
+
/** Reads one run's full report. Rejects non-conforming runIds (path-traversal guard) and corrupt files by returning undefined. */
|
|
82
|
+
export declare function readReport(name: string, runId: string): WatchdogReport | undefined;
|
|
83
|
+
export declare function latestReport(name: string): WatchdogReport | undefined;
|
|
84
|
+
/** Deletes all but the `keep` newest reports (oldest first); returns the number pruned. */
|
|
85
|
+
export declare function pruneReports(name: string, keep: number): number;
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { watchdogsRoot } from '../paths.js';
|
|
4
|
+
// store.ts imports config.ts (ROLE_NAME_RE) but not the reverse — config.ts's
|
|
5
|
+
// import graph (config-yaml, isolation/policy, harness/registry, watchdog/config)
|
|
6
|
+
// never reaches back into watchdog/store.ts, so this does not create a cycle.
|
|
7
|
+
import { ROLE_NAME_RE } from '../config.js';
|
|
8
|
+
const RUN_ID_RE = /^\d{8}T\d{6}Z$/;
|
|
9
|
+
const REPORT_FILE_RE = /^\d{8}T\d{6}Z\.json$/;
|
|
10
|
+
/**
|
|
11
|
+
* A lock directory briefly exists before acquireRunLock can rename its
|
|
12
|
+
* prewritten owner.json into it. Missing/corrupt owner metadata is therefore
|
|
13
|
+
* reclaimable only after this grace period, never while that acquisition
|
|
14
|
+
* window may still be in progress.
|
|
15
|
+
*/
|
|
16
|
+
export const RUN_LOCK_OWNER_GRACE_MS = 10_000;
|
|
17
|
+
/**
|
|
18
|
+
* mkdirSync's `mode` is masked by the process umask and ignored outright when
|
|
19
|
+
* the directory already exists, so an explicit chmod after mkdir is the only
|
|
20
|
+
* way to guarantee 0700 regardless of umask or prior state.
|
|
21
|
+
*/
|
|
22
|
+
function ensureDir(dir) {
|
|
23
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
24
|
+
chmodSync(dir, 0o700);
|
|
25
|
+
return dir;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Choke point every other helper in this module goes through to reach a
|
|
29
|
+
* watchdog's on-disk state. Validates `name` BEFORE any join/mkdir (defense
|
|
30
|
+
* in depth, finding #1): a caller that forgets its own ROLE_NAME_RE guard
|
|
31
|
+
* (as the CLI's `watchdogKnown` once did) must not be able to turn an
|
|
32
|
+
* unvalidated `name` into `join(watchdogsRoot(), '../../victim')` and mkdir
|
|
33
|
+
* an arbitrary path on disk.
|
|
34
|
+
*/
|
|
35
|
+
export function watchdogDir(name) {
|
|
36
|
+
if (!ROLE_NAME_RE.test(name))
|
|
37
|
+
throw new Error(`invalid watchdog name '${name}'`);
|
|
38
|
+
return ensureDir(join(watchdogsRoot(), name));
|
|
39
|
+
}
|
|
40
|
+
export function reportsDir(name) {
|
|
41
|
+
return ensureDir(join(watchdogDir(name), 'reports'));
|
|
42
|
+
}
|
|
43
|
+
function runLockPath(name) {
|
|
44
|
+
return join(watchdogDir(name), '.run-lock');
|
|
45
|
+
}
|
|
46
|
+
function runLockOwnerPath(name) {
|
|
47
|
+
return join(runLockPath(name), 'owner.json');
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* mkdir-as-mutex: atomic across processes, unlike a lock file (open+O_EXCL
|
|
51
|
+
* would work too, but a directory needs no cleanup of file contents and
|
|
52
|
+
* can't be partially written). EEXIST means another run holds it.
|
|
53
|
+
*
|
|
54
|
+
* Stamps `owner.json` with our pid (finding #2): ownership metadata is what
|
|
55
|
+
* lets a later `reclaimStaleRunLock` tell a lock abandoned by a dead process
|
|
56
|
+
* apart from one genuinely held by a live run. A naive "mkdir, then
|
|
57
|
+
* writeFileSync(owner.json)" leaves an unnecessarily wide window —
|
|
58
|
+
* between the mkdir succeeding and the write landing — where the lock dir
|
|
59
|
+
* exists but owner.json doesn't yet. A `reclaimStaleRunLock` call from
|
|
60
|
+
* another process landing in exactly that window would see a lock with no
|
|
61
|
+
* (or corrupt) owner metadata and treat a genuinely live lock as a legacy
|
|
62
|
+
* one, reclaiming it out from under us. Do all slow work (building the JSON,
|
|
63
|
+
* writing it, chmod) to a per-pid temp file BEFORE mkdir, then publish it with
|
|
64
|
+
* one rename. There is still an unavoidable interval between those two
|
|
65
|
+
* syscalls; reclaimStaleRunLock protects it by treating a fresh ownerless lock
|
|
66
|
+
* as held for RUN_LOCK_OWNER_GRACE_MS. The temp file is unique per-pid (this
|
|
67
|
+
* function is synchronous, so there's no same-process concurrent-call hazard
|
|
68
|
+
* either) and is cleaned up if mkdir loses the race.
|
|
69
|
+
*/
|
|
70
|
+
export function acquireRunLock(name) {
|
|
71
|
+
const dir = watchdogDir(name);
|
|
72
|
+
const owner = { pid: process.pid, at: new Date().toISOString() };
|
|
73
|
+
const tempOwnerPath = join(dir, `.owner.${process.pid}.tmp`);
|
|
74
|
+
writeFileSync(tempOwnerPath, JSON.stringify(owner), { mode: 0o600 });
|
|
75
|
+
chmodSync(tempOwnerPath, 0o600);
|
|
76
|
+
try {
|
|
77
|
+
mkdirSync(runLockPath(name));
|
|
78
|
+
}
|
|
79
|
+
catch (e) {
|
|
80
|
+
try {
|
|
81
|
+
unlinkSync(tempOwnerPath);
|
|
82
|
+
}
|
|
83
|
+
catch { /* best effort cleanup */ }
|
|
84
|
+
if (e.code === 'EEXIST')
|
|
85
|
+
return false;
|
|
86
|
+
throw e;
|
|
87
|
+
}
|
|
88
|
+
renameSync(tempOwnerPath, runLockOwnerPath(name));
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Release-tolerant of absence: a lock already gone (or never acquired) is not
|
|
93
|
+
* an error. Recursive because the lock dir now holds `owner.json` alongside
|
|
94
|
+
* the mkdir mutex itself (finding #2) — a plain rmdir would fail ENOTEMPTY.
|
|
95
|
+
*/
|
|
96
|
+
export function releaseRunLock(name) {
|
|
97
|
+
try {
|
|
98
|
+
rmSync(runLockPath(name), { recursive: true });
|
|
99
|
+
}
|
|
100
|
+
catch (e) {
|
|
101
|
+
if (e.code === 'ENOENT')
|
|
102
|
+
return;
|
|
103
|
+
throw e;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/** Reads a run lock's owner metadata; missing or corrupt yields undefined. */
|
|
107
|
+
export function readRunLockOwner(name) {
|
|
108
|
+
try {
|
|
109
|
+
const raw = JSON.parse(readFileSync(runLockOwnerPath(name), 'utf8'));
|
|
110
|
+
if (!Number.isInteger(raw.pid) || raw.pid <= 0 || typeof raw.at !== 'string')
|
|
111
|
+
return undefined;
|
|
112
|
+
return { pid: raw.pid, at: raw.at };
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/** True iff `pid` names a live process. EPERM means the process exists but we can't signal it — still alive. */
|
|
119
|
+
function pidAlive(pid) {
|
|
120
|
+
try {
|
|
121
|
+
process.kill(pid, 0);
|
|
122
|
+
return true;
|
|
123
|
+
}
|
|
124
|
+
catch (e) {
|
|
125
|
+
return e.code === 'EPERM';
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Owner-mandated (finding #2): only a DEMONSTRABLY stale run lock may be
|
|
130
|
+
* reclaimed — a lock is stale iff it isn't held at all, its owner metadata
|
|
131
|
+
* names a dead pid, or its owner metadata is missing/corrupt AND the lock dir
|
|
132
|
+
* is older than RUN_LOCK_OWNER_GRACE_MS. The age gate closes the interprocess
|
|
133
|
+
* interval between acquireRunLock's mkdir and owner.json rename: a concurrent
|
|
134
|
+
* scheduler sees a fresh ownerless lock as held, not stale. A live owner (a
|
|
135
|
+
* foreground `watchdog-run`, another scheduler instance, an in-progress run)
|
|
136
|
+
* is left strictly alone: reclaiming it would let two runs share the same temp
|
|
137
|
+
* dir.
|
|
138
|
+
* Returns true when the lock is (now) not held — whether because it was
|
|
139
|
+
* already absent or because a stale lock was just removed; false when a live
|
|
140
|
+
* lock was found and deliberately left in place.
|
|
141
|
+
*/
|
|
142
|
+
export function reclaimStaleRunLock(name) {
|
|
143
|
+
const path = runLockPath(name);
|
|
144
|
+
if (!existsSync(path))
|
|
145
|
+
return true;
|
|
146
|
+
const owner = readRunLockOwner(name);
|
|
147
|
+
if (owner !== undefined && pidAlive(owner.pid))
|
|
148
|
+
return false;
|
|
149
|
+
if (owner === undefined) {
|
|
150
|
+
try {
|
|
151
|
+
if (Date.now() - statSync(path).mtimeMs < RUN_LOCK_OWNER_GRACE_MS)
|
|
152
|
+
return false;
|
|
153
|
+
}
|
|
154
|
+
catch (e) {
|
|
155
|
+
// The holder may have released between existsSync and statSync.
|
|
156
|
+
if (e.code === 'ENOENT')
|
|
157
|
+
return true;
|
|
158
|
+
throw e;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
releaseRunLock(name);
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
/** Lexical-chronological UTC run id, e.g. '20260731T115000Z'. */
|
|
165
|
+
export function formatRunId(d) {
|
|
166
|
+
return d.toISOString().replace(/\.\d{3}Z$/, 'Z').replace(/[-:]/g, '');
|
|
167
|
+
}
|
|
168
|
+
const CORRUPT_SUMMARY = { checked: 0, healthy: 0, idle: 0, anomalies: 0 };
|
|
169
|
+
function reportPath(name, runId) {
|
|
170
|
+
return join(reportsDir(name), `${runId}.json`);
|
|
171
|
+
}
|
|
172
|
+
/** Newest-first run ids currently on disk, derived from filenames alone. */
|
|
173
|
+
function runIdsDesc(name) {
|
|
174
|
+
return readdirSync(reportsDir(name))
|
|
175
|
+
.filter(f => REPORT_FILE_RE.test(f))
|
|
176
|
+
.sort()
|
|
177
|
+
.reverse()
|
|
178
|
+
.map(f => f.slice(0, -'.json'.length));
|
|
179
|
+
}
|
|
180
|
+
export function writeReport(name, report) {
|
|
181
|
+
const path = reportPath(name, report.run_id);
|
|
182
|
+
writeFileSync(path, JSON.stringify(report, null, 2) + '\n', { mode: 0o600 });
|
|
183
|
+
chmodSync(path, 0o600);
|
|
184
|
+
return path;
|
|
185
|
+
}
|
|
186
|
+
/** Newest-first run listing; a corrupt report file yields a synthetic 'error' entry rather than throwing. */
|
|
187
|
+
export function listRuns(name) {
|
|
188
|
+
return runIdsDesc(name).map((runId) => {
|
|
189
|
+
try {
|
|
190
|
+
const report = JSON.parse(readFileSync(reportPath(name, runId), 'utf8'));
|
|
191
|
+
return {
|
|
192
|
+
runId, status: report.status, startedAt: report.started_at, finishedAt: report.finished_at,
|
|
193
|
+
summary: report.summary, error: report.error,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
return { runId, status: 'error', startedAt: '', finishedAt: '', summary: CORRUPT_SUMMARY, error: 'unreadable report file' };
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
/** Reads one run's full report. Rejects non-conforming runIds (path-traversal guard) and corrupt files by returning undefined. */
|
|
202
|
+
export function readReport(name, runId) {
|
|
203
|
+
if (!RUN_ID_RE.test(runId))
|
|
204
|
+
return undefined;
|
|
205
|
+
try {
|
|
206
|
+
return JSON.parse(readFileSync(reportPath(name, runId), 'utf8'));
|
|
207
|
+
}
|
|
208
|
+
catch {
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
export function latestReport(name) {
|
|
213
|
+
const [newest] = runIdsDesc(name);
|
|
214
|
+
return newest === undefined ? undefined : readReport(name, newest);
|
|
215
|
+
}
|
|
216
|
+
/** Deletes all but the `keep` newest reports (oldest first); returns the number pruned. */
|
|
217
|
+
export function pruneReports(name, keep) {
|
|
218
|
+
const stale = runIdsDesc(name).slice(keep);
|
|
219
|
+
for (const runId of stale) {
|
|
220
|
+
try {
|
|
221
|
+
unlinkSync(reportPath(name, runId));
|
|
222
|
+
}
|
|
223
|
+
catch { /* best effort */ }
|
|
224
|
+
}
|
|
225
|
+
return stale.length;
|
|
226
|
+
}
|
package/dist/web/runtime.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { existsSync, realpathSync } from 'node:fs';
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
|
+
import { loadConfig } from '../config.js';
|
|
4
5
|
import { RoleRepository } from '../application/role-repository.js';
|
|
5
6
|
import { FleetQueryService } from '../application/fleet-query-service.js';
|
|
6
7
|
import { AcpRoleSessionAdapter, TmuxRoleSessionAdapter } from '../application/session-control.js';
|
|
@@ -21,6 +22,24 @@ import { acquireWebServerLock } from './lock.js';
|
|
|
21
22
|
import { TrustedDeviceStore } from './device-store.js';
|
|
22
23
|
import { WebAuth } from './auth.js';
|
|
23
24
|
import { startWebControlServer } from './control.js';
|
|
25
|
+
import { buildWatchdogFindings, cachedWatchdogFindingsProvider, WatchdogQueryService } from '../watchdog/query.js';
|
|
26
|
+
import { latestReport } from '../watchdog/store.js';
|
|
27
|
+
const CONFIG_CACHE_TTL_MS = 5_000;
|
|
28
|
+
/**
|
|
29
|
+
* loadConfig re-parses YAML from disk on every call; the watchdog list/reports
|
|
30
|
+
* routes get polled by the console UI, so cache the resolved config for a
|
|
31
|
+
* short TTL rather than re-parsing per request. Runtime-only concern (not the
|
|
32
|
+
* scheduler's), so a plain Date.now() clock is fine.
|
|
33
|
+
*/
|
|
34
|
+
function cachedConfigProvider(configPath) {
|
|
35
|
+
let cached;
|
|
36
|
+
return () => {
|
|
37
|
+
const now = Date.now();
|
|
38
|
+
if (!cached || now - cached.at >= CONFIG_CACHE_TTL_MS)
|
|
39
|
+
cached = { at: now, cfg: loadConfig(configPath) };
|
|
40
|
+
return cached.cfg;
|
|
41
|
+
};
|
|
42
|
+
}
|
|
24
43
|
export async function startWebConsole(options) {
|
|
25
44
|
const requestedPort = options.port ?? 49_271;
|
|
26
45
|
if (!Number.isInteger(requestedPort) || requestedPort < 0 || requestedPort > 65_535)
|
|
@@ -51,11 +70,35 @@ export async function startWebConsole(options) {
|
|
|
51
70
|
const audit = new AuditSink();
|
|
52
71
|
const terminals = new TerminalBridgeManager({ repository, audit, tmux });
|
|
53
72
|
const terminalAvailable = await terminals.available();
|
|
73
|
+
const watchdogConfigProvider = cachedConfigProvider(options.configPath);
|
|
74
|
+
const log = options.log ?? (() => { });
|
|
75
|
+
let loggedWatchdogFindingsError = false;
|
|
76
|
+
// Needs-attention integration (Task 19): worst per-role watchdog finding,
|
|
77
|
+
// rebuilt from stored reports. A store hiccup (corrupt state, unreadable
|
|
78
|
+
// report) must never break the fleet list, so it degrades to an empty map
|
|
79
|
+
// and logs once rather than repeating on every poll. status() calls this
|
|
80
|
+
// once per role, so list()'s O(roles) sweep would otherwise cost
|
|
81
|
+
// O(roles x watchdogs) disk reads every 1-3s of console polling —
|
|
82
|
+
// cachedWatchdogFindingsProvider memoizes the whole build behind the same
|
|
83
|
+
// TTL as the config cache above.
|
|
84
|
+
const watchdogFindings = cachedWatchdogFindingsProvider(() => {
|
|
85
|
+
try {
|
|
86
|
+
return buildWatchdogFindings(watchdogConfigProvider(), latestReport);
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
if (!loggedWatchdogFindingsError) {
|
|
90
|
+
loggedWatchdogFindingsError = true;
|
|
91
|
+
log(`watchdog findings unavailable: ${error.message}`);
|
|
92
|
+
}
|
|
93
|
+
return new Map();
|
|
94
|
+
}
|
|
95
|
+
}, CONFIG_CACHE_TTL_MS);
|
|
54
96
|
const query = new FleetQueryService({
|
|
55
97
|
repository, supervisor: backend, tmux,
|
|
56
98
|
capabilityContext: { terminalPtyAvailable: terminalAvailable },
|
|
99
|
+
watchdogFindings,
|
|
57
100
|
});
|
|
58
|
-
const ops = { backend, binPath: options.binPath, log
|
|
101
|
+
const ops = { backend, binPath: options.binPath, log };
|
|
59
102
|
const creation = new RoleCreationService({
|
|
60
103
|
configPath: options.configPath, ops, binPath: options.binPath,
|
|
61
104
|
allowedCwdRoots: [realpathSync(home()), realpathSync(process.cwd())],
|
|
@@ -84,10 +127,11 @@ export async function startWebConsole(options) {
|
|
|
84
127
|
},
|
|
85
128
|
});
|
|
86
129
|
const logs = new StructuredLogService(backend, realExec);
|
|
130
|
+
const watchdogs = new WatchdogQueryService(watchdogConfigProvider);
|
|
87
131
|
let server;
|
|
88
132
|
try {
|
|
89
133
|
server = await buildWebServer({
|
|
90
|
-
query, repository, logs, commands, creation, audit, events,
|
|
134
|
+
query, repository, logs, commands, creation, audit, events, watchdogs,
|
|
91
135
|
terminalUpgrade: terminalAvailable
|
|
92
136
|
? async (socket, _request, roleId, _ticket, hello) => terminals.connect(socket, roleId, hello)
|
|
93
137
|
: undefined,
|
package/dist/web/server.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ import type { RoleSessionControl } from '../application/session-control.js';
|
|
|
6
6
|
import type { StructuredLogService } from '../application/log-service.js';
|
|
7
7
|
import type { RoleCommandService } from '../application/role-command-service.js';
|
|
8
8
|
import type { RoleCreationService } from '../application/role-creation-service.js';
|
|
9
|
+
import type { WatchdogQueryService } from '../watchdog/query.js';
|
|
9
10
|
import { AuditSink } from './audit.js';
|
|
10
11
|
import { WebAuth } from './auth.js';
|
|
11
12
|
import { FleetEventBus } from './events.js';
|
|
@@ -18,6 +19,7 @@ export interface WebServices {
|
|
|
18
19
|
creation: RoleCreationService;
|
|
19
20
|
audit?: AuditSink;
|
|
20
21
|
events?: FleetEventBus;
|
|
22
|
+
watchdogs?: WatchdogQueryService;
|
|
21
23
|
terminalUpgrade?: (socket: WebSocket, request: FastifyRequest, roleId: string, ticket: string, hello: Record<string, unknown>) => Promise<void>;
|
|
22
24
|
}
|
|
23
25
|
export interface WebServer {
|
package/dist/web/server.js
CHANGED
|
@@ -186,6 +186,24 @@ export async function buildWebServer(services, boundary, options = {}) {
|
|
|
186
186
|
auth.authenticate(request);
|
|
187
187
|
return { records: audit.list() };
|
|
188
188
|
});
|
|
189
|
+
app.get('/api/v1/watchdogs', async (request) => {
|
|
190
|
+
auth.authenticate(request);
|
|
191
|
+
if (!services.watchdogs)
|
|
192
|
+
throw new FleetError('capability_unavailable', 'watchdogs are unavailable');
|
|
193
|
+
return services.watchdogs.list();
|
|
194
|
+
});
|
|
195
|
+
app.get('/api/v1/watchdogs/:name/reports', async (request) => {
|
|
196
|
+
auth.authenticate(request);
|
|
197
|
+
if (!services.watchdogs)
|
|
198
|
+
throw new FleetError('capability_unavailable', 'watchdogs are unavailable');
|
|
199
|
+
return services.watchdogs.reports(request.params.name, request.query.limit ? Number(request.query.limit) : undefined);
|
|
200
|
+
});
|
|
201
|
+
app.get('/api/v1/watchdogs/:name/reports/:runId', async (request) => {
|
|
202
|
+
auth.authenticate(request);
|
|
203
|
+
if (!services.watchdogs)
|
|
204
|
+
throw new FleetError('capability_unavailable', 'watchdogs are unavailable');
|
|
205
|
+
return services.watchdogs.report(request.params.name, request.params.runId);
|
|
206
|
+
});
|
|
189
207
|
app.get('/api/v1/events', { websocket: true }, (socket, request) => {
|
|
190
208
|
requireSubprotocol(request, 'ours-fleet-events.v1');
|
|
191
209
|
authorizeSocket(socket, request, async (hello) => {
|