@ours.network/fleet 0.10.4 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/application/fleet-query-service.d.ts +11 -0
  2. package/dist/application/fleet-query-service.js +9 -1
  3. package/dist/cli.js +200 -3
  4. package/dist/config.d.ts +2 -0
  5. package/dist/config.js +3 -1
  6. package/dist/docs.d.ts +1 -1
  7. package/dist/docs.js +22 -0
  8. package/dist/duration.d.ts +5 -0
  9. package/dist/duration.js +20 -0
  10. package/dist/ops.d.ts +16 -0
  11. package/dist/ops.js +112 -3
  12. package/dist/paths.d.ts +1 -0
  13. package/dist/paths.js +1 -0
  14. package/dist/resolved-plan.js +7 -0
  15. package/dist/watchdog/alerts.d.ts +34 -0
  16. package/dist/watchdog/alerts.js +78 -0
  17. package/dist/watchdog/briefing.d.ts +65 -0
  18. package/dist/watchdog/briefing.js +181 -0
  19. package/dist/watchdog/config.d.ts +49 -0
  20. package/dist/watchdog/config.js +114 -0
  21. package/dist/watchdog/query.d.ts +78 -0
  22. package/dist/watchdog/query.js +124 -0
  23. package/dist/watchdog/report.d.ts +53 -0
  24. package/dist/watchdog/report.js +126 -0
  25. package/dist/watchdog/run.d.ts +61 -0
  26. package/dist/watchdog/run.js +318 -0
  27. package/dist/watchdog/scheduler.d.ts +105 -0
  28. package/dist/watchdog/scheduler.js +244 -0
  29. package/dist/watchdog/service.d.ts +46 -0
  30. package/dist/watchdog/service.js +179 -0
  31. package/dist/watchdog/store.d.ts +85 -0
  32. package/dist/watchdog/store.js +226 -0
  33. package/dist/web/runtime.js +46 -2
  34. package/dist/web/server.d.ts +2 -0
  35. package/dist/web/server.js +18 -0
  36. package/dist/web-app/assets/{TerminalView-DcImdrI1.js → TerminalView-BvcIkuIF.js} +1 -1
  37. package/dist/web-app/assets/index-B-jtLAkp.css +1 -0
  38. package/dist/web-app/assets/index-CUN7ksTw.js +9 -0
  39. package/dist/web-app/index.html +2 -2
  40. package/package.json +1 -1
  41. package/dist/web-app/assets/index-BokQN1Ao.js +0 -9
  42. package/dist/web-app/assets/index-lAXzaOZM.css +0 -1
@@ -0,0 +1,46 @@
1
+ import { type Exec } from '../exec.js';
2
+ export declare const WATCHDOG_SYSTEMD_UNIT = "ours-fleet-watchdogs.service";
3
+ export declare const WATCHDOG_LAUNCHD_LABEL = "network.ours.fleet.watchdogs";
4
+ /**
5
+ * Supervises the single long-running watchdog-scheduler process (the hidden
6
+ * `_run-watchdogs` command, Task 9's `runScheduler`) the same way
7
+ * `WebServiceManager` (src/web/service.ts) supervises the web console: a
8
+ * private systemd --user unit on Linux, a launchd LaunchAgent on macOS.
9
+ */
10
+ export declare class WatchdogServiceManager {
11
+ private readonly exec;
12
+ private readonly platform;
13
+ constructor(exec?: Exec, platform?: NodeJS.Platform);
14
+ get definitionPath(): string;
15
+ /** false when explicitly disabled (OURS_FLEET_SUPERVISOR=none) or on an unsupported platform. */
16
+ supervised(): boolean;
17
+ /**
18
+ * Writes the unit/plist and returns whether its content actually changed
19
+ * (finding #4): `binPath`/`configPath` are the only inputs that ever
20
+ * change this content, and neither reflects a watchdog's `interval:` or
21
+ * any other config value inside `watchdogs:` — so `changed` here can never
22
+ * by itself justify a restart on every config edit. It's one of the two
23
+ * signals reconcileWatchdogScheduler (ops.ts) combines with a config
24
+ * fingerprint before deciding restart vs. the idempotent start.
25
+ */
26
+ install(binPath: string, configPath?: string): Promise<{
27
+ changed: boolean;
28
+ }>;
29
+ start(): Promise<void>;
30
+ stop(): Promise<void>;
31
+ /**
32
+ * `start()` is a no-op on an already-active unit — systemctl start against
33
+ * a running service just returns 0 without reloading anything, and
34
+ * launchctl kickstart (without -k) behaves the same way. That means a
35
+ * config change (new/changed watchdogs) never reaches a live scheduler
36
+ * process via reconcileWatchdogScheduler's `install` + `start` pair (final
37
+ * review #4). `restart()` mirrors WebServiceManager.restart: an
38
+ * unconditional restart on Linux, and `kickstart -k` (force-restart) on
39
+ * macOS, falling back to stop+start if the kickstart itself fails.
40
+ */
41
+ restart(): Promise<void>;
42
+ status(): Promise<string>;
43
+ uninstall(): Promise<void>;
44
+ private requireInstalled;
45
+ private must;
46
+ }
@@ -0,0 +1,179 @@
1
+ import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync } from 'node:fs';
2
+ import { dirname, isAbsolute, join, resolve } from 'node:path';
3
+ import { replaceFileAtomically } from '../atomic-file.js';
4
+ import { FleetError } from '../application/errors.js';
5
+ import { realExec } from '../exec.js';
6
+ import { home } from '../paths.js';
7
+ export const WATCHDOG_SYSTEMD_UNIT = 'ours-fleet-watchdogs.service';
8
+ export const WATCHDOG_LAUNCHD_LABEL = 'network.ours.fleet.watchdogs';
9
+ /**
10
+ * Supervises the single long-running watchdog-scheduler process (the hidden
11
+ * `_run-watchdogs` command, Task 9's `runScheduler`) the same way
12
+ * `WebServiceManager` (src/web/service.ts) supervises the web console: a
13
+ * private systemd --user unit on Linux, a launchd LaunchAgent on macOS.
14
+ */
15
+ export class WatchdogServiceManager {
16
+ exec;
17
+ platform;
18
+ constructor(exec = realExec, platform = process.platform) {
19
+ this.exec = exec;
20
+ this.platform = platform;
21
+ }
22
+ get definitionPath() {
23
+ return this.platform === 'linux'
24
+ ? join(home(), '.config', 'systemd', 'user', WATCHDOG_SYSTEMD_UNIT)
25
+ : join(home(), 'Library', 'LaunchAgents', `${WATCHDOG_LAUNCHD_LABEL}.plist`);
26
+ }
27
+ /** false when explicitly disabled (OURS_FLEET_SUPERVISOR=none) or on an unsupported platform. */
28
+ supervised() {
29
+ if (process.env.OURS_FLEET_SUPERVISOR === 'none')
30
+ return false;
31
+ return this.platform === 'linux' || this.platform === 'darwin';
32
+ }
33
+ /**
34
+ * Writes the unit/plist and returns whether its content actually changed
35
+ * (finding #4): `binPath`/`configPath` are the only inputs that ever
36
+ * change this content, and neither reflects a watchdog's `interval:` or
37
+ * any other config value inside `watchdogs:` — so `changed` here can never
38
+ * by itself justify a restart on every config edit. It's one of the two
39
+ * signals reconcileWatchdogScheduler (ops.ts) combines with a config
40
+ * fingerprint before deciding restart vs. the idempotent start.
41
+ */
42
+ async install(binPath, configPath) {
43
+ const resolvedScript = resolveExecutable(binPath, 'ours-fleet CLI script');
44
+ const runtime = resolveExecutable(process.execPath, 'Node runtime');
45
+ const config = configPath ? resolve(configPath) : undefined;
46
+ const content = this.platform === 'linux'
47
+ ? watchdogSystemdUnit(runtime, resolvedScript, config)
48
+ : watchdogLaunchdPlist(runtime, resolvedScript, config);
49
+ let previous;
50
+ try {
51
+ previous = readFileSync(this.definitionPath, 'utf8');
52
+ }
53
+ catch { /* absent: definitely changed */ }
54
+ const changed = previous !== content;
55
+ mkdirSync(dirname(this.definitionPath), { recursive: true, mode: 0o700 });
56
+ replaceFileAtomically(this.definitionPath, content, 0o600);
57
+ if (this.platform === 'linux') {
58
+ await this.must('systemctl', ['--user', 'daemon-reload']);
59
+ await this.must('systemctl', ['--user', 'enable', WATCHDOG_SYSTEMD_UNIT]);
60
+ }
61
+ return { changed };
62
+ }
63
+ async start() {
64
+ this.requireInstalled();
65
+ if (this.platform === 'linux') {
66
+ await this.must('systemctl', ['--user', 'start', WATCHDOG_SYSTEMD_UNIT]);
67
+ return;
68
+ }
69
+ const domain = `gui/${uid()}`;
70
+ const loaded = await this.exec('launchctl', ['print', `${domain}/${WATCHDOG_LAUNCHD_LABEL}`]);
71
+ if (loaded.code === 0)
72
+ await this.must('launchctl', ['kickstart', `${domain}/${WATCHDOG_LAUNCHD_LABEL}`]);
73
+ else
74
+ await this.must('launchctl', ['bootstrap', domain, this.definitionPath]);
75
+ }
76
+ async stop() {
77
+ if (this.platform === 'linux') {
78
+ // Tolerate "not loaded" (systemctl exit 5) the same way the launchd
79
+ // branch below tolerates "could not find service": a watchdog-less
80
+ // fleet's `up`/`down` calls stop() defensively even when the unit was
81
+ // never installed, and that must not surface as an error (final
82
+ // review #3).
83
+ const result = await this.exec('systemctl', ['--user', 'stop', WATCHDOG_SYSTEMD_UNIT]);
84
+ if (result.code !== 0 && result.code !== 5 && !/not loaded/i.test(`${result.stdout}\n${result.stderr}`))
85
+ throw new FleetError('control_unavailable', `systemctl --user stop ${WATCHDOG_SYSTEMD_UNIT} failed: ${result.stderr.trim() || `exit ${result.code}`}`);
86
+ return;
87
+ }
88
+ const result = await this.exec('launchctl', ['bootout', `gui/${uid()}/${WATCHDOG_LAUNCHD_LABEL}`]);
89
+ if (result.code !== 0 && !/could not find service|no such process/i.test(`${result.stdout}\n${result.stderr}`))
90
+ throw new FleetError('control_unavailable', `launchctl bootout failed: ${result.stderr.trim()}`);
91
+ }
92
+ /**
93
+ * `start()` is a no-op on an already-active unit — systemctl start against
94
+ * a running service just returns 0 without reloading anything, and
95
+ * launchctl kickstart (without -k) behaves the same way. That means a
96
+ * config change (new/changed watchdogs) never reaches a live scheduler
97
+ * process via reconcileWatchdogScheduler's `install` + `start` pair (final
98
+ * review #4). `restart()` mirrors WebServiceManager.restart: an
99
+ * unconditional restart on Linux, and `kickstart -k` (force-restart) on
100
+ * macOS, falling back to stop+start if the kickstart itself fails.
101
+ */
102
+ async restart() {
103
+ this.requireInstalled();
104
+ if (this.platform === 'linux') {
105
+ await this.must('systemctl', ['--user', 'restart', WATCHDOG_SYSTEMD_UNIT]);
106
+ return;
107
+ }
108
+ const result = await this.exec('launchctl', ['kickstart', '-k', `gui/${uid()}/${WATCHDOG_LAUNCHD_LABEL}`]);
109
+ if (result.code !== 0) {
110
+ await this.stop();
111
+ await this.start();
112
+ }
113
+ }
114
+ async status() {
115
+ if (this.platform === 'linux') {
116
+ const result = await this.exec('systemctl', [
117
+ '--user', 'show', WATCHDOG_SYSTEMD_UNIT, '-p', 'LoadState', '-p', 'ActiveState',
118
+ '-p', 'SubState', '-p', 'ExecMainPID', '--no-pager',
119
+ ]);
120
+ return result.stdout.trim() || result.stderr.trim() || `exit ${result.code}`;
121
+ }
122
+ const result = await this.exec('launchctl', ['print', `gui/${uid()}/${WATCHDOG_LAUNCHD_LABEL}`]);
123
+ return result.code === 0 ? result.stdout.trim() : `not loaded (${WATCHDOG_LAUNCHD_LABEL})`;
124
+ }
125
+ async uninstall() {
126
+ if (this.platform === 'linux') {
127
+ await this.exec('systemctl', ['--user', 'disable', '--now', WATCHDOG_SYSTEMD_UNIT]);
128
+ rmSync(this.definitionPath, { force: true });
129
+ await this.exec('systemctl', ['--user', 'daemon-reload']);
130
+ }
131
+ else {
132
+ await this.stop();
133
+ rmSync(this.definitionPath, { force: true });
134
+ }
135
+ }
136
+ requireInstalled() {
137
+ if (!existsSync(this.definitionPath))
138
+ throw new FleetError('prerequisite_unavailable', 'watchdog scheduler service is not installed');
139
+ }
140
+ async must(command, args) {
141
+ const result = await this.exec(command, args);
142
+ if (result.code !== 0)
143
+ throw new FleetError('control_unavailable', `${command} ${args.join(' ')} failed: ${result.stderr.trim() || `exit ${result.code}`}`);
144
+ }
145
+ }
146
+ function uid() { return process.getuid?.() ?? 501; }
147
+ function resolveExecutable(executable, label) {
148
+ const absolute = isAbsolute(executable) ? executable : resolve(executable);
149
+ try {
150
+ return realpathSync(absolute);
151
+ }
152
+ catch {
153
+ throw new FleetError('prerequisite_unavailable', `${label} does not exist: ${absolute}`);
154
+ }
155
+ }
156
+ function systemdQuote(value) {
157
+ return `"${value.replace(/[%\\"]/g, char => char === '%' ? '%%' : `\\${char}`)}"`;
158
+ }
159
+ function watchdogSystemdUnit(runtime, script, configuration) {
160
+ const config = configuration ? ` -c ${systemdQuote(configuration)}` : '';
161
+ return `[Unit]\nDescription=ours-fleet watchdog scheduler\nAfter=default.target\n\n`
162
+ + `[Service]\nType=simple\nExecStart=${systemdQuote(runtime)} ${systemdQuote(script)} _run-watchdogs${config}\n`
163
+ + `Restart=on-failure\nRestartSec=5\nTimeoutStopSec=15\n\n`
164
+ + `[Install]\nWantedBy=default.target\n`;
165
+ }
166
+ const xml = (value) => value.replace(/[&<>"']/g, char => ({
167
+ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&apos;',
168
+ }[char]));
169
+ function watchdogLaunchdPlist(runtime, script, configuration) {
170
+ const config = configuration
171
+ ? `<string>-c</string><string>${xml(configuration)}</string>` : '';
172
+ return `<?xml version="1.0" encoding="UTF-8"?>\n`
173
+ + `<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n`
174
+ + `<plist version="1.0"><dict>\n<key>Label</key><string>${WATCHDOG_LAUNCHD_LABEL}</string>\n`
175
+ + `<key>ProgramArguments</key><array><string>${xml(runtime)}</string><string>${xml(script)}</string>`
176
+ + `<string>_run-watchdogs</string>${config}</array>\n`
177
+ + `<key>RunAtLoad</key><true/><key>KeepAlive</key><dict><key>SuccessfulExit</key><false/></dict>\n`
178
+ + `<key>ProcessType</key><string>Background</string>\n</dict></plist>\n`;
179
+ }
@@ -0,0 +1,85 @@
1
+ import type { WatchdogReport, WatchdogReportStatus } from './report.js';
2
+ /**
3
+ * A lock directory briefly exists before acquireRunLock can rename its
4
+ * prewritten owner.json into it. Missing/corrupt owner metadata is therefore
5
+ * reclaimable only after this grace period, never while that acquisition
6
+ * window may still be in progress.
7
+ */
8
+ export declare const RUN_LOCK_OWNER_GRACE_MS = 10000;
9
+ /**
10
+ * Choke point every other helper in this module goes through to reach a
11
+ * watchdog's on-disk state. Validates `name` BEFORE any join/mkdir (defense
12
+ * in depth, finding #1): a caller that forgets its own ROLE_NAME_RE guard
13
+ * (as the CLI's `watchdogKnown` once did) must not be able to turn an
14
+ * unvalidated `name` into `join(watchdogsRoot(), '../../victim')` and mkdir
15
+ * an arbitrary path on disk.
16
+ */
17
+ export declare function watchdogDir(name: string): string;
18
+ export declare function reportsDir(name: string): string;
19
+ export interface RunLockOwner {
20
+ pid: number;
21
+ at: string;
22
+ }
23
+ /**
24
+ * mkdir-as-mutex: atomic across processes, unlike a lock file (open+O_EXCL
25
+ * would work too, but a directory needs no cleanup of file contents and
26
+ * can't be partially written). EEXIST means another run holds it.
27
+ *
28
+ * Stamps `owner.json` with our pid (finding #2): ownership metadata is what
29
+ * lets a later `reclaimStaleRunLock` tell a lock abandoned by a dead process
30
+ * apart from one genuinely held by a live run. A naive "mkdir, then
31
+ * writeFileSync(owner.json)" leaves an unnecessarily wide window —
32
+ * between the mkdir succeeding and the write landing — where the lock dir
33
+ * exists but owner.json doesn't yet. A `reclaimStaleRunLock` call from
34
+ * another process landing in exactly that window would see a lock with no
35
+ * (or corrupt) owner metadata and treat a genuinely live lock as a legacy
36
+ * one, reclaiming it out from under us. Do all slow work (building the JSON,
37
+ * writing it, chmod) to a per-pid temp file BEFORE mkdir, then publish it with
38
+ * one rename. There is still an unavoidable interval between those two
39
+ * syscalls; reclaimStaleRunLock protects it by treating a fresh ownerless lock
40
+ * as held for RUN_LOCK_OWNER_GRACE_MS. The temp file is unique per-pid (this
41
+ * function is synchronous, so there's no same-process concurrent-call hazard
42
+ * either) and is cleaned up if mkdir loses the race.
43
+ */
44
+ export declare function acquireRunLock(name: string): boolean;
45
+ /**
46
+ * Release-tolerant of absence: a lock already gone (or never acquired) is not
47
+ * an error. Recursive because the lock dir now holds `owner.json` alongside
48
+ * the mkdir mutex itself (finding #2) — a plain rmdir would fail ENOTEMPTY.
49
+ */
50
+ export declare function releaseRunLock(name: string): void;
51
+ /** Reads a run lock's owner metadata; missing or corrupt yields undefined. */
52
+ export declare function readRunLockOwner(name: string): RunLockOwner | undefined;
53
+ /**
54
+ * Owner-mandated (finding #2): only a DEMONSTRABLY stale run lock may be
55
+ * reclaimed — a lock is stale iff it isn't held at all, its owner metadata
56
+ * names a dead pid, or its owner metadata is missing/corrupt AND the lock dir
57
+ * is older than RUN_LOCK_OWNER_GRACE_MS. The age gate closes the interprocess
58
+ * interval between acquireRunLock's mkdir and owner.json rename: a concurrent
59
+ * scheduler sees a fresh ownerless lock as held, not stale. A live owner (a
60
+ * foreground `watchdog-run`, another scheduler instance, an in-progress run)
61
+ * is left strictly alone: reclaiming it would let two runs share the same temp
62
+ * dir.
63
+ * Returns true when the lock is (now) not held — whether because it was
64
+ * already absent or because a stale lock was just removed; false when a live
65
+ * lock was found and deliberately left in place.
66
+ */
67
+ export declare function reclaimStaleRunLock(name: string): boolean;
68
+ /** Lexical-chronological UTC run id, e.g. '20260731T115000Z'. */
69
+ export declare function formatRunId(d: Date): string;
70
+ export interface RunListEntry {
71
+ runId: string;
72
+ status: WatchdogReportStatus;
73
+ startedAt: string;
74
+ finishedAt: string;
75
+ summary: WatchdogReport['summary'];
76
+ error: string | null;
77
+ }
78
+ export declare function writeReport(name: string, report: WatchdogReport): string;
79
+ /** Newest-first run listing; a corrupt report file yields a synthetic 'error' entry rather than throwing. */
80
+ export declare function listRuns(name: string): RunListEntry[];
81
+ /** Reads one run's full report. Rejects non-conforming runIds (path-traversal guard) and corrupt files by returning undefined. */
82
+ export declare function readReport(name: string, runId: string): WatchdogReport | undefined;
83
+ export declare function latestReport(name: string): WatchdogReport | undefined;
84
+ /** Deletes all but the `keep` newest reports (oldest first); returns the number pruned. */
85
+ export declare function pruneReports(name: string, keep: number): number;
@@ -0,0 +1,226 @@
1
+ import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { watchdogsRoot } from '../paths.js';
4
+ // store.ts imports config.ts (ROLE_NAME_RE) but not the reverse — config.ts's
5
+ // import graph (config-yaml, isolation/policy, harness/registry, watchdog/config)
6
+ // never reaches back into watchdog/store.ts, so this does not create a cycle.
7
+ import { ROLE_NAME_RE } from '../config.js';
8
+ const RUN_ID_RE = /^\d{8}T\d{6}Z$/;
9
+ const REPORT_FILE_RE = /^\d{8}T\d{6}Z\.json$/;
10
+ /**
11
+ * A lock directory briefly exists before acquireRunLock can rename its
12
+ * prewritten owner.json into it. Missing/corrupt owner metadata is therefore
13
+ * reclaimable only after this grace period, never while that acquisition
14
+ * window may still be in progress.
15
+ */
16
+ export const RUN_LOCK_OWNER_GRACE_MS = 10_000;
17
+ /**
18
+ * mkdirSync's `mode` is masked by the process umask and ignored outright when
19
+ * the directory already exists, so an explicit chmod after mkdir is the only
20
+ * way to guarantee 0700 regardless of umask or prior state.
21
+ */
22
+ function ensureDir(dir) {
23
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
24
+ chmodSync(dir, 0o700);
25
+ return dir;
26
+ }
27
+ /**
28
+ * Choke point every other helper in this module goes through to reach a
29
+ * watchdog's on-disk state. Validates `name` BEFORE any join/mkdir (defense
30
+ * in depth, finding #1): a caller that forgets its own ROLE_NAME_RE guard
31
+ * (as the CLI's `watchdogKnown` once did) must not be able to turn an
32
+ * unvalidated `name` into `join(watchdogsRoot(), '../../victim')` and mkdir
33
+ * an arbitrary path on disk.
34
+ */
35
+ export function watchdogDir(name) {
36
+ if (!ROLE_NAME_RE.test(name))
37
+ throw new Error(`invalid watchdog name '${name}'`);
38
+ return ensureDir(join(watchdogsRoot(), name));
39
+ }
40
+ export function reportsDir(name) {
41
+ return ensureDir(join(watchdogDir(name), 'reports'));
42
+ }
43
+ function runLockPath(name) {
44
+ return join(watchdogDir(name), '.run-lock');
45
+ }
46
+ function runLockOwnerPath(name) {
47
+ return join(runLockPath(name), 'owner.json');
48
+ }
49
+ /**
50
+ * mkdir-as-mutex: atomic across processes, unlike a lock file (open+O_EXCL
51
+ * would work too, but a directory needs no cleanup of file contents and
52
+ * can't be partially written). EEXIST means another run holds it.
53
+ *
54
+ * Stamps `owner.json` with our pid (finding #2): ownership metadata is what
55
+ * lets a later `reclaimStaleRunLock` tell a lock abandoned by a dead process
56
+ * apart from one genuinely held by a live run. A naive "mkdir, then
57
+ * writeFileSync(owner.json)" leaves an unnecessarily wide window —
58
+ * between the mkdir succeeding and the write landing — where the lock dir
59
+ * exists but owner.json doesn't yet. A `reclaimStaleRunLock` call from
60
+ * another process landing in exactly that window would see a lock with no
61
+ * (or corrupt) owner metadata and treat a genuinely live lock as a legacy
62
+ * one, reclaiming it out from under us. Do all slow work (building the JSON,
63
+ * writing it, chmod) to a per-pid temp file BEFORE mkdir, then publish it with
64
+ * one rename. There is still an unavoidable interval between those two
65
+ * syscalls; reclaimStaleRunLock protects it by treating a fresh ownerless lock
66
+ * as held for RUN_LOCK_OWNER_GRACE_MS. The temp file is unique per-pid (this
67
+ * function is synchronous, so there's no same-process concurrent-call hazard
68
+ * either) and is cleaned up if mkdir loses the race.
69
+ */
70
+ export function acquireRunLock(name) {
71
+ const dir = watchdogDir(name);
72
+ const owner = { pid: process.pid, at: new Date().toISOString() };
73
+ const tempOwnerPath = join(dir, `.owner.${process.pid}.tmp`);
74
+ writeFileSync(tempOwnerPath, JSON.stringify(owner), { mode: 0o600 });
75
+ chmodSync(tempOwnerPath, 0o600);
76
+ try {
77
+ mkdirSync(runLockPath(name));
78
+ }
79
+ catch (e) {
80
+ try {
81
+ unlinkSync(tempOwnerPath);
82
+ }
83
+ catch { /* best effort cleanup */ }
84
+ if (e.code === 'EEXIST')
85
+ return false;
86
+ throw e;
87
+ }
88
+ renameSync(tempOwnerPath, runLockOwnerPath(name));
89
+ return true;
90
+ }
91
+ /**
92
+ * Release-tolerant of absence: a lock already gone (or never acquired) is not
93
+ * an error. Recursive because the lock dir now holds `owner.json` alongside
94
+ * the mkdir mutex itself (finding #2) — a plain rmdir would fail ENOTEMPTY.
95
+ */
96
+ export function releaseRunLock(name) {
97
+ try {
98
+ rmSync(runLockPath(name), { recursive: true });
99
+ }
100
+ catch (e) {
101
+ if (e.code === 'ENOENT')
102
+ return;
103
+ throw e;
104
+ }
105
+ }
106
+ /** Reads a run lock's owner metadata; missing or corrupt yields undefined. */
107
+ export function readRunLockOwner(name) {
108
+ try {
109
+ const raw = JSON.parse(readFileSync(runLockOwnerPath(name), 'utf8'));
110
+ if (!Number.isInteger(raw.pid) || raw.pid <= 0 || typeof raw.at !== 'string')
111
+ return undefined;
112
+ return { pid: raw.pid, at: raw.at };
113
+ }
114
+ catch {
115
+ return undefined;
116
+ }
117
+ }
118
+ /** True iff `pid` names a live process. EPERM means the process exists but we can't signal it — still alive. */
119
+ function pidAlive(pid) {
120
+ try {
121
+ process.kill(pid, 0);
122
+ return true;
123
+ }
124
+ catch (e) {
125
+ return e.code === 'EPERM';
126
+ }
127
+ }
128
+ /**
129
+ * Owner-mandated (finding #2): only a DEMONSTRABLY stale run lock may be
130
+ * reclaimed — a lock is stale iff it isn't held at all, its owner metadata
131
+ * names a dead pid, or its owner metadata is missing/corrupt AND the lock dir
132
+ * is older than RUN_LOCK_OWNER_GRACE_MS. The age gate closes the interprocess
133
+ * interval between acquireRunLock's mkdir and owner.json rename: a concurrent
134
+ * scheduler sees a fresh ownerless lock as held, not stale. A live owner (a
135
+ * foreground `watchdog-run`, another scheduler instance, an in-progress run)
136
+ * is left strictly alone: reclaiming it would let two runs share the same temp
137
+ * dir.
138
+ * Returns true when the lock is (now) not held — whether because it was
139
+ * already absent or because a stale lock was just removed; false when a live
140
+ * lock was found and deliberately left in place.
141
+ */
142
+ export function reclaimStaleRunLock(name) {
143
+ const path = runLockPath(name);
144
+ if (!existsSync(path))
145
+ return true;
146
+ const owner = readRunLockOwner(name);
147
+ if (owner !== undefined && pidAlive(owner.pid))
148
+ return false;
149
+ if (owner === undefined) {
150
+ try {
151
+ if (Date.now() - statSync(path).mtimeMs < RUN_LOCK_OWNER_GRACE_MS)
152
+ return false;
153
+ }
154
+ catch (e) {
155
+ // The holder may have released between existsSync and statSync.
156
+ if (e.code === 'ENOENT')
157
+ return true;
158
+ throw e;
159
+ }
160
+ }
161
+ releaseRunLock(name);
162
+ return true;
163
+ }
164
+ /** Lexical-chronological UTC run id, e.g. '20260731T115000Z'. */
165
+ export function formatRunId(d) {
166
+ return d.toISOString().replace(/\.\d{3}Z$/, 'Z').replace(/[-:]/g, '');
167
+ }
168
+ const CORRUPT_SUMMARY = { checked: 0, healthy: 0, idle: 0, anomalies: 0 };
169
+ function reportPath(name, runId) {
170
+ return join(reportsDir(name), `${runId}.json`);
171
+ }
172
+ /** Newest-first run ids currently on disk, derived from filenames alone. */
173
+ function runIdsDesc(name) {
174
+ return readdirSync(reportsDir(name))
175
+ .filter(f => REPORT_FILE_RE.test(f))
176
+ .sort()
177
+ .reverse()
178
+ .map(f => f.slice(0, -'.json'.length));
179
+ }
180
+ export function writeReport(name, report) {
181
+ const path = reportPath(name, report.run_id);
182
+ writeFileSync(path, JSON.stringify(report, null, 2) + '\n', { mode: 0o600 });
183
+ chmodSync(path, 0o600);
184
+ return path;
185
+ }
186
+ /** Newest-first run listing; a corrupt report file yields a synthetic 'error' entry rather than throwing. */
187
+ export function listRuns(name) {
188
+ return runIdsDesc(name).map((runId) => {
189
+ try {
190
+ const report = JSON.parse(readFileSync(reportPath(name, runId), 'utf8'));
191
+ return {
192
+ runId, status: report.status, startedAt: report.started_at, finishedAt: report.finished_at,
193
+ summary: report.summary, error: report.error,
194
+ };
195
+ }
196
+ catch {
197
+ return { runId, status: 'error', startedAt: '', finishedAt: '', summary: CORRUPT_SUMMARY, error: 'unreadable report file' };
198
+ }
199
+ });
200
+ }
201
+ /** Reads one run's full report. Rejects non-conforming runIds (path-traversal guard) and corrupt files by returning undefined. */
202
+ export function readReport(name, runId) {
203
+ if (!RUN_ID_RE.test(runId))
204
+ return undefined;
205
+ try {
206
+ return JSON.parse(readFileSync(reportPath(name, runId), 'utf8'));
207
+ }
208
+ catch {
209
+ return undefined;
210
+ }
211
+ }
212
+ export function latestReport(name) {
213
+ const [newest] = runIdsDesc(name);
214
+ return newest === undefined ? undefined : readReport(name, newest);
215
+ }
216
+ /** Deletes all but the `keep` newest reports (oldest first); returns the number pruned. */
217
+ export function pruneReports(name, keep) {
218
+ const stale = runIdsDesc(name).slice(keep);
219
+ for (const runId of stale) {
220
+ try {
221
+ unlinkSync(reportPath(name, runId));
222
+ }
223
+ catch { /* best effort */ }
224
+ }
225
+ return stale.length;
226
+ }
@@ -1,6 +1,7 @@
1
1
  import { spawn } from 'node:child_process';
2
2
  import { existsSync, realpathSync } from 'node:fs';
3
3
  import { resolve } from 'node:path';
4
+ import { loadConfig } from '../config.js';
4
5
  import { RoleRepository } from '../application/role-repository.js';
5
6
  import { FleetQueryService } from '../application/fleet-query-service.js';
6
7
  import { AcpRoleSessionAdapter, TmuxRoleSessionAdapter } from '../application/session-control.js';
@@ -21,6 +22,24 @@ import { acquireWebServerLock } from './lock.js';
21
22
  import { TrustedDeviceStore } from './device-store.js';
22
23
  import { WebAuth } from './auth.js';
23
24
  import { startWebControlServer } from './control.js';
25
+ import { buildWatchdogFindings, cachedWatchdogFindingsProvider, WatchdogQueryService } from '../watchdog/query.js';
26
+ import { latestReport } from '../watchdog/store.js';
27
+ const CONFIG_CACHE_TTL_MS = 5_000;
28
+ /**
29
+ * loadConfig re-parses YAML from disk on every call; the watchdog list/reports
30
+ * routes get polled by the console UI, so cache the resolved config for a
31
+ * short TTL rather than re-parsing per request. Runtime-only concern (not the
32
+ * scheduler's), so a plain Date.now() clock is fine.
33
+ */
34
+ function cachedConfigProvider(configPath) {
35
+ let cached;
36
+ return () => {
37
+ const now = Date.now();
38
+ if (!cached || now - cached.at >= CONFIG_CACHE_TTL_MS)
39
+ cached = { at: now, cfg: loadConfig(configPath) };
40
+ return cached.cfg;
41
+ };
42
+ }
24
43
  export async function startWebConsole(options) {
25
44
  const requestedPort = options.port ?? 49_271;
26
45
  if (!Number.isInteger(requestedPort) || requestedPort < 0 || requestedPort > 65_535)
@@ -51,11 +70,35 @@ export async function startWebConsole(options) {
51
70
  const audit = new AuditSink();
52
71
  const terminals = new TerminalBridgeManager({ repository, audit, tmux });
53
72
  const terminalAvailable = await terminals.available();
73
+ const watchdogConfigProvider = cachedConfigProvider(options.configPath);
74
+ const log = options.log ?? (() => { });
75
+ let loggedWatchdogFindingsError = false;
76
+ // Needs-attention integration (Task 19): worst per-role watchdog finding,
77
+ // rebuilt from stored reports. A store hiccup (corrupt state, unreadable
78
+ // report) must never break the fleet list, so it degrades to an empty map
79
+ // and logs once rather than repeating on every poll. status() calls this
80
+ // once per role, so list()'s O(roles) sweep would otherwise cost
81
+ // O(roles x watchdogs) disk reads every 1-3s of console polling —
82
+ // cachedWatchdogFindingsProvider memoizes the whole build behind the same
83
+ // TTL as the config cache above.
84
+ const watchdogFindings = cachedWatchdogFindingsProvider(() => {
85
+ try {
86
+ return buildWatchdogFindings(watchdogConfigProvider(), latestReport);
87
+ }
88
+ catch (error) {
89
+ if (!loggedWatchdogFindingsError) {
90
+ loggedWatchdogFindingsError = true;
91
+ log(`watchdog findings unavailable: ${error.message}`);
92
+ }
93
+ return new Map();
94
+ }
95
+ }, CONFIG_CACHE_TTL_MS);
54
96
  const query = new FleetQueryService({
55
97
  repository, supervisor: backend, tmux,
56
98
  capabilityContext: { terminalPtyAvailable: terminalAvailable },
99
+ watchdogFindings,
57
100
  });
58
- const ops = { backend, binPath: options.binPath, log: options.log ?? (() => { }) };
101
+ const ops = { backend, binPath: options.binPath, log };
59
102
  const creation = new RoleCreationService({
60
103
  configPath: options.configPath, ops, binPath: options.binPath,
61
104
  allowedCwdRoots: [realpathSync(home()), realpathSync(process.cwd())],
@@ -84,10 +127,11 @@ export async function startWebConsole(options) {
84
127
  },
85
128
  });
86
129
  const logs = new StructuredLogService(backend, realExec);
130
+ const watchdogs = new WatchdogQueryService(watchdogConfigProvider);
87
131
  let server;
88
132
  try {
89
133
  server = await buildWebServer({
90
- query, repository, logs, commands, creation, audit, events,
134
+ query, repository, logs, commands, creation, audit, events, watchdogs,
91
135
  terminalUpgrade: terminalAvailable
92
136
  ? async (socket, _request, roleId, _ticket, hello) => terminals.connect(socket, roleId, hello)
93
137
  : undefined,
@@ -6,6 +6,7 @@ import type { RoleSessionControl } from '../application/session-control.js';
6
6
  import type { StructuredLogService } from '../application/log-service.js';
7
7
  import type { RoleCommandService } from '../application/role-command-service.js';
8
8
  import type { RoleCreationService } from '../application/role-creation-service.js';
9
+ import type { WatchdogQueryService } from '../watchdog/query.js';
9
10
  import { AuditSink } from './audit.js';
10
11
  import { WebAuth } from './auth.js';
11
12
  import { FleetEventBus } from './events.js';
@@ -18,6 +19,7 @@ export interface WebServices {
18
19
  creation: RoleCreationService;
19
20
  audit?: AuditSink;
20
21
  events?: FleetEventBus;
22
+ watchdogs?: WatchdogQueryService;
21
23
  terminalUpgrade?: (socket: WebSocket, request: FastifyRequest, roleId: string, ticket: string, hello: Record<string, unknown>) => Promise<void>;
22
24
  }
23
25
  export interface WebServer {
@@ -186,6 +186,24 @@ export async function buildWebServer(services, boundary, options = {}) {
186
186
  auth.authenticate(request);
187
187
  return { records: audit.list() };
188
188
  });
189
+ app.get('/api/v1/watchdogs', async (request) => {
190
+ auth.authenticate(request);
191
+ if (!services.watchdogs)
192
+ throw new FleetError('capability_unavailable', 'watchdogs are unavailable');
193
+ return services.watchdogs.list();
194
+ });
195
+ app.get('/api/v1/watchdogs/:name/reports', async (request) => {
196
+ auth.authenticate(request);
197
+ if (!services.watchdogs)
198
+ throw new FleetError('capability_unavailable', 'watchdogs are unavailable');
199
+ return services.watchdogs.reports(request.params.name, request.query.limit ? Number(request.query.limit) : undefined);
200
+ });
201
+ app.get('/api/v1/watchdogs/:name/reports/:runId', async (request) => {
202
+ auth.authenticate(request);
203
+ if (!services.watchdogs)
204
+ throw new FleetError('capability_unavailable', 'watchdogs are unavailable');
205
+ return services.watchdogs.report(request.params.name, request.params.runId);
206
+ });
189
207
  app.get('/api/v1/events', { websocket: true }, (socket, request) => {
190
208
  requireSubprotocol(request, 'ours-fleet-events.v1');
191
209
  authorizeSocket(socket, request, async (hello) => {