@ours.network/fleet 0.10.4 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/fleet-query-service.d.ts +11 -0
- package/dist/application/fleet-query-service.js +9 -1
- package/dist/cli.js +202 -3
- package/dist/config.d.ts +2 -0
- package/dist/config.js +3 -1
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +32 -0
- package/dist/duration.d.ts +5 -0
- package/dist/duration.js +20 -0
- package/dist/harness/codex.js +16 -1
- package/dist/isolation/policy.js +5 -0
- package/dist/isolation/runtime.d.ts +16 -0
- package/dist/isolation/runtime.js +136 -0
- package/dist/isolation/types.d.ts +2 -0
- package/dist/ops.d.ts +16 -0
- package/dist/ops.js +112 -3
- package/dist/paths.d.ts +1 -0
- package/dist/paths.js +1 -0
- package/dist/resolved-plan.js +8 -0
- package/dist/runner.js +11 -5
- package/dist/watchdog/alerts.d.ts +34 -0
- package/dist/watchdog/alerts.js +78 -0
- package/dist/watchdog/briefing.d.ts +65 -0
- package/dist/watchdog/briefing.js +181 -0
- package/dist/watchdog/config.d.ts +53 -0
- package/dist/watchdog/config.js +120 -0
- package/dist/watchdog/query.d.ts +78 -0
- package/dist/watchdog/query.js +124 -0
- package/dist/watchdog/report.d.ts +53 -0
- package/dist/watchdog/report.js +126 -0
- package/dist/watchdog/run.d.ts +63 -0
- package/dist/watchdog/run.js +345 -0
- package/dist/watchdog/scheduler.d.ts +105 -0
- package/dist/watchdog/scheduler.js +244 -0
- package/dist/watchdog/service.d.ts +46 -0
- package/dist/watchdog/service.js +179 -0
- package/dist/watchdog/store.d.ts +85 -0
- package/dist/watchdog/store.js +226 -0
- package/dist/web/runtime.js +46 -2
- package/dist/web/server.d.ts +2 -0
- package/dist/web/server.js +18 -0
- package/dist/web-app/assets/{TerminalView-DcImdrI1.js → TerminalView-BvcIkuIF.js} +1 -1
- package/dist/web-app/assets/index-B-jtLAkp.css +1 -0
- package/dist/web-app/assets/index-CUN7ksTw.js +9 -0
- package/dist/web-app/index.html +2 -2
- package/package.json +1 -1
- package/dist/web-app/assets/index-BokQN1Ao.js +0 -9
- package/dist/web-app/assets/index-lAXzaOZM.css +0 -1
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { accessSync, closeSync, constants, existsSync, openSync, readFileSync, readSync, realpathSync, } from 'node:fs';
|
|
2
|
+
import { createRequire } from 'node:module';
|
|
3
|
+
import { delimiter, dirname, isAbsolute, join, resolve, sep } from 'node:path';
|
|
4
|
+
const SYSTEM_ROOTS = ['/usr', '/bin', '/sbin', '/lib', '/lib64', '/etc'];
|
|
5
|
+
const inside = (path, root) => path === root || path.startsWith(root + sep);
|
|
6
|
+
function canonical(path) {
|
|
7
|
+
try {
|
|
8
|
+
return realpathSync.native(path);
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return resolve(path);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
function commandPath(command, pathValue) {
|
|
15
|
+
const candidates = isAbsolute(command) || command.includes(sep)
|
|
16
|
+
? [resolve(command)]
|
|
17
|
+
: pathValue.split(delimiter).filter(Boolean).map(dir => resolve(dir, command));
|
|
18
|
+
for (const candidate of candidates) {
|
|
19
|
+
try {
|
|
20
|
+
accessSync(candidate, constants.X_OK);
|
|
21
|
+
return canonical(candidate);
|
|
22
|
+
}
|
|
23
|
+
catch { /* keep searching PATH */ }
|
|
24
|
+
}
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
function packageRoot(path) {
|
|
28
|
+
let dir = existsSync(path) ? dirname(path) : path;
|
|
29
|
+
for (;;) {
|
|
30
|
+
const manifest = join(dir, 'package.json');
|
|
31
|
+
if (existsSync(manifest))
|
|
32
|
+
return dir;
|
|
33
|
+
const parent = dirname(dir);
|
|
34
|
+
if (parent === dir)
|
|
35
|
+
return undefined;
|
|
36
|
+
dir = parent;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function dependencyManifest(manifestPath, name) {
|
|
40
|
+
const localRequire = createRequire(manifestPath);
|
|
41
|
+
try {
|
|
42
|
+
return localRequire.resolve(`${name}/package.json`);
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// Some packages do not export package.json. Their main entry still gives us
|
|
46
|
+
// a point from which to find the owning package root.
|
|
47
|
+
try {
|
|
48
|
+
const entry = localRequire.resolve(name);
|
|
49
|
+
const root = packageRoot(entry);
|
|
50
|
+
return root ? join(root, 'package.json') : undefined;
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
function addPackageClosure(root, paths, seen) {
|
|
58
|
+
const canonicalRoot = canonical(root);
|
|
59
|
+
if (seen.has(canonicalRoot))
|
|
60
|
+
return;
|
|
61
|
+
seen.add(canonicalRoot);
|
|
62
|
+
paths.add(canonicalRoot);
|
|
63
|
+
const manifestPath = join(canonicalRoot, 'package.json');
|
|
64
|
+
try {
|
|
65
|
+
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
|
66
|
+
const names = new Set([
|
|
67
|
+
...Object.keys(manifest.dependencies ?? {}),
|
|
68
|
+
...Object.keys(manifest.optionalDependencies ?? {}),
|
|
69
|
+
...Object.keys(manifest.peerDependencies ?? {}),
|
|
70
|
+
]);
|
|
71
|
+
for (const name of names) {
|
|
72
|
+
const dependency = dependencyManifest(manifestPath, name);
|
|
73
|
+
if (dependency)
|
|
74
|
+
addPackageClosure(dirname(dependency), paths, seen);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch { /* a malformed/unreadable manifest cannot contribute a closure */ }
|
|
78
|
+
}
|
|
79
|
+
function isNodeScript(path) {
|
|
80
|
+
let fd;
|
|
81
|
+
try {
|
|
82
|
+
fd = openSync(path, 'r');
|
|
83
|
+
const bytes = Buffer.alloc(128);
|
|
84
|
+
const count = readSync(fd, bytes, 0, bytes.length, 0);
|
|
85
|
+
return /^#!.*\bnode\b/.test(bytes.subarray(0, count).toString('utf8'));
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
if (fd !== undefined)
|
|
92
|
+
closeSync(fd);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Resolve the concrete runtime closure for an already-selected harness launch.
|
|
97
|
+
* System roots are already present in every isolation policy; everything else
|
|
98
|
+
* is returned as an exact read-only executable or npm package-root bind.
|
|
99
|
+
*/
|
|
100
|
+
export function resolveLaunchRuntime(argv, options = {}) {
|
|
101
|
+
if (!argv.length)
|
|
102
|
+
return { argv: [], readPaths: [] };
|
|
103
|
+
const nodeExecutable = canonical(options.nodeExecutable ?? process.execPath);
|
|
104
|
+
const executable = commandPath(argv[0], options.path ?? process.env.PATH ?? '');
|
|
105
|
+
const resolvedArgv = executable ? [executable, ...argv.slice(1)] : [...argv];
|
|
106
|
+
const paths = new Set();
|
|
107
|
+
const packages = new Set();
|
|
108
|
+
const consider = (path) => {
|
|
109
|
+
if (!isAbsolute(path) || !existsSync(path))
|
|
110
|
+
return;
|
|
111
|
+
const real = canonical(path);
|
|
112
|
+
const root = packageRoot(real);
|
|
113
|
+
if (root)
|
|
114
|
+
addPackageClosure(root, paths, packages);
|
|
115
|
+
else
|
|
116
|
+
paths.add(real);
|
|
117
|
+
if (real === nodeExecutable || isNodeScript(real))
|
|
118
|
+
paths.add(nodeExecutable);
|
|
119
|
+
};
|
|
120
|
+
if (executable)
|
|
121
|
+
consider(executable);
|
|
122
|
+
// A direct Node launch's first existing absolute non-option argument is its
|
|
123
|
+
// module entrypoint. Other absolute arguments are harness inputs/settings,
|
|
124
|
+
// not runtime code, and already follow the ordinary filesystem policy.
|
|
125
|
+
if (executable === nodeExecutable) {
|
|
126
|
+
const entrypointIndex = argv.findIndex((arg, index) => index > 0 && !arg.startsWith('-') && isAbsolute(arg) && existsSync(arg));
|
|
127
|
+
if (entrypointIndex !== -1) {
|
|
128
|
+
resolvedArgv[entrypointIndex] = canonical(argv[entrypointIndex]);
|
|
129
|
+
consider(resolvedArgv[entrypointIndex]);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
argv: resolvedArgv,
|
|
134
|
+
readPaths: [...paths].filter(path => !SYSTEM_ROOTS.some(root => inside(path, root))),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
@@ -63,6 +63,8 @@ export interface WrapContext {
|
|
|
63
63
|
* them for itself or for its peers.
|
|
64
64
|
*/
|
|
65
65
|
harnessSharedPaths?: string[];
|
|
66
|
+
/** Exact launcher/interpreter/module closure required by the selected command. */
|
|
67
|
+
runtimeReadPaths?: string[];
|
|
66
68
|
brokerEndpoint?: string;
|
|
67
69
|
}
|
|
68
70
|
/**
|
package/dist/ops.d.ts
CHANGED
|
@@ -16,6 +16,22 @@ export interface OpsDeps {
|
|
|
16
16
|
* `ours-fleet up` has no transaction to tell.
|
|
17
17
|
*/
|
|
18
18
|
onInstalled?(outcome: InstallOutcome): void;
|
|
19
|
+
/**
|
|
20
|
+
* Optional: supervises the single watchdog-scheduler process (Task 10).
|
|
21
|
+
* Absent for callers that predate watchdogs — `up`/`down` must never throw
|
|
22
|
+
* just because this hook is missing.
|
|
23
|
+
*/
|
|
24
|
+
watchdogService?: {
|
|
25
|
+
/** `changed` is true when the unit/plist content itself differs from what's on disk (or was absent). */
|
|
26
|
+
install(binPath: string, configPath?: string): Promise<{
|
|
27
|
+
changed: boolean;
|
|
28
|
+
}>;
|
|
29
|
+
start(): Promise<void>;
|
|
30
|
+
stop(): Promise<void>;
|
|
31
|
+
/** Bounces an already-running scheduler so a config change actually reaches it — `start()` is a no-op on an active unit. */
|
|
32
|
+
restart(): Promise<void>;
|
|
33
|
+
supervised(): boolean;
|
|
34
|
+
};
|
|
19
35
|
}
|
|
20
36
|
/** Materialize a role's state dir from config: briefing + markers. Returns the dir. */
|
|
21
37
|
export declare function applyRole(role: ResolvedRole, opts?: {
|
package/dist/ops.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, unlinkSync } from 'node:fs';
|
|
1
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, unlinkSync, } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
-
import { randomUUID } from 'node:crypto';
|
|
4
|
-
import { agentDir, fleetDDir } from './paths.js';
|
|
3
|
+
import { randomUUID, createHash } from 'node:crypto';
|
|
4
|
+
import { agentDir, fleetDDir, watchdogsRoot } from './paths.js';
|
|
5
5
|
import { findRole } from './config.js';
|
|
6
6
|
import { getAdapter } from './harness/registry.js';
|
|
7
7
|
import { generateBriefing } from './briefing.js';
|
|
@@ -75,8 +75,103 @@ export async function up(cfg, names, deps, configPath, identityGuarantee) {
|
|
|
75
75
|
outcomes.push(outcome);
|
|
76
76
|
deps.log(`↑ up: ${role.name} (harness: ${role.harness}, identity: ${role.identity}${role.cwd ? `, cwd: ${role.cwd}` : ''})`);
|
|
77
77
|
}
|
|
78
|
+
await reconcileWatchdogScheduler(cfg, deps, configPath);
|
|
78
79
|
return outcomes;
|
|
79
80
|
}
|
|
81
|
+
const WATCHDOG_FINGERPRINT_FILE = '.config-fingerprint';
|
|
82
|
+
/** Deterministic JSON: object keys sorted recursively, so key-insertion order never affects the hash. */
|
|
83
|
+
function stableStringify(value) {
|
|
84
|
+
return JSON.stringify(value, (_key, v) => {
|
|
85
|
+
if (v !== null && typeof v === 'object' && !Array.isArray(v)) {
|
|
86
|
+
return Object.keys(v).sort()
|
|
87
|
+
.reduce((acc, k) => { acc[k] = v[k]; return acc; }, {});
|
|
88
|
+
}
|
|
89
|
+
return v;
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
/** sha256 over the resolved enabled-watchdog set, stable regardless of config source order (finding #4). */
|
|
93
|
+
function watchdogFingerprint(enabled) {
|
|
94
|
+
const sorted = [...enabled].sort((a, b) => a.name.localeCompare(b.name));
|
|
95
|
+
return createHash('sha256').update(stableStringify(sorted)).digest('hex');
|
|
96
|
+
}
|
|
97
|
+
function fingerprintPath() {
|
|
98
|
+
return join(watchdogsRoot(), WATCHDOG_FINGERPRINT_FILE);
|
|
99
|
+
}
|
|
100
|
+
function readStoredFingerprint() {
|
|
101
|
+
try {
|
|
102
|
+
return readFileSync(fingerprintPath(), 'utf8').trim();
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
return undefined;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
function writeFingerprint(fingerprint) {
|
|
109
|
+
mkdirSync(watchdogsRoot(), { recursive: true, mode: 0o700 });
|
|
110
|
+
writeFileSync(fingerprintPath(), fingerprint + '\n', { mode: 0o600 });
|
|
111
|
+
chmodSync(fingerprintPath(), 0o600); // mkdirSync/writeFileSync's mode is masked by umask; force it
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Install/start (or stop) the single supervised watchdog-scheduler process
|
|
115
|
+
* (Task 10) to match the config's enabled watchdogs. Runs on every `up`,
|
|
116
|
+
* including a named `up <Role>` — cheap and idempotent, and the alternative
|
|
117
|
+
* (only reconciling on a whole-fleet `up`) would leave a newly-enabled
|
|
118
|
+
* watchdog unscheduled until the next bare `up`. Never throws: a scheduler
|
|
119
|
+
* hiccup must not fail the role installs that already succeeded.
|
|
120
|
+
*
|
|
121
|
+
* Restarts ONLY when something the scheduler actually needs to pick up
|
|
122
|
+
* changed (finding #4): an unconditional `restart()` on every `up` —
|
|
123
|
+
* including a named `up <SomeUnrelatedRole>` — interrupted whatever the
|
|
124
|
+
* scheduler was mid-run (a 15s stop timeout can kill a long check without a
|
|
125
|
+
* report). Two independent signals decide: `svc.install()`'s own `changed`
|
|
126
|
+
* (the unit/plist content — binPath/configPath — differs), and a config
|
|
127
|
+
* fingerprint over the resolved enabled-watchdog set (sha256, stored at
|
|
128
|
+
* `<watchdogsRoot>/.config-fingerprint`) — needed because interval/watch/etc.
|
|
129
|
+
* never appear in the unit content itself (the unit just re-execs `-c
|
|
130
|
+
* <configPath>`), so `install()` alone can never see them change. Either
|
|
131
|
+
* signal true -> restart(); neither -> the idempotent start() (a no-op if
|
|
132
|
+
* already active, otherwise brings up a stopped unit).
|
|
133
|
+
*/
|
|
134
|
+
async function reconcileWatchdogScheduler(cfg, deps, configPath) {
|
|
135
|
+
const svc = deps.watchdogService;
|
|
136
|
+
if (!svc)
|
|
137
|
+
return;
|
|
138
|
+
const enabled = cfg.watchdogs.filter(w => w.enabled);
|
|
139
|
+
try {
|
|
140
|
+
// Check supervised() before any stop (final review #3): on an
|
|
141
|
+
// unsupervised platform/config, the scheduler was never installed, so
|
|
142
|
+
// stopping it is not just a no-op — on Linux, stop() throws for a unit
|
|
143
|
+
// that was never loaded. A watchdog-less fleet must not see that
|
|
144
|
+
// failure surface as a spurious "! watchdogs scheduler: ..." warning.
|
|
145
|
+
if (!enabled.length) {
|
|
146
|
+
if (svc.supervised())
|
|
147
|
+
await svc.stop();
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (!svc.supervised()) {
|
|
151
|
+
deps.log("! watchdogs configured but OURS_FLEET_SUPERVISOR=none — run 'ours-fleet _run-watchdogs' in the foreground");
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const { changed: unitChanged } = await svc.install(deps.binPath, configPath);
|
|
155
|
+
const fingerprint = watchdogFingerprint(enabled);
|
|
156
|
+
const fingerprintChanged = readStoredFingerprint() !== fingerprint;
|
|
157
|
+
if (unitChanged || fingerprintChanged) {
|
|
158
|
+
// restart(), not start(): start() is a no-op on an already-active unit,
|
|
159
|
+
// so a config change (new/changed watchdogs) would never reach a
|
|
160
|
+
// scheduler that's already running (final review #4).
|
|
161
|
+
await svc.restart();
|
|
162
|
+
deps.log(`↑ watchdogs scheduler (${enabled.map(w => w.name).join(', ')})`);
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
// Idempotent: a no-op on an already-active unit, but still brings up a
|
|
166
|
+
// stopped one (e.g. the scheduler crashed and systemd gave up retrying).
|
|
167
|
+
await svc.start();
|
|
168
|
+
}
|
|
169
|
+
writeFingerprint(fingerprint);
|
|
170
|
+
}
|
|
171
|
+
catch (e) {
|
|
172
|
+
deps.log(` ! watchdogs scheduler: ${e instanceof Error ? e.message : String(e)}`);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
80
175
|
export async function down(cfg, names, deps) {
|
|
81
176
|
for (const role of selectRoles(cfg, names)) {
|
|
82
177
|
// Never swallow the backend's reason. "maybe not running" hid real stop
|
|
@@ -89,6 +184,20 @@ export async function down(cfg, names, deps) {
|
|
|
89
184
|
deps.log(` ! could not stop ${role.name}: ${e instanceof Error ? e.message : String(e)}`);
|
|
90
185
|
}
|
|
91
186
|
}
|
|
187
|
+
// Only a whole-fleet `down` (no names) stops the scheduler — stopping one
|
|
188
|
+
// named role is not a decision to stop watching the others (tolerate
|
|
189
|
+
// absence/errors: a never-installed scheduler must not fail `down`).
|
|
190
|
+
// supervised() gates the call itself (final review #3): on an
|
|
191
|
+
// unsupervised platform/config there is nothing installed to stop, and
|
|
192
|
+
// Linux's stop() throws for a unit that was never loaded.
|
|
193
|
+
if (names.length === 0 && deps.watchdogService?.supervised()) {
|
|
194
|
+
try {
|
|
195
|
+
await deps.watchdogService.stop();
|
|
196
|
+
}
|
|
197
|
+
catch (e) {
|
|
198
|
+
deps.log(` ! could not stop watchdogs scheduler: ${e instanceof Error ? e.message : String(e)}`);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
92
201
|
}
|
|
93
202
|
/** Re-sync from config + bounce. mode 'keep' resumes context; 'fresh' wipes it. */
|
|
94
203
|
export async function restartRoles(cfg, names, deps, mode, configPath) {
|
package/dist/paths.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export declare const stateRoot: () => string;
|
|
|
14
14
|
export declare const agentsRoot: () => string;
|
|
15
15
|
export declare const tmpRoot: () => string;
|
|
16
16
|
export declare const logsRoot: () => string;
|
|
17
|
+
export declare const watchdogsRoot: () => string;
|
|
17
18
|
export declare const agentDir: (name: string, temp?: boolean) => string;
|
|
18
19
|
export declare const defaultConfigPath: () => string;
|
|
19
20
|
export declare const fleetDDir: () => string;
|
package/dist/paths.js
CHANGED
|
@@ -24,6 +24,7 @@ export const stateRoot = () => join(home(), '.ours-fleet');
|
|
|
24
24
|
export const agentsRoot = () => join(stateRoot(), 'agents');
|
|
25
25
|
export const tmpRoot = () => join(stateRoot(), 'tmp');
|
|
26
26
|
export const logsRoot = () => join(stateRoot(), 'logs');
|
|
27
|
+
export const watchdogsRoot = () => join(stateRoot(), 'watchdogs');
|
|
27
28
|
export const agentDir = (name, temp = false) => join(temp ? tmpRoot() : agentsRoot(), name);
|
|
28
29
|
export const defaultConfigPath = () => join(home(), 'fleet.yaml');
|
|
29
30
|
export const fleetDDir = () => join(home(), 'fleet.d');
|
package/dist/resolved-plan.js
CHANGED
|
@@ -20,6 +20,14 @@ export function resolvedPlan(cfg) {
|
|
|
20
20
|
startStaggerMs: cfg.startStaggerMs,
|
|
21
21
|
diagnostics: cfg.diagnostics.map(diagnostic => ({ ...diagnostic })),
|
|
22
22
|
roles: cfg.roles.map(resolvedRolePlan),
|
|
23
|
+
watchdogs: cfg.watchdogs.map(w => sortedObject({
|
|
24
|
+
name: w.name, sourceFile: w.sourceFile, enabled: w.enabled,
|
|
25
|
+
intervalMs: w.intervalMs, coordinator: w.coordinator, watch: [...w.watch],
|
|
26
|
+
harness: w.harness, session: w.session, model: w.model ?? null,
|
|
27
|
+
identity: w.identity, timeoutMs: w.timeoutMs, keepReports: w.keepReports,
|
|
28
|
+
alertCooldownMs: w.alertCooldownMs, promptFile: w.promptFile ?? null,
|
|
29
|
+
isolation: w.isolation ?? null,
|
|
30
|
+
})),
|
|
23
31
|
};
|
|
24
32
|
}
|
|
25
33
|
export function resolvedRolePlan(role) {
|
package/dist/runner.js
CHANGED
|
@@ -11,6 +11,7 @@ import { realExec, shq } from './exec.js';
|
|
|
11
11
|
import { resolveIsolation } from './isolation/policy.js';
|
|
12
12
|
import { selectIsolationBackend } from './isolation/registry.js';
|
|
13
13
|
import { resourceArgs, cpuControllerDelegated } from './isolation/resources.js';
|
|
14
|
+
import { resolveLaunchRuntime } from './isolation/runtime.js';
|
|
14
15
|
import { AcpSession } from './session/acp.js';
|
|
15
16
|
import { RoleControlServer } from './session/control.js';
|
|
16
17
|
import { TmuxSession } from './session/tmux.js';
|
|
@@ -319,7 +320,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
319
320
|
const runCwd = role.cwd && existsSync(role.cwd) ? role.cwd : dir;
|
|
320
321
|
const prep = await adapter.prepareSession(role, { stateDir: dir, runCwd });
|
|
321
322
|
const sessionBackend = role.session ?? 'tmux';
|
|
322
|
-
|
|
323
|
+
let launch = sessionBackend === 'acp'
|
|
323
324
|
? (() => {
|
|
324
325
|
if (!adapter.buildAcpLaunch)
|
|
325
326
|
throw new Error(`harness '${role.harness}' does not support the ACP session backend`);
|
|
@@ -330,10 +331,15 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
330
331
|
// env prefix + exit capture in buildPaneCommand stay host-side (see §5.3).
|
|
331
332
|
let wrappedArgv = launch.argv;
|
|
332
333
|
if (role.isolation) {
|
|
333
|
-
//
|
|
334
|
-
//
|
|
335
|
-
//
|
|
336
|
-
const
|
|
334
|
+
// Start with the SAME durable context config validation and doctor judged
|
|
335
|
+
// (5.2), then add the selected launch's exact runtime closure. Those paths
|
|
336
|
+
// still pass through resolveIsolation's canonical blocklist enforcement.
|
|
337
|
+
const runtime = resolveLaunchRuntime(launch.argv);
|
|
338
|
+
launch = { ...launch, argv: runtime.argv };
|
|
339
|
+
const ctx = {
|
|
340
|
+
...isolationContextFor(role), stateDir: dir, runCwd,
|
|
341
|
+
runtimeReadPaths: runtime.readPaths,
|
|
342
|
+
};
|
|
337
343
|
const policy = resolveIsolation(role.isolation, ctx);
|
|
338
344
|
const sel = await selectIsolationBackend(policy, deps.exec); // throws on strict + unavailable
|
|
339
345
|
const degradedMarker = join(dir, '.isolation-degraded');
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { WatchdogReport, WatchdogRoleStatus } from './report.js';
|
|
2
|
+
import type { WatchManifest } from './briefing.js';
|
|
3
|
+
/**
|
|
4
|
+
* Severity ordering, lowest to highest (must match the briefing contract's alert-rules
|
|
5
|
+
* sentence verbatim, src/watchdog/briefing.ts): healthy = idle (0) < unknown (1) < stale (2)
|
|
6
|
+
* < blocked = unreachable (3) < off_briefing (4).
|
|
7
|
+
*/
|
|
8
|
+
export declare const WATCHDOG_STATUS_RANK: Record<WatchdogRoleStatus, number>;
|
|
9
|
+
export interface OpenFinding {
|
|
10
|
+
role: string;
|
|
11
|
+
status: WatchdogRoleStatus;
|
|
12
|
+
since: string;
|
|
13
|
+
lastAlertedAt: string | null;
|
|
14
|
+
}
|
|
15
|
+
export interface AlertLedger {
|
|
16
|
+
version: 1;
|
|
17
|
+
open: Record<string, OpenFinding>;
|
|
18
|
+
heldDownAlerted: boolean;
|
|
19
|
+
}
|
|
20
|
+
/** Missing or corrupt ledger file yields a clean empty ledger rather than throwing. */
|
|
21
|
+
export declare function readLedger(name: string): AlertLedger;
|
|
22
|
+
export declare function writeLedger(name: string, l: AlertLedger): void;
|
|
23
|
+
/**
|
|
24
|
+
* Reconcile rules (spec §5): an `error`-status report carries no role evidence, so the ledger
|
|
25
|
+
* is returned unchanged. For each finding with rank > 0: an existing open entry keeps its
|
|
26
|
+
* `since` and updates `status` on escalation/de-escalation; a new one opens with `since = now`.
|
|
27
|
+
* Any role reported `healthy`/`idle` closes (deletes) its open entry. Every role named in
|
|
28
|
+
* `report.alerts` gets `lastAlertedAt = now`. Roles absent from the report keep their entries —
|
|
29
|
+
* a watchdog with a narrowed `watch:` set doesn't silently resolve findings it never inspected.
|
|
30
|
+
* Never mutates `l` — always returns a new ledger object.
|
|
31
|
+
*/
|
|
32
|
+
export declare function reconcileLedger(l: AlertLedger, report: WatchdogReport, now: Date): AlertLedger;
|
|
33
|
+
/** Ledger's open findings + cooldown, in the shape watch.json's `digest` field carries (Task 7 writes it, run reads it back). */
|
|
34
|
+
export declare function computeDigest(l: AlertLedger, cooldownMs: number, now: Date): WatchManifest['digest'];
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { chmodSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { watchdogDir } from './store.js';
|
|
4
|
+
/**
|
|
5
|
+
* Severity ordering, lowest to highest (must match the briefing contract's alert-rules
|
|
6
|
+
* sentence verbatim, src/watchdog/briefing.ts): healthy = idle (0) < unknown (1) < stale (2)
|
|
7
|
+
* < blocked = unreachable (3) < off_briefing (4).
|
|
8
|
+
*/
|
|
9
|
+
export const WATCHDOG_STATUS_RANK = {
|
|
10
|
+
healthy: 0, idle: 0, unknown: 1, stale: 2, blocked: 3, unreachable: 3, off_briefing: 4,
|
|
11
|
+
};
|
|
12
|
+
function emptyLedger() {
|
|
13
|
+
return { version: 1, open: {}, heldDownAlerted: false };
|
|
14
|
+
}
|
|
15
|
+
function ledgerPath(name) {
|
|
16
|
+
return join(watchdogDir(name), 'alerts.json');
|
|
17
|
+
}
|
|
18
|
+
/** Missing or corrupt ledger file yields a clean empty ledger rather than throwing. */
|
|
19
|
+
export function readLedger(name) {
|
|
20
|
+
try {
|
|
21
|
+
return JSON.parse(readFileSync(ledgerPath(name), 'utf8'));
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return emptyLedger();
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export function writeLedger(name, l) {
|
|
28
|
+
const path = ledgerPath(name);
|
|
29
|
+
writeFileSync(path, JSON.stringify(l, null, 2) + '\n', { mode: 0o600 });
|
|
30
|
+
chmodSync(path, 0o600);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Reconcile rules (spec §5): an `error`-status report carries no role evidence, so the ledger
|
|
34
|
+
* is returned unchanged. For each finding with rank > 0: an existing open entry keeps its
|
|
35
|
+
* `since` and updates `status` on escalation/de-escalation; a new one opens with `since = now`.
|
|
36
|
+
* Any role reported `healthy`/`idle` closes (deletes) its open entry. Every role named in
|
|
37
|
+
* `report.alerts` gets `lastAlertedAt = now`. Roles absent from the report keep their entries —
|
|
38
|
+
* a watchdog with a narrowed `watch:` set doesn't silently resolve findings it never inspected.
|
|
39
|
+
* Never mutates `l` — always returns a new ledger object.
|
|
40
|
+
*/
|
|
41
|
+
export function reconcileLedger(l, report, now) {
|
|
42
|
+
if (report.status === 'error')
|
|
43
|
+
return l;
|
|
44
|
+
const open = { ...l.open };
|
|
45
|
+
const nowIso = now.toISOString();
|
|
46
|
+
for (const finding of report.roles) {
|
|
47
|
+
const rank = WATCHDOG_STATUS_RANK[finding.status];
|
|
48
|
+
if (rank > 0) {
|
|
49
|
+
const existing = open[finding.role];
|
|
50
|
+
open[finding.role] = {
|
|
51
|
+
role: finding.role,
|
|
52
|
+
status: finding.status,
|
|
53
|
+
since: existing ? existing.since : nowIso,
|
|
54
|
+
lastAlertedAt: existing ? existing.lastAlertedAt : null,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
delete open[finding.role];
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
for (const alert of report.alerts) {
|
|
62
|
+
if (open[alert.role])
|
|
63
|
+
open[alert.role] = { ...open[alert.role], lastAlertedAt: nowIso };
|
|
64
|
+
}
|
|
65
|
+
return { ...l, open };
|
|
66
|
+
}
|
|
67
|
+
/** Ledger's open findings + cooldown, in the shape watch.json's `digest` field carries (Task 7 writes it, run reads it back). */
|
|
68
|
+
export function computeDigest(l, cooldownMs, now) {
|
|
69
|
+
return {
|
|
70
|
+
cooldown_ms: cooldownMs,
|
|
71
|
+
open: Object.values(l.open).map(f => ({
|
|
72
|
+
role: f.role,
|
|
73
|
+
status: f.status,
|
|
74
|
+
since: f.since,
|
|
75
|
+
realert_after: f.lastAlertedAt ? new Date(new Date(f.lastAlertedAt).getTime() + cooldownMs).toISOString() : null,
|
|
76
|
+
})),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import type { ResolvedWatchdog } from './config.js';
|
|
2
|
+
import type { BriefingVocab } from '../harness/types.js';
|
|
3
|
+
/** One entry in `watch.json`'s `roles` array — where to find a watched role's state. */
|
|
4
|
+
export interface WatchManifestRole {
|
|
5
|
+
name: string;
|
|
6
|
+
stateDir: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* The fixed run manifest a watchdog run reads at `manifestPath` (7.3 §3, Task 7 writes it,
|
|
10
|
+
* Task 12+ digest reads it back). It carries everything the run needs to identify itself in
|
|
11
|
+
* report.json (`watchdog`, `run_id`, `started_at`) plus the suppression digest that makes
|
|
12
|
+
* alerting idempotent across runs.
|
|
13
|
+
*/
|
|
14
|
+
export interface WatchManifest {
|
|
15
|
+
watchdog: string;
|
|
16
|
+
run_id: string;
|
|
17
|
+
coordinator: string;
|
|
18
|
+
started_at: string;
|
|
19
|
+
roles: WatchManifestRole[];
|
|
20
|
+
digest: {
|
|
21
|
+
cooldown_ms: number;
|
|
22
|
+
open: Array<{
|
|
23
|
+
role: string;
|
|
24
|
+
status: string;
|
|
25
|
+
since: string;
|
|
26
|
+
realert_after: string | null;
|
|
27
|
+
}>;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export interface WatchdogBriefingOpts {
|
|
31
|
+
wd: ResolvedWatchdog;
|
|
32
|
+
manifestPath: string;
|
|
33
|
+
reportPath: string;
|
|
34
|
+
vocabulary: BriefingVocab;
|
|
35
|
+
/** What spawn actually established about the watchdog's ours identity (7.3, decision 3). */
|
|
36
|
+
identityGuarantee: 'verified' | 'created' | 'unverified';
|
|
37
|
+
/** Raw prompt_file content, appended as extra focus — never a replacement (owner decision 1). */
|
|
38
|
+
promptFocus?: string;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Render a watchdog run's fixed contract (briefing.md-equivalent for a one-shot clean-context
|
|
42
|
+
* run): bind, observe-only rules, procedure, status vocabulary, evidence rules, alert rules,
|
|
43
|
+
* report schema, and — if the watchdog configures one — an appended prompt_file focus.
|
|
44
|
+
*
|
|
45
|
+
* The contract is fixed for every watchdog (spec §5: "The schema is fixed for every watchdog,
|
|
46
|
+
* including ones with a prompt_file; the override adds focus, never fields"), so every section
|
|
47
|
+
* below is unconditional except the identity-guarantee wording and the trailing focus append.
|
|
48
|
+
*/
|
|
49
|
+
export declare function generateWatchdogBriefing(opts: WatchdogBriefingOpts): string;
|
|
50
|
+
export interface NotifierBriefingOpts {
|
|
51
|
+
wd: ResolvedWatchdog;
|
|
52
|
+
vocabulary: BriefingVocab;
|
|
53
|
+
/** What spawn actually established about the watchdog's ours identity (7.3, decision 3). */
|
|
54
|
+
identityGuarantee: 'verified' | 'created' | 'unverified';
|
|
55
|
+
/** The exact message to relay — the scheduler composed this, the notifier only delivers it. */
|
|
56
|
+
text: string;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Render a scheduler-alert notifier run's entire contract: a minimal one-shot agent whose sole
|
|
60
|
+
* job is to bind its identity, send one exact message to the coordinator, and write `sent.json`
|
|
61
|
+
* as its completion sentinel. Used only for scheduler-level alerts (e.g. held-down) — the fleet
|
|
62
|
+
* process itself cannot send ours messages (deviation 4), so this is how it delegates the send.
|
|
63
|
+
* Unlike `generateWatchdogBriefing`, there is no report.json, no manifest, and no inspection.
|
|
64
|
+
*/
|
|
65
|
+
export declare function generateNotifierBriefing(opts: NotifierBriefingOpts): string;
|