@ours.network/fleet 0.10.4 → 0.11.1

package/dist/watchdog/report.js

@@ -0,0 +1,126 @@
+export const WATCHDOG_ROLE_STATUSES = ['healthy', 'idle', 'stale', 'blocked', 'off_briefing', 'unreachable', 'unknown'];
+const SUMMARY_KEYS = ['checked', 'healthy', 'idle', 'anomalies'];
+function isPlainObject(v) {
+    return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+export function validateWatchdogReport(v) {
+    const errors = [];
+    if (!isPlainObject(v)) {
+        errors.push('report: expected an object');
+        return errors;
+    }
+    const r = v;
+    if (r.schema_version !== 1)
+        errors.push('schema_version: expected 1');
+    for (const key of ['watchdog', 'run_id', 'started_at', 'finished_at'])
+        if (typeof r[key] !== 'string')
+            errors.push(`${key}: expected a string`);
+    if (r.status !== 'ok' && r.status !== 'anomalies' && r.status !== 'error')
+        errors.push(`status: expected ok|anomalies|error, got '${String(r.status)}'`);
+    if (!isPlainObject(r.summary)) {
+        errors.push('summary: expected an object');
+    }
+    else {
+        for (const key of SUMMARY_KEYS)
+            if (typeof r.summary[key] !== 'number')
+                errors.push(`summary.${key}: expected a number`);
+    }
+    if (!Array.isArray(r.roles)) {
+        errors.push('roles: expected an array');
+    }
+    else {
+        r.roles.forEach((role, i) => {
+            if (!isPlainObject(role)) {
+                errors.push(`roles[${i}]: expected an object`);
+                return;
+            }
+            if (typeof role.role !== 'string')
+                errors.push(`roles[${i}].role: expected a string`);
+            if (!WATCHDOG_ROLE_STATUSES.includes(role.status))
+                errors.push(`roles[${i}].status: expected one of ${WATCHDOG_ROLE_STATUSES.join('|')}, got '${String(role.status)}'`);
+            if (role.status !== 'healthy' && role.status !== 'idle' && !role.reason)
+                errors.push(`roles[${i}]: non-healthy finding requires a reason`);
+            if (role.reason !== undefined && typeof role.reason !== 'string')
+                errors.push(`roles[${i}].reason: expected a string`);
+            if (role.evidence !== undefined) {
+                if (!Array.isArray(role.evidence)) {
+                    errors.push(`roles[${i}].evidence: expected an array`);
+                }
+                else {
+                    role.evidence.forEach((ev, j) => {
+                        if (!isPlainObject(ev)) {
+                            errors.push(`roles[${i}].evidence[${j}]: expected an object`);
+                            return;
+                        }
+                        for (const key of ['source', 'detail', 'observed_at'])
+                            if (typeof ev[key] !== 'string')
+                                errors.push(`roles[${i}].evidence[${j}].${key}: expected a string`);
+                    });
+                }
+            }
+        });
+    }
+    if (!Array.isArray(r.alerts)) {
+        errors.push('alerts: expected an array');
+    }
+    else {
+        r.alerts.forEach((alert, i) => {
+            if (!isPlainObject(alert)) {
+                errors.push(`alerts[${i}]: expected an object`);
+                return;
+            }
+            for (const key of ['role', 'code', 'coordinator', 'sent_at'])
+                if (typeof alert[key] !== 'string')
+                    errors.push(`alerts[${i}].${key}: expected a string`);
+        });
+    }
+    if (r.error !== null && typeof r.error !== 'string')
+        errors.push('error: expected string or null');
+    return errors;
+}
+const cleanEvidence = (value, max = 280) => value.replace(/[\0-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '').trim().slice(0, max);
+export function normalizeWatchdogReport(r, ctx) {
+    const copy = JSON.parse(JSON.stringify(r));
+    copy.watchdog = ctx.watchdog;
+    copy.run_id = ctx.run_id;
+    copy.roles = copy.roles.map(role => {
+        // Rebuilt from ONLY the known fields (final review #6a) — the old
+        // `{ ...role }` spread let any unknown per-finding key (e.g. an agent
+        // dumping a huge `pane_dump`) ride unbounded into the store. Report-level
+        // extras (tail/isolation) are untouched here and stay tolerated; this is
+        // a per-finding allowlist only.
+        const next = { role: role.role, status: role.status };
+        if (role.reason !== undefined)
+            next.reason = cleanEvidence(role.reason);
+        if (role.evidence !== undefined) {
+            next.evidence = role.evidence.slice(0, 3).map(ev => ({
+                source: cleanEvidence(ev.source),
+                detail: cleanEvidence(ev.detail),
+                observed_at: cleanEvidence(ev.observed_at),
+            }));
+        }
+        if (role.alerted !== undefined)
+            next.alerted = Boolean(role.alerted);
+        return next;
+    });
+    return copy;
+}
+export function errorReport(ctx) {
+    return {
+        schema_version: 1,
+        watchdog: ctx.watchdog,
+        run_id: ctx.run_id,
+        started_at: ctx.started_at,
+        finished_at: ctx.finished_at,
+        status: 'error',
+        summary: { checked: 0, healthy: 0, idle: 0, anomalies: 0 },
+        roles: [],
+        alerts: [],
+        // Scheduler-built error strings can embed raw agent-controlled bytes
+        // (JSON.parse messages, validator interpolations of agent-supplied
+        // values) and are stored+printed as-is elsewhere — clean them the same
+        // way finding-level text is cleaned (final review #6b).
+        error: cleanEvidence(ctx.error),
+        ...(ctx.tail !== undefined ? { tail: ctx.tail.slice(-4096) } : {}),
+    };
+}

package/dist/watchdog/run.d.ts

@@ -0,0 +1,63 @@
+import type { ResolvedWatchdog } from './config.js';
+import type { WatchdogReport } from './report.js';
+import { type IdentityProvisioner } from '../creation.js';
+import { type FleetConfig } from '../config.js';
+/** A minimal handle over the launched child: kill it, or await its natural exit. */
+export interface WatchdogChildHandle {
+    kill(): void;
+    exited: Promise<void>;
+}
+export interface WatchdogRunDeps {
+    binPath: string;
+    log(line: string): void;
+    now?(): Date;
+    sleep?(ms: number): Promise<void>;
+    identityProvisioner?: IdentityProvisioner;
+    /**
+     * Injectable child launcher for tests. Default: spawn
+     * `node <binPath> _run-watchdog <roleName>` detached:false, stdio to
+     * `<runDir>/run.log`. Returns kill() and an exited promise.
+     */
+    launchChild?(binPath: string, roleName: string, runDir: string): WatchdogChildHandle;
+    /** Pre-loaded config (defaults inheritance). Falls back to `loadConfig()`. */
+    cfg?: FleetConfig;
+    /** Live temporary-role discovery override for focused tests. */
+    discoverLiveTemporaryRoles?(): Promise<string[]>;
+}
+export interface WatchdogRunOutcome {
+    report: WatchdogReport;
+    storedPath: string;
+}
+/**
+ * Run one watchdog agent end-to-end in a clean-context temp state dir: provision
+ * identity, materialize the run's contract (briefing/manifest/role snapshot),
+ * launch the child, enforce a deadline against report.json as the completion
+ * sentinel, harvest whatever it wrote (or a synthetic error report if it
+ * didn't), store the result, and always clean up the temp dir.
+ *
+ * The scheduler's run-lock guarantees only one run per watchdog at a time;
+ * this function itself takes no lock (Task 8).
+ */
+export declare function executeWatchdogRun(wd: ResolvedWatchdog, deps: WatchdogRunDeps): Promise<WatchdogRunOutcome>;
+/**
+ * Deliver one scheduler-level alert (e.g. held-down) by launching a minimal one-shot temp agent
+ * under the watchdog's own identity whose entire mission is: bind identity, send `text` to
+ * `wd.coordinator`, write `sent.json`, exit. The fleet process itself cannot send ours messages
+ * (owner-approved deviation 4), so this is how a scheduler tick that can't reach an operator any
+ * other way still gets a message out.
+ *
+ * Reuses the Task 7/8 run machinery (temp-dir prep, ensureIdentity, ResolvedRole shape, child
+ * launch, kill mechanics) but stores no report, touches no ledger findings, and writes no
+ * watch.json manifest — `sent.json` is this run's only completion sentinel. A fixed 2-minute
+ * deadline applies regardless of `wd.timeoutMs`.
+ *
+ * Unlike executeWatchdogRun (which relies on the scheduler's run lock), this function acquires
+ * and releases the run lock itself — a notifier run shares the same temp dir name
+ * (`agentDir(wd.identity, true)`) as a regular run, and nothing else serializes the two. Refusal
+ * to acquire is logged and treated as a no-op: the held-down state already means no regular runs
+ * are in flight, so a collision is rare. Any other failure (including a timeout) is logged as a
+ * warning and swallowed — a notifier failure must never throw into the scheduler loop.
+ */
+export declare function executeNotifierRun(wd: ResolvedWatchdog, text: string, deps: WatchdogRunDeps): Promise<void>;
+/** What `_run-watchdog` calls: one supervised session, no cleanup — the parent harvests. */
+export declare function runWatchdogAgent(name: string): Promise<void>;

package/dist/watchdog/run.js

@@ -0,0 +1,345 @@
+import { closeSync, existsSync, openSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync, } from 'node:fs';
+import { spawn as spawnChild, execFile } from 'node:child_process';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import { stringify } from 'yaml';
+import { errorReport, normalizeWatchdogReport, validateWatchdogReport } from './report.js';
+import { acquireRunLock, formatRunId, pruneReports, releaseRunLock, writeReport } from './store.js';
+import { generateNotifierBriefing, generateWatchdogBriefing } from './briefing.js';
+import { computeDigest, reconcileLedger, readLedger, writeLedger } from './alerts.js';
+import { applyRole } from '../ops.js';
+import { daemonIdentityProvisioner, ensureIdentity, } from '../creation.js';
+import { runOnce, START_STAGGER_FILE } from '../runner.js';
+import { agentDir, tmpRoot } from '../paths.js';
+import { loadConfig, resolveMonitorConfig, resolveWorklogPolicy, ROLE_NAME_RE, } from '../config.js';
+import { getAdapter } from '../harness/registry.js';
+import { redactLogLine } from '../application/log-service.js';
+import { controlRequest, controlSocketPath } from '../session/control.js';
+import { Tmux } from '../tmux.js';
+const execFileAsync = promisify(execFile);
+/** How often the deadline loop polls for a completed report or a dead child. */
+const POLL_MS = 1000;
+/** A `report.json` this old, with no further writes, is treated as finished. */
+const WRITE_STABLE_MS = 2000;
+/** Grace given to the agent after a stable report before the session is killed. */
+const HARVEST_GRACE_MS = 5000;
+/** Fixed deadline for a one-shot notifier run — always 2 minutes, never wd.timeoutMs (Task 14). */
+const NOTIFIER_TIMEOUT_MS = 120_000;
+/**
+ * Poll for a completion sentinel file (report.json for an inspection run, sent.json for a
+ * notifier run) going write-stable, or the child exiting first, or the deadline passing —
+ * whichever comes first. Shared by executeWatchdogRun and executeNotifierRun so the two run
+ * flavors' wait loops can never drift apart.
+ */
+async function waitForSentinel(sentinelPath, child, deadlineMs, start, now, sleep) {
+    let exited = false;
+    // Attached now so a natural exit is observed even though the loop below never
+    // awaits the promise itself (its resolution races real time in the real
+    // launcher, and is driven purely by test fixtures in tests).
+    child.exited.then(() => { exited = true; }).catch(() => { exited = true; });
+    for (;;) {
+        if (exited)
+            return 'exited';
+        if (existsSync(sentinelPath)) {
+            const age = now().getTime() - statSync(sentinelPath).mtimeMs;
+            if (age >= WRITE_STABLE_MS)
+                return 'stable';
+        }
+        if (now().getTime() - start.getTime() > deadlineMs)
+            return 'timeout';
+        await sleep(POLL_MS);
+    }
+}
+/** After waitForSentinel settles, harvest the grace period (if stable) then kill the child+session. */
+async function killIfNeeded(reason, child, roleName, sleep) {
+    if (reason === 'stable' || reason === 'timeout') {
+        if (reason === 'stable')
+            await sleep(HARVEST_GRACE_MS);
+        child.kill();
+        await killTmuxSession(roleName);
+    }
+}
+function defaultLaunchChild(binPath, roleName, runDir) {
+    const out = openSync(join(runDir, 'run.log'), 'a');
+    const child = spawnChild(process.execPath, [binPath, '_run-watchdog', roleName], {
+        detached: false, stdio: ['ignore', out, out],
+    });
+    // spawn() dup's the fd into the child; Node never closes our copy on its own,
+    // so a long-running scheduler launching many watchdog runs would leak one fd
+    // per run and eventually hit EMFILE. Safe to close immediately — the child
+    // keeps writing to its own dup'd descriptor.
+    closeSync(out);
+    return {
+        kill: () => { child.kill(); },
+        exited: new Promise(resolve => {
+            child.once('exit', () => resolve());
+            child.once('error', () => resolve());
+        }),
+    };
+}
+/** Best-effort: the tmux session outlives the supervisor child (`runOnce` created it). */
+async function killTmuxSession(roleName) {
+    try {
+        await execFileAsync('tmux', ['kill-session', '-t', roleName]);
+    }
+    catch { /* best effort */ }
+}
+/** Last 4096 chars of run.log, redacted — attached to error reports as diagnostic tail. */
+function readTail(runDir) {
+    try {
+        const raw = readFileSync(join(runDir, 'run.log'), 'utf8');
+        return redactLogLine(raw.slice(-4096)).text;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** The temp role every watchdog-family run launches under. Isolation is opt-in. */
+function buildWatchdogRole(wd, cfg) {
+    return {
+        name: wd.identity, sourceFile: '(watchdog)',
+        harness: wd.harness, session: wd.session,
+        identity: wd.identity, model: wd.model,
+        // Watchdogs are observe-only by contract, but their sanctioned status commands
+        // must reach host control sockets. Keep approvals/unattended escalation denied
+        // while disabling the harness's native filesystem/network sandbox.
+        permissions: { approval: 'deny', filesystem: 'unrestricted', unattended: 'deny' },
+        permissionsDeclared: true,
+        monitor: resolveMonitorConfig(cfg.defaults.monitor, undefined),
+        worklog: resolveWorklogPolicy(cfg.defaults.worklog, undefined),
+        isolation: wd.isolation,
+    };
+}
+/** Discover temporary sessions that are live now, not merely stale dirs on disk. */
+async function discoverLiveTemporaryRoles() {
+    let names;
+    try {
+        names = readdirSync(tmpRoot(), { withFileTypes: true })
+            .filter(entry => entry.isDirectory() && !entry.isSymbolicLink() && ROLE_NAME_RE.test(entry.name))
+            .map(entry => entry.name);
+    }
+    catch {
+        return [];
+    }
+    const tmux = new Tmux();
+    const live = await Promise.all(names.map(async (name) => {
+        const dir = agentDir(name, true);
+        const [acpAlive, tmuxAlive] = await Promise.all([
+            existsSync(controlSocketPath(dir))
+                ? controlRequest(dir, { command: 'status' }, 2_000)
+                    .then(response => response.ok
+                    && response.result?.alive === true)
+                    .catch(() => false)
+                : Promise.resolve(false),
+            tmux.has(name).catch(() => false),
+        ]);
+        return acpAlive || tmuxAlive ? name : undefined;
+    }));
+    return live.filter((name) => name !== undefined);
+}
+/**
+ * Run one watchdog agent end-to-end in a clean-context temp state dir: provision
+ * identity, materialize the run's contract (briefing/manifest/role snapshot),
+ * launch the child, enforce a deadline against report.json as the completion
+ * sentinel, harvest whatever it wrote (or a synthetic error report if it
+ * didn't), store the result, and always clean up the temp dir.
+ *
+ * The scheduler's run-lock guarantees only one run per watchdog at a time;
+ * this function itself takes no lock (Task 8).
+ */
+export async function executeWatchdogRun(wd, deps) {
+    const now = deps.now ?? (() => new Date());
+    const sleep = deps.sleep ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)));
+    const start = now();
+    const runId = formatRunId(start);
+    const startedAt = start.toISOString();
+    const roleName = wd.identity;
+    const runDir = agentDir(roleName, true);
+    // Read once, before the manifest is written: the same snapshot both seeds the
+    // digest the agent sees and is the base the post-run reconcile folds into.
+    const ledger = readLedger(wd.name);
+    // A crashed previous run can leave the temp dir behind; start clean.
+    if (existsSync(runDir))
+        rmSync(runDir, { recursive: true, force: true });
+    let report;
+    try {
+        const guarantee = await ensureIdentity(wd.identity, {}, deps.identityProvisioner ?? daemonIdentityProvisioner(), deps.log);
+        const cfg = deps.cfg ?? loadConfig();
+        const discovered = wd.watchExplicit
+            ? []
+            : await (deps.discoverLiveTemporaryRoles ?? discoverLiveTemporaryRoles)();
+        const watch = [...new Set([...wd.watch, ...discovered])]
+            .filter(name => name !== wd.identity);
+        const runWd = { ...wd, watch };
+        const role = buildWatchdogRole(runWd, cfg);
+        const dir = applyRole(role, { temp: true, identityGuarantee: guarantee.state });
+        const reportPath = join(dir, 'report.json');
+        const manifestPath = join(dir, 'watch.json');
+        const promptFocus = wd.promptFile ? readFileSync(wd.promptFile, 'utf8') : undefined;
+        // applyRole wrote the generic role briefing; the watchdog contract replaces it.
+        writeFileSync(join(dir, 'briefing.md'), generateWatchdogBriefing({
+            wd: runWd, manifestPath, reportPath,
+            vocabulary: getAdapter(wd.harness).vocabulary,
+            identityGuarantee: guarantee.state,
+            promptFocus,
+        }));
+        // loadTempRole (runner.ts) needs this to run the child via `_run-watchdog`.
+        writeFileSync(join(dir, 'role.yaml'), stringify(role));
+        const manifest = {
+            watchdog: wd.name, run_id: runId, coordinator: wd.coordinator, started_at: startedAt,
+            roles: runWd.watch.map(r => ({
+                name: r,
+                stateDir: wd.watch.includes(r) ? agentDir(r) : agentDir(r, true),
+            })),
+            digest: computeDigest(ledger, wd.alertCooldownMs, now()),
+        };
+        writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
+        // Snapshot the fleet start-stagger so the detached run (no config path
+        // threaded through `_run-watchdog`) honors the same host-wide launch gate
+        // as every other role (spec §3) — mirrors src/spawn.ts:503-504.
+        if (cfg.startStaggerMs > 0)
+            writeFileSync(join(dir, START_STAGGER_FILE), String(cfg.startStaggerMs));
+        const launch = deps.launchChild ?? defaultLaunchChild;
+        const child = launch(deps.binPath, roleName, dir);
+        const reason = await waitForSentinel(reportPath, child, wd.timeoutMs, start, now, sleep);
+        await killIfNeeded(reason, child, roleName, sleep);
+        const timedOut = reason === 'timeout';
+        if (timedOut) {
+            report = errorReport({
+                watchdog: wd.name, run_id: runId, started_at: startedAt, finished_at: now().toISOString(),
+                error: 'timeout', tail: readTail(dir),
+            });
+        }
+        else {
+            let parsed;
+            let parseError;
+            try {
+                parsed = JSON.parse(readFileSync(reportPath, 'utf8'));
+            }
+            catch (e) {
+                parseError = e instanceof Error ? e.message : String(e);
+            }
+            const validationErrors = parseError === undefined ? validateWatchdogReport(parsed) : [];
+            if (parseError !== undefined || validationErrors.length) {
+                const detail = parseError ?? validationErrors.slice(0, 3).join('; ');
+                report = errorReport({
+                    watchdog: wd.name, run_id: runId, started_at: startedAt, finished_at: now().toISOString(),
+                    error: `invalid report: ${detail}`, tail: readTail(dir),
+                });
+            }
+            else {
+                report = normalizeWatchdogReport(parsed, { watchdog: wd.name, run_id: runId });
+                // Scheduler clock truth overrides whatever the agent wrote.
+                report.started_at = startedAt;
+                report.finished_at = now().toISOString();
+            }
+        }
+        // Isolation degradation (spec §7): a weaker guarantee must never look like
+        // the strong one, so stamp it on every outcome, error reports included.
+        if (existsSync(join(dir, '.isolation-degraded')))
+            report.isolation = 'degraded';
+    }
+    finally {
+        rmSync(runDir, { recursive: true, force: true });
+    }
+    const storedPath = writeReport(wd.name, report);
+    // Reconciles against the FINAL stored report (normalized or error) — reconcileLedger
+    // itself no-ops on error-status reports, so a timeout/invalid run leaves the ledger
+    // byte-for-byte unchanged.
+    writeLedger(wd.name, reconcileLedger(ledger, report, now()));
+    pruneReports(wd.name, wd.keepReports);
+    return { report, storedPath };
+}
+/**
+ * Deliver one scheduler-level alert (e.g. held-down) by launching a minimal one-shot temp agent
+ * under the watchdog's own identity whose entire mission is: bind identity, send `text` to
+ * `wd.coordinator`, write `sent.json`, exit. The fleet process itself cannot send ours messages
+ * (owner-approved deviation 4), so this is how a scheduler tick that can't reach an operator any
+ * other way still gets a message out.
+ *
+ * Reuses the Task 7/8 run machinery (temp-dir prep, ensureIdentity, ResolvedRole shape, child
+ * launch, kill mechanics) but stores no report, touches no ledger findings, and writes no
+ * watch.json manifest — `sent.json` is this run's only completion sentinel. A fixed 2-minute
+ * deadline applies regardless of `wd.timeoutMs`.
+ *
+ * Unlike executeWatchdogRun (which relies on the scheduler's run lock), this function acquires
+ * and releases the run lock itself — a notifier run shares the same temp dir name
+ * (`agentDir(wd.identity, true)`) as a regular run, and nothing else serializes the two. Refusal
+ * to acquire is logged and treated as a no-op: the held-down state already means no regular runs
+ * are in flight, so a collision is rare. Any other failure (including a timeout) is logged as a
+ * warning and swallowed — a notifier failure must never throw into the scheduler loop.
+ */
+export async function executeNotifierRun(wd, text, deps) {
+    const now = deps.now ?? (() => new Date());
+    const sleep = deps.sleep ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)));
+    const fail = (e) => {
+        deps.log(`notifier run for '${wd.name}' failed: ${e instanceof Error ? e.message : String(e)}`);
+    };
+    // acquireRunLock can itself throw (EACCES/EIO, not just EEXIST — store.ts) — guarded
+    // separately from the try/finally below so a throw here never reaches the caller either;
+    // nothing was acquired, so there is nothing to release.
+    let acquired = false;
+    try {
+        acquired = acquireRunLock(wd.name);
+    }
+    catch (e) {
+        fail(e);
+        return;
+    }
+    if (!acquired) {
+        deps.log(`notifier run for '${wd.name}' skipped: run lock held`);
+        return;
+    }
+    const roleName = wd.identity;
+    const runDir = agentDir(roleName, true);
+    try {
+        // A crashed previous run can leave the temp dir behind; start clean.
+        if (existsSync(runDir))
+            rmSync(runDir, { recursive: true, force: true });
+        const guarantee = await ensureIdentity(wd.identity, {}, deps.identityProvisioner ?? daemonIdentityProvisioner(), deps.log);
+        const cfg = deps.cfg ?? loadConfig();
+        const role = buildWatchdogRole(wd, cfg);
+        const dir = applyRole(role, { temp: true, identityGuarantee: guarantee.state });
+        const sentinelPath = join(dir, 'sent.json');
+        // applyRole wrote the generic role briefing; the notifier contract replaces it.
+        writeFileSync(join(dir, 'briefing.md'), generateNotifierBriefing({
+            wd, vocabulary: getAdapter(wd.harness).vocabulary,
+            identityGuarantee: guarantee.state, text,
+        }));
+        // loadTempRole (runner.ts) needs this to run the child via `_run-watchdog`.
+        writeFileSync(join(dir, 'role.yaml'), stringify(role));
+        // Same host-wide launch-gate snapshot executeWatchdogRun writes (spec §3).
+        if (cfg.startStaggerMs > 0)
+            writeFileSync(join(dir, START_STAGGER_FILE), String(cfg.startStaggerMs));
+        const launch = deps.launchChild ?? defaultLaunchChild;
+        const start = now();
+        const child = launch(deps.binPath, roleName, dir);
+        const reason = await waitForSentinel(sentinelPath, child, NOTIFIER_TIMEOUT_MS, start, now, sleep);
+        await killIfNeeded(reason, child, roleName, sleep);
+        if (reason === 'timeout')
+            deps.log(`notifier run for '${wd.name}' timed out`);
+    }
+    catch (e) {
+        fail(e);
+    }
+    finally {
+        // Same "never escape" rule as the lock acquire above: cleanup and release are each
+        // guarded individually so neither can turn a handled failure into an unhandled one.
+        try {
+            rmSync(runDir, { recursive: true, force: true });
+        }
+        catch (e) {
+            fail(e);
+        }
+        try {
+            releaseRunLock(wd.name);
+        }
+        catch (e) {
+            fail(e);
+        }
+    }
+}
+/** What `_run-watchdog` calls: one supervised session, no cleanup — the parent harvests. */
+export async function runWatchdogAgent(name) {
+    await runOnce(name, { temp: true });
+}

package/dist/watchdog/scheduler.d.ts

@@ -0,0 +1,105 @@
+import type { FleetConfig } from '../config.js';
+import type { ResolvedWatchdog } from './config.js';
+import { executeNotifierRun, executeWatchdogRun } from './run.js';
+export interface WatchdogSchedulerState {
+    version: 1;
+    consecutiveFailures: number;
+    heldDown: boolean;
+    heldSince?: string;
+    lastRunAt?: string;
+    nextRunAt?: string;
+    lastError?: string;
+}
+/** Read a watchdog's scheduler state; a missing or corrupt file starts clean (mirrors readRestartLedger). */
+export declare function readSchedulerState(name: string): WatchdogSchedulerState;
+/** Write never throws: scheduler diagnostics must never take the loop down. */
+export declare function writeSchedulerState(name: string, s: WatchdogSchedulerState): void;
+/**
+ * Operator release (Task 15): clears failures and heldDown so a held-down
+ * loop's next held-down poll sees a clean state and resumes running. Also
+ * clears the ledger's `heldDownAlerted` flag — that flag is what makes "alert
+ * once per hold-down" durable across scheduler restarts (it lives in
+ * alerts.json, not state.json), so a release must reset it too or a
+ * subsequent hold-down would silently alert zero times.
+ *
+ * Also reclaims a STALE run lock (final review #2, tightened by finding #2):
+ * a watchdog SIGKILLed mid-run (e.g. systemd's TimeoutStopSec on a
+ * fleet-wide stop) leaves `.run-lock` behind forever — the loop's `finally`
+ * that would normally release it never runs. Without this, every future tick
+ * sees the lock held and reports `skipped_overlap` indefinitely, and skips
+ * never alert. `ours-fleet restart <watchdog>` is the documented recovery, so
+ * it must clear a stale lock too, not just the failure/hold-down bookkeeping.
+ * But only a DEMONSTRABLY stale lock (dead owner pid, or legacy lock with no
+ * owner metadata) — a lock genuinely held by a live run (foreground
+ * `watchdog-run`, another scheduler instance) must survive an operator's
+ * `restart` of a DIFFERENT problem (e.g. releasing hold-down) unrelated to
+ * that live run; two runs sharing the same temp dir would corrupt each
+ * other's output. Best-effort: a reclaim failure here must not turn an
+ * operator's recovery action into a crash.
+ */
+export declare function resetSchedulerState(name: string): void;
+export declare const WATCHDOG_HOLD_THRESHOLD = 3;
+export declare const WATCHDOG_BACKOFF_MAX_MS = 3600000;
+/** Bounded exponential backoff: 1x, 2x, 4x, ... capped at WATCHDOG_BACKOFF_MAX_MS (spec §3). */
+export declare function watchdogBackoffMs(intervalMs: number, failures: number): number;
+export interface SchedulerDeps {
+    now(): Date;
+    sleep(ms: number): Promise<void>;
+    log(line: string): void;
+    binPath: string;
+    /**
+     * The fleet config `runScheduler` loaded from the `-c FILE`/default path
+     * (final review #1). Threaded into both `runOnceFor`'s and the notifier's
+     * deps so a run under a non-default config doesn't silently fall back to
+     * `loadConfig()`'s default `~/fleet.yaml`. `runWatchdogLoop` callers that
+     * bypass `runScheduler` (tests) may omit it — the run/notifier machinery
+     * falls back to `loadConfig()` itself when `cfg` is undefined.
+     */
+    cfg?: FleetConfig;
+    /** Injectable for tests. */
+    runOnceFor?: typeof executeWatchdogRun;
+    /** Loop exit for tests + SIGTERM (wired by the CLI, not here). */
+    shouldStop?(): boolean;
+    /**
+     * Scheduler-level alert hook (Task 14, spec §5.5): fired once per hold-down
+     * transition (guarded in `settle` by `ledger.heldDownAlerted`, cleared by
+     * `resetSchedulerState`). Default: `executeNotifierRun` — the fleet
+     * process can't message on its own (deviation 4), so the default delivers
+     * the alert via a one-shot notifier agent under the watchdog's identity.
+     */
+    onSchedulerAlert?(wd: ResolvedWatchdog, text: string): Promise<void>;
+    /**
+     * Injectable notifier launcher backing the default `onSchedulerAlert` —
+     * consulted only when `onSchedulerAlert` itself is not supplied. Default:
+     * `executeNotifierRun`. Lets tests observe/short-circuit the default alert
+     * path (binPath/log/now/sleep wiring) without replacing onSchedulerAlert
+     * wholesale.
+     */
+    notifierRun?: typeof executeNotifierRun;
+    /** Poll cadence while held down. Default 5000. */
+    heldPollMs?: number;
+    /**
+     * Injectable run-lock primitives. Default: store's acquireRunLock /
+     * releaseRunLock. Both are mkdir/rmdir-backed and can throw (EACCES, EIO,
+     * ENOTEMPTY on release) — the loop always classifies such a throw as a
+     * failed tick rather than letting it escape (review finding #1).
+     */
+    locks?: {
+        acquire(name: string): boolean;
+        release(name: string): void;
+    };
+}
+/**
+ * One watchdog's scheduling loop: run immediately, then repeatedly sleep the
+ * backed-off interval and run again, until `shouldStop()`. No overlap (a run
+ * lock guards each attempt), bounded exponential backoff on failure, and a
+ * hold-down circuit breaker after WATCHDOG_HOLD_THRESHOLD consecutive
+ * failures (released externally via resetSchedulerState).
+ */
+export declare function runWatchdogLoop(wd: ResolvedWatchdog, deps: SchedulerDeps): Promise<void>;
+/**
+ * Run every enabled watchdog's loop concurrently until deps.shouldStop().
+ * SIGTERM wiring into shouldStop is the CLI's job (Task 10), not this
+ * function's.
+ */
+export declare function runScheduler(configPath: string | undefined, deps: SchedulerDeps): Promise<void>;