@ours.network/fleet 0.10.4 → 0.11.0

package/dist/watchdog/run.js

@@ -0,0 +1,318 @@
+import { closeSync, existsSync, openSync, readFileSync, rmSync, statSync, writeFileSync, } from 'node:fs';
+import { spawn as spawnChild, execFile } from 'node:child_process';
+import { join } from 'node:path';
+import { promisify } from 'node:util';
+import { stringify } from 'yaml';
+import { errorReport, normalizeWatchdogReport, validateWatchdogReport } from './report.js';
+import { acquireRunLock, formatRunId, pruneReports, releaseRunLock, writeReport } from './store.js';
+import { generateNotifierBriefing, generateWatchdogBriefing } from './briefing.js';
+import { computeDigest, reconcileLedger, readLedger, writeLedger } from './alerts.js';
+import { applyRole } from '../ops.js';
+import { daemonIdentityProvisioner, ensureIdentity, } from '../creation.js';
+import { runOnce, START_STAGGER_FILE } from '../runner.js';
+import { agentDir } from '../paths.js';
+import { loadConfig, resolveMonitorConfig, resolveWorklogPolicy, } from '../config.js';
+import { getAdapter } from '../harness/registry.js';
+import { redactLogLine } from '../application/log-service.js';
+const execFileAsync = promisify(execFile);
+/** How often the deadline loop polls for a completed report or a dead child. */
+const POLL_MS = 1000;
+/** A `report.json` this old, with no further writes, is treated as finished. */
+const WRITE_STABLE_MS = 2000;
+/** Grace given to the agent after a stable report before the session is killed. */
+const HARVEST_GRACE_MS = 5000;
+/** Fixed deadline for a one-shot notifier run — always 2 minutes, never wd.timeoutMs (Task 14). */
+const NOTIFIER_TIMEOUT_MS = 120_000;
+/**
+ * Poll for a completion sentinel file (report.json for an inspection run, sent.json for a
+ * notifier run) going write-stable, or the child exiting first, or the deadline passing —
+ * whichever comes first. Shared by executeWatchdogRun and executeNotifierRun so the two run
+ * flavors' wait loops can never drift apart.
+ */
+async function waitForSentinel(sentinelPath, child, deadlineMs, start, now, sleep) {
+    let exited = false;
+    // Attached now so a natural exit is observed even though the loop below never
+    // awaits the promise itself (its resolution races real time in the real
+    // launcher, and is driven purely by test fixtures in tests).
+    child.exited.then(() => { exited = true; }).catch(() => { exited = true; });
+    for (;;) {
+        if (exited)
+            return 'exited';
+        if (existsSync(sentinelPath)) {
+            const age = now().getTime() - statSync(sentinelPath).mtimeMs;
+            if (age >= WRITE_STABLE_MS)
+                return 'stable';
+        }
+        if (now().getTime() - start.getTime() > deadlineMs)
+            return 'timeout';
+        await sleep(POLL_MS);
+    }
+}
+/** After waitForSentinel settles, harvest the grace period (if stable) then kill the child+session. */
+async function killIfNeeded(reason, child, roleName, sleep) {
+    if (reason === 'stable' || reason === 'timeout') {
+        if (reason === 'stable')
+            await sleep(HARVEST_GRACE_MS);
+        child.kill();
+        await killTmuxSession(roleName);
+    }
+}
+function defaultLaunchChild(binPath, roleName, runDir) {
+    const out = openSync(join(runDir, 'run.log'), 'a');
+    const child = spawnChild(process.execPath, [binPath, '_run-watchdog', roleName], {
+        detached: false, stdio: ['ignore', out, out],
+    });
+    // spawn() dup's the fd into the child; Node never closes our copy on its own,
+    // so a long-running scheduler launching many watchdog runs would leak one fd
+    // per run and eventually hit EMFILE. Safe to close immediately — the child
+    // keeps writing to its own dup'd descriptor.
+    closeSync(out);
+    return {
+        kill: () => { child.kill(); },
+        exited: new Promise(resolve => {
+            child.once('exit', () => resolve());
+            child.once('error', () => resolve());
+        }),
+    };
+}
+/** Best-effort: the tmux session outlives the supervisor child (`runOnce` created it). */
+async function killTmuxSession(roleName) {
+    try {
+        await execFileAsync('tmux', ['kill-session', '-t', roleName]);
+    }
+    catch { /* best effort */ }
+}
+/** Last 4096 chars of run.log, redacted — attached to error reports as diagnostic tail. */
+function readTail(runDir) {
+    try {
+        const raw = readFileSync(join(runDir, 'run.log'), 'utf8');
+        return redactLogLine(raw.slice(-4096)).text;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * The temp role every watchdog-family run (inspection and notifier alike) launches under.
+ * `network: 'broker'` keeps ours messaging available; no write binds beyond stateDir/cwd, which
+ * resolveIsolation adds itself. ~/fleet.yaml and fleet.d are deliberately NOT bound — they're on
+ * the isolation blocklist, and everything either run flavor needs is written into its own dir.
+ *
+ * Read access is scoped to exactly `wd.watch` (finding #3): a watchdog configured to watch one
+ * role must not be able to read every other role's state dir just because they all live under
+ * the same agents root. `wd.watch` defaults to every role (watchdog/config.ts's
+ * resolveWatchdogs), so a watchdog that watches everything still sees everything — this only
+ * narrows visibility for a watchdog scoped to fewer roles. A watched role whose state dir doesn't
+ * exist (e.g. a role removed after the watchdog was configured) is simply absent from the
+ * bwrap ro-bind-try set — the agent reports it unreachable from status evidence, same as any
+ * other missing state dir.
+ */
+function buildWatchdogRole(wd, cfg) {
+    return {
+        name: wd.identity, sourceFile: '(watchdog)',
+        harness: wd.harness, session: wd.session,
+        identity: wd.identity, model: wd.model,
+        permissions: { approval: 'allow', filesystem: 'workspace', unattended: 'deny' },
+        permissionsDeclared: true,
+        monitor: resolveMonitorConfig(cfg.defaults.monitor, undefined),
+        worklog: resolveWorklogPolicy(cfg.defaults.worklog, undefined),
+        isolation: { fs: { read: wd.watch.map(r => agentDir(r)) }, network: 'broker' },
+    };
+}
+/**
+ * Run one watchdog agent end-to-end in a clean-context temp state dir: provision
+ * identity, materialize the run's contract (briefing/manifest/role snapshot),
+ * launch the child, enforce a deadline against report.json as the completion
+ * sentinel, harvest whatever it wrote (or a synthetic error report if it
+ * didn't), store the result, and always clean up the temp dir.
+ *
+ * The scheduler's run-lock guarantees only one run per watchdog at a time;
+ * this function itself takes no lock (Task 8).
+ */
+export async function executeWatchdogRun(wd, deps) {
+    const now = deps.now ?? (() => new Date());
+    const sleep = deps.sleep ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)));
+    const start = now();
+    const runId = formatRunId(start);
+    const startedAt = start.toISOString();
+    const roleName = wd.identity;
+    const runDir = agentDir(roleName, true);
+    // Read once, before the manifest is written: the same snapshot both seeds the
+    // digest the agent sees and is the base the post-run reconcile folds into.
+    const ledger = readLedger(wd.name);
+    // A crashed previous run can leave the temp dir behind; start clean.
+    if (existsSync(runDir))
+        rmSync(runDir, { recursive: true, force: true });
+    let report;
+    try {
+        const guarantee = await ensureIdentity(wd.identity, {}, deps.identityProvisioner ?? daemonIdentityProvisioner(), deps.log);
+        const cfg = deps.cfg ?? loadConfig();
+        const role = buildWatchdogRole(wd, cfg);
+        const dir = applyRole(role, { temp: true, identityGuarantee: guarantee.state });
+        const reportPath = join(dir, 'report.json');
+        const manifestPath = join(dir, 'watch.json');
+        const promptFocus = wd.promptFile ? readFileSync(wd.promptFile, 'utf8') : undefined;
+        // applyRole wrote the generic role briefing; the watchdog contract replaces it.
+        writeFileSync(join(dir, 'briefing.md'), generateWatchdogBriefing({
+            wd, manifestPath, reportPath,
+            vocabulary: getAdapter(wd.harness).vocabulary,
+            identityGuarantee: guarantee.state,
+            promptFocus,
+        }));
+        // loadTempRole (runner.ts) needs this to run the child via `_run-watchdog`.
+        writeFileSync(join(dir, 'role.yaml'), stringify(role));
+        const manifest = {
+            watchdog: wd.name, run_id: runId, coordinator: wd.coordinator, started_at: startedAt,
+            roles: wd.watch.map(r => ({ name: r, stateDir: agentDir(r) })),
+            digest: computeDigest(ledger, wd.alertCooldownMs, now()),
+        };
+        writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
+        // Snapshot the fleet start-stagger so the detached run (no config path
+        // threaded through `_run-watchdog`) honors the same host-wide launch gate
+        // as every other role (spec §3) — mirrors src/spawn.ts:503-504.
+        if (cfg.startStaggerMs > 0)
+            writeFileSync(join(dir, START_STAGGER_FILE), String(cfg.startStaggerMs));
+        const launch = deps.launchChild ?? defaultLaunchChild;
+        const child = launch(deps.binPath, roleName, dir);
+        const reason = await waitForSentinel(reportPath, child, wd.timeoutMs, start, now, sleep);
+        await killIfNeeded(reason, child, roleName, sleep);
+        const timedOut = reason === 'timeout';
+        if (timedOut) {
+            report = errorReport({
+                watchdog: wd.name, run_id: runId, started_at: startedAt, finished_at: now().toISOString(),
+                error: 'timeout', tail: readTail(dir),
+            });
+        }
+        else {
+            let parsed;
+            let parseError;
+            try {
+                parsed = JSON.parse(readFileSync(reportPath, 'utf8'));
+            }
+            catch (e) {
+                parseError = e instanceof Error ? e.message : String(e);
+            }
+            const validationErrors = parseError === undefined ? validateWatchdogReport(parsed) : [];
+            if (parseError !== undefined || validationErrors.length) {
+                const detail = parseError ?? validationErrors.slice(0, 3).join('; ');
+                report = errorReport({
+                    watchdog: wd.name, run_id: runId, started_at: startedAt, finished_at: now().toISOString(),
+                    error: `invalid report: ${detail}`, tail: readTail(dir),
+                });
+            }
+            else {
+                report = normalizeWatchdogReport(parsed, { watchdog: wd.name, run_id: runId });
+                // Scheduler clock truth overrides whatever the agent wrote.
+                report.started_at = startedAt;
+                report.finished_at = now().toISOString();
+            }
+        }
+        // Isolation degradation (spec §7): a weaker guarantee must never look like
+        // the strong one, so stamp it on every outcome, error reports included.
+        if (existsSync(join(dir, '.isolation-degraded')))
+            report.isolation = 'degraded';
+    }
+    finally {
+        rmSync(runDir, { recursive: true, force: true });
+    }
+    const storedPath = writeReport(wd.name, report);
+    // Reconciles against the FINAL stored report (normalized or error) — reconcileLedger
+    // itself no-ops on error-status reports, so a timeout/invalid run leaves the ledger
+    // byte-for-byte unchanged.
+    writeLedger(wd.name, reconcileLedger(ledger, report, now()));
+    pruneReports(wd.name, wd.keepReports);
+    return { report, storedPath };
+}
+/**
+ * Deliver one scheduler-level alert (e.g. held-down) by launching a minimal one-shot temp agent
+ * under the watchdog's own identity whose entire mission is: bind identity, send `text` to
+ * `wd.coordinator`, write `sent.json`, exit. The fleet process itself cannot send ours messages
+ * (owner-approved deviation 4), so this is how a scheduler tick that can't reach an operator any
+ * other way still gets a message out.
+ *
+ * Reuses the Task 7/8 run machinery (temp-dir prep, ensureIdentity, ResolvedRole shape, child
+ * launch, kill mechanics) but stores no report, touches no ledger findings, and writes no
+ * watch.json manifest — `sent.json` is this run's only completion sentinel. A fixed 2-minute
+ * deadline applies regardless of `wd.timeoutMs`.
+ *
+ * Unlike executeWatchdogRun (which relies on the scheduler's run lock), this function acquires
+ * and releases the run lock itself — a notifier run shares the same temp dir name
+ * (`agentDir(wd.identity, true)`) as a regular run, and nothing else serializes the two. Refusal
+ * to acquire is logged and treated as a no-op: the held-down state already means no regular runs
+ * are in flight, so a collision is rare. Any other failure (including a timeout) is logged as a
+ * warning and swallowed — a notifier failure must never throw into the scheduler loop.
+ */
+export async function executeNotifierRun(wd, text, deps) {
+    const now = deps.now ?? (() => new Date());
+    const sleep = deps.sleep ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)));
+    const fail = (e) => {
+        deps.log(`notifier run for '${wd.name}' failed: ${e instanceof Error ? e.message : String(e)}`);
+    };
+    // acquireRunLock can itself throw (EACCES/EIO, not just EEXIST — store.ts) — guarded
+    // separately from the try/finally below so a throw here never reaches the caller either;
+    // nothing was acquired, so there is nothing to release.
+    let acquired = false;
+    try {
+        acquired = acquireRunLock(wd.name);
+    }
+    catch (e) {
+        fail(e);
+        return;
+    }
+    if (!acquired) {
+        deps.log(`notifier run for '${wd.name}' skipped: run lock held`);
+        return;
+    }
+    const roleName = wd.identity;
+    const runDir = agentDir(roleName, true);
+    try {
+        // A crashed previous run can leave the temp dir behind; start clean.
+        if (existsSync(runDir))
+            rmSync(runDir, { recursive: true, force: true });
+        const guarantee = await ensureIdentity(wd.identity, {}, deps.identityProvisioner ?? daemonIdentityProvisioner(), deps.log);
+        const cfg = deps.cfg ?? loadConfig();
+        const role = buildWatchdogRole(wd, cfg);
+        const dir = applyRole(role, { temp: true, identityGuarantee: guarantee.state });
+        const sentinelPath = join(dir, 'sent.json');
+        // applyRole wrote the generic role briefing; the notifier contract replaces it.
+        writeFileSync(join(dir, 'briefing.md'), generateNotifierBriefing({
+            wd, vocabulary: getAdapter(wd.harness).vocabulary,
+            identityGuarantee: guarantee.state, text,
+        }));
+        // loadTempRole (runner.ts) needs this to run the child via `_run-watchdog`.
+        writeFileSync(join(dir, 'role.yaml'), stringify(role));
+        // Same host-wide launch-gate snapshot executeWatchdogRun writes (spec §3).
+        if (cfg.startStaggerMs > 0)
+            writeFileSync(join(dir, START_STAGGER_FILE), String(cfg.startStaggerMs));
+        const launch = deps.launchChild ?? defaultLaunchChild;
+        const start = now();
+        const child = launch(deps.binPath, roleName, dir);
+        const reason = await waitForSentinel(sentinelPath, child, NOTIFIER_TIMEOUT_MS, start, now, sleep);
+        await killIfNeeded(reason, child, roleName, sleep);
+        if (reason === 'timeout')
+            deps.log(`notifier run for '${wd.name}' timed out`);
+    }
+    catch (e) {
+        fail(e);
+    }
+    finally {
+        // Same "never escape" rule as the lock acquire above: cleanup and release are each
+        // guarded individually so neither can turn a handled failure into an unhandled one.
+        try {
+            rmSync(runDir, { recursive: true, force: true });
+        }
+        catch (e) {
+            fail(e);
+        }
+        try {
+            releaseRunLock(wd.name);
+        }
+        catch (e) {
+            fail(e);
+        }
+    }
+}
+/** What `_run-watchdog` calls: one supervised session, no cleanup — the parent harvests. */
+export async function runWatchdogAgent(name) {
+    await runOnce(name, { temp: true });
+}

package/dist/watchdog/scheduler.d.ts

@@ -0,0 +1,105 @@
+import type { FleetConfig } from '../config.js';
+import type { ResolvedWatchdog } from './config.js';
+import { executeNotifierRun, executeWatchdogRun } from './run.js';
+export interface WatchdogSchedulerState {
+    version: 1;
+    consecutiveFailures: number;
+    heldDown: boolean;
+    heldSince?: string;
+    lastRunAt?: string;
+    nextRunAt?: string;
+    lastError?: string;
+}
+/** Read a watchdog's scheduler state; a missing or corrupt file starts clean (mirrors readRestartLedger). */
+export declare function readSchedulerState(name: string): WatchdogSchedulerState;
+/** Write never throws: scheduler diagnostics must never take the loop down. */
+export declare function writeSchedulerState(name: string, s: WatchdogSchedulerState): void;
+/**
+ * Operator release (Task 15): clears failures and heldDown so a held-down
+ * loop's next held-down poll sees a clean state and resumes running. Also
+ * clears the ledger's `heldDownAlerted` flag — that flag is what makes "alert
+ * once per hold-down" durable across scheduler restarts (it lives in
+ * alerts.json, not state.json), so a release must reset it too or a
+ * subsequent hold-down would silently alert zero times.
+ *
+ * Also reclaims a STALE run lock (final review #2, tightened by finding #2):
+ * a watchdog SIGKILLed mid-run (e.g. systemd's TimeoutStopSec on a
+ * fleet-wide stop) leaves `.run-lock` behind forever — the loop's `finally`
+ * that would normally release it never runs. Without this, every future tick
+ * sees the lock held and reports `skipped_overlap` indefinitely, and skips
+ * never alert. `ours-fleet restart <watchdog>` is the documented recovery, so
+ * it must clear a stale lock too, not just the failure/hold-down bookkeeping.
+ * But only a DEMONSTRABLY stale lock (dead owner pid, or legacy lock with no
+ * owner metadata) — a lock genuinely held by a live run (foreground
+ * `watchdog-run`, another scheduler instance) must survive an operator's
+ * `restart` of a DIFFERENT problem (e.g. releasing hold-down) unrelated to
+ * that live run; two runs sharing the same temp dir would corrupt each
+ * other's output. Best-effort: a reclaim failure here must not turn an
+ * operator's recovery action into a crash.
+ */
+export declare function resetSchedulerState(name: string): void;
+export declare const WATCHDOG_HOLD_THRESHOLD = 3;
+export declare const WATCHDOG_BACKOFF_MAX_MS = 3600000;
+/** Bounded exponential backoff: 1x, 2x, 4x, ... capped at WATCHDOG_BACKOFF_MAX_MS (spec §3). */
+export declare function watchdogBackoffMs(intervalMs: number, failures: number): number;
+export interface SchedulerDeps {
+    now(): Date;
+    sleep(ms: number): Promise<void>;
+    log(line: string): void;
+    binPath: string;
+    /**
+     * The fleet config `runScheduler` loaded from the `-c FILE`/default path
+     * (final review #1). Threaded into both `runOnceFor`'s and the notifier's
+     * deps so a run under a non-default config doesn't silently fall back to
+     * `loadConfig()`'s default `~/fleet.yaml`. `runWatchdogLoop` callers that
+     * bypass `runScheduler` (tests) may omit it — the run/notifier machinery
+     * falls back to `loadConfig()` itself when `cfg` is undefined.
+     */
+    cfg?: FleetConfig;
+    /** Injectable for tests. */
+    runOnceFor?: typeof executeWatchdogRun;
+    /** Loop exit for tests + SIGTERM (wired by the CLI, not here). */
+    shouldStop?(): boolean;
+    /**
+     * Scheduler-level alert hook (Task 14, spec §5.5): fired once per hold-down
+     * transition (guarded in `settle` by `ledger.heldDownAlerted`, cleared by
+     * `resetSchedulerState`). Default: `executeNotifierRun` — the fleet
+     * process can't message on its own (deviation 4), so the default delivers
+     * the alert via a one-shot notifier agent under the watchdog's identity.
+     */
+    onSchedulerAlert?(wd: ResolvedWatchdog, text: string): Promise<void>;
+    /**
+     * Injectable notifier launcher backing the default `onSchedulerAlert` —
+     * consulted only when `onSchedulerAlert` itself is not supplied. Default:
+     * `executeNotifierRun`. Lets tests observe/short-circuit the default alert
+     * path (binPath/log/now/sleep wiring) without replacing onSchedulerAlert
+     * wholesale.
+     */
+    notifierRun?: typeof executeNotifierRun;
+    /** Poll cadence while held down. Default 5000. */
+    heldPollMs?: number;
+    /**
+     * Injectable run-lock primitives. Default: store's acquireRunLock /
+     * releaseRunLock. Both are mkdir/rmdir-backed and can throw (EACCES, EIO,
+     * ENOTEMPTY on release) — the loop always classifies such a throw as a
+     * failed tick rather than letting it escape (review finding #1).
+     */
+    locks?: {
+        acquire(name: string): boolean;
+        release(name: string): void;
+    };
+}
+/**
+ * One watchdog's scheduling loop: run immediately, then repeatedly sleep the
+ * backed-off interval and run again, until `shouldStop()`. No overlap (a run
+ * lock guards each attempt), bounded exponential backoff on failure, and a
+ * hold-down circuit breaker after WATCHDOG_HOLD_THRESHOLD consecutive
+ * failures (released externally via resetSchedulerState).
+ */
+export declare function runWatchdogLoop(wd: ResolvedWatchdog, deps: SchedulerDeps): Promise<void>;
+/**
+ * Run every enabled watchdog's loop concurrently until deps.shouldStop().
+ * SIGTERM wiring into shouldStop is the CLI's job (Task 10), not this
+ * function's.
+ */
+export declare function runScheduler(configPath: string | undefined, deps: SchedulerDeps): Promise<void>;

package/dist/watchdog/scheduler.js

@@ -0,0 +1,244 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { loadConfig } from '../config.js';
+import { errorReport } from './report.js';
+import { executeNotifierRun, executeWatchdogRun } from './run.js';
+import { acquireRunLock, formatRunId, readRunLockOwner, reclaimStaleRunLock, releaseRunLock, watchdogDir, writeReport, } from './store.js';
+import { readLedger, writeLedger } from './alerts.js';
+const STATE_FILE = 'state.json';
+const cleanState = () => ({ version: 1, consecutiveFailures: 0, heldDown: false });
+/** Read a watchdog's scheduler state; a missing or corrupt file starts clean (mirrors readRestartLedger). */
+export function readSchedulerState(name) {
+    try {
+        const raw = JSON.parse(readFileSync(join(watchdogDir(name), STATE_FILE), 'utf8'));
+        if (raw.version !== 1)
+            return cleanState();
+        return { ...cleanState(), ...raw, version: 1 };
+    }
+    catch {
+        return cleanState();
+    }
+}
+/** Write never throws: scheduler diagnostics must never take the loop down. */
+export function writeSchedulerState(name, s) {
+    try {
+        writeFileSync(join(watchdogDir(name), STATE_FILE), JSON.stringify(s, null, 2) + '\n', { mode: 0o600 });
+    }
+    catch { /* diagnostics must never take the loop down */ }
+}
+/**
+ * Operator release (Task 15): clears failures and heldDown so a held-down
+ * loop's next held-down poll sees a clean state and resumes running. Also
+ * clears the ledger's `heldDownAlerted` flag — that flag is what makes "alert
+ * once per hold-down" durable across scheduler restarts (it lives in
+ * alerts.json, not state.json), so a release must reset it too or a
+ * subsequent hold-down would silently alert zero times.
+ *
+ * Also reclaims a STALE run lock (final review #2, tightened by finding #2):
+ * a watchdog SIGKILLed mid-run (e.g. systemd's TimeoutStopSec on a
+ * fleet-wide stop) leaves `.run-lock` behind forever — the loop's `finally`
+ * that would normally release it never runs. Without this, every future tick
+ * sees the lock held and reports `skipped_overlap` indefinitely, and skips
+ * never alert. `ours-fleet restart <watchdog>` is the documented recovery, so
+ * it must clear a stale lock too, not just the failure/hold-down bookkeeping.
+ * But only a DEMONSTRABLY stale lock (dead owner pid, or legacy lock with no
+ * owner metadata) — a lock genuinely held by a live run (foreground
+ * `watchdog-run`, another scheduler instance) must survive an operator's
+ * `restart` of a DIFFERENT problem (e.g. releasing hold-down) unrelated to
+ * that live run; two runs sharing the same temp dir would corrupt each
+ * other's output. Best-effort: a reclaim failure here must not turn an
+ * operator's recovery action into a crash.
+ */
+export function resetSchedulerState(name) {
+    writeSchedulerState(name, cleanState());
+    const ledger = readLedger(name);
+    if (ledger.heldDownAlerted)
+        writeLedger(name, { ...ledger, heldDownAlerted: false });
+    try {
+        reclaimStaleRunLock(name);
+    }
+    catch { /* best effort */ }
+}
+export const WATCHDOG_HOLD_THRESHOLD = 3;
+export const WATCHDOG_BACKOFF_MAX_MS = 3_600_000;
+/** Bounded exponential backoff: 1x, 2x, 4x, ... capped at WATCHDOG_BACKOFF_MAX_MS (spec §3). */
+export function watchdogBackoffMs(intervalMs, failures) {
+    if (failures <= 0)
+        return intervalMs;
+    return Math.min(intervalMs * 2 ** (failures - 1), WATCHDOG_BACKOFF_MAX_MS);
+}
+/**
+ * One watchdog's scheduling loop: run immediately, then repeatedly sleep the
+ * backed-off interval and run again, until `shouldStop()`. No overlap (a run
+ * lock guards each attempt), bounded exponential backoff on failure, and a
+ * hold-down circuit breaker after WATCHDOG_HOLD_THRESHOLD consecutive
+ * failures (released externally via resetSchedulerState).
+ */
+export async function runWatchdogLoop(wd, deps) {
+    const runOnceFor = deps.runOnceFor ?? executeWatchdogRun;
+    const shouldStop = deps.shouldStop ?? (() => false);
+    const notifierRun = deps.notifierRun ?? executeNotifierRun;
+    const onSchedulerAlert = deps.onSchedulerAlert ?? ((wd, text) => notifierRun(wd, text, {
+        binPath: deps.binPath, log: deps.log, now: deps.now, sleep: deps.sleep, cfg: deps.cfg,
+    }));
+    const heldPollMs = deps.heldPollMs ?? 5_000;
+    const locks = deps.locks ?? { acquire: acquireRunLock, release: releaseRunLock };
+    /**
+     * Apply one tick's outcome to state.json: update the failure streak,
+     * transition into hold-down on the Nth consecutive failure (firing the
+     * alert at most once per transition via the ledger's `heldDownAlerted`
+     * guard — durable across restarts, unlike an in-memory flag), then sleep
+     * before the next tick. On the tick that transitions into hold-down, sleep
+     * `heldPollMs` rather than the (possibly hour-long) backoff delay — an
+     * operator's resetSchedulerState must be noticed within one poll cycle,
+     * not after the last backoff finishes (review finding #2).
+     */
+    const settle = async (isFailure, errorMessage, startedAt) => {
+        const s = readSchedulerState(wd.name);
+        if (isFailure) {
+            s.consecutiveFailures += 1;
+            s.lastError = errorMessage;
+        }
+        else {
+            s.consecutiveFailures = 0;
+            s.lastError = undefined;
+        }
+        s.lastRunAt = startedAt.toISOString();
+        let justHeld = false;
+        if (s.consecutiveFailures >= WATCHDOG_HOLD_THRESHOLD && !s.heldDown) {
+            s.heldDown = true;
+            s.heldSince = deps.now().toISOString();
+            justHeld = true;
+        }
+        const delay = justHeld ? heldPollMs : watchdogBackoffMs(wd.intervalMs, s.consecutiveFailures);
+        s.nextRunAt = new Date(deps.now().getTime() + delay).toISOString();
+        writeSchedulerState(wd.name, s);
+        if (justHeld) {
+            const msg = `watchdog ${wd.name} held down after ${WATCHDOG_HOLD_THRESHOLD} consecutive failed runs: ${s.lastError}`;
+            deps.log(msg);
+            // "Once per state change" (spec §5.5) must survive a scheduler restart, so the guard
+            // lives in the ledger (alerts.json), not in memory — resetSchedulerState clears it.
+            const ledger = readLedger(wd.name);
+            if (!ledger.heldDownAlerted) {
+                await onSchedulerAlert(wd, msg);
+                writeLedger(wd.name, { ...ledger, heldDownAlerted: true });
+            }
+        }
+        await deps.sleep(delay);
+    };
+    for (;;) {
+        if (shouldStop())
+            return;
+        if (readSchedulerState(wd.name).heldDown) {
+            await deps.sleep(heldPollMs);
+            if (shouldStop())
+                return;
+            continue;
+        }
+        const tickStart = deps.now();
+        // The run-lock is a filesystem mutex (mkdir/rmdir) and can throw
+        // (EACCES, EIO, ENOTEMPTY on release — store.ts's release deliberately
+        // rethrows non-ENOENT failures). Never let that throw escape this loop:
+        // runScheduler drives every watchdog's loop via Promise.all, so an
+        // uncaught throw here would kill every OTHER watchdog's loop too.
+        // Classify it as this tick's failure instead (review finding #1).
+        let acquired = false;
+        let acquireError;
+        try {
+            acquired = locks.acquire(wd.name);
+        }
+        catch (e) {
+            acquireError = e instanceof Error ? e.message : String(e);
+        }
+        if (acquireError !== undefined) {
+            await settle(true, acquireError, tickStart);
+            if (shouldStop())
+                return;
+            continue;
+        }
+        if (!acquired) {
+            writeReport(wd.name, errorReport({
+                watchdog: wd.name, run_id: formatRunId(tickStart),
+                started_at: tickStart.toISOString(), finished_at: tickStart.toISOString(),
+                error: 'skipped_overlap',
+            }));
+            deps.log(`watchdog ${wd.name}: skipped run (previous run still holds the lock)`);
+            // Failure count is untouched by a skip; only lastRunAt/nextRunAt move.
+            // The backoff cadence (not the raw interval) still governs during an
+            // active failure streak: a skip isn't a finished run, so it shouldn't
+            // reset the retry cadence to full speed either (spec §3).
+            const skipState = readSchedulerState(wd.name);
+            const delay = watchdogBackoffMs(wd.intervalMs, skipState.consecutiveFailures);
+            writeSchedulerState(wd.name, {
+                ...skipState,
+                lastRunAt: tickStart.toISOString(),
+                nextRunAt: new Date(tickStart.getTime() + delay).toISOString(),
+            });
+            await deps.sleep(delay);
+            if (shouldStop())
+                return;
+            continue;
+        }
+        let outcome;
+        let tickError;
+        try {
+            outcome = await runOnceFor(wd, {
+                binPath: deps.binPath, log: deps.log, now: deps.now, sleep: deps.sleep, cfg: deps.cfg,
+            });
+        }
+        catch (e) {
+            tickError = e instanceof Error ? e.message : String(e);
+        }
+        finally {
+            // Same "never escape" rule applies to release: fold a throw into this
+            // tick's failure rather than letting it propagate out of the loop.
+            try {
+                locks.release(wd.name);
+            }
+            catch (e) {
+                tickError = tickError ?? (e instanceof Error ? e.message : String(e));
+            }
+        }
+        let errorMessage;
+        if (outcome === undefined) {
+            errorMessage = tickError;
+        }
+        else if (outcome.report.status === 'error' && outcome.report.error !== 'skipped_overlap') {
+            errorMessage = outcome.report.error ?? tickError ?? 'unknown error';
+        }
+        else {
+            errorMessage = tickError;
+        }
+        await settle(errorMessage !== undefined, errorMessage, tickStart);
+        if (shouldStop())
+            return;
+    }
+}
+/**
+ * Run every enabled watchdog's loop concurrently until deps.shouldStop().
+ * SIGTERM wiring into shouldStop is the CLI's job (Task 10), not this
+ * function's.
+ */
+export async function runScheduler(configPath, deps) {
+    const cfg = loadConfig(configPath);
+    const watchdogs = cfg.watchdogs.filter(w => w.enabled);
+    // Recover from a SIGKILLed prior run (final review #2a, tightened by
+    // finding #2): a scheduler restart is EVIDENCE, not proof, that no
+    // scheduler-owned run is in flight — a foreground `watchdog-run`, or
+    // another scheduler instance, may genuinely still hold a watchdog's lock.
+    // Only a DEMONSTRABLY stale lock (dead owner pid, or a legacy lock with no
+    // owner metadata) may be reclaimed; a live lock is left alone so the two
+    // runs never share the same temp dir — the loop then simply observes
+    // skipped_overlap on its first tick, same as any other overlap. Best-effort:
+    // a reclaim failure here must not prevent the scheduler from starting.
+    for (const wd of watchdogs) {
+        try {
+            if (!reclaimStaleRunLock(wd.name)) {
+                const owner = readRunLockOwner(wd.name);
+                deps.log(`watchdog '${wd.name}': run lock held by live pid ${owner?.pid ?? 'unknown'} — not reclaimed`);
+            }
+        }
+        catch { /* best effort */ }
+    }
+    await Promise.all(watchdogs.map(wd => runWatchdogLoop(wd, { ...deps, cfg })));
+}