@ours.network/fleet 0.10.3 → 0.11.0

package/dist/watchdog/scheduler.js

@@ -0,0 +1,244 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { loadConfig } from '../config.js';
+import { errorReport } from './report.js';
+import { executeNotifierRun, executeWatchdogRun } from './run.js';
+import { acquireRunLock, formatRunId, readRunLockOwner, reclaimStaleRunLock, releaseRunLock, watchdogDir, writeReport, } from './store.js';
+import { readLedger, writeLedger } from './alerts.js';
+const STATE_FILE = 'state.json';
+const cleanState = () => ({ version: 1, consecutiveFailures: 0, heldDown: false });
+/** Read a watchdog's scheduler state; a missing or corrupt file starts clean (mirrors readRestartLedger). */
+export function readSchedulerState(name) {
+    try {
+        const raw = JSON.parse(readFileSync(join(watchdogDir(name), STATE_FILE), 'utf8'));
+        if (raw.version !== 1)
+            return cleanState();
+        return { ...cleanState(), ...raw, version: 1 };
+    }
+    catch {
+        return cleanState();
+    }
+}
+/** Write never throws: scheduler diagnostics must never take the loop down. */
+export function writeSchedulerState(name, s) {
+    try {
+        writeFileSync(join(watchdogDir(name), STATE_FILE), JSON.stringify(s, null, 2) + '\n', { mode: 0o600 });
+    }
+    catch { /* diagnostics must never take the loop down */ }
+}
+/**
+ * Operator release (Task 15): clears failures and heldDown so a held-down
+ * loop's next held-down poll sees a clean state and resumes running. Also
+ * clears the ledger's `heldDownAlerted` flag — that flag is what makes "alert
+ * once per hold-down" durable across scheduler restarts (it lives in
+ * alerts.json, not state.json), so a release must reset it too or a
+ * subsequent hold-down would silently alert zero times.
+ *
+ * Also reclaims a STALE run lock (final review #2, tightened by finding #2):
+ * a watchdog SIGKILLed mid-run (e.g. systemd's TimeoutStopSec on a
+ * fleet-wide stop) leaves `.run-lock` behind forever — the loop's `finally`
+ * that would normally release it never runs. Without this, every future tick
+ * sees the lock held and reports `skipped_overlap` indefinitely, and skips
+ * never alert. `ours-fleet restart <watchdog>` is the documented recovery, so
+ * it must clear a stale lock too, not just the failure/hold-down bookkeeping.
+ * But only a DEMONSTRABLY stale lock (dead owner pid, or legacy lock with no
+ * owner metadata) — a lock genuinely held by a live run (foreground
+ * `watchdog-run`, another scheduler instance) must survive an operator's
+ * `restart` of a DIFFERENT problem (e.g. releasing hold-down) unrelated to
+ * that live run; two runs sharing the same temp dir would corrupt each
+ * other's output. Best-effort: a reclaim failure here must not turn an
+ * operator's recovery action into a crash.
+ */
+export function resetSchedulerState(name) {
+    writeSchedulerState(name, cleanState());
+    const ledger = readLedger(name);
+    if (ledger.heldDownAlerted)
+        writeLedger(name, { ...ledger, heldDownAlerted: false });
+    try {
+        reclaimStaleRunLock(name);
+    }
+    catch { /* best effort */ }
+}
+export const WATCHDOG_HOLD_THRESHOLD = 3;
+export const WATCHDOG_BACKOFF_MAX_MS = 3_600_000;
+/** Bounded exponential backoff: 1x, 2x, 4x, ... capped at WATCHDOG_BACKOFF_MAX_MS (spec §3). */
+export function watchdogBackoffMs(intervalMs, failures) {
+    if (failures <= 0)
+        return intervalMs;
+    return Math.min(intervalMs * 2 ** (failures - 1), WATCHDOG_BACKOFF_MAX_MS);
+}
+/**
+ * One watchdog's scheduling loop: run immediately, then repeatedly sleep the
+ * backed-off interval and run again, until `shouldStop()`. No overlap (a run
+ * lock guards each attempt), bounded exponential backoff on failure, and a
+ * hold-down circuit breaker after WATCHDOG_HOLD_THRESHOLD consecutive
+ * failures (released externally via resetSchedulerState).
+ */
+export async function runWatchdogLoop(wd, deps) {
+    const runOnceFor = deps.runOnceFor ?? executeWatchdogRun;
+    const shouldStop = deps.shouldStop ?? (() => false);
+    const notifierRun = deps.notifierRun ?? executeNotifierRun;
+    const onSchedulerAlert = deps.onSchedulerAlert ?? ((wd, text) => notifierRun(wd, text, {
+        binPath: deps.binPath, log: deps.log, now: deps.now, sleep: deps.sleep, cfg: deps.cfg,
+    }));
+    const heldPollMs = deps.heldPollMs ?? 5_000;
+    const locks = deps.locks ?? { acquire: acquireRunLock, release: releaseRunLock };
+    /**
+     * Apply one tick's outcome to state.json: update the failure streak,
+     * transition into hold-down on the Nth consecutive failure (firing the
+     * alert at most once per transition via the ledger's `heldDownAlerted`
+     * guard — durable across restarts, unlike an in-memory flag), then sleep
+     * before the next tick. On the tick that transitions into hold-down, sleep
+     * `heldPollMs` rather than the (possibly hour-long) backoff delay — an
+     * operator's resetSchedulerState must be noticed within one poll cycle,
+     * not after the last backoff finishes (review finding #2).
+     */
+    const settle = async (isFailure, errorMessage, startedAt) => {
+        const s = readSchedulerState(wd.name);
+        if (isFailure) {
+            s.consecutiveFailures += 1;
+            s.lastError = errorMessage;
+        }
+        else {
+            s.consecutiveFailures = 0;
+            s.lastError = undefined;
+        }
+        s.lastRunAt = startedAt.toISOString();
+        let justHeld = false;
+        if (s.consecutiveFailures >= WATCHDOG_HOLD_THRESHOLD && !s.heldDown) {
+            s.heldDown = true;
+            s.heldSince = deps.now().toISOString();
+            justHeld = true;
+        }
+        const delay = justHeld ? heldPollMs : watchdogBackoffMs(wd.intervalMs, s.consecutiveFailures);
+        s.nextRunAt = new Date(deps.now().getTime() + delay).toISOString();
+        writeSchedulerState(wd.name, s);
+        if (justHeld) {
+            const msg = `watchdog ${wd.name} held down after ${WATCHDOG_HOLD_THRESHOLD} consecutive failed runs: ${s.lastError}`;
+            deps.log(msg);
+            // "Once per state change" (spec §5.5) must survive a scheduler restart, so the guard
+            // lives in the ledger (alerts.json), not in memory — resetSchedulerState clears it.
+            const ledger = readLedger(wd.name);
+            if (!ledger.heldDownAlerted) {
+                await onSchedulerAlert(wd, msg);
+                writeLedger(wd.name, { ...ledger, heldDownAlerted: true });
+            }
+        }
+        await deps.sleep(delay);
+    };
+    for (;;) {
+        if (shouldStop())
+            return;
+        if (readSchedulerState(wd.name).heldDown) {
+            await deps.sleep(heldPollMs);
+            if (shouldStop())
+                return;
+            continue;
+        }
+        const tickStart = deps.now();
+        // The run-lock is a filesystem mutex (mkdir/rmdir) and can throw
+        // (EACCES, EIO, ENOTEMPTY on release — store.ts's release deliberately
+        // rethrows non-ENOENT failures). Never let that throw escape this loop:
+        // runScheduler drives every watchdog's loop via Promise.all, so an
+        // uncaught throw here would kill every OTHER watchdog's loop too.
+        // Classify it as this tick's failure instead (review finding #1).
+        let acquired = false;
+        let acquireError;
+        try {
+            acquired = locks.acquire(wd.name);
+        }
+        catch (e) {
+            acquireError = e instanceof Error ? e.message : String(e);
+        }
+        if (acquireError !== undefined) {
+            await settle(true, acquireError, tickStart);
+            if (shouldStop())
+                return;
+            continue;
+        }
+        if (!acquired) {
+            writeReport(wd.name, errorReport({
+                watchdog: wd.name, run_id: formatRunId(tickStart),
+                started_at: tickStart.toISOString(), finished_at: tickStart.toISOString(),
+                error: 'skipped_overlap',
+            }));
+            deps.log(`watchdog ${wd.name}: skipped run (previous run still holds the lock)`);
+            // Failure count is untouched by a skip; only lastRunAt/nextRunAt move.
+            // The backoff cadence (not the raw interval) still governs during an
+            // active failure streak: a skip isn't a finished run, so it shouldn't
+            // reset the retry cadence to full speed either (spec §3).
+            const skipState = readSchedulerState(wd.name);
+            const delay = watchdogBackoffMs(wd.intervalMs, skipState.consecutiveFailures);
+            writeSchedulerState(wd.name, {
+                ...skipState,
+                lastRunAt: tickStart.toISOString(),
+                nextRunAt: new Date(tickStart.getTime() + delay).toISOString(),
+            });
+            await deps.sleep(delay);
+            if (shouldStop())
+                return;
+            continue;
+        }
+        let outcome;
+        let tickError;
+        try {
+            outcome = await runOnceFor(wd, {
+                binPath: deps.binPath, log: deps.log, now: deps.now, sleep: deps.sleep, cfg: deps.cfg,
+            });
+        }
+        catch (e) {
+            tickError = e instanceof Error ? e.message : String(e);
+        }
+        finally {
+            // Same "never escape" rule applies to release: fold a throw into this
+            // tick's failure rather than letting it propagate out of the loop.
+            try {
+                locks.release(wd.name);
+            }
+            catch (e) {
+                tickError = tickError ?? (e instanceof Error ? e.message : String(e));
+            }
+        }
+        let errorMessage;
+        if (outcome === undefined) {
+            errorMessage = tickError;
+        }
+        else if (outcome.report.status === 'error' && outcome.report.error !== 'skipped_overlap') {
+            errorMessage = outcome.report.error ?? tickError ?? 'unknown error';
+        }
+        else {
+            errorMessage = tickError;
+        }
+        await settle(errorMessage !== undefined, errorMessage, tickStart);
+        if (shouldStop())
+            return;
+    }
+}
+/**
+ * Run every enabled watchdog's loop concurrently until deps.shouldStop().
+ * SIGTERM wiring into shouldStop is the CLI's job (Task 10), not this
+ * function's.
+ */
+export async function runScheduler(configPath, deps) {
+    const cfg = loadConfig(configPath);
+    const watchdogs = cfg.watchdogs.filter(w => w.enabled);
+    // Recover from a SIGKILLed prior run (final review #2a, tightened by
+    // finding #2): a scheduler restart is EVIDENCE, not proof, that no
+    // scheduler-owned run is in flight — a foreground `watchdog-run`, or
+    // another scheduler instance, may genuinely still hold a watchdog's lock.
+    // Only a DEMONSTRABLY stale lock (dead owner pid, or a legacy lock with no
+    // owner metadata) may be reclaimed; a live lock is left alone so the two
+    // runs never share the same temp dir — the loop then simply observes
+    // skipped_overlap on its first tick, same as any other overlap. Best-effort:
+    // a reclaim failure here must not prevent the scheduler from starting.
+    for (const wd of watchdogs) {
+        try {
+            if (!reclaimStaleRunLock(wd.name)) {
+                const owner = readRunLockOwner(wd.name);
+                deps.log(`watchdog '${wd.name}': run lock held by live pid ${owner?.pid ?? 'unknown'} — not reclaimed`);
+            }
+        }
+        catch { /* best effort */ }
+    }
+    await Promise.all(watchdogs.map(wd => runWatchdogLoop(wd, { ...deps, cfg })));
+}

package/dist/watchdog/service.d.ts

@@ -0,0 +1,46 @@
+import { type Exec } from '../exec.js';
+export declare const WATCHDOG_SYSTEMD_UNIT = "ours-fleet-watchdogs.service";
+export declare const WATCHDOG_LAUNCHD_LABEL = "network.ours.fleet.watchdogs";
+/**
+ * Supervises the single long-running watchdog-scheduler process (the hidden
+ * `_run-watchdogs` command, Task 9's `runScheduler`) the same way
+ * `WebServiceManager` (src/web/service.ts) supervises the web console: a
+ * private systemd --user unit on Linux, a launchd LaunchAgent on macOS.
+ */
+export declare class WatchdogServiceManager {
+    private readonly exec;
+    private readonly platform;
+    constructor(exec?: Exec, platform?: NodeJS.Platform);
+    get definitionPath(): string;
+    /** false when explicitly disabled (OURS_FLEET_SUPERVISOR=none) or on an unsupported platform. */
+    supervised(): boolean;
+    /**
+     * Writes the unit/plist and returns whether its content actually changed
+     * (finding #4): `binPath`/`configPath` are the only inputs that ever
+     * change this content, and neither reflects a watchdog's `interval:` or
+     * any other config value inside `watchdogs:` — so `changed` here can never
+     * by itself justify a restart on every config edit. It's one of the two
+     * signals reconcileWatchdogScheduler (ops.ts) combines with a config
+     * fingerprint before deciding restart vs. the idempotent start.
+     */
+    install(binPath: string, configPath?: string): Promise<{
+        changed: boolean;
+    }>;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    /**
+     * `start()` is a no-op on an already-active unit — systemctl start against
+     * a running service just returns 0 without reloading anything, and
+     * launchctl kickstart (without -k) behaves the same way. That means a
+     * config change (new/changed watchdogs) never reaches a live scheduler
+     * process via reconcileWatchdogScheduler's `install` + `start` pair (final
+     * review #4). `restart()` mirrors WebServiceManager.restart: an
+     * unconditional restart on Linux, and `kickstart -k` (force-restart) on
+     * macOS, falling back to stop+start if the kickstart itself fails.
+     */
+    restart(): Promise<void>;
+    status(): Promise<string>;
+    uninstall(): Promise<void>;
+    private requireInstalled;
+    private must;
+}

package/dist/watchdog/service.js

@@ -0,0 +1,179 @@
+import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync } from 'node:fs';
+import { dirname, isAbsolute, join, resolve } from 'node:path';
+import { replaceFileAtomically } from '../atomic-file.js';
+import { FleetError } from '../application/errors.js';
+import { realExec } from '../exec.js';
+import { home } from '../paths.js';
+export const WATCHDOG_SYSTEMD_UNIT = 'ours-fleet-watchdogs.service';
+export const WATCHDOG_LAUNCHD_LABEL = 'network.ours.fleet.watchdogs';
+/**
+ * Supervises the single long-running watchdog-scheduler process (the hidden
+ * `_run-watchdogs` command, Task 9's `runScheduler`) the same way
+ * `WebServiceManager` (src/web/service.ts) supervises the web console: a
+ * private systemd --user unit on Linux, a launchd LaunchAgent on macOS.
+ */
+export class WatchdogServiceManager {
+    exec;
+    platform;
+    constructor(exec = realExec, platform = process.platform) {
+        this.exec = exec;
+        this.platform = platform;
+    }
+    get definitionPath() {
+        return this.platform === 'linux'
+            ? join(home(), '.config', 'systemd', 'user', WATCHDOG_SYSTEMD_UNIT)
+            : join(home(), 'Library', 'LaunchAgents', `${WATCHDOG_LAUNCHD_LABEL}.plist`);
+    }
+    /** false when explicitly disabled (OURS_FLEET_SUPERVISOR=none) or on an unsupported platform. */
+    supervised() {
+        if (process.env.OURS_FLEET_SUPERVISOR === 'none')
+            return false;
+        return this.platform === 'linux' || this.platform === 'darwin';
+    }
+    /**
+     * Writes the unit/plist and returns whether its content actually changed
+     * (finding #4): `binPath`/`configPath` are the only inputs that ever
+     * change this content, and neither reflects a watchdog's `interval:` or
+     * any other config value inside `watchdogs:` — so `changed` here can never
+     * by itself justify a restart on every config edit. It's one of the two
+     * signals reconcileWatchdogScheduler (ops.ts) combines with a config
+     * fingerprint before deciding restart vs. the idempotent start.
+     */
+    async install(binPath, configPath) {
+        const resolvedScript = resolveExecutable(binPath, 'ours-fleet CLI script');
+        const runtime = resolveExecutable(process.execPath, 'Node runtime');
+        const config = configPath ? resolve(configPath) : undefined;
+        const content = this.platform === 'linux'
+            ? watchdogSystemdUnit(runtime, resolvedScript, config)
+            : watchdogLaunchdPlist(runtime, resolvedScript, config);
+        let previous;
+        try {
+            previous = readFileSync(this.definitionPath, 'utf8');
+        }
+        catch { /* absent: definitely changed */ }
+        const changed = previous !== content;
+        mkdirSync(dirname(this.definitionPath), { recursive: true, mode: 0o700 });
+        replaceFileAtomically(this.definitionPath, content, 0o600);
+        if (this.platform === 'linux') {
+            await this.must('systemctl', ['--user', 'daemon-reload']);
+            await this.must('systemctl', ['--user', 'enable', WATCHDOG_SYSTEMD_UNIT]);
+        }
+        return { changed };
+    }
+    async start() {
+        this.requireInstalled();
+        if (this.platform === 'linux') {
+            await this.must('systemctl', ['--user', 'start', WATCHDOG_SYSTEMD_UNIT]);
+            return;
+        }
+        const domain = `gui/${uid()}`;
+        const loaded = await this.exec('launchctl', ['print', `${domain}/${WATCHDOG_LAUNCHD_LABEL}`]);
+        if (loaded.code === 0)
+            await this.must('launchctl', ['kickstart', `${domain}/${WATCHDOG_LAUNCHD_LABEL}`]);
+        else
+            await this.must('launchctl', ['bootstrap', domain, this.definitionPath]);
+    }
+    async stop() {
+        if (this.platform === 'linux') {
+            // Tolerate "not loaded" (systemctl exit 5) the same way the launchd
+            // branch below tolerates "could not find service": a watchdog-less
+            // fleet's `up`/`down` calls stop() defensively even when the unit was
+            // never installed, and that must not surface as an error (final
+            // review #3).
+            const result = await this.exec('systemctl', ['--user', 'stop', WATCHDOG_SYSTEMD_UNIT]);
+            if (result.code !== 0 && result.code !== 5 && !/not loaded/i.test(`${result.stdout}\n${result.stderr}`))
+                throw new FleetError('control_unavailable', `systemctl --user stop ${WATCHDOG_SYSTEMD_UNIT} failed: ${result.stderr.trim() || `exit ${result.code}`}`);
+            return;
+        }
+        const result = await this.exec('launchctl', ['bootout', `gui/${uid()}/${WATCHDOG_LAUNCHD_LABEL}`]);
+        if (result.code !== 0 && !/could not find service|no such process/i.test(`${result.stdout}\n${result.stderr}`))
+            throw new FleetError('control_unavailable', `launchctl bootout failed: ${result.stderr.trim()}`);
+    }
+    /**
+     * `start()` is a no-op on an already-active unit — systemctl start against
+     * a running service just returns 0 without reloading anything, and
+     * launchctl kickstart (without -k) behaves the same way. That means a
+     * config change (new/changed watchdogs) never reaches a live scheduler
+     * process via reconcileWatchdogScheduler's `install` + `start` pair (final
+     * review #4). `restart()` mirrors WebServiceManager.restart: an
+     * unconditional restart on Linux, and `kickstart -k` (force-restart) on
+     * macOS, falling back to stop+start if the kickstart itself fails.
+     */
+    async restart() {
+        this.requireInstalled();
+        if (this.platform === 'linux') {
+            await this.must('systemctl', ['--user', 'restart', WATCHDOG_SYSTEMD_UNIT]);
+            return;
+        }
+        const result = await this.exec('launchctl', ['kickstart', '-k', `gui/${uid()}/${WATCHDOG_LAUNCHD_LABEL}`]);
+        if (result.code !== 0) {
+            await this.stop();
+            await this.start();
+        }
+    }
+    async status() {
+        if (this.platform === 'linux') {
+            const result = await this.exec('systemctl', [
+                '--user', 'show', WATCHDOG_SYSTEMD_UNIT, '-p', 'LoadState', '-p', 'ActiveState',
+                '-p', 'SubState', '-p', 'ExecMainPID', '--no-pager',
+            ]);
+            return result.stdout.trim() || result.stderr.trim() || `exit ${result.code}`;
+        }
+        const result = await this.exec('launchctl', ['print', `gui/${uid()}/${WATCHDOG_LAUNCHD_LABEL}`]);
+        return result.code === 0 ? result.stdout.trim() : `not loaded (${WATCHDOG_LAUNCHD_LABEL})`;
+    }
+    async uninstall() {
+        if (this.platform === 'linux') {
+            await this.exec('systemctl', ['--user', 'disable', '--now', WATCHDOG_SYSTEMD_UNIT]);
+            rmSync(this.definitionPath, { force: true });
+            await this.exec('systemctl', ['--user', 'daemon-reload']);
+        }
+        else {
+            await this.stop();
+            rmSync(this.definitionPath, { force: true });
+        }
+    }
+    requireInstalled() {
+        if (!existsSync(this.definitionPath))
+            throw new FleetError('prerequisite_unavailable', 'watchdog scheduler service is not installed');
+    }
+    async must(command, args) {
+        const result = await this.exec(command, args);
+        if (result.code !== 0)
+            throw new FleetError('control_unavailable', `${command} ${args.join(' ')} failed: ${result.stderr.trim() || `exit ${result.code}`}`);
+    }
+}
+function uid() { return process.getuid?.() ?? 501; }
+function resolveExecutable(executable, label) {
+    const absolute = isAbsolute(executable) ? executable : resolve(executable);
+    try {
+        return realpathSync(absolute);
+    }
+    catch {
+        throw new FleetError('prerequisite_unavailable', `${label} does not exist: ${absolute}`);
+    }
+}
+function systemdQuote(value) {
+    return `"${value.replace(/[%\\"]/g, char => char === '%' ? '%%' : `\\${char}`)}"`;
+}
+function watchdogSystemdUnit(runtime, script, configuration) {
+    const config = configuration ? ` -c ${systemdQuote(configuration)}` : '';
+    return `[Unit]\nDescription=ours-fleet watchdog scheduler\nAfter=default.target\n\n`
+        + `[Service]\nType=simple\nExecStart=${systemdQuote(runtime)} ${systemdQuote(script)} _run-watchdogs${config}\n`
+        + `Restart=on-failure\nRestartSec=5\nTimeoutStopSec=15\n\n`
+        + `[Install]\nWantedBy=default.target\n`;
+}
+const xml = (value) => value.replace(/[&<>"']/g, char => ({
+    '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&apos;',
+}[char]));
+function watchdogLaunchdPlist(runtime, script, configuration) {
+    const config = configuration
+        ? `<string>-c</string><string>${xml(configuration)}</string>` : '';
+    return `<?xml version="1.0" encoding="UTF-8"?>\n`
+        + `<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n`
+        + `<plist version="1.0"><dict>\n<key>Label</key><string>${WATCHDOG_LAUNCHD_LABEL}</string>\n`
+        + `<key>ProgramArguments</key><array><string>${xml(runtime)}</string><string>${xml(script)}</string>`
+        + `<string>_run-watchdogs</string>${config}</array>\n`
+        + `<key>RunAtLoad</key><true/><key>KeepAlive</key><dict><key>SuccessfulExit</key><false/></dict>\n`
+        + `<key>ProcessType</key><string>Background</string>\n</dict></plist>\n`;
+}

package/dist/watchdog/store.d.ts

@@ -0,0 +1,85 @@
+import type { WatchdogReport, WatchdogReportStatus } from './report.js';
+/**
+ * A lock directory briefly exists before acquireRunLock can rename its
+ * prewritten owner.json into it. Missing/corrupt owner metadata is therefore
+ * reclaimable only after this grace period, never while that acquisition
+ * window may still be in progress.
+ */
+export declare const RUN_LOCK_OWNER_GRACE_MS = 10000;
+/**
+ * Choke point every other helper in this module goes through to reach a
+ * watchdog's on-disk state. Validates `name` BEFORE any join/mkdir (defense
+ * in depth, finding #1): a caller that forgets its own ROLE_NAME_RE guard
+ * (as the CLI's `watchdogKnown` once did) must not be able to turn an
+ * unvalidated `name` into `join(watchdogsRoot(), '../../victim')` and mkdir
+ * an arbitrary path on disk.
+ */
+export declare function watchdogDir(name: string): string;
+export declare function reportsDir(name: string): string;
+export interface RunLockOwner {
+    pid: number;
+    at: string;
+}
+/**
+ * mkdir-as-mutex: atomic across processes, unlike a lock file (open+O_EXCL
+ * would work too, but a directory needs no cleanup of file contents and
+ * can't be partially written). EEXIST means another run holds it.
+ *
+ * Stamps `owner.json` with our pid (finding #2): ownership metadata is what
+ * lets a later `reclaimStaleRunLock` tell a lock abandoned by a dead process
+ * apart from one genuinely held by a live run. A naive "mkdir, then
+ * writeFileSync(owner.json)" leaves an unnecessarily wide window —
+ * between the mkdir succeeding and the write landing — where the lock dir
+ * exists but owner.json doesn't yet. A `reclaimStaleRunLock` call from
+ * another process landing in exactly that window would see a lock with no
+ * (or corrupt) owner metadata and treat a genuinely live lock as a legacy
+ * one, reclaiming it out from under us. Do all slow work (building the JSON,
+ * writing it, chmod) to a per-pid temp file BEFORE mkdir, then publish it with
+ * one rename. There is still an unavoidable interval between those two
+ * syscalls; reclaimStaleRunLock protects it by treating a fresh ownerless lock
+ * as held for RUN_LOCK_OWNER_GRACE_MS. The temp file is unique per-pid (this
+ * function is synchronous, so there's no same-process concurrent-call hazard
+ * either) and is cleaned up if mkdir loses the race.
+ */
+export declare function acquireRunLock(name: string): boolean;
+/**
+ * Release-tolerant of absence: a lock already gone (or never acquired) is not
+ * an error. Recursive because the lock dir now holds `owner.json` alongside
+ * the mkdir mutex itself (finding #2) — a plain rmdir would fail ENOTEMPTY.
+ */
+export declare function releaseRunLock(name: string): void;
+/** Reads a run lock's owner metadata; missing or corrupt yields undefined. */
+export declare function readRunLockOwner(name: string): RunLockOwner | undefined;
+/**
+ * Owner-mandated (finding #2): only a DEMONSTRABLY stale run lock may be
+ * reclaimed — a lock is stale iff it isn't held at all, its owner metadata
+ * names a dead pid, or its owner metadata is missing/corrupt AND the lock dir
+ * is older than RUN_LOCK_OWNER_GRACE_MS. The age gate closes the interprocess
+ * interval between acquireRunLock's mkdir and owner.json rename: a concurrent
+ * scheduler sees a fresh ownerless lock as held, not stale. A live owner (a
+ * foreground `watchdog-run`, another scheduler instance, an in-progress run)
+ * is left strictly alone: reclaiming it would let two runs share the same temp
+ * dir.
+ * Returns true when the lock is (now) not held — whether because it was
+ * already absent or because a stale lock was just removed; false when a live
+ * lock was found and deliberately left in place.
+ */
+export declare function reclaimStaleRunLock(name: string): boolean;
+/** Lexical-chronological UTC run id, e.g. '20260731T115000Z'. */
+export declare function formatRunId(d: Date): string;
+export interface RunListEntry {
+    runId: string;
+    status: WatchdogReportStatus;
+    startedAt: string;
+    finishedAt: string;
+    summary: WatchdogReport['summary'];
+    error: string | null;
+}
+export declare function writeReport(name: string, report: WatchdogReport): string;
+/** Newest-first run listing; a corrupt report file yields a synthetic 'error' entry rather than throwing. */
+export declare function listRuns(name: string): RunListEntry[];
+/** Reads one run's full report. Rejects non-conforming runIds (path-traversal guard) and corrupt files by returning undefined. */
+export declare function readReport(name: string, runId: string): WatchdogReport | undefined;
+export declare function latestReport(name: string): WatchdogReport | undefined;
+/** Deletes all but the `keep` newest reports (oldest first); returns the number pruned. */
+export declare function pruneReports(name: string, keep: number): number;