@ours.network/fleet 0.10.4 → 0.11.0

package/dist/watchdog/query.js

@@ -0,0 +1,124 @@
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { FleetError } from '../application/errors.js';
+import { ROLE_NAME_RE } from '../config.js';
+import { watchdogsRoot } from '../paths.js';
+import { WATCHDOG_STATUS_RANK } from './alerts.js';
+import { readSchedulerState } from './scheduler.js';
+import { listRuns, readReport } from './store.js';
+/**
+ * Needs-attention integration (Task 19): worst current finding per role across
+ * every configured watchdog, for FleetQueryService.status() to fold into a
+ * role's problems. An `error`-status report carries no role evidence (the run
+ * itself failed) so it's skipped outright, matching alerts.ts's
+ * reconcileLedger rule. Healthy/idle findings (rank 0) never surface here —
+ * only actionable anomalies do. "Worst" is decided by WATCHDOG_STATUS_RANK;
+ * ties keep whichever watchdog was seen first. Takes `latestReport` as a
+ * parameter (rather than importing store.ts directly) so it stays a pure,
+ * disk-free function for unit testing; runtime.ts wires the real store.
+ *
+ * Skips disabled watchdogs (finding #5): a watchdog turned off in config
+ * still has its last stored report sitting on disk, and without this check
+ * that stale report's findings would pin roles in "Needs attention" forever
+ * — an operator disabling a noisy/broken watchdog has no way to make the
+ * findings it already produced go away.
+ */
+export function buildWatchdogFindings(cfg, latestReport) {
+    const findings = new Map();
+    for (const wd of cfg.watchdogs) {
+        if (!wd.enabled)
+            continue;
+        const report = latestReport(wd.name);
+        if (!report || report.status === 'error')
+            continue;
+        for (const finding of report.roles) {
+            const rank = WATCHDOG_STATUS_RANK[finding.status];
+            if (rank <= 0)
+                continue;
+            const existing = findings.get(finding.role);
+            if (existing && WATCHDOG_STATUS_RANK[existing.status] >= rank)
+                continue;
+            findings.set(finding.role, { watchdog: wd.name, status: finding.status, reason: finding.reason ?? '' });
+        }
+    }
+    return findings;
+}
+/**
+ * Memoizes a findings-map builder with a short TTL (mirrors runtime.ts's
+ * cachedConfigProvider pattern). status() calls its injected watchdogFindings
+ * provider once per role, so an unmemoized thunk turns a single list() sweep
+ * into O(roles x watchdogs) disk reads, repeated every 1-3s of console
+ * polling. `now` is injectable (defaults to Date.now) so the TTL boundary is
+ * unit-testable without real timers.
+ */
+export function cachedWatchdogFindingsProvider(build, ttlMs, now = Date.now) {
+    let cached;
+    return () => {
+        const at = now();
+        if (!cached || at - cached.at >= ttlMs)
+            cached = { at, value: build() };
+        return cached.value;
+    };
+}
+const DEFAULT_REPORTS_LIMIT = 50;
+const MAX_REPORTS_LIMIT = 500;
+/**
+ * Read-only view over watchdog config + on-disk scheduler state + stored
+ * reports, for the authenticated web console. Never mutates: `list()` reads
+ * config plus `readSchedulerState`/`listRuns` (both already tolerant of a
+ * corrupt/missing state.json or an empty/nonexistent reports dir, per
+ * store.ts and scheduler.ts), and `reports()`/`report()` reject unknown
+ * watchdog names before touching disk.
+ */
+export class WatchdogQueryService {
+    cfgProvider;
+    constructor(cfgProvider) {
+        this.cfgProvider = cfgProvider;
+    }
+    list() {
+        const cfg = this.cfgProvider();
+        const watchdogs = cfg.watchdogs.map((wd) => {
+            const state = readSchedulerState(wd.name);
+            const [latest = null] = listRuns(wd.name);
+            return {
+                name: wd.name, enabled: wd.enabled, heldDown: state.heldDown, heldSince: state.heldSince ?? null,
+                intervalMs: wd.intervalMs,
+                coordinator: wd.coordinator, watch: wd.watch,
+                lastRunAt: state.lastRunAt ?? null, nextRunAt: state.nextRunAt ?? null,
+                latest,
+            };
+        });
+        return { watchdogs };
+    }
+    reports(name, limit = DEFAULT_REPORTS_LIMIT) {
+        this.requireKnown(name);
+        const n = Number.isFinite(limit) ? Math.floor(limit) : DEFAULT_REPORTS_LIMIT;
+        const clamped = Math.min(Math.max(n, 1), MAX_REPORTS_LIMIT);
+        return { runs: listRuns(name).slice(0, clamped) };
+    }
+    report(name, runId) {
+        this.requireKnown(name);
+        // readReport parses the stored file with JSON.parse and returns the parsed object;
+        // fastify re-serializes it for the HTTP response, so what's actually verifiable (and
+        // what the tests assert) is deep equality with the stored JSON, not byte-for-byte
+        // identity of the response bytes against the file on disk.
+        const found = readReport(name, runId);
+        if (!found)
+            throw new FleetError('role_not_found', `no such report '${runId}' for watchdog '${name}'`);
+        return found;
+    }
+    /**
+     * Known means: configured today, OR a store directory already exists for a
+     * watchdog that used to be configured (its history should stay readable).
+     * The regex check runs before any filesystem lookup so a hostile `name`
+     * (path separators, '..', etc.) can never reach `join(watchdogsRoot(), name)`.
+     */
+    requireKnown(name) {
+        const cfg = this.cfgProvider();
+        if (cfg.watchdogs.some(wd => wd.name === name))
+            return;
+        if (ROLE_NAME_RE.test(name) && existsSync(join(watchdogsRoot(), name)))
+            return;
+        throw new FleetError('role_not_found', `no such watchdog '${name}'`);
+    }
+}

package/dist/watchdog/report.d.ts

@@ -0,0 +1,53 @@
+export declare const WATCHDOG_ROLE_STATUSES: readonly ["healthy", "idle", "stale", "blocked", "off_briefing", "unreachable", "unknown"];
+export type WatchdogRoleStatus = typeof WATCHDOG_ROLE_STATUSES[number];
+export interface WatchdogEvidence {
+    source: string;
+    detail: string;
+    observed_at: string;
+}
+export interface WatchdogFinding {
+    role: string;
+    status: WatchdogRoleStatus;
+    reason?: string;
+    evidence?: WatchdogEvidence[];
+    alerted?: boolean;
+}
+export interface WatchdogAlert {
+    role: string;
+    code: string;
+    coordinator: string;
+    sent_at: string;
+}
+export type WatchdogReportStatus = 'ok' | 'anomalies' | 'error';
+export interface WatchdogReport {
+    schema_version: 1;
+    watchdog: string;
+    run_id: string;
+    started_at: string;
+    finished_at: string;
+    status: WatchdogReportStatus;
+    summary: {
+        checked: number;
+        healthy: number;
+        idle: number;
+        anomalies: number;
+    };
+    roles: WatchdogFinding[];
+    alerts: WatchdogAlert[];
+    error: string | null;
+}
+export declare function validateWatchdogReport(v: unknown): string[];
+export declare function normalizeWatchdogReport(r: WatchdogReport, ctx: {
+    watchdog: string;
+    run_id: string;
+}): WatchdogReport;
+export declare function errorReport(ctx: {
+    watchdog: string;
+    run_id: string;
+    started_at: string;
+    finished_at: string;
+    error: string;
+    tail?: string;
+}): WatchdogReport & {
+    tail?: string;
+};

package/dist/watchdog/report.js

@@ -0,0 +1,126 @@
+export const WATCHDOG_ROLE_STATUSES = ['healthy', 'idle', 'stale', 'blocked', 'off_briefing', 'unreachable', 'unknown'];
+const SUMMARY_KEYS = ['checked', 'healthy', 'idle', 'anomalies'];
+function isPlainObject(v) {
+    return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+export function validateWatchdogReport(v) {
+    const errors = [];
+    if (!isPlainObject(v)) {
+        errors.push('report: expected an object');
+        return errors;
+    }
+    const r = v;
+    if (r.schema_version !== 1)
+        errors.push('schema_version: expected 1');
+    for (const key of ['watchdog', 'run_id', 'started_at', 'finished_at'])
+        if (typeof r[key] !== 'string')
+            errors.push(`${key}: expected a string`);
+    if (r.status !== 'ok' && r.status !== 'anomalies' && r.status !== 'error')
+        errors.push(`status: expected ok|anomalies|error, got '${String(r.status)}'`);
+    if (!isPlainObject(r.summary)) {
+        errors.push('summary: expected an object');
+    }
+    else {
+        for (const key of SUMMARY_KEYS)
+            if (typeof r.summary[key] !== 'number')
+                errors.push(`summary.${key}: expected a number`);
+    }
+    if (!Array.isArray(r.roles)) {
+        errors.push('roles: expected an array');
+    }
+    else {
+        r.roles.forEach((role, i) => {
+            if (!isPlainObject(role)) {
+                errors.push(`roles[${i}]: expected an object`);
+                return;
+            }
+            if (typeof role.role !== 'string')
+                errors.push(`roles[${i}].role: expected a string`);
+            if (!WATCHDOG_ROLE_STATUSES.includes(role.status))
+                errors.push(`roles[${i}].status: expected one of ${WATCHDOG_ROLE_STATUSES.join('|')}, got '${String(role.status)}'`);
+            if (role.status !== 'healthy' && role.status !== 'idle' && !role.reason)
+                errors.push(`roles[${i}]: non-healthy finding requires a reason`);
+            if (role.reason !== undefined && typeof role.reason !== 'string')
+                errors.push(`roles[${i}].reason: expected a string`);
+            if (role.evidence !== undefined) {
+                if (!Array.isArray(role.evidence)) {
+                    errors.push(`roles[${i}].evidence: expected an array`);
+                }
+                else {
+                    role.evidence.forEach((ev, j) => {
+                        if (!isPlainObject(ev)) {
+                            errors.push(`roles[${i}].evidence[${j}]: expected an object`);
+                            return;
+                        }
+                        for (const key of ['source', 'detail', 'observed_at'])
+                            if (typeof ev[key] !== 'string')
+                                errors.push(`roles[${i}].evidence[${j}].${key}: expected a string`);
+                    });
+                }
+            }
+        });
+    }
+    if (!Array.isArray(r.alerts)) {
+        errors.push('alerts: expected an array');
+    }
+    else {
+        r.alerts.forEach((alert, i) => {
+            if (!isPlainObject(alert)) {
+                errors.push(`alerts[${i}]: expected an object`);
+                return;
+            }
+            for (const key of ['role', 'code', 'coordinator', 'sent_at'])
+                if (typeof alert[key] !== 'string')
+                    errors.push(`alerts[${i}].${key}: expected a string`);
+        });
+    }
+    if (r.error !== null && typeof r.error !== 'string')
+        errors.push('error: expected string or null');
+    return errors;
+}
+const cleanEvidence = (value, max = 280) => value.replace(/[\0-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '').trim().slice(0, max);
+export function normalizeWatchdogReport(r, ctx) {
+    const copy = JSON.parse(JSON.stringify(r));
+    copy.watchdog = ctx.watchdog;
+    copy.run_id = ctx.run_id;
+    copy.roles = copy.roles.map(role => {
+        // Rebuilt from ONLY the known fields (final review #6a) — the old
+        // `{ ...role }` spread let any unknown per-finding key (e.g. an agent
+        // dumping a huge `pane_dump`) ride unbounded into the store. Report-level
+        // extras (tail/isolation) are untouched here and stay tolerated; this is
+        // a per-finding allowlist only.
+        const next = { role: role.role, status: role.status };
+        if (role.reason !== undefined)
+            next.reason = cleanEvidence(role.reason);
+        if (role.evidence !== undefined) {
+            next.evidence = role.evidence.slice(0, 3).map(ev => ({
+                source: cleanEvidence(ev.source),
+                detail: cleanEvidence(ev.detail),
+                observed_at: cleanEvidence(ev.observed_at),
+            }));
+        }
+        if (role.alerted !== undefined)
+            next.alerted = Boolean(role.alerted);
+        return next;
+    });
+    return copy;
+}
+export function errorReport(ctx) {
+    return {
+        schema_version: 1,
+        watchdog: ctx.watchdog,
+        run_id: ctx.run_id,
+        started_at: ctx.started_at,
+        finished_at: ctx.finished_at,
+        status: 'error',
+        summary: { checked: 0, healthy: 0, idle: 0, anomalies: 0 },
+        roles: [],
+        alerts: [],
+        // Scheduler-built error strings can embed raw agent-controlled bytes
+        // (JSON.parse messages, validator interpolations of agent-supplied
+        // values) and are stored+printed as-is elsewhere — clean them the same
+        // way finding-level text is cleaned (final review #6b).
+        error: cleanEvidence(ctx.error),
+        ...(ctx.tail !== undefined ? { tail: ctx.tail.slice(-4096) } : {}),
+    };
+}

package/dist/watchdog/run.d.ts

@@ -0,0 +1,61 @@
+import type { ResolvedWatchdog } from './config.js';
+import type { WatchdogReport } from './report.js';
+import { type IdentityProvisioner } from '../creation.js';
+import { type FleetConfig } from '../config.js';
+/** A minimal handle over the launched child: kill it, or await its natural exit. */
+export interface WatchdogChildHandle {
+    kill(): void;
+    exited: Promise<void>;
+}
+export interface WatchdogRunDeps {
+    binPath: string;
+    log(line: string): void;
+    now?(): Date;
+    sleep?(ms: number): Promise<void>;
+    identityProvisioner?: IdentityProvisioner;
+    /**
+     * Injectable child launcher for tests. Default: spawn
+     * `node <binPath> _run-watchdog <roleName>` detached:false, stdio to
+     * `<runDir>/run.log`. Returns kill() and an exited promise.
+     */
+    launchChild?(binPath: string, roleName: string, runDir: string): WatchdogChildHandle;
+    /** Pre-loaded config (defaults inheritance). Falls back to `loadConfig()`. */
+    cfg?: FleetConfig;
+}
+export interface WatchdogRunOutcome {
+    report: WatchdogReport;
+    storedPath: string;
+}
+/**
+ * Run one watchdog agent end-to-end in a clean-context temp state dir: provision
+ * identity, materialize the run's contract (briefing/manifest/role snapshot),
+ * launch the child, enforce a deadline against report.json as the completion
+ * sentinel, harvest whatever it wrote (or a synthetic error report if it
+ * didn't), store the result, and always clean up the temp dir.
+ *
+ * The scheduler's run-lock guarantees only one run per watchdog at a time;
+ * this function itself takes no lock (Task 8).
+ */
+export declare function executeWatchdogRun(wd: ResolvedWatchdog, deps: WatchdogRunDeps): Promise<WatchdogRunOutcome>;
+/**
+ * Deliver one scheduler-level alert (e.g. held-down) by launching a minimal one-shot temp agent
+ * under the watchdog's own identity whose entire mission is: bind identity, send `text` to
+ * `wd.coordinator`, write `sent.json`, exit. The fleet process itself cannot send ours messages
+ * (owner-approved deviation 4), so this is how a scheduler tick that can't reach an operator any
+ * other way still gets a message out.
+ *
+ * Reuses the Task 7/8 run machinery (temp-dir prep, ensureIdentity, ResolvedRole shape, child
+ * launch, kill mechanics) but stores no report, touches no ledger findings, and writes no
+ * watch.json manifest — `sent.json` is this run's only completion sentinel. A fixed 2-minute
+ * deadline applies regardless of `wd.timeoutMs`.
+ *
+ * Unlike executeWatchdogRun (which relies on the scheduler's run lock), this function acquires
+ * and releases the run lock itself — a notifier run shares the same temp dir name
+ * (`agentDir(wd.identity, true)`) as a regular run, and nothing else serializes the two. Refusal
+ * to acquire is logged and treated as a no-op: the held-down state already means no regular runs
+ * are in flight, so a collision is rare. Any other failure (including a timeout) is logged as a
+ * warning and swallowed — a notifier failure must never throw into the scheduler loop.
+ */
+export declare function executeNotifierRun(wd: ResolvedWatchdog, text: string, deps: WatchdogRunDeps): Promise<void>;
+/** What `_run-watchdog` calls: one supervised session, no cleanup — the parent harvests. */
+export declare function runWatchdogAgent(name: string): Promise<void>;