@dsdenes/devboard 0.1.560 → 0.1.562

package/watchdog/collect.ts

@@ -0,0 +1,311 @@
+import { readFileSync, readdirSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { join, relative, resolve } from "node:path";
+import { JOURNAL_KINDS, journalFiles, type JournalKind } from "../server/journal";
+import { processSnapshot, type Instance } from "./discovery";
+
+/** The evidence window is the ten minutes before cycle start, both ends inclusive. */
+export const WINDOW_MS = 10 * 60 * 1000;
+
+export interface Window {
+  start: Date;
+  end: Date;
+}
+
+export interface EvidenceEvent {
+  /** File the evidence came from. */
+  source: string;
+  /** A journal kind, or `omp-log` for omp's own process log. */
+  kind: string;
+  /** ISO timestamp used for the window decision. */
+  ts: string;
+  text: string;
+  /** Logical run, lane, or peer the journal record named. */
+  scope?: string;
+  /** Severity token an `omp-log` record carried. */
+  level?: "error" | "warn" | "fatal";
+  /** Project the evidence belongs to, when attribution resolved it. */
+  instance?: Instance;
+}
+
+export interface Collection {
+  events: EvidenceEvent[];
+  /** Records with an unparseable timestamp or malformed shape. */
+  malformed: number;
+  /** omp log lines outside every instance's working tree. */
+  unattributed: number;
+}
+
+export interface WindowOptions {
+  window: Window;
+}
+
+/** Returns the parsed instant, or undefined when the timestamp is unusable. */
+export function parseTimestamp(value: unknown): number | undefined {
+  if (typeof value !== "string" || value.length === 0) return undefined;
+  const parsed = Date.parse(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+function normalizedPath(root: string, path: string): string {
+  const resolved = resolve(path);
+  const local = relative(resolve(root), resolved);
+  return local.length === 0 || local.startsWith("..") ? resolved : local;
+}
+
+/** In-window events of one journal file, with the identity used to merge copies. */
+interface JournalFileEvidence {
+  events: EvidenceEvent[];
+  /** Record identity parallel to `events`. */
+  identities: string[];
+  malformed: number;
+}
+
+/**
+ * Reads one JSONL journal file. Records inside the window are returned;
+ * malformed records are counted, never guessed. An unreadable file is empty.
+ */
+function readJournalFile(
+  path: string,
+  options: WindowOptions & { root: string },
+  instance: Instance,
+): JournalFileEvidence {
+  let source: string;
+  try {
+    source = readFileSync(path, "utf8");
+  } catch {
+    return { events: [], identities: [], malformed: 0 };
+  }
+  const events: EvidenceEvent[] = [];
+  const identities: string[] = [];
+  let malformed = 0;
+  for (const line of source.split("\n")) {
+    if (line.trim().length === 0) continue;
+    let record: Record<string, unknown>;
+    try {
+      const parsed: unknown = JSON.parse(line);
+      if (typeof parsed !== "object" || parsed === null) throw new Error("not an object");
+      record = parsed as Record<string, unknown>;
+    } catch {
+      malformed += 1;
+      continue;
+    }
+    const ts = parseTimestamp(record.ts);
+    if (ts === undefined) {
+      malformed += 1;
+      continue;
+    }
+    if (ts < options.window.start.getTime() || ts > options.window.end.getTime()) continue;
+    const kind = record.kind;
+    if (typeof kind !== "string" || !JOURNAL_KINDS.includes(kind as JournalKind)) continue;
+    if (typeof record.text !== "string") continue;
+    const scope = typeof record.scope === "string" && record.scope.length > 0 ? record.scope : "";
+    events.push({
+      source: normalizedPath(options.root, path),
+      kind,
+      ts: new Date(ts).toISOString(),
+      text: record.text,
+      ...(scope.length > 0 ? { scope } : {}),
+      instance,
+    });
+    identities.push(JSON.stringify([ts, kind, scope, record.text]));
+  }
+  return { events, identities, malformed };
+}
+
+/**
+ * Reads one project's JSONL journal, live and rotated. The files are the
+ * serving harness's, so a dashboard from another checkout contributes its own
+ * records. Rotated journals are read oldest first, and a record present in
+ * both a rotated file and the live one is counted once: rotation moves records
+ * rather than copying them, so a copy is the same record. Malformed records
+ * count across every file read.
+ */
+export function readJournalEvidence(
+  instance: Instance,
+  options: WindowOptions & { root: string },
+): { events: EvidenceEvent[]; malformed: number } {
+  const events: EvidenceEvent[] = [];
+  let malformed = 0;
+  /** Most occurrences of one identity any earlier file held. */
+  const seen = new Map<string, number>();
+  for (const path of journalFiles(instance.journal, options.window.start)) {
+    const file = readJournalFile(path, options, instance);
+    malformed += file.malformed;
+    const occurrences = new Map<string, number>();
+    file.events.forEach((event, index) => {
+      const identity = file.identities[index] ?? "";
+      const occurrence = occurrences.get(identity) ?? 0;
+      occurrences.set(identity, occurrence + 1);
+      // Copies beyond what an earlier file already held are the same record.
+      if (occurrence < (seen.get(identity) ?? 0)) return;
+      events.push(event);
+    });
+    for (const [identity, count] of occurrences) {
+      seen.set(identity, Math.max(seen.get(identity) ?? 0, count));
+    }
+  }
+  return { events, malformed };
+}
+
+export interface ProcessInfo {
+  /** Process ids from the parent upward, nearest first. */
+  ancestors: number[];
+}
+
+export interface OmpOptions extends WindowOptions {
+  logsDir?: string;
+  instances: Instance[];
+  /** Pids of the watchdog and its descendants. */
+  selfPids: Set<number>;
+  /** Injectable process lookup; defaults to `/proc`. */
+  processInfo?: (pid: number) => ProcessInfo | undefined;
+}
+
+interface OmpRecord {
+  timestamp?: unknown;
+  level?: unknown;
+  pid?: unknown;
+  message?: unknown;
+  error?: unknown;
+  [key: string]: unknown;
+}
+
+function describeOmpLine(record: OmpRecord): string {
+  const parts = [record.message, record.error]
+    .filter((part): part is string => typeof part === "string" && part.length > 0)
+    .join(": ");
+  return parts.length > 0 ? parts : JSON.stringify(record);
+}
+
+/**
+ * An omp line belongs to an instance only when its pid descends from that
+ * dashboard's server process. A shared working directory never attributes a
+ * line: an operator's own omp session in the project runs from the same
+ * directory while being unrelated to the dashboard, and its warnings are not
+ * dashboard evidence.
+ */
+function attribution(
+  pid: number,
+  instance: Instance[],
+  processInfo: (pid: number) => ProcessInfo | undefined,
+): Instance | undefined {
+  const info = processInfo(pid);
+  if (info === undefined) return undefined;
+  return instance.find((candidate) => info.ancestors.includes(candidate.pid));
+}
+
+/**
+ * Reads omp's own process logs. A line is evidence for an instance only when
+ * its pid descends from that instance's server; a shared working directory
+ * never attributes a line, and the watchdog's own pids never become evidence.
+ */
+export function readOmpEvidence(options: OmpOptions): {
+  events: EvidenceEvent[];
+  malformed: number;
+  unattributed: number;
+} {
+  const logsDir = options.logsDir ?? join(homedir(), ".omp", "logs");
+  const processInfo =
+    options.processInfo ??
+    ((pid: number) => {
+      const snapshot = processSnapshot(pid);
+      return snapshot === undefined ? undefined : { ancestors: snapshot.ancestors };
+    });
+  let names: string[];
+  try {
+    names = readdirSync(logsDir).filter((name) => /^omp\..*\.log$/.test(name));
+  } catch {
+    return { events: [], malformed: 0, unattributed: 0 };
+  }
+  let malformed = 0;
+  let unattributed = 0;
+  const events: EvidenceEvent[] = [];
+  for (const name of names) {
+    const path = join(logsDir, name);
+    try {
+      if (statSync(path).mtimeMs < options.window.start.getTime()) continue;
+    } catch {
+      continue;
+    }
+    let source: string;
+    try {
+      source = readFileSync(path, "utf8");
+    } catch {
+      continue;
+    }
+    for (const line of source.split("\n")) {
+      if (line.trim().length === 0) continue;
+      let record: OmpRecord;
+      try {
+        const parsed: unknown = JSON.parse(line);
+        if (typeof parsed !== "object" || parsed === null) throw new Error("not an object");
+        record = parsed as OmpRecord;
+      } catch {
+        malformed += 1;
+        continue;
+      }
+      const ts = parseTimestamp(record.timestamp);
+      if (ts === undefined) {
+        malformed += 1;
+        continue;
+      }
+      if (ts < options.window.start.getTime() || ts > options.window.end.getTime()) continue;
+      const level = typeof record.level === "string" ? record.level.toLowerCase() : "";
+      if (level !== "error" && level !== "warn" && level !== "fatal") continue;
+      const pid = Number(record.pid);
+      if (!Number.isInteger(pid) || pid <= 0) {
+        malformed += 1;
+        continue;
+      }
+      if (options.selfPids.has(pid)) continue;
+      const ancestors = processInfo(pid)?.ancestors ?? [];
+      if (ancestors.some((ancestor) => options.selfPids.has(ancestor))) continue;
+      const owner = attribution(pid, options.instances, processInfo);
+      if (owner === undefined) {
+        unattributed += 1;
+        continue;
+      }
+      events.push({
+        source: path,
+        kind: "omp-log",
+        ts: new Date(ts).toISOString(),
+        text: describeOmpLine(record),
+        level,
+        instance: owner,
+      });
+    }
+  }
+  return { events, malformed, unattributed };
+}
+
+export interface CollectOptions extends WindowOptions {
+  root: string;
+  instances: Instance[];
+  selfPids: Set<number>;
+  logsDir?: string;
+  processInfo?: (pid: number) => ProcessInfo | undefined;
+}
+
+/** Collects the journal and omp evidence for one cycle window. */
+export function collectEvidence(options: CollectOptions): Collection {
+  const events: EvidenceEvent[] = [];
+  let malformed = 0;
+  for (const instance of options.instances) {
+    const journal = readJournalEvidence(instance, options);
+    events.push(...journal.events);
+    malformed += journal.malformed;
+  }
+  const omp = readOmpEvidence(options);
+  events.push(...omp.events);
+  malformed += omp.malformed;
+  return { events, malformed, unattributed: omp.unattributed };
+}
+
+/** An evidence line without its timestamp or incidental whitespace. */
+export function normalizeEvidenceLine(text: string): string {
+  return text
+    .replace(/^\d{4}-\d{2}-\d{2}T[^\s]+/, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}

package/watchdog/coordinator.ts

@@ -0,0 +1,397 @@
+import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { pruneJournal } from "../server/journal";
+import { collectEvidence, WINDOW_MS, type Collection, type Window } from "./collect";
+import { coveredStateDirs, runCleanup, type CleanupReport } from "./cleanup";
+import { discoverInstances, type Instance } from "./discovery";
+import { analyzeEvidence } from "./findings";
+import { runGh, type GhRun, type IssueInput } from "./github";
+import { createWatchdogDelivery, type WatchdogDelivery } from "./delivery";
+import {
+  loadLedger,
+  markCommented,
+  markFiled,
+  markPending,
+  recordFindings,
+  saveLedger,
+  type LedgerEntry,
+} from "./ledger";
+
+/**
+ * One watchdog cycle: discover live dashboards, collect the previous ten
+ * minutes of machine-checkable evidence, and file or comment one GitHub issue
+ * per distinct signal. It never spawns an agent, creates a worktree, commits,
+ * merges, pushes, or publishes.
+ */
+
+/** How long an existing issue waits before a recurrence comments on it. */
+export const COMMENT_INTERVAL_MS = 24 * 60 * 60 * 1000;
+
+export interface CycleContext {
+  root: string;
+  harness: string;
+  stateDir: string;
+  runDir: string;
+  now: Date;
+  window: Window;
+  log: (line: string) => void;
+}
+
+export interface CycleDeps {
+  discover: (root: string) => Instance[];
+  collect: (root: string, instances: Instance[], window: Window) => Collection;
+  gh: GhRun;
+  delivery?: WatchdogDelivery;
+}
+
+export interface ReportInstance {
+  pid: number;
+  port: number;
+  repoDir: string;
+  harness: string;
+  instrumented: boolean;
+  journal: string;
+  legacyLog?: string;
+}
+
+export interface ReportFinding {
+  category: string;
+  ts: string;
+  quote: string;
+  fingerprint: string;
+  instances: { repoDir: string; port: number; pid: number; journal: string }[];
+}
+
+export interface ReportIssue {
+  url: string;
+  category: string;
+  fingerprint: string;
+  line: string;
+}
+
+export type CycleAction = "none" | "duplicate" | "filed" | "commented" | "failed";
+
+export interface Report {
+  cycle: string;
+  window: { start: string; end: string; minutes: number };
+  instances: ReportInstance[];
+  findings: ReportFinding[];
+  action: CycleAction;
+  issues: ReportIssue[];
+  /** Fingerprints still waiting for a successful filing. */
+  pending: number;
+  /** This cycle's report; the issue body points at it. */
+  reportPath: string;
+  reason?: string;
+  malformed: number;
+  unattributed: number;
+  /** Age-based retention over every covered state directory. */
+  cleanup: CleanupReport;
+}
+
+export interface CycleOptions {
+  root?: string;
+  now?: Date;
+  stateDir?: string;
+  dryRun?: boolean;
+  deps?: Partial<CycleDeps>;
+}
+
+export interface CycleResult {
+  report: Report;
+  exitCode: number;
+  runDir?: string;
+}
+
+function stamp(now: Date): string {
+  return now.toISOString().replace(/[:.]/g, "-");
+}
+
+function isFiled(entry: LedgerEntry | undefined): boolean {
+  return entry !== undefined && entry.disposition.startsWith("filed:");
+}
+
+/** Runs one watchdog cycle. The caller owns the singleton lock. */
+export async function runCycle(options: CycleOptions = {}): Promise<CycleResult> {
+  const root = resolve(options.root ?? join(import.meta.dir, "..", ".."));
+  const harness = join(root, ".devboard");
+  const now = options.now ?? new Date();
+  const window: Window = {
+    start: new Date(now.getTime() - WINDOW_MS),
+    end: now,
+  };
+  const stateDir = resolve(options.stateDir ?? join(harness, ".tmp", "watchdog"));
+  const runDir = join(stateDir, "runs", stamp(now));
+  const reportPath = join(runDir, "report.json");
+  const deps: CycleDeps = {
+    discover: () => discoverInstances({ selfPids: new Set([process.pid]) }),
+    collect: (target, instances, collectedWindow) =>
+      collectEvidence({
+        root: target,
+        instances,
+        window: collectedWindow,
+        selfPids: new Set([process.pid]),
+      }),
+    gh: runGh,
+    ...options.deps,
+  };
+  const dryRun = options.dryRun === true;
+  if (!dryRun) mkdirSync(runDir, { recursive: true });
+  const coordinatorLog = dryRun ? undefined : join(runDir, "coordinator.log");
+  const context: CycleContext = {
+    root,
+    harness,
+    stateDir,
+    runDir,
+    now,
+    window,
+    log(line: string) {
+      const stamped = `${new Date().toISOString()} ${line}\n`;
+      if (coordinatorLog !== undefined) {
+        try {
+          appendFileSync(coordinatorLog, stamped);
+        } catch {
+          // Reporting must not fail the cycle.
+        }
+      }
+      process.stderr.write(`watchdog: ${line}\n`);
+    },
+  };
+
+  const ledger = loadLedger(stateDir);
+  const instances = deps.discover(root);
+  context.log(`discovered ${instances.length} instance(s)`);
+
+  const covered = coveredStateDirs(root, instances);
+  if (!dryRun) {
+    for (const entry of covered) {
+      const removed = pruneJournal(entry.stateDir, {
+        now,
+        windowStart: window.start,
+      });
+      if (removed.length > 0) {
+        context.log(`pruned ${removed.length} old journal file(s) in ${entry.stateDir}`);
+      }
+    }
+  }
+
+  const cleanup = runCleanup({
+    root,
+    now,
+    dryRun,
+    instances,
+    covered,
+    watchdogRunDir: runDir,
+  });
+  for (const entry of cleanup.roots) {
+    context.log(
+      `cleanup ${entry.stateDir}: ${dryRun ? "would remove" : "removed"} ` +
+        `${entry.files} file(s), ${entry.directories} directory/directories, ` +
+        `${entry.bytes} byte(s)` +
+        (entry.errors.length > 0 ? `, ${entry.errors.length} error(s)` : ""),
+    );
+    for (const error of entry.errors) context.log(`cleanup ${entry.stateDir}: ${error}`);
+  }
+
+  const collection = deps.collect(root, instances, window);
+  const findings = analyzeEvidence(collection.events, { window });
+  context.log(
+    `collected ${collection.events.length} event(s), ${findings.length} finding(s), ` +
+      `${collection.malformed} malformed, ${collection.unattributed} unattributed`,
+  );
+  recordFindings(ledger, findings, now);
+
+  const report: Report = {
+    cycle: now.toISOString(),
+    window: {
+      start: window.start.toISOString(),
+      end: window.end.toISOString(),
+      minutes: WINDOW_MS / 60 / 1000,
+    },
+    instances: instances.map((instance) => ({
+      pid: instance.pid,
+      port: instance.port,
+      repoDir: instance.repoDir,
+      harness: instance.harness,
+      instrumented: instance.instrumented,
+      journal: instance.journal,
+      ...(instance.legacyLog === undefined ? {} : { legacyLog: instance.legacyLog }),
+    })),
+    findings: findings.map((finding) => ({
+      category: finding.category,
+      ts: finding.ts,
+      quote: finding.quote,
+      fingerprint: finding.fingerprint,
+      instances: finding.instances.map((instance) => ({
+        repoDir: instance.repoDir,
+        port: instance.port,
+        pid: instance.pid,
+        journal: instance.journal,
+      })),
+    })),
+    action: "none",
+    issues: [],
+    pending: ledger.pendingFiling.length,
+    reportPath,
+    malformed: collection.malformed,
+    unattributed: collection.unattributed,
+    cleanup,
+  };
+
+  if (dryRun) {
+    const settled = findings.every((finding) => isFiled(ledger.findings[finding.fingerprint]));
+    if (findings.length === 0 && ledger.pendingFiling.length === 0) {
+      report.reason = "no finding in the window";
+    } else if (settled && ledger.pendingFiling.length === 0) {
+      report.action = "duplicate";
+      report.reason = "every finding already has an issue";
+    } else {
+      report.reason =
+        `dry run: ${findings.length} finding(s) and ` +
+        `${ledger.pendingFiling.length} pending filing(s) not attempted`;
+    }
+    return { report, exitCode: 0 };
+  }
+
+  let delivery = deps.delivery;
+  const deliveryFor = () => {
+    delivery ??= createWatchdogDelivery({ root, gh: deps.gh });
+    return delivery;
+  };
+  const queue = [
+    ...ledger.pendingFiling.filter((fingerprint) => !isFiled(ledger.findings[fingerprint])),
+    ...findings
+      .map((finding) => finding.fingerprint)
+      .filter(
+        (fingerprint) =>
+          !isFiled(ledger.findings[fingerprint]) &&
+          !ledger.pendingFiling.includes(fingerprint),
+      ),
+  ];
+
+  const issueInput = (fingerprint: string, entry: LedgerEntry): IssueInput => ({
+    category: entry.category,
+    fingerprint,
+    line: entry.line,
+    firstSeen: entry.firstSeen,
+    lastSeen: entry.lastSeen,
+    count: entry.count,
+    window,
+    instances: entry.instances,
+    reportPath,
+  });
+
+  let failure: string | undefined;
+  let commented = 0;
+
+  // Pending filings are retried first: a failed `gh` call never loses evidence.
+  for (const fingerprint of queue) {
+    const entry = ledger.findings[fingerprint];
+    if (entry === undefined) {
+      ledger.pendingFiling = ledger.pendingFiling.filter((pending) => pending !== fingerprint);
+      continue;
+    }
+    const filed = await deliveryFor().create(issueInput(fingerprint, entry));
+    if (filed.ok) {
+      markFiled(ledger, fingerprint, filed.url, now);
+      report.issues.push({
+        url: filed.url,
+        category: entry.category,
+        fingerprint,
+        line: entry.line,
+      });
+      context.log(`filed ${filed.url} for ${entry.category}: ${entry.line}`);
+      continue;
+    }
+    failure ??= filed.reason;
+    markPending(ledger, fingerprint);
+    context.log(`filing ${entry.category} failed: ${filed.reason}`);
+  }
+
+  const filedThisCycle = new Set(report.issues.map((issue) => issue.fingerprint));
+  for (const finding of findings) {
+    const entry = ledger.findings[finding.fingerprint];
+    if (entry === undefined || entry.url === undefined) continue;
+    if (filedThisCycle.has(finding.fingerprint)) continue;
+    const baseline = Date.parse(entry.lastCommentAt ?? entry.filedAt ?? "");
+    if (!Number.isFinite(baseline) || now.getTime() - baseline < COMMENT_INTERVAL_MS) continue;
+    const comment = await deliveryFor().comment(
+      entry.url,
+      issueInput(finding.fingerprint, entry),
+    );
+    if (comment.ok) {
+      markCommented(ledger, finding.fingerprint, now);
+      commented += 1;
+      context.log(`commented on ${entry.url}`);
+      continue;
+    }
+    failure ??= comment.reason;
+    context.log(`comment on ${entry.url} failed: ${comment.reason}`);
+  }
+
+  if (failure !== undefined) {
+    report.action = "failed";
+    report.reason = failure;
+  } else if (report.issues.length > 0) {
+    report.action = "filed";
+    report.reason = `filed ${report.issues.length} issue(s)`;
+  } else if (commented > 0) {
+    report.action = "commented";
+    report.reason = `commented on ${commented} issue(s)`;
+  } else if (findings.length > 0 || ledger.pendingFiling.length > 0) {
+    report.action = "duplicate";
+    report.reason = "every finding already has an issue";
+  } else {
+    report.reason = "no finding in the window";
+  }
+  report.pending = ledger.pendingFiling.length;
+
+  saveLedger(context.stateDir, ledger);
+  writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+  writeFileSync(join(context.stateDir, "latest.json"), `${JSON.stringify(report, null, 2)}\n`);
+  writeFileSync(join(context.runDir, "report.md"), renderMarkdown(report));
+  context.log(`cycle ${report.action}: ${report.reason ?? "no reason"}`);
+  return {
+    report,
+    exitCode: report.action === "failed" ? 1 : 0,
+    runDir: context.runDir,
+  };
+}
+
+function renderMarkdown(report: Report): string {
+  const lines = [
+    `# Watchdog cycle ${report.cycle}`,
+    "",
+    `- Window: ${report.window.start} .. ${report.window.end} (${report.window.minutes} minutes)`,
+    `- Action: ${report.action}`,
+    `- Reason: ${report.reason ?? "none"}`,
+    `- Findings: ${report.findings.length}`,
+    `- Pending filings: ${report.pending}`,
+    "",
+    "## Instances",
+    "",
+    ...(report.instances.length === 0
+      ? ["No dashboard was running."]
+      : report.instances.map(
+          (instance) =>
+            `- port ${instance.port}: ${instance.repoDir} (pid ${instance.pid}, ` +
+            `${instance.instrumented ? "instrumented" : "uninstrumented"})`,
+        )),
+    "",
+    "## Findings",
+    "",
+    ...(report.findings.length === 0
+      ? ["None."]
+      : report.findings.map(
+          (finding) => `- ${finding.category} at ${finding.ts}: ${finding.quote}`,
+        )),
+    "",
+    "## Issues",
+    "",
+    ...(report.issues.length === 0
+      ? ["None."]
+      : report.issues.map((issue) => `- ${issue.url} (${issue.category})`)),
+    "",
+  ];
+  return `${lines.join("\n")}\n`;
+}