@dsdenes/devboard 0.1.560 → 0.1.562

package/watchdog/github.ts

@@ -0,0 +1,189 @@
+import type { Window } from "./collect";
+import type { LedgerInstance } from "./ledger";
+
+/**
+ * GitHub issue delivery for the watchdog. Every call goes through the
+ * operator's authenticated `gh`; nothing here reads or prints an environment
+ * value, and nothing closes an issue.
+ */
+
+/** GitHub's title limit for this watchdog; the line is truncated to fit. */
+export const TITLE_MAX = 80;
+/** Longest first stderr line kept as a failure reason. */
+export const REASON_MAX = 300;
+
+export interface GhOutcome {
+  code: number;
+  stdout: string;
+  stderr: string;
+  timedOut?: boolean;
+}
+
+/** Runs one `gh` invocation with a body on standard input. */
+export type GhRun = (argv: string[], stdin: string) => GhOutcome;
+
+export function runGh(argv: string[], stdin: string): GhOutcome {
+  try {
+    const result = Bun.spawnSync({
+      cmd: ["gh", ...argv],
+      stdin: Buffer.from(stdin),
+      stdout: "pipe",
+      stderr: "pipe",
+      timeout: 120_000,
+      env: { ...process.env, GH_PROMPT_DISABLED: "1", GH_NO_UPDATE_NOTIFIER: "1" },
+    });
+    return {
+      code: result.exitCode ?? 1,
+      stdout: result.stdout.toString(),
+      stderr: result.stderr.toString(),
+      ...(result.exitedDueToTimeout === true ? { timedOut: true } : {}),
+    };
+  } catch (error) {
+    // A missing or unspawnable `gh` is a filing failure, not a crashed cycle.
+    return {
+      code: 1,
+      stdout: "",
+      stderr: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+function truncate(value: string, limit: number): string {
+  const single = value.replace(/\s+/g, " ").trim();
+  if (single.length <= limit) return single;
+  return `${single.slice(0, limit - 1).trimEnd()}…`;
+}
+
+/** `[watchdog] <category>: <line>`, one line, never longer than 80 characters. */
+export function watchdogTitle(category: string, line: string): string {
+  const title = `[watchdog] ${category}: ${line.replace(/\s+/g, " ").trim()}`;
+  if (title.length <= TITLE_MAX) return title;
+  return `${title.slice(0, TITLE_MAX - 1).trimEnd()}…`;
+}
+
+/** The first diagnostic line of a failed `gh` call, bounded for a report. */
+export function ghFailureReason(outcome: GhOutcome): string {
+  const line = [...outcome.stderr.split("\n"), ...outcome.stdout.split("\n")]
+    .map((entry) => entry.trim())
+    .find((entry) => entry.length > 0);
+  if (line !== undefined) return truncate(line, REASON_MAX);
+  return outcome.timedOut === true ? "gh timed out" : `gh exited with code ${outcome.code}`;
+}
+
+export interface IssueInput {
+  category: string;
+  fingerprint: string;
+  line: string;
+  firstSeen: string;
+  lastSeen: string;
+  count: number;
+  window: Window;
+  instances: LedgerInstance[];
+  /** The cycle report that filed this issue. */
+  reportPath: string;
+}
+
+function fence(text: string): string {
+  const fenceText = text.includes("```") ? "````" : "```";
+  return `${fenceText}text\n${text}\n${fenceText}`;
+}
+
+/** Category, identity, sightings, every contributing instance, and the report. */
+export function watchdogBody(input: IssueInput): string {
+  const instances =
+    input.instances.length === 0
+      ? ["No dashboard is recorded."]
+      : input.instances.map(
+          (instance) =>
+            `- \`${instance.repoDir}\` on port ${instance.port} ` +
+            `(pid ${instance.pid}, journal \`${instance.journal}\`)`,
+        );
+  return [
+    "## Watchdog finding",
+    "",
+    `- Category: \`${input.category}\``,
+    `- Fingerprint: \`${input.fingerprint}\``,
+    `- First seen: ${input.firstSeen}`,
+    `- Last seen: ${input.lastSeen}`,
+    `- Count: ${input.count}`,
+    `- Window: ${input.window.start.toISOString()} .. ${input.window.end.toISOString()}`,
+    "",
+    "### Quote",
+    "",
+    fence(input.line),
+    "",
+    "### Contributing instances",
+    "",
+    ...instances,
+    "",
+    "### Cycle report",
+    "",
+    `\`${input.reportPath}\``,
+    "",
+    "Filed by the Devboard watchdog from machine-checkable evidence. " +
+      "It never fixes, merges, pushes, or publishes.",
+    "",
+  ].join("\n");
+}
+
+/** A recurrence: the new window, the latest sighting, and the running count. */
+export function watchdogComment(input: IssueInput): string {
+  return [
+    `Another sighting of \`${input.fingerprint}\` (\`${input.category}\`).`,
+    "",
+    `- Last seen: ${input.lastSeen}`,
+    `- Count: ${input.count}`,
+    `- Window: ${input.window.start.toISOString()} .. ${input.window.end.toISOString()}`,
+    "",
+    "### Quote",
+    "",
+    fence(input.line),
+    "",
+    "Filed by the Devboard watchdog from machine-checkable evidence.",
+    "",
+  ].join("\n");
+}
+
+function printedUrl(outcome: GhOutcome): string | undefined {
+  return outcome.stdout
+    .split("\n")
+    .map((line) => line.trim())
+    .reverse()
+    .find((line) => /^https?:\/\//.test(line));
+}
+
+/** Files one issue in `repository`; only `gh` can fail it. */
+export function createIssue(
+  gh: GhRun,
+  repository: string,
+  input: IssueInput,
+): { ok: true; url: string } | { ok: false; reason: string } {
+  const outcome = gh(
+    [
+      "issue",
+      "create",
+      "--repo",
+      repository,
+      "--title",
+      watchdogTitle(input.category, input.line),
+      "--body-file",
+      "-",
+    ],
+    watchdogBody(input),
+  );
+  if (outcome.code !== 0) return { ok: false, reason: ghFailureReason(outcome) };
+  const url = printedUrl(outcome);
+  if (url === undefined) return { ok: false, reason: "gh printed no issue URL" };
+  return { ok: true, url };
+}
+
+/** Comments on an existing issue. The URL is the only locator. */
+export function commentIssue(
+  gh: GhRun,
+  url: string,
+  input: IssueInput,
+): { ok: true } | { ok: false; reason: string } {
+  const outcome = gh(["issue", "comment", url, "--body-file", "-"], watchdogComment(input));
+  if (outcome.code !== 0) return { ok: false, reason: ghFailureReason(outcome) };
+  return { ok: true };
+}

package/watchdog/history.ts

@@ -0,0 +1,138 @@
+import { readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+
+/**
+ * The watchdog's retained cycle history, read from the run directories the
+ * coordinator writes. It is a read-only projection: no file is created,
+ * repaired, or locked while the dashboard renders it.
+ */
+
+/** The dashboard never lists more than the newest 100 run directories. */
+export const CYCLE_HISTORY_LIMIT = 100;
+
+export interface WatchdogCycleWindow {
+  start: string | null;
+  end: string | null;
+}
+
+export interface WatchdogCycle {
+  /** The cycle's start, UTC ISO 8601, taken from its report. */
+  at: string;
+  window: WatchdogCycleWindow;
+  action: string;
+  findings: number;
+  /** The GitHub issues this cycle's findings map to, in finding order. */
+  issues: string[];
+  /** The report's optional reason, absent when the report carries none. */
+  reason?: string;
+}
+
+/** Resolves a finding fingerprint to the GitHub issue that covers it. */
+export type WatchdogIssueResolver = (fingerprint: string) => string | undefined;
+
+/** The cycle rows never repeat an issue link. */
+function connectedIssues(
+  report: Record<string, unknown>,
+  resolve?: WatchdogIssueResolver,
+): string[] {
+  const urls: string[] = [];
+  const add = (value: unknown) => {
+    if (typeof value === "string" && value.length > 0 && !urls.includes(value)) urls.push(value);
+  };
+  if (Array.isArray(report.findings)) {
+    for (const value of report.findings) {
+      const finding = record(value);
+      if (typeof finding?.fingerprint !== "string") continue;
+      add(resolve?.(finding.fingerprint));
+    }
+  }
+  if (Array.isArray(report.issues)) {
+    for (const value of report.issues) add(record(value)?.url);
+  }
+  return urls;
+}
+
+function record(value: unknown): Record<string, unknown> | undefined {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+function windowOf(value: unknown): WatchdogCycleWindow {
+  const window = record(value);
+  return {
+    start: typeof window?.start === "string" ? window.start : null,
+    end: typeof window?.end === "string" ? window.end : null,
+  };
+}
+
+/** One report, or undefined when it cannot be rendered as a cycle row. */
+function cycleFromReport(
+  value: unknown,
+  resolve?: WatchdogIssueResolver,
+): WatchdogCycle | undefined {
+  const report = record(value);
+  if (report === undefined) return undefined;
+  if (typeof report.cycle !== "string" || report.cycle.length === 0) return undefined;
+  return {
+    at: report.cycle,
+    window: windowOf(report.window),
+    action: typeof report.action === "string" ? report.action : "none",
+    findings: Array.isArray(report.findings) ? report.findings.length : 0,
+    issues: connectedIssues(report, resolve),
+    ...(typeof report.reason === "string" ? { reason: report.reason } : {}),
+  };
+}
+
+function readJson(path: string): unknown {
+  try {
+    return JSON.parse(readFileSync(path, "utf8"));
+  } catch {
+    return undefined;
+  }
+}
+
+/** The newest run directory names first, capped before any file is read. */
+function runDirectories(watchdogDir: string): string[] {
+  try {
+    return readdirSync(join(watchdogDir, "runs"), { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name)
+      .sort((left, right) => (left < right ? 1 : left > right ? -1 : 0))
+      .slice(0, CYCLE_HISTORY_LIMIT);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * The newest-first cycle history. When no run directory exists at all, the
+ * cycle already projected from `latest.json` is the single row; a run
+ * directory whose `report.json` is missing or unparsable is skipped without
+ * hiding every other row.
+ */
+export function readWatchdogCycles(
+  watchdogDir: string,
+  latestCycle: WatchdogCycle | undefined,
+  resolve?: WatchdogIssueResolver,
+): WatchdogCycle[] {
+  const directories = runDirectories(watchdogDir);
+  if (directories.length === 0) return latestCycle === undefined ? [] : [latestCycle];
+  const cycles: WatchdogCycle[] = [];
+  for (const directory of directories) {
+    const cycle = cycleFromReport(
+      readJson(join(watchdogDir, "runs", directory, "report.json")),
+      resolve,
+    );
+    if (cycle !== undefined) cycles.push(cycle);
+  }
+  return cycles;
+}
+
+/** The cycle `latest.json` carries, when it carries a usable one. */
+export function latestWatchdogCycle(
+  latest: unknown,
+  resolve?: WatchdogIssueResolver,
+): WatchdogCycle | undefined {
+  return cycleFromReport(latest, resolve);
+}

package/watchdog/install.ts

@@ -0,0 +1,103 @@
+import { realpathSync } from "node:fs";
+import { join, resolve } from "node:path";
+import {
+  requireLinger,
+  renderUnit,
+  systemctl,
+  uniqueDirectories,
+  unitPaths,
+  writeUnit,
+} from "./systemd";
+
+/**
+ * Installs and starts the user timer. Idempotent: rerunning replaces the two
+ * units and re-enables the timer without duplicating anything.
+ */
+
+const harness = resolve(import.meta.dir, "..");
+const top = Bun.spawnSync({
+  cmd: ["git", "-C", harness, "rev-parse", "--show-toplevel"],
+  stdin: "ignore",
+  stdout: "pipe",
+  stderr: "pipe",
+});
+const root = top.stdout.toString().trim();
+if (top.exitCode !== 0 || root.length === 0) {
+  fail("missing prerequisite: the harness is not inside a git checkout");
+}
+
+function fail(message: string): never {
+  console.error(`watchdog:install: ${message}`);
+  process.exit(1);
+}
+
+const absoluteGitDir = Bun.spawnSync({
+  cmd: ["git", "-C", harness, "rev-parse", "--absolute-git-dir"],
+  stdin: "ignore",
+  stdout: "pipe",
+  stderr: "pipe",
+}).stdout.toString().trim();
+const commonGitDir = Bun.spawnSync({
+  cmd: ["git", "-C", harness, "rev-parse", "--path-format=absolute", "--git-common-dir"],
+  stdin: "ignore",
+  stdout: "pipe",
+  stderr: "pipe",
+}).stdout.toString().trim();
+if (absoluteGitDir.length > 0 && absoluteGitDir !== commonGitDir) {
+  fail("install runs from the canonical checkout, not a worktree");
+}
+const canonicalHarness = join(root, ".devboard");
+try {
+  if (realpathSync(harness) !== realpathSync(canonicalHarness)) {
+    fail(`the harness must be ${canonicalHarness}`);
+  }
+} catch {
+  fail(`missing prerequisite: ${canonicalHarness} is not readable`);
+}
+
+try {
+  requireLinger();
+} catch (error) {
+  fail(error instanceof Error ? error.message : String(error));
+}
+
+const bun = process.execPath;
+const omp = Bun.which("omp");
+const git = Bun.which("git");
+if (omp === undefined) fail("missing prerequisite: omp is not on PATH");
+if (git === undefined) fail("missing prerequisite: git is not on PATH");
+
+const values = {
+  ROOT: root,
+  HARNESS: canonicalHarness,
+  BUN: bun,
+  PATH: uniqueDirectories([bun, omp, git]),
+};
+const paths = unitPaths();
+writeUnit(
+  paths.service,
+  renderUnit(join(import.meta.dir, "units", "devboard-watchdog.service"), values),
+);
+writeUnit(
+  paths.timer,
+  renderUnit(join(import.meta.dir, "units", "devboard-watchdog.timer"), values),
+);
+
+const reload = systemctl("daemon-reload");
+if (reload.code !== 0) fail(`daemon-reload failed: ${reload.stderr || reload.stdout}`);
+const enabled = systemctl("enable", "--now", "devboard-watchdog.timer");
+if (enabled.code !== 0) fail(`could not enable the timer: ${enabled.stderr || enabled.stdout}`);
+
+const isEnabled = systemctl("is-enabled", "devboard-watchdog.timer");
+if (isEnabled.stdout !== "enabled") fail(`the timer is ${isEnabled.stdout || "not enabled"}`);
+const isActive = systemctl("is-active", "devboard-watchdog.timer");
+if (isActive.stdout !== "active") fail(`the timer is ${isActive.stdout || "not active"}`);
+const linger = requireLinger();
+const timers = systemctl("list-timers", "devboard-watchdog.timer", "--all");
+if (!/devboard-watchdog\.timer/.test(timers.stdout)) {
+  fail("the timer did not appear in list-timers");
+}
+
+console.log(`watchdog:install: installed ${paths.service} and ${paths.timer}`);
+console.log(`watchdog:install: timer enabled, active, ${linger}`);
+console.log(timers.stdout);

package/watchdog/ledger.ts

@@ -0,0 +1,188 @@
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { Finding } from "./findings";
+
+/**
+ * The ledger remembers every distinct signal and the GitHub issue it owns. It
+ * is harness state under `.tmp`, never committed, and it survives cycles so a
+ * recurrence comments on the existing issue instead of filing another one.
+ */
+
+export const LEDGER_VERSION = 2;
+
+export interface LedgerInstance {
+  repoDir: string;
+  port: number;
+  pid: number;
+  journal: string;
+}
+
+export interface LedgerEntry {
+  category: string;
+  /** Last observed normalized evidence line. */
+  line: string;
+  firstSeen: string;
+  lastSeen: string;
+  count: number;
+  /** `filed:<url>` once the issue exists, `none` before the first filing. */
+  disposition: string;
+  url?: string;
+  filedAt?: string;
+  lastCommentAt?: string;
+  /** Every dashboard that has produced this signal. */
+  instances: LedgerInstance[];
+}
+
+export interface Ledger {
+  version: typeof LEDGER_VERSION;
+  findings: Record<string, LedgerEntry>;
+  /** Fingerprints whose `gh issue create` failed and are retried first. */
+  pendingFiling: string[];
+}
+
+/** The version 2 ledger, used when no file exists or the file is unusable. */
+export function emptyLedger(): Ledger {
+  return { version: LEDGER_VERSION, findings: {}, pendingFiling: [] };
+}
+
+function text(value: unknown, fallback: string): string {
+  return typeof value === "string" && value.length > 0 ? value : fallback;
+}
+
+function instances(value: unknown): LedgerInstance[] {
+  if (!Array.isArray(value)) return [];
+  const parsed: LedgerInstance[] = [];
+  for (const candidate of value) {
+    if (typeof candidate !== "object" || candidate === null) continue;
+    const record = candidate as Record<string, unknown>;
+    if (typeof record.repoDir !== "string" || typeof record.journal !== "string") continue;
+    const port = Number(record.port);
+    const pid = Number(record.pid);
+    if (!Number.isInteger(port) || !Number.isInteger(pid)) continue;
+    parsed.push({ repoDir: record.repoDir, port, pid, journal: record.journal });
+  }
+  return parsed;
+}
+
+/** One entry, or undefined when the record cannot be trusted. */
+function parseEntry(fingerprint: string, value: unknown): LedgerEntry | undefined {
+  if (typeof value !== "object" || value === null) return undefined;
+  const record = value as Record<string, unknown>;
+  const disposition = text(record.disposition, "");
+  if (disposition !== "none" && !disposition.startsWith("filed:")) return undefined;
+  const url =
+    typeof record.url === "string"
+      ? record.url
+      : disposition.startsWith("filed:")
+        ? disposition.slice("filed:".length)
+        : undefined;
+  if (disposition.startsWith("filed:") && (url === undefined || url.length === 0)) return undefined;
+  if (typeof record.category !== "string" || typeof record.line !== "string") return undefined;
+  if (typeof record.firstSeen !== "string" || typeof record.lastSeen !== "string") return undefined;
+  const count = Number(record.count);
+  return {
+    category: record.category,
+    line: record.line,
+    firstSeen: record.firstSeen,
+    lastSeen: record.lastSeen,
+    count: Number.isFinite(count) && count > 0 ? count : 1,
+    disposition,
+    ...(url === undefined ? {} : { url }),
+    ...(typeof record.filedAt === "string" ? { filedAt: record.filedAt } : {}),
+    ...(typeof record.lastCommentAt === "string" ? { lastCommentAt: record.lastCommentAt } : {}),
+    instances: instances(record.instances),
+  };
+}
+
+/**
+ * Reads the ledger. A version 1 file recorded `blocked:*` and `published:<sha>`
+ * dispositions from the retired fix pipeline: those entries describe prose, not
+ * issues, so they are discarded instead of migrated.
+ */
+export function loadLedger(stateDir: string): Ledger {
+  try {
+    const parsed: unknown = JSON.parse(readFileSync(join(stateDir, "state.json"), "utf8"));
+    if (typeof parsed !== "object" || parsed === null) return emptyLedger();
+    const record = parsed as Record<string, unknown>;
+    if (record.version !== LEDGER_VERSION) return emptyLedger();
+    const findings: Record<string, LedgerEntry> = {};
+    const source = record.findings;
+    if (typeof source === "object" && source !== null) {
+      for (const [fingerprint, value] of Object.entries(source)) {
+        const entry = parseEntry(fingerprint, value);
+        if (entry !== undefined) findings[fingerprint] = entry;
+      }
+    }
+    const pending = Array.isArray(record.pendingFiling)
+      ? record.pendingFiling.filter(
+          (value): value is string => typeof value === "string" && findings[value] !== undefined,
+        )
+      : [];
+    return { version: LEDGER_VERSION, findings, pendingFiling: [...new Set(pending)] };
+  } catch {
+    return emptyLedger();
+  }
+}
+
+export function saveLedger(stateDir: string, ledger: Ledger): void {
+  mkdirSync(stateDir, { recursive: true });
+  const path = join(stateDir, "state.json");
+  const temporary = `${path}.${process.pid}.tmp`;
+  writeFileSync(temporary, `${JSON.stringify(ledger, null, 2)}\n`);
+  renameSync(temporary, path);
+}
+
+/** Records every finding in the window, creating or extending its entry. */
+export function recordFindings(ledger: Ledger, findings: Finding[], now: Date): void {
+  const stamp = now.toISOString();
+  for (const finding of findings) {
+    const entry = ledger.findings[finding.fingerprint];
+    const contributed = finding.instances.map((instance) => ({
+      repoDir: instance.repoDir,
+      port: instance.port,
+      pid: instance.pid,
+      journal: instance.journal,
+    }));
+    if (entry === undefined) {
+      ledger.findings[finding.fingerprint] = {
+        category: finding.category,
+        line: finding.quote,
+        firstSeen: stamp,
+        lastSeen: stamp,
+        count: 1,
+        disposition: "none",
+        instances: contributed,
+      };
+      continue;
+    }
+    entry.category = finding.category;
+    entry.line = finding.quote;
+    entry.lastSeen = stamp;
+    entry.count += 1;
+    for (const instance of contributed) {
+      if (!entry.instances.some((known) => known.journal === instance.journal)) {
+        entry.instances.push(instance);
+      }
+    }
+  }
+}
+
+export function markFiled(ledger: Ledger, fingerprint: string, url: string, now: Date): void {
+  const entry = ledger.findings[fingerprint];
+  if (entry === undefined) return;
+  entry.disposition = `filed:${url}`;
+  entry.url = url;
+  entry.filedAt = now.toISOString();
+  ledger.pendingFiling = ledger.pendingFiling.filter((pending) => pending !== fingerprint);
+}
+
+export function markCommented(ledger: Ledger, fingerprint: string, now: Date): void {
+  const entry = ledger.findings[fingerprint];
+  if (entry === undefined) return;
+  entry.lastCommentAt = now.toISOString();
+}
+
+/** The fingerprint is waiting for a retry until its filing succeeds. */
+export function markPending(ledger: Ledger, fingerprint: string): void {
+  if (!ledger.pendingFiling.includes(fingerprint)) ledger.pendingFiling.push(fingerprint);
+}

package/watchdog/lock.ts

@@ -0,0 +1,63 @@
+import { mkdirSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs";
+import { dirname } from "node:path";
+
+/**
+ * One machine-wide lock serialises every watchdog invocation. The lock file
+ * records the owner's pid, so a crashed run cannot wedge the timer forever.
+ */
+
+export type LockResult =
+  | { acquired: true; release: () => void }
+  | { acquired: false; owner?: number };
+
+function pidAlive(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return (error as NodeJS.ErrnoException).code === "EPERM";
+  }
+}
+
+function owner(path: string): number | undefined {
+  try {
+    const pid = Number.parseInt(readFileSync(path, "utf8").trim(), 10);
+    return Number.isInteger(pid) && pid > 0 ? pid : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export function acquireLock(path: string): LockResult {
+  mkdirSync(dirname(path), { recursive: true });
+  for (let attempt = 0; attempt < 2; attempt += 1) {
+    try {
+      const descriptor = openSync(path, "wx");
+      writeSync(descriptor, `${process.pid}\n`);
+      return {
+        acquired: true,
+        release() {
+          try {
+            unlinkSync(path);
+          } catch {
+            // Another cycle or the operator already removed it.
+          }
+        },
+      };
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      const holder = owner(path);
+      if (holder !== undefined && pidAlive(holder)) {
+        return { acquired: false, owner: holder };
+      }
+      // The owner exited without releasing; reclaim the lock once.
+      try {
+        unlinkSync(path);
+      } catch {
+        // A concurrent invocation won the race.
+      }
+    }
+  }
+  return { acquired: false };
+}