@dsdenes/devboard 0.1.559 → 0.1.562

package/watchdog/delivery.ts

@@ -0,0 +1,107 @@
+import {
+  configuredGiteaToken,
+  giteaBaseUrl,
+} from "../lib/forge";
+import {
+  giteaCommentIssue,
+  giteaCreateIssue,
+  issueNumberFromUrl,
+} from "../lib/gitea";
+import { originForge } from "../issues/repository";
+import {
+  commentIssue as commentGithubIssue,
+  createIssue as createGithubIssue,
+  watchdogBody,
+  watchdogComment,
+  watchdogTitle,
+  type GhRun,
+  type IssueInput,
+} from "./github";
+
+/**
+ * Watchdog issue delivery for the checkout's own forge. Every URL it returns
+ * is linked in the cycle history; no issue is ever closed and no repository is
+ * named in source. GitHub uses the operator's `gh`; Gitea uses its public API
+ * and the configured token.
+ */
+
+export type DeliveryResult = { ok: true; url: string } | { ok: false; reason: string };
+export type CommentResult = { ok: true } | { ok: false; reason: string };
+
+export interface WatchdogDelivery {
+  /** `owner/name` that receives the issue, for diagnostics. */
+  repository: string;
+  create(input: IssueInput): Promise<DeliveryResult>;
+  comment(url: string, input: IssueInput): Promise<CommentResult>;
+}
+
+export interface DeliveryOptions {
+  root: string;
+  environment?: Record<string, string | undefined>;
+  /** The GitHub command; ignored for a Gitea origin. */
+  gh: GhRun;
+  /** Injectable Gitea transport. */
+  fetchImpl?: typeof fetch;
+}
+
+function firstLine(error: unknown): string {
+  const text = error instanceof Error ? error.message : String(error);
+  return (
+    text
+      .split("\n")
+      .map((line) => line.trim())
+      .find((line) => line.length > 0) ?? "no diagnostic"
+  );
+}
+
+/** The delivery for the checkout's origin, or an explicit refusal. */
+export function createWatchdogDelivery(options: DeliveryOptions): WatchdogDelivery {
+  const environment = options.environment ?? process.env;
+  const remote = originForge(options.root, environment);
+  if (remote === undefined) {
+    throw new Error("the checkout's origin does not name a supported repository");
+  }
+  if (remote.forge === "gitea") {
+    const baseUrl = giteaBaseUrl(remote, environment);
+    if (baseUrl === undefined || baseUrl.length === 0) {
+      throw new Error("the watchdog needs DEVBOARD_GITEA_URL for a Gitea origin");
+    }
+    const token = configuredGiteaToken(environment);
+    if (token === undefined) throw new Error("the watchdog needs GITEA_TOKEN for a Gitea origin");
+    const target = { baseUrl, project: remote.project, token };
+    return {
+      repository: remote.project,
+      async create(input) {
+        try {
+          const issue = await giteaCreateIssue(
+            target,
+            { title: watchdogTitle(input.category, input.line), body: watchdogBody(input) },
+            options.fetchImpl,
+          );
+          return { ok: true, url: issue.url };
+        } catch (error) {
+          return { ok: false, reason: firstLine(error) };
+        }
+      },
+      async comment(url, input) {
+        const number = issueNumberFromUrl(url);
+        if (number === undefined) return { ok: false, reason: "the issue URL has no number" };
+        try {
+          await giteaCommentIssue(target, number, watchdogComment(input), options.fetchImpl);
+          return { ok: true };
+        } catch (error) {
+          return { ok: false, reason: firstLine(error) };
+        }
+      },
+    };
+  }
+  return {
+    repository: remote.project,
+    async create(input) {
+      return createGithubIssue(options.gh, remote.project, input);
+    },
+    async comment(url, input) {
+      return commentGithubIssue(options.gh, url, input);
+    },
+  };
+}

package/watchdog/discovery.ts

@@ -0,0 +1,209 @@
+import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { journalFile } from "../server/journal";
+
+/**
+ * Instance discovery is deliberately independent of `scripts/publish.ts`: it
+ * sees every dashboard on the machine, whichever harness checkout serves it,
+ * and it never depends on a dashboard's state file.
+ */
+
+export interface ProcessSnapshot {
+  pid: number;
+  cmdline: string;
+  env: Map<string, string>;
+  cwd?: string;
+  /** Process ids from the parent upward, nearest first. */
+  ancestors: number[];
+}
+
+export interface Instance {
+  pid: number;
+  port: number;
+  repoDir: string;
+  host: string;
+  /** Harness directory that serves this dashboard. */
+  harness: string;
+  /** The dashboard's own state directory, from `STATE_DIR`. */
+  stateDir: string;
+  journal: string;
+  /** False when the serving harness never wrote a journal for this project. */
+  instrumented: boolean;
+  /** Legacy `.tmp/devboard-<cksum>.log` the starter script wrote, when present. */
+  legacyLog?: string;
+}
+
+/**
+ * POSIX `cksum` (CRC-32 with the length trailer) as the legacy starter script
+ * computed it for the old log name. The checksum is only a file name.
+ */
+export function posixCksum(data: Buffer): number {
+  let crc = 0;
+  for (const byte of data) {
+    crc ^= byte << 24;
+    for (let bit = 0; bit < 8; bit += 1) {
+      crc = (crc & 0x80000000) !== 0 ? (crc << 1) ^ 0x04c11db7 : crc << 1;
+      crc >>>= 0;
+    }
+  }
+  for (let length = data.length; length > 0; length = Math.floor(length / 256)) {
+    crc ^= (length & 0xff) << 24;
+    for (let bit = 0; bit < 8; bit += 1) {
+      crc = (crc & 0x80000000) !== 0 ? (crc << 1) ^ 0x04c11db7 : crc << 1;
+      crc >>>= 0;
+    }
+  }
+  return ~crc >>> 0;
+}
+
+export function legacyLogPath(harness: string, repoDir: string): string {
+  return join(harness, ".tmp", `devboard-${posixCksum(Buffer.from(repoDir))}.log`);
+}
+
+function real(path: string | undefined): string | undefined {
+  if (path === undefined || path.length === 0) return undefined;
+  try {
+    return realpathSync(path);
+  } catch {
+    return undefined;
+  }
+}
+
+/** Resolves the harness that serves an instance from its STATIC_DIR or cwd. */
+export function servingHarness(process: ProcessSnapshot): string | undefined {
+  const staticDir = process.env.get("STATIC_DIR");
+  if (staticDir !== undefined) {
+    const harness = real(dirname(staticDir));
+    if (harness !== undefined) return harness;
+  }
+  return real(process.cwd);
+}
+
+/**
+ * Builds instances from a process table. Excludes the watchdog and its
+ * descendants, de-duplicates by `(REPO_DIR, PORT)`, and resolves the journal
+ * and legacy log through the serving harness.
+ */
+export function instancesFromProcesses(
+  processes: ProcessSnapshot[],
+  options: { selfPids: Set<number> },
+): Instance[] {
+  const ordered = [...processes].sort((left, right) => left.pid - right.pid);
+  const seen = new Set<string>();
+  const instances: Instance[] = [];
+  for (const process of ordered) {
+    if (options.selfPids.has(process.pid)) continue;
+    if (process.ancestors.some((pid) => options.selfPids.has(pid))) continue;
+    if (!process.cmdline.includes("server/index.ts")) continue;
+    const repoDir = process.env.get("REPO_DIR");
+    const port = Number(process.env.get("PORT"));
+    if (repoDir === undefined || !Number.isInteger(port) || port <= 0) continue;
+    const harness = servingHarness(process);
+    if (harness === undefined) continue;
+    const key = `${repoDir}\0${port}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    // An installed dashboard keeps its journal beside its npm prefix, named by
+    // STATE_DIR; the serving package's own `.tmp` is read-only to it.
+    const stateDir = process.env.get("STATE_DIR") ?? join(harness, ".tmp");
+    const journal = journalFile(stateDir, repoDir);
+    const legacy = legacyLogPath(harness, repoDir);
+    instances.push({
+      pid: process.pid,
+      port,
+      repoDir,
+      host: process.env.get("HOST") ?? "127.0.0.1",
+      harness,
+      stateDir,
+      journal,
+      instrumented: existsSync(join(stateDir, "journal")),
+      ...(existsSync(legacy) ? { legacyLog: legacy } : {}),
+    });
+  }
+  return instances;
+}
+
+function readStatus(pid: number): { ppid?: number } {
+  try {
+    for (const line of readFileSync(`/proc/${pid}/status`, "utf8").split("\n")) {
+      if (line.startsWith("PPid:")) return { ppid: Number(line.slice(5).trim()) };
+    }
+  } catch {
+    // The process exited.
+  }
+  return {};
+}
+
+/** The parent chain of a pid, nearest first, stopping at pid 1 or a cycle. */
+export function ancestry(pid: number, limit = 64): number[] {
+  const chain: number[] = [];
+  const seen = new Set<number>([pid]);
+  let current = readStatus(pid).ppid;
+  while (current !== undefined && current > 1 && !seen.has(current) && chain.length < limit) {
+    chain.push(current);
+    seen.add(current);
+    current = readStatus(current).ppid;
+  }
+  return chain;
+}
+
+export function processSnapshot(pid: number): ProcessSnapshot | undefined {
+  let cmdline: string;
+  let env = new Map<string, string>();
+  try {
+    cmdline = readFileSync(`/proc/${pid}/cmdline`, "utf8").replace(/\0/g, " ");
+  } catch {
+    return undefined;
+  }
+  try {
+    for (const entry of readFileSync(`/proc/${pid}/environ`, "utf8").split("\0")) {
+      const split = entry.indexOf("=");
+      if (split > 0) env.set(entry.slice(0, split), entry.slice(split + 1));
+    }
+  } catch {
+    env = new Map();
+  }
+  let cwd: string | undefined;
+  try {
+    cwd = realpathSync(`/proc/${pid}/cwd`);
+  } catch {
+    cwd = undefined;
+  }
+  return { pid, cmdline, env, cwd, ancestors: ancestry(pid) };
+}
+
+/** Every process on the machine, read from `/proc`. */
+export function processTable(): ProcessSnapshot[] {
+  const processes: ProcessSnapshot[] = [];
+  let entries: string[];
+  try {
+    entries = readdirSync("/proc");
+  } catch {
+    return processes;
+  }
+  for (const entry of entries) {
+    if (!/^[0-9]+$/.test(entry)) continue;
+    const snapshot = processSnapshot(Number(entry));
+    if (snapshot !== undefined) processes.push(snapshot);
+  }
+  return processes;
+}
+
+export interface DiscoveryOptions {
+  /** Pids that are the watchdog itself and its descendants. */
+  selfPids?: Set<number>;
+  /** Injectable process table for tests. */
+  processes?: ProcessSnapshot[];
+}
+
+export function discoverInstances(options: DiscoveryOptions = {}): Instance[] {
+  const selfPids = options.selfPids ?? new Set([process.pid]);
+  const processes = options.processes ?? processTable();
+  return instancesFromProcesses(processes, { selfPids });
+}
+
+/** True when the pid, or any ancestor, belongs to the watchdog. */
+export function isOwnProcess(pid: number, selfPids: Set<number>): boolean {
+  if (selfPids.has(pid)) return true;
+  return ancestry(pid).some((ancestor) => selfPids.has(ancestor));
+}

package/watchdog/findings.ts

@@ -0,0 +1,211 @@
+import { createHash } from "node:crypto";
+import { resolve } from "node:path";
+import {
+  normalizeEvidenceLine,
+  parseTimestamp,
+  type EvidenceEvent,
+  type Window,
+} from "./collect";
+import type { Instance } from "./discovery";
+
+/**
+ * Findings are machine-checkable signals, never prose. A category names the
+ * evidence kind that produced it, so one text produces one fingerprint no
+ * matter which dashboard or which checkout observed it.
+ */
+export const FINDING_CATEGORIES = [
+  "server-error",
+  "update-error",
+  "acp-stderr",
+  "lane-failed",
+  "omp-error",
+  "omp-warn",
+  "omp-fatal",
+  "stall",
+  "question-loop",
+] as const;
+
+export type FindingCategory = (typeof FINDING_CATEGORIES)[number];
+
+/** Journal kinds that are a finding on their own. */
+const SIGNAL_KINDS: Record<string, FindingCategory> = {
+  "server-error": "server-error",
+  "update-error": "update-error",
+  "acp-stderr": "acp-stderr",
+};
+
+/** A start this long before window end with no matching stop is a stall. */
+export const STALL_MS = 5 * 60 * 1000;
+
+/** This many questions for one instance in one window is a loop. */
+export const QUESTION_LOOP_COUNT = 3;
+
+export interface Finding {
+  category: FindingCategory;
+  /** The normalized evidence line, truncated for a report body. */
+  quote: string;
+  /** Earliest event timestamp the finding was derived from, UTC ISO 8601. */
+  ts: string;
+  /** Every dashboard that contributed the same signal. */
+  instances: Instance[];
+  /** Stable identity across cycles and hosts: category and normalized line. */
+  fingerprint: string;
+}
+
+/** Source paths are reported relative to the repository when they live inside it. */
+export function normalizeSource(root: string, path: string): string {
+  const resolved = resolve(path);
+  const prefix = `${resolve(root)}/`;
+  return resolved.startsWith(prefix) ? resolved.slice(prefix.length) : resolved;
+}
+
+/**
+ * `sha256(category + "\n" + normalized line)`, first 32 hex characters. The
+ * source path is deliberately absent: the same error on two dashboards is one
+ * fingerprint, and so is the same error after a dashboard moves.
+ */
+export function findingFingerprint(category: string, line: string): string {
+  return createHash("sha256")
+    .update(`${category}\n${normalizeEvidenceLine(line)}`)
+    .digest("hex")
+    .slice(0, 32);
+}
+
+/** The last whitespace-separated token of a `lane-stop` line is its status. */
+export function laneStatus(text: string): string | undefined {
+  const tokens = text.trim().split(/\s+/);
+  return tokens.length < 2 ? undefined : tokens[tokens.length - 1];
+}
+
+function eventKey(event: EvidenceEvent): string {
+  return event.instance === undefined
+    ? event.source
+    : `${event.instance.repoDir}:${event.instance.port}`;
+}
+
+/**
+ * Questions belong to one run, lane, or peer when the journal recorded a
+ * scope. Separate work streams on one dashboard are separate conversations,
+ * never one loop.
+ */
+function questionKey(event: EvidenceEvent): string {
+  return event.scope === undefined ? eventKey(event) : `${eventKey(event)}\n${event.scope}`;
+}
+
+/**
+ * A start is finished only by a stop of the same work stream on the same
+ * dashboard. Lane records begin with the lane id; run records begin with the
+ * run file before the `file: title` separator. A lane and a run never share a
+ * stream, so a `run-stop` cannot end a lane and a sibling lane's stop cannot
+ * end another lane's stall.
+ */
+function streamKey(event: EvidenceEvent): string {
+  const text = normalizeEvidenceLine(event.text);
+  if (event.kind.startsWith("lane-")) {
+    const lane = text.split(/\s+/, 1)[0] ?? "";
+    return `${eventKey(event)}\nlane\n${lane}`;
+  }
+  const separator = text.indexOf(":");
+  const file = separator === -1 ? "" : text.slice(0, separator).trim();
+  return `${eventKey(event)}\nrun\n${file}`;
+}
+
+/**
+ * Turns a window of evidence into one finding per distinct signal. Only the
+ * evidence kinds in `FINDING_CATEGORIES` can produce one: activity, lifecycle,
+ * update, run and lane records, and questions below the loop threshold are
+ * context, never findings.
+ */
+export function analyzeEvidence(
+  events: EvidenceEvent[],
+  options: { window: Window },
+): Finding[] {
+  const findings = new Map<string, Finding>();
+  const questions = new Map<string, EvidenceEvent[]>();
+  const starts = new Map<string, EvidenceEvent[]>();
+  const stops = new Map<string, number[]>();
+
+  const add = (category: FindingCategory, event: EvidenceEvent) => {
+    const quote = normalizeEvidenceLine(event.text);
+    if (quote.length === 0) return;
+    const fingerprint = findingFingerprint(category, quote);
+    const existing = findings.get(fingerprint);
+    if (existing !== undefined) {
+      if (event.ts < existing.ts) existing.ts = event.ts;
+      if (
+        event.instance !== undefined &&
+        !existing.instances.some(
+          (instance) =>
+            instance.repoDir === event.instance?.repoDir && instance.port === event.instance.port,
+        )
+      ) {
+        existing.instances.push(event.instance);
+      }
+      return;
+    }
+    findings.set(fingerprint, {
+      category,
+      quote: quote.slice(0, 300),
+      ts: event.ts,
+      instances: event.instance === undefined ? [] : [event.instance],
+      fingerprint,
+    });
+  };
+
+  for (const event of events) {
+    switch (event.kind) {
+      case "question":
+        questions.set(questionKey(event), [...(questions.get(questionKey(event)) ?? []), event]);
+        continue;
+      case "run-start":
+      case "lane-start":
+        starts.set(streamKey(event), [...(starts.get(streamKey(event)) ?? []), event]);
+        continue;
+      case "run-stop":
+      case "lane-stop": {
+        const parsed = parseTimestamp(event.ts);
+        stops.set(streamKey(event), [
+          ...(stops.get(streamKey(event)) ?? []),
+          ...(parsed === undefined ? [] : [parsed]),
+        ]);
+        // A stop is only a stall terminator; a failed lane is a finding.
+        if (event.kind === "lane-stop" && laneStatus(event.text) === "failed") {
+          add("lane-failed", event);
+        }
+        continue;
+      }
+      default:
+        break;
+    }
+    if (event.kind === "omp-log" && event.level !== undefined) {
+      add(`omp-${event.level}` as FindingCategory, event);
+      continue;
+    }
+    const category = SIGNAL_KINDS[event.kind];
+    if (category !== undefined) add(category, event);
+  }
+
+  for (const list of questions.values()) {
+    if (list.length < QUESTION_LOOP_COUNT) continue;
+    const last = list[list.length - 1];
+    if (last !== undefined) add("question-loop", last);
+  }
+
+  const stallCutoff = options.window.end.getTime() - STALL_MS;
+  for (const [key, list] of starts) {
+    const finished = stops.get(key) ?? [];
+    for (const event of list) {
+      const started = parseTimestamp(event.ts);
+      if (started === undefined || started > stallCutoff) continue;
+      if (finished.some((stopped) => stopped >= started)) continue;
+      add("stall", event);
+    }
+  }
+
+  return [...findings.values()].sort(
+    (left, right) =>
+      left.ts.localeCompare(right.ts) ||
+      left.category.localeCompare(right.category) ||
+      left.fingerprint.localeCompare(right.fingerprint),
+  );
+}