@dsdenes/devboard 0.1.559 → 0.1.562

package/issues/scout/scan.ts

@@ -0,0 +1,143 @@
+import { mkdirSync, readdirSync, readFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { GOAL_PROPOSALS_DIRECTORY } from "../../server/goal-proposal";
+import { runIdentifier } from "../worker";
+import {
+  CATEGORIES,
+  EXCLUDED_ROOTS,
+  SIGNIFICANCE_THRESHOLD,
+  recommendationsPath,
+  requestPath,
+  scanDirectory,
+  scansDirectory,
+} from "./paths";
+import { writeAtomic } from "./state";
+
+/**
+ * One scan's request: written before the prompt and read by the agent, so the
+ * contract between the worker and `/scout-recommendations` is a file, not a
+ * convention. The request names how many issues may be filed, what to ignore,
+ * and which titles already exist, so the agent never restates a known finding.
+ */
+
+export const SCAN_PROMPT = "/scout-recommendations";
+
+export interface ScanRequest {
+  version: 2;
+  scanId: string;
+  repository: string;
+  repositoryPath: string;
+  /** How many significant findings this scan may turn into issues. */
+  requested: number;
+  categories: string[];
+  excludedRoots: string[];
+  /** Normalized titles of existing Auto Scout issues and open issues. */
+  knownTitles: string[];
+  draftGoalHeadings: string[];
+  /** Findings below this score are not filed. */
+  significanceThreshold: number;
+  createdAt: string;
+}
+
+export interface CreatedScan {
+  scanId: string;
+  directory: string;
+  startedAt: string;
+}
+
+/** A UTC timestamp name, made unique when two scans start in one millisecond. */
+export function createScanDirectory(repoDir: string, now: Date): CreatedScan {
+  const base = scansDirectory(repoDir);
+  mkdirSync(base, { recursive: true });
+  const stamp = runIdentifier(now);
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    const scanId = attempt === 0 ? stamp : `${stamp}-${attempt}`;
+    const directory = scanDirectory(repoDir, scanId);
+    try {
+      mkdirSync(directory);
+      return { scanId, directory, startedAt: now.toISOString() };
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+    }
+  }
+  throw new Error(`could not create a scan directory under ${base}`);
+}
+
+export function buildRequest(options: {
+  scan: CreatedScan;
+  repository: string;
+  repoDir: string;
+  requested: number;
+  knownTitles: string[];
+  draftGoalHeadings: string[];
+}): ScanRequest {
+  return {
+    version: 2,
+    scanId: options.scan.scanId,
+    repository: options.repository,
+    repositoryPath: resolve(options.repoDir),
+    requested: options.requested,
+    categories: [...CATEGORIES],
+    excludedRoots: [...EXCLUDED_ROOTS],
+    knownTitles: [...options.knownTitles],
+    draftGoalHeadings: [...options.draftGoalHeadings],
+    significanceThreshold: SIGNIFICANCE_THRESHOLD,
+    createdAt: options.scan.startedAt,
+  };
+}
+
+export function writeRequest(directory: string, request: ScanRequest): string {
+  const path = requestPath(directory);
+  writeAtomic(path, `${JSON.stringify(request, null, 2)}\n`);
+  return path;
+}
+
+export function promptFor(directory: string): string {
+  return `${SCAN_PROMPT} ${directory}`;
+}
+
+/**
+ * The one correction turn a scan gets after its answer is rejected. It quotes
+ * the rejection and restates the two output rules the model most often
+ * misses, so a fixable mistake costs a turn instead of the whole finding.
+ */
+export function repairPrompt(directory: string, rejection: string): string {
+  return [
+    `Auto Scout rejected the scan answer at ${recommendationsPath(directory)}: ${rejection}`,
+    "",
+    `Rewrite ${recommendationsPath(directory)} so it satisfies the scan prompt's output ` +
+      "contract, changing only what the rejection names.",
+    `Every \`evidence[].path\` and every \`files\` entry is relative to the \`repositoryPath\` ` +
+      `in ${requestPath(directory)}: a file nested under a directory keeps those leading ` +
+      "segments in its path. Each cited path must name an existing regular file and each " +
+      "line span must lie inside that file.",
+    "Never modify the repository and write nothing else. Finish in this turn.",
+  ].join("\n");
+}
+
+/** The `# heading` of every draft under `<REPO_DIR>/.devboard/goal-proposals/`. */
+export function readDraftGoalHeadings(repoDir: string): string[] {
+  const directory = join(resolve(repoDir), GOAL_PROPOSALS_DIRECTORY);
+  let names: string[];
+  try {
+    names = readdirSync(directory).filter((name) => name.endsWith(".md")).sort();
+  } catch {
+    return [];
+  }
+  const headings: string[] = [];
+  for (const name of names) {
+    let text: string;
+    try {
+      text = readFileSync(join(directory, name), "utf8");
+    } catch {
+      continue;
+    }
+    for (const line of text.split("\n")) {
+      const trimmed = line.trim();
+      if (trimmed.length === 0) continue;
+      if (trimmed.startsWith("# ")) headings.push(trimmed.slice(2).trim());
+      break;
+    }
+  }
+  return headings;
+}

package/issues/scout/state.ts

@@ -0,0 +1,448 @@
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { isInteger, isRecord, nullableString } from "../guards";
+import {
+  DEFAULT_MAX_PER_HOUR,
+  DEFAULT_MAX_PENDING,
+  LIMIT_MAX,
+  LIMIT_MIN,
+  SCAN_VERSION,
+  STATE_VERSION,
+  scanFilePath,
+} from "./paths";
+
+/**
+ * The scout worker's published state. It is the whole contract between the
+ * detached worker and whatever dashboard is attached: the worker writes it
+ * atomically before publishing anything, and a dashboard only ever reads it.
+ * A file this module cannot understand is renamed aside, never deleted; a
+ * version-1 file is migrated instead, discarding every staged recommendation.
+ */
+
+export type ScoutLifecycle = "stopped" | "idle" | "scanning" | "failed";
+export type ScoutScanStatus = "running" | "succeeded" | "failed" | "stopped";
+export type ScoutIssueState = "pending" | "implemented";
+export type ScoutSyncState = "ok" | "error" | "stale";
+
+export interface ScoutWorkerRecord {
+  pid: number;
+  startedAt: string;
+  stateDir: string;
+  harnessVersion: string | null;
+  argv: string[];
+}
+
+export interface ScoutAcpChild {
+  pid: number;
+  argv: string[];
+  cwd: string;
+}
+
+/** The adjustable caps the dashboard persists and the worker enforces. */
+export interface ScoutLimits {
+  maxPerHour: number;
+  maxPending: number;
+}
+
+/** One Auto Scout issue, as GitHub last reported it. */
+export interface ScoutIssueRow {
+  number: number;
+  title: string;
+  url: string;
+  createdAt: string;
+  /** An open issue is `pending`; a closed one is `implemented`. */
+  state: ScoutIssueState;
+  /** The recommendation's category, the body's `## Category`, or null. */
+  category: string | null;
+  /** The earliest published release that shipped the closing commit, or null. */
+  version: string | null;
+}
+
+/** The last periodic GitHub read and whether it succeeded. */
+export interface ScoutSync {
+  state: ScoutSyncState;
+  /** The last successful read, or null before one. */
+  at: string | null;
+  /** Why the last read failed, or null. */
+  error: string | null;
+}
+
+export interface ScoutScanRecord {
+  scanId: string;
+  startedAt: string;
+  /** How many recommendations the request asked for. */
+  requested: number;
+}
+
+export interface ScoutState {
+  version: 2;
+  repo: string;
+  root: string;
+  lifecycle: ScoutLifecycle;
+  enabled: boolean;
+  worker: ScoutWorkerRecord | null;
+  limits: ScoutLimits;
+  /** Every Auto Scout issue known from GitHub, newest creation first. */
+  issues: ScoutIssueRow[];
+  sync: ScoutSync;
+  scan: ScoutScanRecord | null;
+  /** The newest scan, running or finished, so its activity can be replayed. */
+  newestScanId: string | null;
+  nextAttemptAt: string | null;
+  lastError: string | null;
+  notice: string | null;
+  updatedAt: string;
+}
+
+export const DEFAULT_LIMITS: ScoutLimits = Object.freeze({
+  maxPerHour: DEFAULT_MAX_PER_HOUR,
+  maxPending: DEFAULT_MAX_PENDING,
+});
+
+export function isValidLimit(value: unknown): value is number {
+  return isInteger(value) && value >= LIMIT_MIN && value <= LIMIT_MAX;
+}
+
+export function isLimits(value: unknown): value is ScoutLimits {
+  return isRecord(value) && isValidLimit(value.maxPerHour) && isValidLimit(value.maxPending);
+}
+
+function isIssueState(value: unknown): value is ScoutIssueState {
+  return value === "pending" || value === "implemented";
+}
+
+function isIssueRow(value: unknown): value is ScoutIssueRow {
+  return (
+    isRecord(value) &&
+    isInteger(value.number) &&
+    value.number > 0 &&
+    typeof value.title === "string" &&
+    typeof value.url === "string" &&
+    typeof value.createdAt === "string" &&
+    isIssueState(value.state) &&
+    nullableString(value.category) &&
+    nullableString(value.version)
+  );
+}
+
+function isSync(value: unknown): value is ScoutSync {
+  return (
+    isRecord(value) &&
+    (value.state === "ok" || value.state === "error" || value.state === "stale") &&
+    nullableString(value.at) &&
+    nullableString(value.error)
+  );
+}
+
+function isWorker(value: unknown): value is ScoutWorkerRecord {
+  return (
+    isRecord(value) &&
+    isInteger(value.pid) &&
+    typeof value.startedAt === "string" &&
+    typeof value.stateDir === "string" &&
+    nullableString(value.harnessVersion) &&
+    Array.isArray(value.argv) &&
+    value.argv.every((entry) => typeof entry === "string")
+  );
+}
+
+function isScan(value: unknown): value is ScoutScanRecord {
+  return (
+    isRecord(value) &&
+    typeof value.scanId === "string" &&
+    typeof value.startedAt === "string" &&
+    isInteger(value.requested)
+  );
+}
+
+function isLifecycle(value: unknown): value is ScoutLifecycle {
+  return value === "stopped" || value === "idle" || value === "scanning" || value === "failed";
+}
+
+export function emptyState(options: {
+  repo: string;
+  root: string;
+  lifecycle?: ScoutLifecycle;
+  limit?: ScoutLimits;
+  now?: Date;
+}): ScoutState {
+  return {
+    version: STATE_VERSION,
+    repo: options.repo,
+    root: options.root,
+    lifecycle: options.lifecycle ?? "stopped",
+    enabled: false,
+    worker: null,
+    limits: options.limit ?? { ...DEFAULT_LIMITS },
+    issues: [],
+    sync: { state: "stale", at: null, error: null },
+    scan: null,
+    newestScanId: null,
+    nextAttemptAt: null,
+    lastError: null,
+    notice: null,
+    updatedAt: (options.now ?? new Date()).toISOString(),
+  };
+}
+
+/** Parses one version-2 document; `undefined` means the bytes are not ours. */
+export function parseState(text: string): ScoutState | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+  if (!isRecord(parsed) || parsed.version !== STATE_VERSION) return undefined;
+  if (typeof parsed.repo !== "string" || typeof parsed.root !== "string") return undefined;
+  if (!isLifecycle(parsed.lifecycle)) return undefined;
+  if (typeof parsed.enabled !== "boolean") return undefined;
+  if (parsed.worker !== null && !isWorker(parsed.worker)) return undefined;
+  if (!isLimits(parsed.limits)) return undefined;
+  if (!Array.isArray(parsed.issues) || !parsed.issues.every(isIssueRow)) return undefined;
+  if (!isSync(parsed.sync)) return undefined;
+  if (parsed.scan !== null && !isScan(parsed.scan)) return undefined;
+  if (!nullableString(parsed.newestScanId)) return undefined;
+  if (!nullableString(parsed.nextAttemptAt) || !nullableString(parsed.lastError)) return undefined;
+  if (!nullableString(parsed.notice)) return undefined;
+  if (typeof parsed.updatedAt !== "string") return undefined;
+
+  return {
+    version: STATE_VERSION,
+    repo: parsed.repo,
+    root: parsed.root,
+    lifecycle: parsed.lifecycle,
+    enabled: parsed.enabled,
+    worker: parsed.worker as ScoutWorkerRecord | null,
+    limits: parsed.limits,
+    issues: parsed.issues,
+    sync: parsed.sync,
+    scan: parsed.scan as ScoutScanRecord | null,
+    newestScanId: parsed.newestScanId,
+    nextAttemptAt: parsed.nextAttemptAt,
+    lastError: parsed.lastError,
+    notice: parsed.notice,
+    updatedAt: parsed.updatedAt,
+  };
+}
+
+/**
+ * A version-1 board kept only its staging array and its decisions. The worker
+ * and scan facts are carried forward, the staged recommendations are discarded
+ * without filing anything, and both limits take their defaults. The notice
+ * names the discard so the operator sees what the upgrade dropped.
+ */
+export function migrateState(text: string): ScoutState | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+  if (!isRecord(parsed) || parsed.version !== 1) return undefined;
+  if (typeof parsed.repo !== "string" || typeof parsed.root !== "string") return undefined;
+  if (!isLifecycle(parsed.lifecycle)) return undefined;
+  if (typeof parsed.enabled !== "boolean") return undefined;
+  if (parsed.worker !== null && !isWorker(parsed.worker)) return undefined;
+  if (!Array.isArray(parsed.staged)) return undefined;
+  if (parsed.scan !== null && !isScan(parsed.scan)) return undefined;
+  if (!nullableString(parsed.newestScanId)) return undefined;
+  if (!nullableString(parsed.lastError) || !nullableString(parsed.notice)) return undefined;
+  if (typeof parsed.updatedAt !== "string") return undefined;
+
+  const discarded = parsed.staged.length;
+  return {
+    version: STATE_VERSION,
+    repo: parsed.repo,
+    root: parsed.root,
+    lifecycle: parsed.lifecycle,
+    enabled: parsed.enabled,
+    worker: parsed.worker as ScoutWorkerRecord | null,
+    limits: { ...DEFAULT_LIMITS },
+    issues: [],
+    sync: { state: "stale", at: null, error: null },
+    scan: parsed.scan as ScoutScanRecord | null,
+    newestScanId: parsed.newestScanId,
+    nextAttemptAt: null,
+    lastError: parsed.lastError,
+    notice:
+      discarded === 0
+        ? parsed.notice
+        : `Discarded ${discarded} staged recommendation${discarded === 1 ? "" : "s"} from the ` +
+          "previous Auto Scout version; nothing was filed.",
+    updatedAt: parsed.updatedAt,
+  };
+}
+
+/**
+ * Reads without side effects; the dashboard uses this. A version-1 file is
+ * migrated in memory, so a dashboard upgraded ahead of its worker still reads
+ * the worker's lifecycle and can stop it.
+ */
+export function readState(path: string): ScoutState | undefined {
+  try {
+    const text = readFileSync(path, "utf8");
+    return parseState(text) ?? migrateState(text);
+  } catch {
+    return undefined;
+  }
+}
+
+/** Reads and migrates, moving an unusable file aside, preserved. */
+export function loadState(path: string): ScoutState | undefined {
+  if (!existsSync(path)) return undefined;
+  let text: string;
+  try {
+    text = readFileSync(path, "utf8");
+  } catch {
+    return undefined;
+  }
+  const state = parseState(text) ?? migrateState(text);
+  if (state !== undefined) return state;
+  quarantine(path);
+  return undefined;
+}
+
+export function quarantine(path: string): string | undefined {
+  const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+  for (let attempt = 0; attempt < 10; attempt += 1) {
+    const target = `${path}.invalid-${stamp}${attempt === 0 ? "" : `-${attempt}`}`;
+    if (existsSync(target)) continue;
+    try {
+      renameSync(path, target);
+      return target;
+    } catch {
+      return undefined;
+    }
+  }
+  return undefined;
+}
+
+/** Temporary file plus rename: no reader ever sees a prefix of a state. */
+export function writeState(path: string, state: ScoutState): void {
+  writeAtomic(path, `${JSON.stringify(state, null, 2)}\n`);
+}
+
+export function writeAtomic(path: string, content: string): void {
+  mkdirSync(dirname(path), { recursive: true });
+  const temporary = `${path}.${process.pid}.tmp`;
+  writeFileSync(temporary, content);
+  renameSync(temporary, path);
+}
+
+/* ------------------------------------------------------------------ */
+/* Persisted limits                                                    */
+/* ------------------------------------------------------------------ */
+
+export function parseSettings(text: string): ScoutLimits | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+  return isLimits(parsed) ? { maxPerHour: parsed.maxPerHour, maxPending: parsed.maxPending } : undefined;
+}
+
+export function readSettings(path: string): ScoutLimits | undefined {
+  try {
+    return parseSettings(readFileSync(path, "utf8"));
+  } catch {
+    return undefined;
+  }
+}
+
+export function writeSettings(path: string, limits: ScoutLimits): void {
+  writeAtomic(path, `${JSON.stringify(limits, null, 2)}\n`);
+}
+
+/* ------------------------------------------------------------------ */
+/* Scan records                                                        */
+/* ------------------------------------------------------------------ */
+
+export interface ScoutScanFile {
+  version: 2;
+  scanId: string;
+  repo: string;
+  root: string;
+  status: ScoutScanStatus;
+  startedAt: string;
+  finishedAt: string | null;
+  requested: number;
+  filed: number;
+  prompt: string;
+  acp: ScoutAcpChild | null;
+  error: string | null;
+  shortfall: string | null;
+  workerPid: number;
+  harnessVersion: string | null;
+}
+
+function isAcp(value: unknown): value is ScoutAcpChild {
+  return (
+    isRecord(value) &&
+    isInteger(value.pid) &&
+    Array.isArray(value.argv) &&
+    value.argv.every((entry) => typeof entry === "string") &&
+    typeof value.cwd === "string"
+  );
+}
+
+/** Reads a version-2 record; a version-1 record's `staged` becomes `filed`. */
+export function parseScanFile(text: string): ScoutScanFile | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+  if (!isRecord(parsed)) return undefined;
+  if (parsed.version !== 1 && parsed.version !== SCAN_VERSION) return undefined;
+  if (typeof parsed.scanId !== "string" || typeof parsed.repo !== "string") return undefined;
+  if (typeof parsed.root !== "string") return undefined;
+  if (
+    parsed.status !== "running" &&
+    parsed.status !== "succeeded" &&
+    parsed.status !== "failed" &&
+    parsed.status !== "stopped"
+  ) {
+    return undefined;
+  }
+  if (typeof parsed.startedAt !== "string" || !nullableString(parsed.finishedAt)) return undefined;
+  if (!isInteger(parsed.requested)) return undefined;
+  const filed = parsed.version === 1 ? parsed.staged : parsed.filed;
+  if (!isInteger(filed)) return undefined;
+  if (typeof parsed.prompt !== "string") return undefined;
+  if (parsed.acp !== null && !isAcp(parsed.acp)) return undefined;
+  if (!nullableString(parsed.error) || !nullableString(parsed.shortfall)) return undefined;
+  if (!isInteger(parsed.workerPid) || !nullableString(parsed.harnessVersion)) return undefined;
+  return {
+    version: SCAN_VERSION,
+    scanId: parsed.scanId,
+    repo: parsed.repo,
+    root: parsed.root,
+    status: parsed.status,
+    startedAt: parsed.startedAt,
+    finishedAt: parsed.finishedAt,
+    requested: parsed.requested,
+    filed,
+    prompt: parsed.prompt,
+    acp: parsed.acp as ScoutAcpChild | null,
+    error: parsed.error,
+    shortfall: parsed.shortfall,
+    workerPid: parsed.workerPid,
+    harnessVersion: parsed.harnessVersion,
+  };
+}
+
+export function readScanFile(directory: string): ScoutScanFile | undefined {
+  try {
+    return parseScanFile(readFileSync(scanFilePath(directory), "utf8"));
+  } catch {
+    return undefined;
+  }
+}
+
+export function writeScanFile(directory: string, scan: ScoutScanFile): void {
+  writeAtomic(scanFilePath(directory), `${JSON.stringify(scan, null, 2)}\n`);
+}