@coldtea/abide 0.0.1

package/src/lib/opencodePlugin.ts

@@ -0,0 +1,38 @@
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { packageRoot } from "./packageRoot.js";
+
+export const OPENCODE_PLUGIN_MARKER = "abide-opencode-plugin";
+
+/**
+ * The plugin as shipped in the package. The installed file only points at it,
+ * so an upgrade needs no reinstall. OpenCode discovers `plugins/*.js` and
+ * `*.ts` only, so the installed file is `.js` whatever the package uses.
+ */
+export const pluginSourcePath = (): string => path.join(packageRoot(), "opencode", "abide.mjs");
+
+const shim = (source: string): string =>
+  [
+    `// ${OPENCODE_PLUGIN_MARKER}: written by \`abide init opencode\`; remove with \`abide uninstall opencode\`.`,
+    `export { default } from ${JSON.stringify(pathToFileURL(source).href)};`,
+    "",
+  ].join("\n");
+
+export const installOpencodePlugin = (target: string): void => {
+  mkdirSync(path.dirname(target), { recursive: true });
+  writeFileSync(target, shim(pluginSourcePath()));
+};
+
+export const uninstallOpencodePlugin = (target: string): boolean => {
+  if (!existsSync(target)) return false;
+  let text: string;
+  try {
+    text = readFileSync(target, "utf8");
+  } catch {
+    return false;
+  }
+  if (!text.includes(OPENCODE_PLUGIN_MARKER)) return false;
+  rmSync(target);
+  return true;
+};

package/src/lib/output.ts

@@ -0,0 +1,44 @@
+import { assertNever, type HookOutput } from "@coldtea/abide-schema";
+
+export type HookEventName = "SessionStart" | "UserPromptSubmit" | "PostToolUse" | "Stop";
+
+const write = (payload: Record<string, unknown>, done: () => void): void => {
+  process.stdout.write(`${JSON.stringify(payload)}\n`, () => done());
+};
+
+/** The only path to stdout. Everything else goes to stderr, which the host keeps out of the transcript. */
+export const emit = (output: HookOutput, event: HookEventName, done: () => void): void => {
+  switch (output.kind) {
+    case "silent":
+      done();
+      return;
+    case "session-context":
+      write(
+        {
+          ...(output.systemMessage === undefined ? {} : { systemMessage: output.systemMessage }),
+          hookSpecificOutput: { hookEventName: event, additionalContext: output.additionalContext },
+        },
+        done,
+      );
+      return;
+    case "block":
+      write(
+        {
+          decision: "block",
+          reason: output.reason,
+          ...(output.systemMessage === undefined ? {} : { systemMessage: output.systemMessage }),
+        },
+        done,
+      );
+      return;
+    case "notice":
+      write({ systemMessage: output.systemMessage }, done);
+      return;
+    default:
+      return assertNever(output);
+  }
+};
+
+export const debug = (message: string): void => {
+  if (process.env.ABIDE_DEBUG) process.stderr.write(`abide: ${message}\n`);
+};

package/src/lib/packageRoot.ts

@@ -0,0 +1,48 @@
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { readRegularText, writeRegularFile } from "./regularFile.js";
+
+/** Directory of the installed @coldtea/abide package (the one holding package.json). */
+export const packageRoot = (): string => {
+  let dir = path.dirname(fileURLToPath(import.meta.url));
+  for (;;) {
+    const candidate = path.join(dir, "package.json");
+    if (existsSync(candidate)) {
+      try {
+        const name: unknown = JSON.parse(readFileSync(candidate, "utf8")).name;
+        if (name === "@coldtea/abide") return dir;
+      } catch {
+        // keep walking
+      }
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) return dir;
+    dir = parent;
+  }
+};
+
+export const hookScriptPath = (): string => path.join(packageRoot(), "dist", "abide-hook.js");
+export const binScriptPath = (): string => path.join(packageRoot(), "dist", "bin.js");
+export const compileSkillPath = (): string =>
+  path.join(packageRoot(), "skills", "abide-compile", "SKILL.md");
+
+/**
+ * Copies the packaged skill into the repo so the agent can read it without
+ * leaving the project. A FIFO there would hold the hook and a symlink would
+ * redirect the copy, so either falls back to the packaged path.
+ */
+export const placeCompileSkill = (root: string): string => {
+  const packaged = compileSkillPath();
+  const target = path.join(root, ".abide", "compile-skill.md");
+  const skill = readRegularText(packaged);
+  if (skill === undefined) return packaged;
+  try {
+    mkdirSync(path.dirname(target), { recursive: true });
+  } catch {
+    return packaged;
+  }
+  return writeRegularFile(target, skill, { use: "replace", followSymlinks: false })
+    ? target
+    : packaged;
+};

package/src/lib/paths.ts

@@ -0,0 +1,96 @@
+import { existsSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import path from "node:path";
+import picomatch from "picomatch";
+
+export const homeDir = (): string => process.env.ABIDE_HOME_DIR ?? homedir();
+
+export const globalAbideDir = (): string => path.join(homeDir(), ".abide");
+export const globalRubricPath = (): string => path.join(globalAbideDir(), "global.json");
+export const sessionsDir = (): string => path.join(globalAbideDir(), "sessions");
+
+export const abideDir = (root: string): string => path.join(root, ".abide");
+export const rubricPath = (root: string): string => path.join(abideDir(root), "rubric.json");
+export const eventsPath = (root: string): string => path.join(abideDir(root), "events.jsonl");
+
+const isDir = (p: string): boolean => {
+  try {
+    return statSync(p).isDirectory();
+  } catch {
+    return false;
+  }
+};
+
+const ROOT_MARKERS = [".git", ".abide", "AGENTS.md", "CLAUDE.md"];
+
+/** The nearest ancestor that looks like a repository root, else the start directory. */
+export const findRepoRoot = (start: string): string => {
+  let dir = path.resolve(start);
+  if (!isDir(dir)) dir = path.dirname(dir);
+  let fallback: string | undefined;
+  for (;;) {
+    if (existsSync(path.join(dir, ".git"))) return dir;
+    if (fallback === undefined && ROOT_MARKERS.some((m) => existsSync(path.join(dir, m)))) {
+      fallback = dir;
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return fallback ?? path.resolve(start);
+};
+
+export const expandHome = (p: string): string =>
+  p === "~" ? homeDir() : p.startsWith("~/") ? path.join(homeDir(), p.slice(2)) : p;
+
+/** Absolute location of a rubric source path ("~/x" or repo-relative). */
+export const resolveSourcePath = (root: string, sourcePath: string): string =>
+  sourcePath.startsWith("~") ? expandHome(sourcePath) : path.resolve(root, sourcePath);
+
+const toPosix = (p: string): string => p.split(path.sep).join("/");
+
+/** The rubric's spelling of an absolute path: repo-relative, "~/..." inside home, else absolute. */
+export const toSourcePath = (root: string, absolute: string): string => {
+  const rel = path.relative(root, absolute);
+  if (
+    path.resolve(root) !== homeDir() &&
+    rel !== "" &&
+    !rel.startsWith("..") &&
+    !path.isAbsolute(rel)
+  ) {
+    return toPosix(rel);
+  }
+  const fromHome = path.relative(homeDir(), absolute);
+  if (fromHome !== "" && !fromHome.startsWith("..") && !path.isAbsolute(fromHome)) {
+    return `~/${toPosix(fromHome)}`;
+  }
+  return toPosix(absolute);
+};
+
+/** One spelling for a source path however the agent wrote it. */
+export const canonicalSourcePath = (root: string, sourcePath: string): string =>
+  toSourcePath(root, resolveSourcePath(root, sourcePath));
+
+export const relativeToRoot = (root: string, absolute: string): string =>
+  toPosix(path.relative(root, absolute));
+
+/** Files abide owns are never checked; the agent writes the rubric under supervision of the compile skill. */
+export const isAbideOwned = (relativePath: string): boolean =>
+  relativePath === ".abide" || relativePath.startsWith(".abide/");
+
+/** Basename globs; also fed to git as pathspecs. */
+export const SECRET_FILE_PATTERNS: readonly string[] = [
+  ".env",
+  ".env.*",
+  ".envrc",
+  "*.pem",
+  "*.key",
+];
+
+const isSecretName = picomatch([...SECRET_FILE_PATTERNS], { dot: true });
+
+export const isSecretFile = (relativePath: string): boolean =>
+  isSecretName(path.posix.basename(relativePath));
+
+export const isExcludedPath = (relativePath: string): boolean =>
+  isAbideOwned(relativePath) || isSecretFile(relativePath);

package/src/lib/reason.ts

@@ -0,0 +1,46 @@
+import type { Rule, Verdict } from "@coldtea/abide-schema";
+
+const where = (rule: Rule): string =>
+  rule.source.line === undefined
+    ? rule.source.path
+    : `${rule.source.path} line ${rule.source.line}`;
+
+const quote = (text: string): string => {
+  const trimmed = text.trim().replace(/\s+/g, " ");
+  return trimmed.length > 220 ? `${trimmed.slice(0, 217)}...` : trimmed;
+};
+
+/** One sentence per violation, naming the rule. Never the whole instruction file. */
+export const repairReason = (
+  phase: "edit" | "turn",
+  violations: readonly { rule: Rule; verdict: Verdict }[],
+  files: readonly string[],
+): string => {
+  const lines = violations.map(({ rule, verdict }) => {
+    const evidence =
+      rule.check.type === "lint" && verdict.answer !== undefined
+        ? ` Matched: ${verdict.answer}`
+        : verdict.answer !== undefined
+          ? ` Judged: ${verdict.answer} (${verdict.probability.toFixed(2)}).`
+          : ` (${verdict.probability.toFixed(2)})`;
+    return `Rule "${rule.id}" from ${where(rule)}: "${quote(rule.text)}".${evidence}`;
+  });
+  const subject = phase === "edit" ? "This edit" : "The changes in this turn";
+  const target = files.length === 1 ? files[0] : `${files.length} files (${files.join(", ")})`;
+  const ask =
+    phase === "edit"
+      ? `Repair ${target} now, then continue with the task.`
+      : `Repair ${target} before you finish. Keep the fix to what the rule asks.`;
+  return `Abide: ${subject} appears to break ${lines.length === 1 ? "a rule" : `${lines.length} rules`} from this repository's instructions.\n${lines.map((l) => `- ${l}`).join("\n")}\n${ask}`;
+};
+
+export const flagNotice = (
+  phase: "edit" | "turn",
+  flagged: readonly { rule: Rule; verdict: Verdict }[],
+  files: readonly string[],
+): string => {
+  const list = flagged
+    .map(({ rule, verdict }) => `${rule.id} ${verdict.probability.toFixed(2)}`)
+    .join(", ");
+  return `Abide: uncertain about ${list} on ${files.join(", ")} (${phase}). Not sent to the agent. Details in .abide/events.jsonl.`;
+};

package/src/lib/regularFile.ts

@@ -0,0 +1,96 @@
+import {
+  closeSync,
+  constants,
+  fstatSync,
+  openSync,
+  readSync,
+  writeSync,
+  type Stats,
+} from "node:fs";
+import { MAX_FILE_READ_BYTES } from "./constants.js";
+
+type Use = "read" | "append" | "replace";
+
+type OpenOptions = {
+  followSymlinks?: boolean;
+  use?: Use;
+};
+
+type ReadOptions = Omit<OpenOptions, "use"> & { maxBytes?: number };
+
+type WriteOptions = Omit<OpenOptions, "use"> & { use: Exclude<Use, "read"> };
+
+const FLAGS: Record<Use, number> = {
+  read: constants.O_RDONLY,
+  append: constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT,
+  replace: constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC,
+};
+
+/**
+ * A FIFO or device would hold a synchronous read or write for as long as the
+ * other side likes, past any hook deadline. Opening non-blocking and judging
+ * the descriptor leaves no window between the check and the use. The caller
+ * closes it.
+ */
+export const openRegular = (
+  file: string,
+  options: OpenOptions = {},
+): { fd: number; size: number } | undefined => {
+  const { followSymlinks = true, use = "read" } = options;
+  let fd: number;
+  try {
+    fd = openSync(
+      file,
+      FLAGS[use] | constants.O_NONBLOCK | (followSymlinks ? 0 : constants.O_NOFOLLOW),
+    );
+  } catch {
+    return undefined;
+  }
+  let stat: Stats | undefined;
+  try {
+    stat = fstatSync(fd);
+  } catch {
+    stat = undefined;
+  }
+  if (stat?.isFile()) return { fd, size: stat.size };
+  closeSync(fd);
+  return undefined;
+};
+
+export const readRegularFile = (file: string, options: ReadOptions = {}): Buffer | undefined => {
+  const { maxBytes = MAX_FILE_READ_BYTES, ...open } = options;
+  const opened = openRegular(file, open);
+  if (opened === undefined) return undefined;
+  const { fd, size } = opened;
+  try {
+    if (size > maxBytes) return undefined;
+    const buffer = Buffer.alloc(size);
+    let read = 0;
+    while (read < size) {
+      const count = readSync(fd, buffer, read, size - read, read);
+      if (count === 0) break;
+      read += count;
+    }
+    return buffer.subarray(0, read);
+  } catch {
+    return undefined;
+  } finally {
+    closeSync(fd);
+  }
+};
+
+export const readRegularText = (file: string, options: ReadOptions = {}): string | undefined =>
+  readRegularFile(file, options)?.toString("utf8");
+
+export const writeRegularFile = (file: string, text: string, options: WriteOptions): boolean => {
+  const opened = openRegular(file, options);
+  if (opened === undefined) return false;
+  try {
+    writeSync(opened.fd, text);
+    return true;
+  } catch {
+    return false;
+  } finally {
+    closeSync(opened.fd);
+  }
+};

package/src/lib/replay.ts

@@ -0,0 +1,314 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import {
+  postToolUseInputSchema,
+  type PostToolUseInput,
+  type Rule,
+  type Thresholds,
+  type Verdict,
+} from "@coldtea/abide-schema";
+import { pool } from "./audit.js";
+import { runCheck } from "./checkRunner.js";
+import { EDIT_CHECK_TIMEOUT_MS, MAX_TASK_CHARS, TURN_CHECK_TIMEOUT_MS } from "./constants.js";
+import { boundState, editsFromPostToolUse } from "./diff.js";
+import type { FileDiff } from "./git.js";
+import { findRepoRoot, isExcludedPath, relativeToRoot } from "./paths.js";
+
+/**
+ * A past coding session, read back from the host's transcript, so every edit
+ * it made can be judged as if abide had been installed at the time. Nothing
+ * here touches the working tree: the diffs come from the transcript alone.
+ */
+export type ReplayEdit = { turn: number; input: PostToolUseInput };
+export type ReplayTurn = { index: number; prompt: string | undefined; edits: ReplayEdit[] };
+export type ReplaySession = { file: string; cwd: string; turns: ReplayTurn[] };
+
+const EDIT_TOOLS = new Set(["Edit", "Write", "MultiEdit"]);
+
+type ToolUse = { id: string; name: string; input: unknown; turn: number };
+
+const asRecord = (value: unknown): Record<string, unknown> | undefined => {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
+  const record: Record<string, unknown> = { ...value };
+  return record;
+};
+
+const asList = (value: unknown): unknown[] => (Array.isArray(value) ? value : []);
+
+const promptText = (content: unknown): string | undefined => {
+  if (typeof content === "string") return content.trim() === "" ? undefined : content;
+  const parts = asList(content).flatMap((p) => {
+    const part = asRecord(p);
+    return part?.type === "text" && typeof part.text === "string" ? [part.text] : [];
+  });
+  const hasToolResult = asList(content).some((p) => asRecord(p)?.type === "tool_result");
+  if (hasToolResult || parts.length === 0) return undefined;
+  return parts.join("\n");
+};
+
+export const parseTranscript = (file: string): ReplaySession => {
+  const turns: ReplayTurn[] = [];
+  const uses = new Map<string, ToolUse>();
+  let cwd: string | undefined;
+  let turnIndex = 0;
+  const current = (): ReplayTurn => {
+    const last = turns.at(-1);
+    if (last !== undefined) return last;
+    const first: ReplayTurn = { index: 0, prompt: undefined, edits: [] };
+    turns.push(first);
+    return first;
+  };
+  for (const line of readFileSync(file, "utf8").split("\n")) {
+    if (line.trim() === "") continue;
+    let entry: Record<string, unknown> | undefined;
+    try {
+      entry = asRecord(JSON.parse(line));
+    } catch {
+      // a torn line, still being written
+      continue;
+    }
+    if (entry === undefined) continue;
+    if (typeof entry.cwd === "string") cwd ??= entry.cwd;
+    const message = asRecord(entry.message);
+    if (entry.type === "user" && entry.isMeta !== true) {
+      const prompt = promptText(message?.content);
+      if (prompt !== undefined) {
+        turnIndex += 1;
+        turns.push({ index: turnIndex, prompt: prompt.slice(0, MAX_TASK_CHARS), edits: [] });
+      }
+      const result = asRecord(entry.toolUseResult);
+      for (const p of asList(message?.content)) {
+        const part = asRecord(p);
+        if (part?.type !== "tool_result" || typeof part.tool_use_id !== "string") continue;
+        const use = uses.get(part.tool_use_id);
+        if (use === undefined) continue;
+        uses.delete(part.tool_use_id);
+        const parsed = postToolUseInputSchema.safeParse({
+          session_id: path.basename(file, ".jsonl"),
+          cwd: cwd ?? process.cwd(),
+          hook_event_name: "PostToolUse",
+          tool_name: use.name,
+          tool_use_id: use.id,
+          tool_input: use.input,
+          ...(result === undefined ? {} : { tool_response: result }),
+        });
+        if (!parsed.success) continue;
+        const turn = turns.find((t) => t.index === use.turn) ?? current();
+        turn.edits.push({ turn: turn.index, input: parsed.data });
+      }
+    }
+    if (entry.type === "assistant") {
+      for (const p of asList(message?.content)) {
+        const part = asRecord(p);
+        if (part?.type !== "tool_use" || typeof part.id !== "string") continue;
+        if (typeof part.name !== "string" || !EDIT_TOOLS.has(part.name)) continue;
+        uses.set(part.id, { id: part.id, name: part.name, input: part.input, turn: turnIndex });
+      }
+    }
+  }
+  return { file, cwd: cwd ?? process.cwd(), turns: turns.filter((t) => t.edits.length > 0) };
+};
+
+export type ReplayEditResult = {
+  session: string;
+  turn: number;
+  file: string;
+  verdicts: Verdict[];
+  costUsd: number;
+  error?: string;
+  diff?: string;
+  task?: string;
+};
+
+export type ReplayTurnResult = {
+  session: string;
+  turn: number;
+  files: string[];
+  verdicts: Verdict[];
+  costUsd: number;
+  error?: string;
+};
+
+export type ReplayResult = { edits: ReplayEditResult[]; turns: ReplayTurnResult[] };
+
+export type ReplayProgress = (done: number, total: number, spendUsd: number) => void;
+
+const failed = (error: unknown): string => (error instanceof Error ? error.message : String(error));
+
+/**
+ * Judges every edit of every session as the edit hook would have, then every
+ * turn as the stop hook would have, from the edits alone. Shell-made changes
+ * are not in a transcript and are not judged.
+ */
+export const replaySessions = async (
+  sessions: readonly ReplaySession[],
+  rules: readonly Rule[],
+  thresholds: Thresholds,
+  concurrency: number,
+  progress: ReplayProgress,
+  keepDiffs = false,
+): Promise<ReplayResult> => {
+  type Job = { session: ReplaySession; turn: ReplayTurn; edit: ReplayEdit };
+  const jobs: Job[] = sessions.flatMap((session) =>
+    session.turns.flatMap((turn) => turn.edits.map((edit) => ({ session, turn, edit }))),
+  );
+  const edits: ReplayEditResult[] = [];
+  const turnDiffs = new Map<
+    string,
+    { session: ReplaySession; turn: ReplayTurn; files: Map<string, string[]> }
+  >();
+  let done = 0;
+  let spendUsd = 0;
+  await pool(jobs, concurrency, async ({ session, turn, edit }) => {
+    const root = findRepoRoot(session.cwd);
+    const name = path.basename(session.file, ".jsonl");
+    for (const hunk of editsFromPostToolUse(edit.input)) {
+      const relative = relativeToRoot(root, hunk.filePath);
+      if (relative.startsWith("..") || isExcludedPath(relative) || hunk.text === undefined)
+        continue;
+      const { text } = boundState(hunk.text);
+      if (text.trim() === "") continue;
+      const key = `${name}:${turn.index}`;
+      const bucket = turnDiffs.get(key) ?? { session, turn, files: new Map<string, string[]>() };
+      bucket.files.set(relative, [...(bucket.files.get(relative) ?? []), text]);
+      turnDiffs.set(key, bucket);
+      try {
+        const out = await runCheck({
+          phase: "edit",
+          fileDiffs: [{ file: relative, text }],
+          task: turn.prompt,
+          rules,
+          thresholds,
+          timeoutMs: EDIT_CHECK_TIMEOUT_MS,
+          retries: 2,
+        });
+        spendUsd += out.usage.costUsd ?? 0;
+        edits.push({
+          session: name,
+          turn: turn.index,
+          file: relative,
+          verdicts: out.verdicts,
+          costUsd: out.usage.costUsd ?? 0,
+          ...(keepDiffs
+            ? { diff: text, ...(turn.prompt === undefined ? {} : { task: turn.prompt }) }
+            : {}),
+        });
+      } catch (error) {
+        edits.push({
+          session: name,
+          turn: turn.index,
+          file: relative,
+          verdicts: [],
+          costUsd: 0,
+          error: failed(error),
+        });
+      }
+    }
+    done += 1;
+    progress(done, jobs.length, spendUsd);
+  });
+
+  const turns: ReplayTurnResult[] = [];
+  await pool([...turnDiffs.values()], concurrency, async ({ session, turn, files }) => {
+    const name = path.basename(session.file, ".jsonl");
+    const fileDiffs: FileDiff[] = [...files.entries()].map(([file, texts]) => ({
+      file,
+      text: boundState(texts.join("\n"), 8_000).text,
+    }));
+    try {
+      const out = await runCheck({
+        phase: "turn",
+        fileDiffs,
+        task: turn.prompt,
+        rules,
+        thresholds,
+        timeoutMs: TURN_CHECK_TIMEOUT_MS,
+        retries: 2,
+      });
+      spendUsd += out.usage.costUsd ?? 0;
+      turns.push({
+        session: name,
+        turn: turn.index,
+        files: fileDiffs.map((f) => f.file),
+        verdicts: out.verdicts,
+        costUsd: out.usage.costUsd ?? 0,
+      });
+    } catch (error) {
+      turns.push({
+        session: name,
+        turn: turn.index,
+        files: fileDiffs.map((f) => f.file),
+        verdicts: [],
+        costUsd: 0,
+        error: failed(error),
+      });
+    }
+    progress(done, jobs.length, spendUsd);
+  });
+  return { edits, turns };
+};
+
+/** Turn ranges the drift is reported over. The first covers the session's opening, where instructions are fresh. */
+export const DRIFT_BUCKETS: readonly { label: string; from: number; to: number }[] = [
+  { label: "turns 1 to 5", from: 1, to: 5 },
+  { label: "turns 6 to 15", from: 6, to: 15 },
+  { label: "turns 16 and later", from: 16, to: Infinity },
+];
+
+export type DriftRow = {
+  label: string;
+  edits: number;
+  broken: number;
+  /** Share of judged edits with at least one rule above the act line. */
+  rate: number;
+};
+
+export const driftByTurn = (edits: readonly ReplayEditResult[]): DriftRow[] =>
+  DRIFT_BUCKETS.map((b) => {
+    const inBucket = edits.filter(
+      (e) => e.error === undefined && e.turn >= b.from && e.turn <= b.to,
+    );
+    const broken = inBucket.filter((e) => e.verdicts.some((v) => v.band === "act")).length;
+    return {
+      label: b.label,
+      edits: inBucket.length,
+      broken,
+      rate: inBucket.length === 0 ? 0 : broken / inBucket.length,
+    };
+  });
+
+export type RuleTally = {
+  rule: string;
+  phase: "edit" | "turn";
+  broken: number;
+  flagged: number;
+  of: number;
+  example: string;
+};
+
+export const tallyRules = (result: ReplayResult, rules: readonly Rule[]): RuleTally[] => {
+  const tallies: RuleTally[] = [];
+  for (const rule of rules) {
+    if (rule.check.type !== "model") continue;
+    const phase = rule.when ?? "edit";
+    const items: { file: string; verdicts: Verdict[] }[] =
+      phase === "edit"
+        ? result.edits.map((e) => ({ file: e.file, verdicts: e.verdicts }))
+        : result.turns.map((t) => ({ file: t.files.join(", "), verdicts: t.verdicts }));
+    let broken = 0;
+    let flagged = 0;
+    let of = 0;
+    let example = "";
+    for (const item of items) {
+      const v = item.verdicts.find((x) => x.ruleId === rule.id);
+      if (v === undefined) continue;
+      of += 1;
+      if (v.band === "act") {
+        broken += 1;
+        if (example === "") example = item.file;
+      } else if (v.band === "flag") flagged += 1;
+    }
+    if (of > 0) tallies.push({ rule: rule.id, phase, broken, flagged, of, example });
+  }
+  return tallies.sort((a, b) => b.broken - a.broken || b.flagged - a.flagged);
+};