diffcop 0.0.0-stage → 0.1.0

package/src/reporter/review-layout.ts

@@ -0,0 +1,359 @@
+import type { Explored, Search } from "@/agent/explored";
+import { SEVERITIES } from "@/review/findings";
+import type {
+  DroppedFinding,
+  Finding,
+  ReviewResult,
+  Severity,
+} from "@/review/findings";
+
+const SEVERITY_LABELS: Record<Severity, string> = {
+  high: "High",
+  medium: "Medium",
+  low: "Low",
+  nit: "Nit",
+};
+
+const MAX_LISTED = 30;
+
+export interface Headline {
+  icon: string;
+  text: string;
+}
+
+export interface ReviewSection {
+  title: string;
+  count?: number;
+  // Markdown, rendered as is by both the console and the PR comment.
+  body: string;
+  open: boolean;
+}
+
+export interface ReviewLayout {
+  headline: Headline;
+  sections: ReviewSection[];
+}
+
+function lines({ file, startLine, endLine }: Finding) {
+  if (endLine <= startLine) {
+    return `${file}:${startLine}`;
+  }
+  return `${file}:${startLine}-${endLine}`;
+}
+
+export interface LayoutOptions {
+  // GitHub renders <br> inside a table cell; the terminal renderer prints it
+  // literally and wraps long cells, so it gets a narrower table.
+  html?: boolean;
+}
+
+function cell(text: string, html: boolean) {
+  return text
+    .trim()
+    .replaceAll("|", "\\|")
+    .replaceAll(/\r?\n/g, html ? "<br>" : " ");
+}
+
+function bySeverity(a: Finding, b: Finding) {
+  return SEVERITIES.indexOf(a.severity) - SEVERITIES.indexOf(b.severity);
+}
+
+function table(header: string[], rows: string[][]) {
+  return [
+    header,
+    header.map(() => {
+      return "---";
+    }),
+    ...rows,
+  ]
+    .map((row) => {
+      return `| ${row.join(" | ")} |`;
+    })
+    .join("\n");
+}
+
+interface Row {
+  finding: Finding;
+  // The verifier's reason, for dropped findings.
+  reason?: string;
+}
+
+function severityCell({ severity }: Finding) {
+  return `**${SEVERITY_LABELS[severity]}**`;
+}
+
+function htmlTable(rows: Row[]) {
+  const withReason = rows.some(({ reason }) => {
+    return reason !== undefined;
+  });
+  const header = ["Severity", "Location", "Issue"];
+  if (withReason) {
+    header.push("Verifier");
+  }
+  return table(
+    header,
+    rows.map(({ finding, reason }) => {
+      const row = [
+        severityCell(finding),
+        cell(code(lines(finding)), true),
+        `**${cell(finding.title, true)}**<br>${cell(finding.explanation, true)}`,
+      ];
+      if (reason !== undefined) {
+        row.push(cell(reason, true));
+      }
+      return row;
+    }),
+  );
+}
+
+// A terminal table wraps long cells into tall columns, so it keeps the short
+// fields and the explanations follow as a list keyed by row number.
+function terminalTable(rows: Row[]) {
+  const summary = table(
+    ["#", "Severity", "Location", "Title"],
+    rows.map(({ finding }, index) => {
+      return [
+        String(index + 1),
+        severityCell(finding),
+        cell(code(lines(finding)), false),
+        cell(finding.title, false),
+      ];
+    }),
+  );
+  const details = rows.map(({ finding, reason }, index) => {
+    const paragraphs = [finding.explanation.trim()];
+    if (reason !== undefined) {
+      paragraphs.push(`Verifier: ${reason.trim()}`);
+    }
+    // Indented to the list marker so each paragraph stays in the item.
+    return `${index + 1}. ${paragraphs.join("\n\n").replaceAll(/\n(?=.)/g, "\n   ")}`;
+  });
+  return `${summary}\n\n${details.join("\n")}`;
+}
+
+function rowsBody(rows: Row[], html: boolean) {
+  const sorted = rows.toSorted((a, b) => {
+    return bySeverity(a.finding, b.finding);
+  });
+  if (html) {
+    return htmlTable(sorted);
+  }
+  return terminalTable(sorted);
+}
+
+function findingsTable(findings: Finding[], html: boolean) {
+  return rowsBody(
+    findings.map((finding) => {
+      return { finding: finding };
+    }),
+    html,
+  );
+}
+
+function droppedTable(dropped: DroppedFinding[], html: boolean) {
+  return rowsBody(dropped, html);
+}
+
+// Patterns can contain backticks, so fence with a longer run than the text has.
+export function code(text: string) {
+  const runs = text.match(/`+/g) ?? [];
+  if (runs.length === 0) {
+    return `\`${text}\``;
+  }
+  const fence = "`".repeat(
+    Math.max(
+      ...runs.map((run) => {
+        return run.length;
+      }),
+    ) + 1,
+  );
+  return `${fence} ${text} ${fence}`;
+}
+
+function capped(items: string[]) {
+  const shown = items.slice(0, MAX_LISTED).join(", ");
+  if (items.length <= MAX_LISTED) {
+    return shown;
+  }
+  return `${shown} and ${items.length - MAX_LISTED} more`;
+}
+
+function search({ pattern, path }: Search) {
+  if (path === undefined) {
+    return code(pattern);
+  }
+  return `${code(pattern)} in ${code(path)}`;
+}
+
+function exploredBody(summary: string, { filesRead, searches }: Explored) {
+  const paragraphs: string[] = [];
+  if (summary.trim()) {
+    paragraphs.push(summary.trim());
+  }
+  if (filesRead.length > 0) {
+    paragraphs.push(
+      `Read (${filesRead.length}): ${capped(filesRead.map(code))}`,
+    );
+  }
+  if (searches.length > 0) {
+    paragraphs.push(
+      `Searched (${searches.length}): ${capped(searches.map(search))}`,
+    );
+  }
+  return paragraphs.join("\n\n") || "Nothing recorded.";
+}
+
+function plural(count: number, noun: string) {
+  return `${count} ${noun}${count === 1 ? "" : "s"}`;
+}
+
+function isIssue({ severity }: Finding) {
+  return severity === "high" || severity === "medium";
+}
+
+type FindingGroup = "issues" | "mismatches" | "nits" | "preexisting";
+
+function findingGroup(finding: Finding): FindingGroup {
+  switch (finding.scope) {
+    case "description":
+      return "mismatches";
+    case "preexisting":
+      return "preexisting";
+    case "introduced":
+      if (isIssue(finding)) {
+        return "issues";
+      }
+      return "nits";
+  }
+}
+
+interface GroupLayout {
+  group: FindingGroup;
+  title: string;
+  open: boolean;
+  // Shown above the group's table.
+  note?: string;
+  headline?: (count: number) => Headline;
+}
+
+// In section order, which is also the headline's priority: the first group
+// with findings and a headline sets it. Preexisting bugs don't block the PR.
+const GROUP_LAYOUTS: GroupLayout[] = [
+  {
+    group: "issues",
+    title: "Issues",
+    open: true,
+    headline: (count) => {
+      return {
+        icon: "🔴",
+        text: `${plural(count, "issue")} found`,
+      };
+    },
+  },
+  {
+    group: "mismatches",
+    title: "Description doesn't match the diff",
+    open: true,
+    headline: () => {
+      return {
+        icon: "🟡",
+        text: "Description doesn't match the diff",
+      };
+    },
+  },
+  {
+    group: "nits",
+    title: "Nits",
+    open: false,
+    headline: () => {
+      return {
+        icon: "🟡",
+        text: "Only nits",
+      };
+    },
+  },
+  {
+    group: "preexisting",
+    title: "Outside this PR's scope",
+    open: false,
+    note: "Already wrong before this change, so not blocking it.",
+  },
+];
+
+interface GroupedFindings {
+  layout: GroupLayout;
+  findings: Finding[];
+}
+
+function groupSection({ layout, findings }: GroupedFindings, html: boolean) {
+  let body = findingsTable(findings, html);
+  if (layout.note) {
+    body = `${layout.note}\n\n${body}`;
+  }
+  return {
+    title: layout.title,
+    count: findings.length,
+    body: body,
+    open: layout.open,
+  };
+}
+
+function headline(groups: GroupedFindings[]): Headline {
+  for (const { layout, findings } of groups) {
+    if (layout.headline) {
+      return layout.headline(findings.length);
+    }
+  }
+  return {
+    icon: "✅",
+    text: "All good",
+  };
+}
+
+export function sectionHeading({ title, count }: ReviewSection) {
+  if (count === undefined) {
+    return title;
+  }
+  return `${title} (${count})`;
+}
+
+export function reviewLayout(
+  {
+    summary,
+    findings,
+    dropped,
+    explored,
+  }: Pick<ReviewResult, "summary" | "findings" | "dropped" | "explored">,
+  { html = true }: LayoutOptions = {},
+): ReviewLayout {
+  const groups = GROUP_LAYOUTS.map((layout) => {
+    return {
+      layout: layout,
+      findings: findings.filter((f) => {
+        return findingGroup(f) === layout.group;
+      }),
+    };
+  }).filter((group) => {
+    return group.findings.length > 0;
+  });
+  const sections: ReviewSection[] = groups.map((group) => {
+    return groupSection(group, html);
+  });
+  if (dropped.length > 0) {
+    sections.push({
+      title: "Dropped by verification",
+      count: dropped.length,
+      body: droppedTable(dropped, html),
+      open: false,
+    });
+  }
+  sections.push({
+    title: "What we looked at",
+    body: exploredBody(summary, explored),
+    open: false,
+  });
+  return {
+    headline: headline(groups),
+    sections: sections,
+  };
+}

package/src/reporter/review-reporter.ts

@@ -0,0 +1,19 @@
+import type { ReviewResult } from "@/review/findings";
+
+export interface TextSink {
+  write(text: string): void;
+}
+
+// `started`, `warning`, `verifying` and `failed` must not throw: a report
+// shouldn't stop the review or hide the error that ended it.
+export interface ReviewReporter {
+  started(): Promise<void>;
+  // Async so the GitHub reporter can update its status comment before the
+  // review starts.
+  warning(message: string): Promise<void>;
+  toolCall(toolName: string, input: string): void;
+  thinking(durationMs: number): void;
+  verifying(findings: number): Promise<void>;
+  completed(result: ReviewResult): Promise<void>;
+  failed(error: Error, options: { cancelled: boolean }): Promise<void>;
+}

package/src/reporter/stats.ts

@@ -0,0 +1,41 @@
+import type { RunStats } from "@/agent/reviewer-agent";
+
+const compact = new Intl.NumberFormat("en", {
+  notation: "compact",
+  maximumFractionDigits: 1,
+});
+
+const percent = new Intl.NumberFormat("en", { style: "percent" });
+
+export function formatDuration(ms: number) {
+  const seconds = Math.round(ms / 1000);
+  if (seconds < 60) {
+    return `${seconds}s`;
+  }
+  return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
+}
+
+function formatInputTokens(inputTokens: number, cacheReadTokens = 0) {
+  const tokens = `${compact.format(inputTokens)} input tokens`;
+  if (inputTokens === 0 || cacheReadTokens === 0) {
+    return tokens;
+  }
+  return `${tokens} (${percent.format(cacheReadTokens / inputTokens)} cached)`;
+}
+
+export function formatStats(stats: RunStats) {
+  const parts = [
+    formatDuration(stats.durationMs),
+    `${stats.toolCalls} tool call${stats.toolCalls === 1 ? "" : "s"}`,
+  ];
+  if (stats.inputTokens !== undefined) {
+    parts.push(formatInputTokens(stats.inputTokens, stats.cacheReadTokens));
+  }
+  if (stats.outputTokens !== undefined) {
+    parts.push(`${compact.format(stats.outputTokens)} output`);
+  }
+  if (stats.costUsd !== undefined) {
+    parts.push(`~$${stats.costUsd.toFixed(2)}`);
+  }
+  return parts.join(" · ");
+}

package/src/review/change-size-limit.ts

@@ -0,0 +1,92 @@
+import { basename } from "node:path";
+import type { ChangedFile } from "@/context/repository";
+
+// Generated, often huge, and not worth the reviewer's attention.
+const LOCKFILES = new Set([
+  "bun.lock",
+  "bun.lockb",
+  "package-lock.json",
+  "yarn.lock",
+  "pnpm-lock.yaml",
+]);
+
+export interface ChangeSize {
+  lines: number;
+  files: number;
+}
+
+export interface ChangeSizeLimits {
+  maxLines: number;
+  maxFiles: number;
+}
+
+// Binary files count as files but add no lines: `git diff --numstat` gives
+// them `-` for both counts.
+export function measureChangeSize(changedFiles: ChangedFile[]): ChangeSize {
+  const counted = changedFiles.filter((file) => {
+    return !LOCKFILES.has(basename(file.path));
+  });
+  return {
+    lines: counted.reduce((sum, file) => {
+      return sum + file.additions + file.deletions;
+    }, 0),
+    files: counted.length,
+  };
+}
+
+function count(n: number, unit: string) {
+  if (n === 1) {
+    return `1 ${unit}`;
+  }
+  return `${n.toLocaleString("en-US")} ${unit}s`;
+}
+
+function overLimit(actual: number, limit: number, unit: string) {
+  const text = count(actual, unit);
+  if (actual <= limit) {
+    return text;
+  }
+  return `${text} (over the ${limit.toLocaleString("en-US")}-${unit} limit)`;
+}
+
+export class ChangeSizeLimit {
+  readonly #limits: ChangeSizeLimits;
+
+  constructor(limits: ChangeSizeLimits) {
+    this.#limits = limits;
+  }
+
+  warning(changedFiles: ChangedFile[]): string | undefined {
+    const { maxLines, maxFiles } = this.#limits;
+    const { lines, files } = measureChangeSize(changedFiles);
+    if (lines <= maxLines && files <= maxFiles) {
+      return undefined;
+    }
+    return (
+      `This change is large: ${overLimit(lines, maxLines, "line")} across ` +
+      `${overLimit(files, maxFiles, "file")}, not counting lockfiles. ` +
+      "Large changes get less accurate reviews; consider splitting it into stacked PRs."
+    );
+  }
+}
+
+// Far more files than GitHub lists for the PR means the diff used another base,
+// e.g. one changed after the run started, so splitting the PR wouldn't help.
+export function staleBaseWarning(
+  base: string,
+  localFiles: number,
+  githubFiles: number | undefined,
+): string | undefined {
+  if (
+    githubFiles === undefined ||
+    localFiles <= githubFiles * 2 ||
+    localFiles - githubFiles <= 10
+  ) {
+    return undefined;
+  }
+  return (
+    `The diff against ${base} has ${count(localFiles, "file")}, but GitHub lists ` +
+    `${count(githubFiles, "file")} for this PR, so its base may have changed after ` +
+    "this run started. Re-run the review to diff against the current base."
+  );
+}

package/src/review/code-reviewer.ts

@@ -0,0 +1,120 @@
+import { mergeExplored } from "@/agent/explored";
+import { sumStats } from "@/agent/reviewer-agent";
+import type { ReviewerAgent } from "@/agent/reviewer-agent";
+import type {
+  ReviewContext,
+  ReviewContextCollector,
+} from "@/context/review-context-collector";
+import { toError } from "@/util/errors";
+import type { ReviewPromptBuilder } from "@/prompt/review-prompt-builder";
+import type { ReviewReporter } from "@/reporter/review-reporter";
+import { staleBaseWarning } from "@/review/change-size-limit";
+import type { ChangeSizeLimit } from "@/review/change-size-limit";
+import {
+  findingsSchema,
+  splitByVerdict,
+  verdictsSchema,
+} from "@/review/findings";
+import type { ReviewResult } from "@/review/findings";
+
+export interface ReviewRequest {
+  request: string;
+  base?: string;
+  abortSignal?: AbortSignal;
+}
+
+// Modeled on claude-code-action's context → prompt → agent pipeline, with a
+// second pass that tries to refute each finding before it's reported.
+export class CodeReviewer {
+  readonly #collector: ReviewContextCollector;
+  readonly #promptBuilder: ReviewPromptBuilder;
+  readonly #agent: ReviewerAgent;
+  readonly #reporter: ReviewReporter;
+  readonly #sizeLimit: ChangeSizeLimit;
+
+  constructor(
+    collector: ReviewContextCollector,
+    promptBuilder: ReviewPromptBuilder,
+    agent: ReviewerAgent,
+    reporter: ReviewReporter,
+    sizeLimit: ChangeSizeLimit,
+  ) {
+    this.#collector = collector;
+    this.#promptBuilder = promptBuilder;
+    this.#agent = agent;
+    this.#reporter = reporter;
+    this.#sizeLimit = sizeLimit;
+  }
+
+  async review({
+    request,
+    base,
+    abortSignal,
+  }: ReviewRequest): Promise<ReviewResult> {
+    await this.#reporter.started();
+    try {
+      const context = await this.#collector.collect(base);
+      const sizeWarning =
+        staleBaseWarning(
+          context.base,
+          context.changedFiles.length,
+          context.pullRequest?.data.files?.totalCount,
+        ) ?? this.#sizeLimit.warning(context.changedFiles);
+      if (sizeWarning) {
+        await this.#reporter.warning(sizeWarning);
+      }
+      const result = await this.#findAndVerify(context, request, abortSignal);
+      await this.#reporter.completed(result);
+      return result;
+    } catch (error) {
+      await this.#reporter.failed(toError(error), {
+        cancelled: abortSignal?.aborted ?? false,
+      });
+      throw error;
+    }
+  }
+
+  async #findAndVerify(
+    context: ReviewContext,
+    request: string,
+    abortSignal: AbortSignal | undefined,
+  ): Promise<ReviewResult> {
+    const found = await this.#agent.run({
+      cwd: context.repoRoot,
+      prompt: this.#promptBuilder.build(context, request),
+      output: findingsSchema,
+      reporter: this.#reporter,
+      abortSignal: abortSignal,
+    });
+    const { summary, findings } = found.output;
+    if (findings.length === 0) {
+      return {
+        summary: summary,
+        findings: findings,
+        dropped: [],
+        explored: found.explored,
+        stats: found.stats,
+      };
+    }
+
+    await this.#reporter.verifying(findings.length);
+    const verified = await this.#agent.run({
+      cwd: context.repoRoot,
+      prompt: this.#promptBuilder.buildVerification(context, findings),
+      output: verdictsSchema,
+      reporter: this.#reporter,
+      abortSignal: abortSignal,
+    });
+    const { confirmed, dropped } = splitByVerdict(
+      findings,
+      verified.output.verdicts,
+    );
+    return {
+      summary: summary,
+      findings: confirmed,
+      dropped: dropped,
+      explored: mergeExplored(found.explored, verified.explored),
+      stats: sumStats(found.stats, verified.stats),
+    };
+  }
+}