@reviewflow/sdk 0.0.0-stage → 6.22.0

package/src/git/pullRequestDiff.ts

@@ -0,0 +1,159 @@
+import type { PullRequestRef } from "../api/pullRequest.ts";
+import { diffPaths } from "../diff/diffPaths.ts";
+import type {
+  DiffStep,
+  DiffWhitespace,
+  PullRequestDiffResponse,
+} from "../diff/types.ts";
+import { fetchCompareDiff } from "../github/fetchCompareDiff.ts";
+import type { GitHubTransport } from "../github/transport.ts";
+import {
+  groupDiffPaths,
+  noPackages,
+  packagesAtTree,
+} from "./groupDiffPaths.ts";
+import type { IsPackage } from "./groupDiffPaths.ts";
+import { readBranchDiff } from "./readBranchDiff.ts";
+import { git, gitOrNull } from "./run.ts";
+
+export const isSha = (value: string): boolean => /^[\da-f]{40}$/i.test(value);
+
+const hasCommit = async (checkout: string, sha: string): Promise<boolean> =>
+  (await gitOrNull(checkout, ["cat-file", "-e", `${sha}^{commit}`])) !== null;
+
+/**
+ * Fetches the commits the checkout lacks, by sha: GitHub serves a pull
+ * request's head that way even when no branch of the user's names it.
+ * Objects only — `--no-write-fetch-head` and a refspec with no destination
+ * leave every ref and `FETCH_HEAD` as they were, so whatever works in the
+ * checkout sees nothing move. Nothing is asked when both are already there,
+ * the common case for the user's own pull request.
+ */
+export const ensureCommits = async (
+  checkout: string,
+  shas: string[],
+  onStep: ((step: DiffStep) => void) | undefined,
+  signal?: AbortSignal,
+): Promise<void> => {
+  const missing: string[] = [];
+  for (const sha of shas) {
+    if (!(await hasCommit(checkout, sha))) missing.push(sha);
+  }
+  if (missing.length === 0) return;
+  onStep?.("fetching");
+  await git(
+    checkout,
+    ["fetch", "--no-write-fetch-head", "--no-tags", "origin", ...missing],
+    { network: true, ...(signal === undefined ? {} : { signal }) },
+  );
+};
+
+export interface PullRequestDiffRequest {
+  /** the registered checkout, resolved by the caller; null when there is none */
+  checkout: string | null;
+  transport: GitHubTransport;
+  ref: PullRequestRef;
+  head: string;
+  base: string;
+  /** `show` unless set */
+  whitespace?: DiffWhitespace;
+  /** called with "fetching" before a fetch that runs, and "diffing" before the diff */
+  onStep?: (step: DiffStep) => void;
+  /** the reader left: the fetch is killed and GitHub is not asked instead */
+  signal?: AbortSignal;
+}
+
+const readFromGit = async (
+  checkout: string,
+  { head, base, whitespace = "show", onStep, signal }: PullRequestDiffRequest,
+): Promise<PullRequestDiffResponse> => {
+  await ensureCommits(checkout, [head, base], onStep, signal);
+  onStep?.("diffing");
+  const mergeBaseOutput = await git(checkout, ["merge-base", base, head]);
+  const mergeBase = mergeBaseOutput.trim();
+  const { patch, paths, omitted } = await readBranchDiff({
+    cwd: checkout,
+    mergeBase,
+    scope: "committed",
+    head,
+    whitespace,
+  });
+  return {
+    source: "git",
+    fallbackReason: null,
+    whitespace,
+    patch,
+    sides: { old: mergeBase, new: head },
+    omitted,
+    groups: groupDiffPaths(await packagesAtTree(checkout, head), paths),
+  };
+};
+
+/**
+ * The compare endpoint rather than the pull request's own diff: it takes any
+ * range. Three dots, from the merge base, as the local path does. Grouped by
+ * the head's manifests when a checkout holds the head, else by first directory.
+ */
+const readFromGitHub = async (
+  { checkout, transport, ref, head, base }: PullRequestDiffRequest,
+  fallbackReason: string,
+): Promise<PullRequestDiffResponse> => {
+  const patch = await fetchCompareDiff(transport, {
+    ownerRepo: ref.ownerRepo,
+    base,
+    head,
+  });
+  let isPackage: IsPackage = noPackages;
+  if (checkout !== null) {
+    // the head may be absent from the checkout (that is a reason to be here)
+    isPackage = await packagesAtTree(checkout, head).catch(() => noPackages);
+  }
+  return {
+    source: "github",
+    fallbackReason,
+    whitespace: "show",
+    patch,
+    sides: null,
+    omitted: [],
+    groups: groupDiffPaths(isPackage, diffPaths(patch)),
+  };
+};
+
+/**
+ * Local git first, for its size bounds and no file cap, over any range of two
+ * commits; GitHub's compare endpoint only when there is no checkout or git
+ * fails — and the reason travels with the answer, so a fallback is never
+ * silent: "no checkout" when none was given, git's own message otherwise.
+ */
+export const readPullRequestDiff = async (
+  request: PullRequestDiffRequest,
+): Promise<PullRequestDiffResponse> => {
+  if (!isSha(request.head) || !isSha(request.base)) {
+    throw new Error("head and base must be commit shas");
+  }
+  if (request.checkout === null) return readFromGitHub(request, "no checkout");
+  try {
+    return await readFromGit(request.checkout, request);
+  } catch (error: unknown) {
+    if (request.signal?.aborted) throw error;
+    const reason = error instanceof Error ? error.message : String(error);
+    return readFromGitHub(request, reason);
+  }
+};
+
+/**
+ * `cat-file blob` rather than `show`: the bytes as committed, with no textconv
+ * or filter, which is what the patch's lines were read from. The commits are
+ * already in the checkout, fetched when its diff was read.
+ */
+export const readFileAtSha = async (
+  checkout: string,
+  sha: string,
+  path: string,
+): Promise<string> => {
+  if (!isSha(sha)) throw new Error("sha must be a commit sha");
+  if (path === "") throw new Error("path is required");
+  return git(checkout, ["cat-file", "blob", `${sha}:${path}`], {
+    maxBuffer: 64 * 1024 * 1024,
+  });
+};

package/src/git/readBranchDiff.ts

@@ -0,0 +1,359 @@
+import { lstat } from "node:fs/promises";
+import { join } from "node:path";
+import type { DiffOmittedFile, DiffWhitespace } from "../diff/types.ts";
+import { git } from "./run.ts";
+
+/**
+ * The patch of a branch since its merge base, in the unified format any diff
+ * viewer parses.
+ *
+ * **The operator's git config is neutralised flag by flag**, since every one of
+ * these would break the parse in the viewer: `diff.noprefix` and
+ * `diff.mnemonicPrefix` (explicit prefixes), `color.ui=always` (`--no-color`),
+ * `diff.external` (`--no-ext-diff`), a textconv driver (`--no-textconv`),
+ * `diff.relative` (`--no-relative`) and `core.quotePath` (octal-escaped names).
+ *
+ * **Bounded twice.** `--numstat` runs first and is cheap: a binary file, a file
+ * past `maxLinesPerFile` (a lockfile) and everything past the total budget are
+ * listed as omitted and excluded from the patch by pathspec, so one generated
+ * file never costs the whole diff. `maxBuffer` is the backstop for what line
+ * counts cannot see — a minified file is one line.
+ *
+ * **Untracked files are diffed against `/dev/null` one by one**, never through
+ * `git add -N`: that writes the index, and a read must not change what the user
+ * will commit. Whitespace is never hidden there: every line of a new file is an
+ * addition.
+ *
+ * **Hiding whitespace is `-w` on both reads**, so the budget counts the lines
+ * shown. `git diff -w` drops a file whose every change is whitespace, so a
+ * third read without it names those files, listed as omitted.
+ */
+
+export interface DiffLimits {
+  maxLinesPerFile: number;
+  maxTotalLines: number;
+  maxFiles: number;
+  /** untracked files read under the `all` scope; the rest are omitted */
+  maxUntrackedFiles: number;
+  /** an untracked file larger than this is omitted as too large */
+  maxUntrackedBytes: number;
+}
+
+export const defaultDiffLimits: DiffLimits = {
+  maxLinesPerFile: 3000,
+  maxTotalLines: 30_000,
+  maxFiles: 500,
+  maxUntrackedFiles: 100,
+  maxUntrackedBytes: 512 * 1024,
+};
+
+const maxPatchBytes = 64 * 1024 * 1024;
+
+const neutralConfig = ["-c", "core.quotePath=false"];
+
+const diffFlags = [
+  "--no-color",
+  "--no-ext-diff",
+  "--no-textconv",
+  "--src-prefix=a/",
+  "--dst-prefix=b/",
+];
+
+export interface NumstatEntry {
+  /** the path after the change, the one a viewer names */
+  path: string;
+  /** both sides of a rename, which must be excluded together */
+  paths: string[];
+  /** null for a binary file */
+  lines: number | null;
+}
+
+/**
+ * `-z` keeps names verbatim. A rename leaves the path field empty and puts
+ * both names in the next two records.
+ */
+export const parseNumstat = (output: string): NumstatEntry[] => {
+  const records = output.split("\0");
+  const entries: NumstatEntry[] = [];
+  for (let index = 0; index < records.length; index++) {
+    const record = records[index] ?? "";
+    if (record === "") continue;
+    const [added = "", deleted = "", ...rest] = record.split("\t");
+    const lines =
+      added === "-" || deleted === "-" ? null : Number(added) + Number(deleted);
+    const path = rest.join("\t");
+    if (path !== "") {
+      entries.push({ path, paths: [path], lines });
+      continue;
+    }
+    const from = records[index + 1] ?? "";
+    const to = records[index + 2] ?? "";
+    index += 2;
+    entries.push({ path: to, paths: [from, to], lines });
+  }
+  return entries;
+};
+
+interface Budget {
+  files: number;
+  lines: number;
+}
+
+const omissionReason = (
+  lines: number | null,
+  budget: Budget,
+  limits: DiffLimits,
+): DiffOmittedFile["reason"] | null => {
+  if (lines === null) return "binary";
+  if (lines > limits.maxLinesPerFile) return "tooLarge";
+  if (
+    budget.files + 1 > limits.maxFiles ||
+    budget.lines + lines > limits.maxTotalLines
+  ) {
+    return "budget";
+  }
+  return null;
+};
+
+/**
+ * One pathspec per path makes the command line grow with the diff: naming
+ * the shorter side keeps it under the OS argument limit for a codemod touching
+ * tens of thousands of files, since what is shown is capped by `maxFiles`.
+ */
+const pathspecs = (included: string[], excluded: string[]): string[] => {
+  if (excluded.length === 0) return [];
+  if (included.length < excluded.length) {
+    return ["--", ...included.map((path) => `:(literal)${path}`)];
+  }
+  return ["--", ...excluded.map((path) => `:(exclude,literal)${path}`)];
+};
+
+interface TrackedDiff {
+  patch: string;
+  /** the files in `patch`, after any rename */
+  paths: string[];
+  omitted: DiffOmittedFile[];
+}
+
+export const readNumstat = async (
+  cwd: string,
+  range: string[],
+  whitespaceFlags: string[],
+): Promise<NumstatEntry[]> =>
+  parseNumstat(
+    await git(cwd, [
+      ...neutralConfig,
+      "diff",
+      "--no-relative",
+      "-M",
+      ...whitespaceFlags,
+      "--numstat",
+      "-z",
+      ...range,
+    ]),
+  );
+
+const whitespaceOnlyFiles = async (
+  cwd: string,
+  range: string[],
+  shown: NumstatEntry[],
+): Promise<DiffOmittedFile[]> => {
+  const shownPaths = new Set(shown.map(({ path }) => path));
+  const all = await readNumstat(cwd, range, []);
+  return all
+    .filter(({ path }) => !shownPaths.has(path))
+    .map(({ path }) => ({ path, reason: "whitespace" }));
+};
+
+interface ReadTrackedDiffParams {
+  cwd: string;
+  range: string[];
+  whitespace: DiffWhitespace;
+  limits: DiffLimits;
+  budget: Budget;
+}
+
+const readTrackedDiff = async ({
+  cwd,
+  range,
+  whitespace,
+  limits,
+  budget,
+}: ReadTrackedDiffParams): Promise<TrackedDiff> => {
+  const whitespaceFlags = whitespace === "hide" ? ["-w"] : [];
+  const entries = await readNumstat(cwd, range, whitespaceFlags);
+  const omitted: DiffOmittedFile[] =
+    whitespace === "hide" ? await whitespaceOnlyFiles(cwd, range, entries) : [];
+  const excluded: string[] = [];
+  const included: string[] = [];
+  const paths: string[] = [];
+  for (const entry of entries) {
+    const reason = omissionReason(entry.lines, budget, limits);
+    if (reason === null) {
+      budget.files += 1;
+      budget.lines += entry.lines ?? 0;
+      paths.push(entry.path);
+      included.push(...entry.paths);
+      continue;
+    }
+    omitted.push({ path: entry.path, reason });
+    excluded.push(...entry.paths);
+  }
+  if (paths.length === 0) return { patch: "", paths, omitted };
+
+  const patch = await git(
+    cwd,
+    [
+      ...neutralConfig,
+      "diff",
+      ...diffFlags,
+      "--no-relative",
+      "-M",
+      ...whitespaceFlags,
+      ...range,
+      ...pathspecs(included, excluded),
+    ],
+    { maxBuffer: maxPatchBytes },
+  );
+  return { patch, paths, omitted };
+};
+
+const countAddedLines = (patch: string): number => {
+  const lines = patch.split("\n");
+  const firstHunk = lines.findIndex((line) => line.startsWith("@@"));
+  if (firstHunk === -1) return 0;
+  return lines.slice(firstHunk).filter((line) => line.startsWith("+")).length;
+};
+
+const isBinaryPatch = (patch: string): boolean => /^Binary files /m.test(patch);
+
+interface UntrackedDiff {
+  patches: string[];
+  /** the files in `patches`, in the same order */
+  paths: string[];
+  omitted: DiffOmittedFile[];
+}
+
+const readUntrackedDiff = async (
+  cwd: string,
+  limits: DiffLimits,
+  budget: Budget,
+): Promise<UntrackedDiff> => {
+  const listing = await git(cwd, [
+    "ls-files",
+    "--others",
+    "--exclude-standard",
+    "-z",
+  ]);
+  const listed = listing.split("\0").filter((path) => path !== "");
+  const patches: string[] = [];
+  const paths: string[] = [];
+  const omitted: DiffOmittedFile[] = [];
+
+  for (const [index, path] of listed.entries()) {
+    if (index >= limits.maxUntrackedFiles) {
+      omitted.push({ path, reason: "budget" });
+      continue;
+    }
+    const entry = await lstat(join(cwd, path)).catch(() => null);
+    // gone between the listing and this read, or a nested repository
+    if (entry === null || entry.isDirectory()) continue;
+    if (entry.size > limits.maxUntrackedBytes) {
+      omitted.push({ path, reason: "tooLarge" });
+      continue;
+    }
+    const patch = await git(
+      cwd,
+      [
+        ...neutralConfig,
+        "diff",
+        ...diffFlags,
+        "--no-index",
+        "--",
+        "/dev/null",
+        path,
+      ],
+      { exitOneIsAnswer: true, maxBuffer: maxPatchBytes },
+    );
+    const lines = isBinaryPatch(patch) ? null : countAddedLines(patch);
+    const reason = omissionReason(lines, budget, limits);
+    if (reason !== null) {
+      omitted.push({ path, reason });
+      continue;
+    }
+    budget.files += 1;
+    budget.lines += lines ?? 0;
+    patches.push(patch);
+    paths.push(path);
+  }
+  return { patches, paths, omitted };
+};
+
+/**
+ * `committed`: the merge base to `head`. `all`: the merge base to the working
+ * tree, which adds the staged and unstaged changes and the untracked files.
+ */
+export type BranchDiffScope = "all" | "committed";
+
+export interface BranchDiff {
+  patch: string;
+  /** the files in `patch`, in its order */
+  paths: string[];
+  omitted: DiffOmittedFile[];
+  /** the untracked files in `patch`; always 0 under `committed` */
+  untrackedCount: number;
+}
+
+interface ReadBranchDiffParams {
+  cwd: string;
+  mergeBase: string;
+  /** `committed` unless set: the pull request path reads no working tree */
+  scope?: BranchDiffScope;
+  /**
+   * The commit the `committed` scope ends at: a pull request's head, which no
+   * worktree need be on, or `HEAD` unless set. Ignored under `all`, which is
+   * the working tree.
+   */
+  head?: string;
+  /** `show` unless set */
+  whitespace?: DiffWhitespace;
+  limits?: DiffLimits;
+}
+
+export const readBranchDiff = async ({
+  cwd,
+  mergeBase,
+  scope = "committed",
+  head = "HEAD",
+  whitespace = "show",
+  limits = defaultDiffLimits,
+}: ReadBranchDiffParams): Promise<BranchDiff> => {
+  if (mergeBase.startsWith("-") || head.startsWith("-")) {
+    throw new Error(
+      `not a revision: ${mergeBase.startsWith("-") ? mergeBase : head}`,
+    );
+  }
+  const budget: Budget = { files: 0, lines: 0 };
+  const tracked = await readTrackedDiff({
+    cwd,
+    range: scope === "committed" ? [mergeBase, head] : [mergeBase],
+    whitespace,
+    limits,
+    budget,
+  });
+  if (scope === "committed") {
+    return {
+      patch: tracked.patch,
+      paths: tracked.paths,
+      omitted: tracked.omitted,
+      untrackedCount: 0,
+    };
+  }
+  const untracked = await readUntrackedDiff(cwd, limits, budget);
+  return {
+    patch: [tracked.patch, ...untracked.patches].join(""),
+    paths: [...tracked.paths, ...untracked.paths],
+    omitted: [...tracked.omitted, ...untracked.omitted],
+    untrackedCount: untracked.patches.length,
+  };
+};