@shortlink-org/portolan 0.2.4 → 0.3.0

package/scripts/history.mjs

@@ -1,5 +1,6 @@
 // When each file was first committed and last changed, read by the host for a
-// plugin that asks (portolan.0007).
+// plugin that asks (portolan.0007), and for the provenance of every catalog
+// source (portolan.0010).
 //
 // One `git log` over the whole checkout, walked oldest first with renames
 // followed, gives every path its first commit and its last. A plugin used to
@@ -8,8 +9,8 @@
 // reads it once and hands each step the part under its root.
 
 import { execFileSync } from "node:child_process";
-import { realpathSync } from "node:fs";
-import { relative, resolve, sep } from "node:path";
+import { realpathSync, statSync } from "node:fs";
+import { dirname, relative, resolve, sep } from "node:path";
 
 const RECORD = "\x1e";
 const FIELD = "\x1f";
@@ -17,6 +18,188 @@ const FIELD = "\x1f";
 /** Per repository, per process: the history does not change during a run. */
 const cache = new Map();
 
+/** Repositories already told they are shallow; it is said once. */
+const warnedShallow = new Set();
+
+/**
+ * When each of `paths` last changed, as the history says: the commit that
+ * last touched the file, and that commit's date.
+ *
+ * This is where a source's provenance comes from (portolan.0010). It is read
+ * off the checkout that holds the file and never written into the file, so a
+ * fragment is content and nothing else, and a regeneration that changes no
+ * fact changes no byte. The stamp names the commit the change landed in,
+ * which a stamp written into the file never could: that one was the parent
+ * at best, and it took a second commit to catch up.
+ *
+ * A file with uncommitted changes, or one git does not track, is
+ * `uncommitted` and dated by its mtime - an observation of the working tree,
+ * shown and never committed. A path outside any repository is the same. A
+ * shallow clone makes every path look as though it changed in the one commit
+ * that was fetched; that is said once, and the stamps are what the truncated
+ * history says, which is at least the checkout.
+ *
+ * @param {string} workspace
+ * @param {string[]} paths  relative to `workspace`, forward slashes
+ * @returns {Map<string, {commit: string, generatedAt: string}>}
+ */
+export function stampsFor(workspace, paths) {
+  const workspaceReal = realpathSync(resolve(workspace));
+  const stamps = new Map();
+  const repoOfDir = new Map();
+  const byRepo = new Map();
+
+  for (const path of paths) {
+    const absolute = resolve(workspaceReal, path);
+    const dir = dirname(absolute);
+    if (!repoOfDir.has(dir)) repoOfDir.set(dir, repositoryOf(dir));
+    const repo = repoOfDir.get(dir);
+    if (!repo) {
+      stamps.set(path, uncommitted(absolute));
+      continue;
+    }
+    if (!byRepo.has(repo)) byRepo.set(repo, []);
+    byRepo.get(repo).push({ path, absolute, key: toPosix(relative(repo, absolute)) });
+  }
+
+  for (const [repo, files] of byRepo) {
+    warnIfShallow(repo);
+    const history = fileHistory(repo);
+    const dirty = dirtyIn(repo);
+    for (const file of files) {
+      const entry = history.get(file.key);
+      const last = entry?.revised ?? entry?.created;
+      stamps.set(
+        file.path,
+        !last || dirty.has(file.key)
+          ? uncommitted(file.absolute)
+          : { commit: last.commit.slice(0, 7), generatedAt: last.date },
+      );
+    }
+  }
+
+  return stamps;
+}
+
+/**
+ * The commit that last touched any of `paths`, and its date, or null when
+ * none of them has been committed. For a step's output this is when it was
+ * last generated: the history remembers it, so nothing has to write it down
+ * anywhere else (portolan.0010).
+ *
+ * @param {string} workspace
+ * @param {string[]} paths  relative to `workspace`
+ * @returns {{commit: string, date: string} | null}
+ */
+export function lastCommitTouching(workspace, paths) {
+  const { repo, keys } = inRepo(workspace, paths);
+  if (!repo || keys.length === 0) return null;
+  const answer = git(repo, ["log", "-1", "--format=%H %cI", "--", ...keys]);
+  if (!answer) return null;
+  const [sha, date] = answer.split(" ");
+  // Seven characters, the width every stamp has, so the two read alike.
+  return sha && date ? { commit: sha.slice(0, 7), date } : null;
+}
+
+/**
+ * Which files under `paths` differ from what `commit` held, working tree
+ * included: modified, added, deleted and untracked, less anything under
+ * `excludes`. Paths in and out are relative to `workspace`.
+ *
+ * @param {string} workspace
+ * @param {string} commit
+ * @param {string[]} paths
+ * @param {string[]} [excludes]
+ * @returns {string[]}
+ */
+export function changedSince(workspace, commit, paths, excludes = []) {
+  const { repo, keys, toWorkspace } = inRepo(workspace, paths);
+  if (!repo || keys.length === 0) return [];
+  const spec = [...keys, ...inRepo(workspace, excludes).keys.map((key) => `:(exclude)${key}`)];
+  const tracked = git(repo, ["diff", "--name-only", commit, "--", ...spec]);
+  const untracked = git(repo, ["ls-files", "--others", "--exclude-standard", "--", ...spec]);
+  const changed = new Set();
+  for (const line of `${tracked}\n${untracked}`.split("\n")) {
+    if (line) changed.add(toWorkspace(line));
+  }
+  return [...changed].sort();
+}
+
+/**
+ * The text of `path` as `commit` held it, or "" when the commit did not hold
+ * it or there is no history to ask.
+ *
+ * @param {string} workspace
+ * @param {string} commit
+ * @param {string} path  relative to `workspace`
+ * @returns {string}
+ */
+export function fileAt(workspace, commit, path) {
+  const { repo, keys } = inRepo(workspace, [path]);
+  if (!repo || keys.length === 0) return "";
+  return git(repo, ["show", `${commit}:${keys[0]}`]);
+}
+
+/** The paths as the workspace's repository spells them, and the way back. */
+function inRepo(workspace, paths) {
+  const workspaceReal = realpathSync(resolve(workspace));
+  const repo = repositoryOf(workspaceReal);
+  if (!repo) return { repo: "", keys: [], toWorkspace: (line) => line };
+  const keys = [];
+  for (const path of paths) {
+    const key = toPosix(relative(repo, resolve(workspaceReal, path)));
+    if (key.startsWith("..")) continue;
+    keys.push(key || ".");
+  }
+  return {
+    repo,
+    keys,
+    toWorkspace: (line) => toPosix(relative(workspaceReal, resolve(repo, line))),
+  };
+}
+
+function uncommitted(absolute) {
+  let generatedAt = "";
+  try {
+    generatedAt = statSync(absolute).mtime.toISOString();
+  } catch {
+    // Gone, or never there: undated rather than dated by a guess.
+  }
+  return { commit: "uncommitted", generatedAt };
+}
+
+/** Every path with a change the history does not hold yet, relative to the repository. */
+function dirtyIn(repo) {
+  const dirty = new Set();
+  // Read raw: an entry begins with two status letters, and a space is one of
+  // them, which the trimming reader would eat off the first line.
+  let status = "";
+  try {
+    status = execFileSync("git", ["-C", repo, "status", "--porcelain", "-z", "--no-renames", "--untracked-files=all"], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+      maxBuffer: 64 * 1024 * 1024,
+    });
+  } catch {
+    return dirty;
+  }
+  for (const entry of status.split("\0")) {
+    if (entry.length > 3) dirty.add(entry.slice(3));
+  }
+  return dirty;
+}
+
+function warnIfShallow(repo) {
+  if (warnedShallow.has(repo)) return;
+  warnedShallow.add(repo);
+  if (git(repo, ["rev-parse", "--is-shallow-repository"]) !== "true") return;
+  console.warn(
+    `${repo} is a shallow clone, where every source looks as though it last changed in ` +
+      "the one commit that was fetched. Fetch the full history (git fetch --unshallow, " +
+      "or actions/checkout with fetch-depth: 0) for the provenance to say anything.",
+  );
+}
+
 /**
  * History for every file under `root`, keyed by the path a plugin would name
  * it - relative to `workspace`, forward slashes. `undefined` when the root is

package/scripts/history.test.mjs

@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
 
-import { fileHistory, forgetHistory, historyFor, parseLog } from "./history.mjs";
+import { changedSince, fileHistory, forgetHistory, historyFor, lastCommitTouching, parseLog, stampsFor } from "./history.mjs";
 
 const created = [];
 afterEach(() => {

package/scripts/host-plugins/fetch-git.mjs

@@ -44,6 +44,18 @@ export const LOCK_NAME = "git.lock.json";
 export const PIN_NAME = "git.repo.json";
 export const OFFLINE_ENV = "PORTOLAN_OFFLINE";
 
+const SNIFF_BYTES = 8 * 1024;
+const BINARY_SUFFIXES = [
+  ".tar.gz", ".tar.bz2", ".tar.xz",
+  ".7z", ".a", ".apk", ".avif", ".bin", ".bmp", ".bz2", ".class",
+  ".deb", ".dll", ".dmg", ".doc", ".docx", ".eot", ".exe", ".gif",
+  ".gz", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lib", ".mmdb",
+  ".mov", ".mp3", ".mp4", ".o", ".otf", ".pdf", ".png", ".pyc",
+  ".rar", ".so", ".tar", ".tgz", ".tif", ".tiff", ".ttf", ".war",
+  ".wasm", ".webm", ".webp", ".woff", ".woff2", ".xls", ".xlsx",
+  ".xz", ".zip",
+];
+
 export function describe() {
   return {
     name: "fetch-git",
@@ -105,7 +117,7 @@ export function run(request, { env = process.env } = {}) {
       }
       continue;
     }
-    emitFetched(out, dir, want, fetched.commit, fetched.files, generatedAt);
+    emitFetched(out, dir, want, fetched.commit, fetched.files, fetched.skipped, generatedAt);
   }
 
   return out.response();
@@ -172,9 +184,11 @@ export function pin(repo, commit, generatedAt = "") {
 /** The lock, written the way every generated file here is written. */
 export function encodeLock(entry) {
   const files = [...entry.files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const skipped = [...(entry.skipped ?? [])].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
   const record = { repo: entry.repo, commit: entry.commit };
   if (entry.paths?.length) record.paths = [...entry.paths].sort();
   record.files = files.map((file) => ({ path: file.path, sha256: file.sha256, size: file.size }));
+  if (skipped.length) record.skipped = skipped.map((file) => ({ path: file.path, size: file.size, reason: file.reason }));
   return `${JSON.stringify({ repos: [record] }, null, 2)}\n`;
 }
 
@@ -191,13 +205,15 @@ function live(url, want, out, env) {
     commit = resolve(url, want.ref, env);
     out.warn(want.repo, `is not pinned; "${want.ref || "HEAD"}" resolved to ${commit}. Pin it in portolan.json or every run is a lottery.`);
   }
-  const files = download(url, commit, want.paths ?? [], env);
-  if (files.size === 0) throw new Error(`${want.repo} at ${commit} holds no files under ${(want.paths ?? []).join(", ")}`);
-  return { commit, files };
+  const fetched = download(url, commit, want.paths ?? [], env);
+  if (fetched.files.size === 0 && fetched.skipped.length === 0) {
+    throw new Error(`${want.repo} at ${commit} holds no files under ${(want.paths ?? []).join(", ")}`);
+  }
+  return { commit, ...fetched };
 }
 
-function emitFetched(out, dir, want, commit, files, generatedAt) {
-  const entry = { repo: want.repo, commit, paths: want.paths ?? [], files: [] };
+function emitFetched(out, dir, want, commit, files, skipped, generatedAt) {
+  const entry = { repo: want.repo, commit, paths: want.paths ?? [], files: [], skipped };
   for (const path of [...files.keys()].sort()) {
     const contents = files.get(path);
     out.file(posix.join(dir, path), contents);
@@ -205,6 +221,7 @@ function emitFetched(out, dir, want, commit, files, generatedAt) {
   }
   out.file(posix.join(dir, LOCK_NAME), encodeLock(entry));
   out.file(posix.join(dir, PIN_NAME), pin(want.repo, commit, generatedAt));
+  warnSkipped(out, want.repo, skipped);
 }
 
 /**
@@ -225,6 +242,13 @@ function emitCached(out, dir, at, want, why, generatedAt = "") {
   // what is actually on disk rather than what was asked for.
   out.file(posix.join(dir, PIN_NAME), pin(want.repo, held.lock.commit, generatedAt));
   out.warn(want.repo, `not fetched (${why}); the copy committed in this repository is used unchanged`);
+  warnSkipped(out, want.repo, held.lock.skipped);
+}
+
+function warnSkipped(out, repo, skipped = []) {
+  if (skipped.length === 0) return;
+  const bytes = skipped.reduce((total, file) => total + file.size, 0);
+  out.warn(repo, `skipped ${skipped.length} binary ${skipped.length === 1 ? "file" : "files"} (${bytes} bytes); metadata is recorded in ${LOCK_NAME}`);
 }
 
 /**
@@ -252,7 +276,14 @@ function replay(dir) {
   if (repos.length !== 1) throw new Error(`${lockPath} names ${repos.length} repositories; expected exactly one`);
   const entry = repos[0];
   const files = new Map();
+  const kept = [];
+  const skipped = [...(Array.isArray(entry.skipped) ? entry.skipped : [])];
   for (const want of entry.files ?? []) {
+    const known = knownBinaryReason(want.path);
+    if (known) {
+      skipped.push({ path: want.path, size: want.size, reason: known });
+      continue;
+    }
     let contents;
     try {
       contents = readFileSync(join(dir, ...String(want.path).split("/")));
@@ -261,9 +292,15 @@ function replay(dir) {
       throw cause;
     }
     if (digestOf(contents) !== want.sha256) throw new Error(`${want.path} does not match its digest; the vendored copy was edited by hand`);
+    const sniffed = sniffedBinaryReason(contents);
+    if (sniffed) {
+      skipped.push({ path: want.path, size: contents.length, reason: sniffed });
+      continue;
+    }
     files.set(want.path, contents);
+    kept.push(want);
   }
-  return { lock: { repo: entry.repo, commit: entry.commit, paths: entry.paths ?? [], files: entry.files ?? [] }, files };
+  return { lock: { repo: entry.repo, commit: entry.commit, paths: entry.paths ?? [], files: kept, skipped }, files };
 }
 
 // --- git --------------------------------------------------------------------
@@ -305,7 +342,7 @@ function resolve(url, ref, env) {
  * its branches where it does not, which is the case for a plain file://
  * remote; either way what is read is verified to be that commit.
  *
- * @returns {Map<string, Buffer>} path → contents
+ * @returns {{files: Map<string, Buffer>, skipped: {path: string, size: number, reason: string}[]}}
  */
 function download(url, commit, paths, env) {
   const tmp = mkdtempSync(join(tmpdir(), "portolan-fetch-git-"));
@@ -326,19 +363,24 @@ function download(url, commit, paths, env) {
       throw new Error(`${commit} is not a commit ${url} has, or not one reachable from a branch`);
     }
 
-    const listing = git(tmp, ["ls-tree", "-r", "-z", "--format=%(objectname) %(objecttype) %(path)", commit, "--", ...paths], env);
+    const listing = git(tmp, ["ls-tree", "-r", "-z", "--format=%(objectname) %(objecttype) %(objectsize) %(path)", commit, "--", ...paths], env);
     const blobs = [];
+    const skipped = [];
     for (const line of listing.split("\0")) {
       if (!line) continue;
-      const [sha, type, ...rest] = line.split(" ");
+      const [sha, type, rawSize, ...rest] = line.split(" ");
       if (type !== "blob") continue;
-      blobs.push({ sha, path: posix.normalize(rest.join(" ")) });
+      const path = posix.normalize(rest.join(" "));
+      const size = Number(rawSize);
+      const reason = knownBinaryReason(path);
+      if (reason) skipped.push({ path, size, reason });
+      else blobs.push({ sha, path, size });
     }
     const files = new Map();
-    if (blobs.length === 0) return files;
+    if (blobs.length === 0) return { files, skipped };
 
-    // One process for every blob: `cat-file --batch` answers each sha with a
-    // header line and the bytes.
+    // One `cat-file --batch` process answers every remaining sha with a header
+    // line and the bytes. Known binaries never enter this batch.
     const batch = execFileSync("git", ["cat-file", "--batch"], {
       cwd: tmp,
       env: { ...env, GIT_TERMINAL_PROMPT: "0" },
@@ -352,15 +394,30 @@ function download(url, commit, paths, env) {
       if (header[1] === "missing") throw new Error(`${blob.path} is missing from ${commit}`);
       const size = Number(header[2]);
       const start = newline + 1;
-      files.set(blob.path, Buffer.from(batch.subarray(start, start + size)));
+      const contents = batch.subarray(start, start + size);
+      const reason = sniffedBinaryReason(contents);
+      if (reason) skipped.push({ path: blob.path, size, reason });
+      else files.set(blob.path, Buffer.from(contents));
       offset = start + size + 1;
     }
-    return files;
+    return { files, skipped };
   } finally {
     rmSync(tmp, { recursive: true, force: true });
   }
 }
 
+function knownBinaryReason(path) {
+  const lower = String(path).toLowerCase();
+  const suffix = BINARY_SUFFIXES.find((candidate) => lower.endsWith(candidate));
+  return suffix ? `known binary extension ${suffix}` : "";
+}
+
+function sniffedBinaryReason(contents) {
+  if (contents.subarray(0, SNIFF_BYTES).includes(0)) return `NUL byte in first ${SNIFF_BYTES} bytes`;
+  const text = contents.toString("utf8");
+  return Buffer.from(text, "utf8").equals(contents) ? "" : "invalid UTF-8";
+}
+
 /** Accumulates a response the way plugin.Builder does. */
 class Builder {
   files = [];
@@ -372,11 +429,10 @@ class Builder {
       return;
     }
     const text = contents.toString("utf8");
-    this.files.push(
-      Buffer.from(text, "utf8").equals(contents)
-        ? { name, contents: text }
-        : { name, contents: contents.toString("base64"), encoding: "base64" },
-    );
+    if (!Buffer.from(text, "utf8").equals(contents) || contents.subarray(0, SNIFF_BYTES).includes(0)) {
+      throw new Error(`${name} is binary and cannot be emitted by fetch-git`);
+    }
+    this.files.push({ name, contents: text });
   }
 
   warn(ref, message) {

package/scripts/host-plugins/fetch-git.test.mjs

@@ -1,11 +1,12 @@
 import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
 import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { basename, dirname, join } from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
 
 import { runPlugin } from "../plugin-host.mjs";
-import { OFFLINE_ENV, encodeLock, pin, run, splitRepo, webRepo } from "./fetch-git.mjs";
+import { LOCK_NAME, OFFLINE_ENV, encodeLock, pin, run, splitRepo, webRepo } from "./fetch-git.mjs";
 
 const created = [];
 afterEach(() => {
@@ -105,18 +106,71 @@ describe("fetch-git", () => {
     expect(inCi.warnings).toHaveLength(1);
   });
 
-  it("preserves binary blobs online and from the offline cache", () => {
+  it("skips binary blobs online and from the offline cache", () => {
     const remote = repository();
     const cacheDir = cache();
     const first = fetch(options(remote, cacheDir, ["docs/example.png"]), online);
     const name = `${remote.copyDir}/docs/example.png`;
-    const image = first.files.find((file) => file.name === name);
-    expect(image?.encoding).toBe("base64");
+    expect(first.files.find((file) => file.name === name)).toBeUndefined();
+    expect(first.files.every((file) => file.encoding === undefined)).toBe(true);
+    const lockName = `${remote.copyDir}/${LOCK_NAME}`;
+    expect(JSON.parse(contentsOf(first, lockName)).repos[0].skipped).toEqual([
+      { path: "docs/example.png", size: 10, reason: "known binary extension .png" },
+    ]);
+    expect(first.warnings[0].message).toContain("skipped 1 binary file (10 bytes)");
     write(cacheDir, first);
-    expect(readFileSync(join(cacheDir, name))).toEqual(readFileSync(join(remote.dir, "docs/example.png")));
+    expect(() => readFileSync(join(cacheDir, name))).toThrow();
 
     const replayed = fetch(options(remote, cacheDir, ["docs/example.png"]), offline);
-    expect(replayed.files.find((file) => file.name === name)).toEqual(image);
+    expect(replayed.files.find((file) => file.name === name)).toBeUndefined();
+    expect(JSON.parse(contentsOf(replayed, lockName)).repos[0].skipped).toEqual([
+      { path: "docs/example.png", size: 10, reason: "known binary extension .png" },
+    ]);
+  });
+
+  it("does not read a large known binary into the plugin response and sniffs unknown binary files", () => {
+    const remote = repository();
+    const mmdb = "tests/integration/resources/GeoLite2-Country.mmdb";
+    const unknown = "tests/integration/resources/blob.data";
+    mkdirSync(dirname(join(remote.dir, mmdb)), { recursive: true });
+    writeFileSync(join(remote.dir, mmdb), Buffer.alloc(10 * 1024 * 1024, 0xab));
+    writeFileSync(join(remote.dir, unknown), Buffer.from([1, 0, 2]));
+    remote.git(["add", "."]);
+    remote.git(["commit", "--quiet", "-m", "binary fixtures"]);
+    remote.commit = remote.git(["rev-parse", "HEAD"]);
+
+    const response = fetch(options(remote, cache(), [mmdb, unknown]), online);
+    expect(response.files.every((file) => file.encoding === undefined)).toBe(true);
+    expect(JSON.stringify(response).length).toBeLessThan(20_000);
+    const lock = JSON.parse(contentsOf(response, `${remote.copyDir}/${LOCK_NAME}`)).repos[0];
+    expect(lock.files).toEqual([]);
+    expect(lock.skipped).toEqual([
+      { path: mmdb, size: 10 * 1024 * 1024, reason: "known binary extension .mmdb" },
+      { path: unknown, size: 3, reason: "NUL byte in first 8192 bytes" },
+    ]);
+  });
+
+  it("drops binary files from a cache written by an older Portolan", () => {
+    const remote = repository();
+    const cacheDir = cache();
+    const at = join(cacheDir, remote.copyDir);
+    const path = "docs/example.png";
+    const contents = readFileSync(join(remote.dir, path));
+    mkdirSync(dirname(join(at, path)), { recursive: true });
+    writeFileSync(join(at, path), contents);
+    writeFileSync(join(at, LOCK_NAME), encodeLock({
+      repo: remote.url,
+      commit: remote.commit,
+      paths: [path],
+      files: [{ path, size: contents.length, sha256: createHash("sha256").update(contents).digest("hex") }],
+    }));
+
+    const replayed = fetch(options(remote, cacheDir, [path]), offline);
+    expect(replayed.files.some((file) => file.name.endsWith(path))).toBe(false);
+    expect(JSON.parse(contentsOf(replayed, `${remote.copyDir}/${LOCK_NAME}`)).repos[0]).toMatchObject({
+      files: [],
+      skipped: [{ path, size: contents.length, reason: "known binary extension .png" }],
+    });
   });
 
   it("names the edited file when a vendored copy no longer matches its lock", () => {
@@ -223,8 +277,8 @@ describe("names", () => {
       repos: [{ repo: "github.com/acme/shop", commit: "c1d2e3f" }],
     });
 
-    const lock = encodeLock({ repo: "github.com/acme/shop", commit: "abc", paths: ["services/oms", "proto"], files: [{ path: "b", sha256: "2", size: 1 }, { path: "a", sha256: "1", size: 1 }] });
-    expect(JSON.parse(lock)).toEqual({ repos: [{ repo: "github.com/acme/shop", commit: "abc", paths: ["proto", "services/oms"], files: [{ path: "a", sha256: "1", size: 1 }, { path: "b", sha256: "2", size: 1 }] }] });
+    const lock = encodeLock({ repo: "github.com/acme/shop", commit: "abc", paths: ["services/oms", "proto"], files: [{ path: "b", sha256: "2", size: 1 }, { path: "a", sha256: "1", size: 1 }], skipped: [{ path: "z.png", size: 2, reason: "known binary extension .png" }] });
+    expect(JSON.parse(lock)).toEqual({ repos: [{ repo: "github.com/acme/shop", commit: "abc", paths: ["proto", "services/oms"], files: [{ path: "a", sha256: "1", size: 1 }, { path: "b", sha256: "2", size: 1 }], skipped: [{ path: "z.png", size: 2, reason: "known binary extension .png" }] }] });
     expect(JSON.parse(encodeLock({ repo: "r", commit: "c", paths: [], files: [] })).repos[0]).not.toHaveProperty("paths");
   });
 });

package/scripts/local-api.mjs

@@ -20,6 +20,7 @@ import { basename, dirname, join, posix, relative, resolve, sep } from "node:pat
 
 import { loadManifest, readManifest, readManifestText } from "./manifest.mjs";
 import { builtinPluginNames } from "./builtin-plugins.mjs";
+import { djangoAggregateCandidates } from "../src/lib/django-aggregates.mjs";
 import { installDeliveryPreset, planDeliveryPreset, publicDeliveryPreset } from "./delivery-presets.mjs";
 import {
   discoverProject,
@@ -325,7 +326,7 @@ function pluginOptions(plugin, project, detectedOptions = {}) {
   if (plugin === "graphql") return { ...common, ...detectedOptions, out: "graphql.json" };
   if (plugin === "proto") return { ...common, ...detectedOptions, out: "proto.json" };
   if (plugin === "glossary") return { context: group, ...detectedOptions, out: "glossary.json" };
-  if (plugin === "adr") return { ...detectedOptions, out: "adr.json" };
+  if (plugin === "adr") return { scope: [group, component].filter(Boolean).join(".") || "org", ...detectedOptions, out: "adr.json" };
   return {};
 }
 
@@ -617,6 +618,52 @@ export function writeManifest(path, manifest) {
   }
 }
 
+export function djangoAggregateProposals(workspace) {
+  const path = join(workspace, "portolan.json");
+  const text = readFileSync(path, "utf8");
+  const manifest = readManifestText(text, path);
+  const revision = createHash("sha256").update(text).digest("hex");
+  let report;
+  try { report = JSON.parse(readFileSync(join(workspace, ".portolan/build-report.json"), "utf8")); } catch {}
+  const proposals = [];
+  for (const step of report?.steps ?? []) {
+    if (step.phase !== "extract") continue;
+    const matches = (manifest.extract ?? []).map((entry, index) => ({ entry, index }))
+      .filter(({ entry }) => entry.plugin === step.plugin && entry.in === step.input && entry.out === step.output);
+    if (matches.length !== 1) continue;
+    const { entry, index } = matches[0];
+    for (const message of step.warnings ?? []) {
+      const candidates = djangoAggregateCandidates(message);
+      if (!candidates) continue;
+      const id = `${index}:${candidates.app}`;
+      if (proposals.some((proposal) => proposal.id === id)) continue;
+      proposals.push({ id, step: index, plugin: entry.plugin, input: entry.in, output: entry.out, message, ...candidates });
+    }
+  }
+  return { revision, stale: !report || report.manifestSha256 !== revision || report.status === "running", proposals };
+}
+
+export function saveDjangoAggregates(workspace, request) {
+  const current = djangoAggregateProposals(workspace);
+  if (current.stale || request.revision !== current.revision) {
+    throw new Error("The manifest or extraction report has changed. Regenerate and review the candidates again.");
+  }
+  if (!Array.isArray(request.selections) || !request.selections.length) throw new Error("Choose at least one aggregate root.");
+  const manifest = readManifest(join(workspace, "portolan.json"));
+  const seen = new Set();
+  for (const choice of request.selections) {
+    const proposal = current.proposals.find((candidate) => candidate.id === choice?.id);
+    if (!proposal || !proposal.models.some((model) => model.name === choice.model) || seen.has(choice.id)) {
+      throw new Error("Choose one of the reported models for each application.");
+    }
+    seen.add(choice.id);
+    const step = manifest.extract[proposal.step];
+    step.options = { ...step.options, aggregates: { ...step.options?.aggregates, [proposal.app]: choice.model } };
+  }
+  writeManifest(join(workspace, "portolan.json"), manifest);
+  return { saved: request.selections.length };
+}
+
 export function writeProject(workspace, request) {
   const manifestPath = join(workspace, "portolan.json");
   const before = readFileSync(manifestPath, "utf8");
@@ -747,11 +794,21 @@ function feed(job, stream, chunk) {
   }
 }
 
-function workspaceFingerprint(workspace) {
+// The tree with nothing in it, which every repository has. Before the first
+// commit there is no HEAD to diff against, and the index diffed against this
+// tree is exactly what has been staged.
+const EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
+
+export function workspaceFingerprint(workspace) {
   const hash = createHash("sha256");
   try {
-    const options = { cwd: workspace, encoding: "buffer", maxBuffer: 128 * 1024 * 1024 };
-    hash.update(execFileSync("git", ["diff", "--binary", "HEAD", "--", "."], options));
+    // Git speaks to nobody here: a workspace that is not a checkout, or one
+    // without a commit yet, is a case this function handles, not an error to
+    // print from the dev server.
+    const options = { cwd: workspace, encoding: "buffer", maxBuffer: 128 * 1024 * 1024, stdio: ["ignore", "pipe", "ignore"] };
+    let base = EMPTY_TREE;
+    try { base = execFileSync("git", ["rev-parse", "--verify", "--quiet", "HEAD^{commit}"], options).toString().trim() || EMPTY_TREE; } catch {}
+    hash.update(execFileSync("git", ["diff", "--binary", base, "--", "."], options));
     const untracked = execFileSync("git", ["ls-files", "--others", "--exclude-standard", "-z"], options)
       .toString().split("\0").filter(Boolean).sort();
     for (const name of untracked) {
@@ -865,6 +922,7 @@ export function summarizeProjectTrial(snapshot, plan, events) {
       fileCount: event.fileCount,
       changedCount: event.changedCount,
       warnings: event.warnings ?? [],
+      diagnostics: event.diagnostics ?? [],
       ...(event.message ? { message: event.message } : {}),
     }));
   const facts = new Map(TRIAL_FACTS.map(([key]) => [key, new Set()]));
@@ -912,10 +970,12 @@ export function summarizeProjectTrial(snapshot, plan, events) {
     for (const term of fragment.terms ?? []) add("terms", term.id ?? term.slug ?? term.name);
   }
   const warnings = steps.flatMap((step) => step.warnings.map((message) => ({ plugin: step.plugin, message })));
+  const diagnostics = steps.flatMap((step) => step.diagnostics);
   return {
     steps,
     facts: TRIAL_FACTS.map(([key, label]) => ({ key, label, count: facts.get(key)?.size ?? 0 })).filter((fact) => fact.count > 0),
     warnings,
+    diagnostics,
     generatedFiles: files.length,
   };
 }
@@ -1077,6 +1137,9 @@ export function localApiPlugin(workspace = process.cwd(), publicSetupFrom) {
             const active = [...jobs.values()].find((job) => job.status === "running");
             return send(res, 200, { local: true, workspace: realpathSync(workspace), setup: setup(workspace, publicSetupFrom), activeRun: active ? { id: active.id, mode: active.mode } : null });
           }
+          if (req.method === "GET" && url.pathname === `${LOCAL_API_PREFIX}/django-aggregates`) {
+            return send(res, 200, djangoAggregateProposals(workspace));
+          }
           if (req.method === "GET" && url.pathname === `${LOCAL_API_PREFIX}/delivery-presets`) {
             const features = url.searchParams.has("features")
               ? url.searchParams.get("features").split(",").filter(Boolean)
@@ -1100,6 +1163,10 @@ export function localApiPlugin(workspace = process.cwd(), publicSetupFrom) {
             return send(res, 405, { error: "Use a local JSON request." });
           }
           const input = await body(req);
+          if (url.pathname === `${LOCAL_API_PREFIX}/django-aggregates`) {
+            if ([...jobs.values()].some((job) => job.status === "running")) throw new Error("Wait for the current generation to finish before saving aggregate roots.");
+            return send(res, 200, saveDjangoAggregates(workspace, input));
+          }
           if (url.pathname === `${LOCAL_API_PREFIX}/delivery-presets/install`) {
             return send(res, 201, installDeliveryPreset(workspace, input));
           }