@shortlink-org/portolan 0.2.4 → 0.4.0

package/scripts/host-plugins/fetch-argocd.test.mjs

@@ -0,0 +1,274 @@
+// The fetcher, against a recorded Argo CD. NEVER against a live one: the
+// whole point of this plugin's design is that the build does not depend on
+// a control plane being up, or on anyone holding a token.
+import { createHash } from "node:crypto";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { createServer } from "node:http";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+
+import { runPlugin } from "../plugin-host.mjs";
+import { FRAGMENT_NAME, LIST_PATH, LOCK_NAME, OFFLINE_ENV, TOKEN_ENV, baseUrl, deploymentOf, run, toolOf } from "./fetch-argocd.mjs";
+
+const REVISION = "a1b2c3d4e5f60718293a4b5c6d7e8f9012345678";
+
+/** Two applications the way the API lists them, with everything a page must not keep. */
+const CART = {
+  metadata: { name: "shop-cart", namespace: "argocd", labels: { env: "prod", team: "shop", "app.kubernetes.io/part-of": "shop", "app.kubernetes.io/name": "cart" } },
+  spec: {
+    project: "shop",
+    source: { repoURL: "https://github.com/shortlink-org/portolan.git", path: "./examples/shop/cart/deploy/k8s/", targetRevision: "main", kustomize: { namePrefix: "prod-" } },
+    destination: { server: "https://kubernetes.default.svc", namespace: "shop" },
+  },
+  status: {
+    sync: { status: "Synced", revision: REVISION },
+    health: { status: "Healthy" },
+    summary: { images: ["ghcr.io/shortlink-org/cart:1.4.2", "ghcr.io/shortlink-org/cart:1.4.2", "redis:7"] },
+    sourceType: "Kustomize",
+    operationState: { finishedAt: "2026-09-11T10:00:00Z" },
+  },
+};
+const PRICING = {
+  metadata: { name: "shop-pricing", namespace: "argocd" },
+  spec: {
+    project: "shop",
+    sources: [
+      { repoURL: "https://charts.example.com", chart: "service", targetRevision: "2.1.0", helm: { valueFiles: ["$values/pricing.yaml"], parameters: [{ name: "db.password", value: "hunter2" }] } },
+      { repoURL: "git@github.com:shortlink-org/portolan.git", path: "examples/shop/pricing/deploy", targetRevision: "HEAD", ref: "values" },
+    ],
+    destination: { name: "eu-west-1", namespace: "shop" },
+  },
+  status: { sync: { revisions: ["2.1.0", REVISION] }, summary: { images: ["ghcr.io/shortlink-org/pricing:0.9.0"] } },
+};
+
+const cleanups = [];
+afterEach(async () => {
+  for (const cleanup of cleanups.splice(0)) await cleanup();
+});
+
+/** Serves the one call the plugin makes, from recorded shapes. */
+async function argocd(handle = null) {
+  const seen = [];
+  const urls = [];
+  const server = createServer((request, response) => {
+    seen.push(request.headers.authorization ?? "");
+    urls.push(request.url);
+    if (handle) return handle(request, response);
+    response.setHeader("Content-Type", "application/json");
+    if (request.url.startsWith(LIST_PATH)) {
+      response.end(JSON.stringify({ metadata: { resourceVersion: "12345" }, items: [PRICING, CART] }));
+    } else {
+      response.statusCode = 404;
+      response.end(JSON.stringify({ error: "not found", code: 5, message: "no such route" }));
+    }
+  });
+  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+  cleanups.push(() => new Promise((resolve) => server.close(resolve)));
+  return { url: `http://127.0.0.1:${server.address().port}`, seen, urls };
+}
+
+function cache() {
+  const dir = mkdtempSync(join(tmpdir(), "portolan-fetch-argocd-"));
+  cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
+  return dir;
+}
+
+// A test must never inherit a real credential or a real offline setting.
+const isolated = { [TOKEN_ENV]: "", [OFFLINE_ENV]: "", CI: "" };
+const options = (server, cacheDir, extra = {}) => ({ server, cache: cacheDir, ...extra });
+const fetch = (opts, env = {}) => run({ options: opts }, { env: { ...isolated, ...env } });
+const names = (response) => response.files.map((file) => file.name).sort();
+const contentsOf = (response, name) => response.files.find((file) => file.name === name)?.contents ?? "";
+const write = (dir, response) => {
+  for (const file of response.files) {
+    mkdirSync(dirname(join(dir, file.name)), { recursive: true });
+    writeFileSync(join(dir, file.name), file.contents);
+  }
+};
+
+describe("fetch-argocd", () => {
+  it("writes the snapshot and a lock beside it, sorted, keeping what a deploy changes and nothing else", async () => {
+    const server = await argocd();
+    const response = await fetch(options(server.url, cache(), { projects: ["shop"] }));
+    expect(names(response)).toEqual([FRAGMENT_NAME, LOCK_NAME]);
+    expect(server.urls).toEqual([`${LIST_PATH}?projects=shop`]);
+
+    const catalog = JSON.parse(contentsOf(response, FRAGMENT_NAME));
+    expect(catalog).toMatchObject({ contexts: [], defs: {}, flows: [], adrs: [] });
+    expect(catalog.deployments.map((d) => d.id)).toEqual(["argocd/shop-cart", "argocd/shop-pricing"]);
+    expect(catalog.deployments[0]).toEqual({
+      id: "argocd/shop-cart",
+      name: "shop-cart",
+      project: "shop",
+      environment: "prod",
+      cluster: "in-cluster",
+      namespace: "shop",
+      repo: "github.com/shortlink-org/portolan",
+      path: "examples/shop/cart/deploy/k8s",
+      targetRevision: "main",
+      revision: REVISION,
+      tool: "kustomize",
+      url: `${server.url}/applications/argocd/shop-cart`,
+      basis: "api",
+      service: "shop.cart",
+      images: ["ghcr.io/shortlink-org/cart:1.4.2", "redis:7"],
+    });
+    // No labels, no service: the catalog places it by repository and path.
+    expect(catalog.deployments[1].service).toBeUndefined();
+    // A multi-source application follows the source that carries the
+    // manifests, and its cluster names the environment when no label does.
+    expect(catalog.deployments[1]).toMatchObject({
+      environment: "eu-west-1",
+      cluster: "eu-west-1",
+      repo: "github.com/shortlink-org/portolan",
+      path: "examples/shop/pricing/deploy",
+      targetRevision: "HEAD",
+      revision: REVISION,
+      tool: "helm",
+    });
+    expect(catalog.deployments[1].chart).toBeUndefined();
+    // Nothing that moves without a commit, and no value, reaches the tree.
+    const text = contentsOf(response, FRAGMENT_NAME);
+    for (const leaked of ["Synced", "Healthy", "finishedAt", "hunter2", "namePrefix", "resourceVersion"]) expect(text).not.toContain(leaked);
+
+    const lock = JSON.parse(contentsOf(response, LOCK_NAME));
+    expect(lock).toEqual({
+      server: server.url,
+      applications: ["argocd/shop-cart", "argocd/shop-pricing"],
+      sha256: createHash("sha256").update(text).digest("hex"),
+    });
+    expect(response.warnings).toEqual([]);
+  });
+
+  it("asks with the projects, the selector and the token, and lets none of them reach the output", async () => {
+    const server = await argocd();
+    const anonymous = await fetch(options(server.url, cache()));
+    const authorised = await fetch(options(server.url, cache(), { projects: ["shop", "platform"], selector: "team=shop" }), { [TOKEN_ENV]: "a-real-looking-secret" });
+    expect(server.urls).toEqual([LIST_PATH, `${LIST_PATH}?projects=shop&projects=platform&selector=team%3Dshop`]);
+    expect(server.seen).toEqual(["", "Bearer a-real-looking-secret"]);
+    for (const file of anonymous.files) {
+      expect(contentsOf(authorised, file.name)).toBe(file.contents);
+      expect(contentsOf(authorised, file.name)).not.toContain("a-real-looking-secret");
+    }
+  });
+
+  it("replays the committed snapshot offline, byte for byte, and in CI", async () => {
+    const server = await argocd();
+    const cacheDir = cache();
+    const online = await fetch(options(server.url, cacheDir));
+    write(cacheDir, online);
+
+    const replayed = await fetch(options(server.url, cacheDir), { [OFFLINE_ENV]: "1" });
+    expect(names(replayed)).toEqual(names(online));
+    for (const file of online.files) expect(contentsOf(replayed, file.name)).toBe(file.contents);
+    expect(replayed.warnings).toHaveLength(1);
+    expect(replayed.warnings[0].message).toContain("not fetched (offline)");
+
+    const inCi = await fetch(options(server.url, cacheDir), { CI: "true" });
+    expect(names(inCi)).toEqual(names(online));
+    expect(server.urls).toHaveLength(1);
+  });
+
+  it("names a snapshot edited by hand", async () => {
+    const server = await argocd();
+    const cacheDir = cache();
+    write(cacheDir, await fetch(options(server.url, cacheDir)));
+    writeFileSync(join(cacheDir, FRAGMENT_NAME), '{ "deployments": [] }\n');
+    await expect(fetch(options(server.url, cacheDir), { [OFFLINE_ENV]: "1" })).rejects.toThrow(/edited by hand/);
+  });
+
+  it("fails rather than emitting nothing when there is no snapshot to fall back to", async () => {
+    await expect(fetch(options("http://127.0.0.1:1", cache()), { [OFFLINE_ENV]: "1" })).rejects.toThrow(/no argocd\.lock\.json/);
+
+    const broken = await argocd((request, response) => {
+      response.statusCode = 500;
+      response.end(JSON.stringify({ error: "the control plane is having a day", code: 13 }));
+    });
+    await expect(fetch(options(broken.url, cache()))).rejects.toThrow(/the control plane is having a day.*no usable snapshot/);
+
+    const empty = await argocd((request, response) => {
+      response.end(JSON.stringify({ items: null }));
+    });
+    await expect(fetch(options(empty.url, cache()))).rejects.toThrow(/lists no applications/);
+  });
+
+  it("falls back to the committed snapshot when the server fails", async () => {
+    const server = await argocd();
+    const cacheDir = cache();
+    const online = await fetch(options(server.url, cacheDir));
+    write(cacheDir, online);
+
+    const broken = await argocd((request, response) => {
+      response.statusCode = 502;
+      response.end();
+    });
+    const fallback = await fetch(options(broken.url, cacheDir));
+    expect(names(fallback)).toEqual(names(online));
+    expect(fallback.warnings[0].message).toMatch(/not fetched \(.*http 502\)/);
+  });
+
+  it("refuses a missing server and a missing cache", async () => {
+    await expect(fetch(options("", cache()))).rejects.toThrow(/server/);
+    await expect(fetch(options("argocd.example.com", ""))).rejects.toThrow(/cache/);
+  });
+
+  it("runs through the host like any plugin", async () => {
+    const server = await argocd();
+    const cacheDir = cache();
+    write(cacheDir, await fetch(options(server.url, cacheDir)));
+    const saved = { CI: process.env.CI, [OFFLINE_ENV]: process.env[OFFLINE_ENV] };
+    process.env.CI = "";
+    process.env[OFFLINE_ENV] = "1";
+    try {
+      const result = await runPlugin({ name: "argocd", host: "fetch-argocd" }, { portolanVersion: "0.1.0", options: options(server.url, cacheDir) });
+      expect(result.files).toHaveLength(2);
+      expect(result.warnings).toEqual([`${server.url}: not fetched (offline); the snapshot committed in this repository is used unchanged`]);
+      const described = await runPlugin({ name: "argocd", host: "fetch-argocd" }, { portolanVersion: "0.1.0", kind: "describe" });
+      expect(described.describe).toMatchObject({ name: "fetch-argocd", phases: ["extract"] });
+    } finally {
+      for (const [key, value] of Object.entries(saved)) {
+        if (value === undefined) delete process.env[key];
+        else process.env[key] = value;
+      }
+    }
+  });
+});
+
+describe("one application", () => {
+  it("spells the server as a URL, https unless told otherwise", () => {
+    expect(baseUrl("argocd.example.com")).toBe("https://argocd.example.com");
+    expect(baseUrl("https://argocd.example.com/")).toBe("https://argocd.example.com");
+    expect(baseUrl("http://127.0.0.1:8080")).toBe("http://127.0.0.1:8080");
+  });
+
+  it("reads the environment off the label the manifest names, else off the cluster", () => {
+    const base = "https://argocd.example.com";
+    expect(deploymentOf(CART, base, "env").environment).toBe("prod");
+    expect(deploymentOf(CART, base, "tier").environment).toBe("in-cluster");
+    expect(deploymentOf({ ...CART, spec: { ...CART.spec, destination: { server: "https://1.2.3.4", namespace: "x" } } }, base, "tier").environment).toBe("https://1.2.3.4");
+  });
+
+  it("infers the tool from the source until the status has said", () => {
+    expect(toolOf("Kustomize", {})).toBe("kustomize");
+    expect(toolOf("", { chart: "x" })).toBe("helm");
+    expect(toolOf("", { helm: {} })).toBe("helm");
+    expect(toolOf("", { kustomize: {} })).toBe("kustomize");
+    expect(toolOf("", { plugin: {} })).toBe("plugin");
+    expect(toolOf("", { directory: {} })).toBe("directory");
+    expect(toolOf("", {})).toBe("");
+  });
+
+  it("places by the labels the manifest names, and by nothing when only one of the two is there", () => {
+    const base = "https://argocd.example.com";
+    const labelled = (labels) => ({ ...CART, metadata: { ...CART.metadata, labels } });
+    expect(deploymentOf(labelled({ team: "shop", component: "cart" }), base, "env", { context: "team", service: "component" }).service).toBe("shop.cart");
+    expect(deploymentOf(labelled({ "app.kubernetes.io/name": "cart" }), base).service).toBeUndefined();
+    expect(deploymentOf(labelled({}), base).service).toBeUndefined();
+  });
+
+  it("refuses an application without a name, and defaults the namespace and project", () => {
+    expect(() => deploymentOf({ metadata: {} }, "https://a")).toThrow(/no name/);
+    expect(deploymentOf({ metadata: { name: "x" } }, "https://a")).toMatchObject({ id: "argocd/x", project: "default", tool: "", url: "https://a/applications/argocd/x" });
+  });
+});

package/scripts/host-plugins/fetch-bsr.mjs

@@ -46,6 +46,7 @@ export function describe() {
   return {
     name: "fetch-bsr",
     summary: "Fetches pinned modules from a Buf Schema Registry into the tree, with a lock beside each, so the proto extractor can read a published contract.",
+    category: "sources",
     phases: ["extract"],
     options: optionsSchema,
   };

package/scripts/host-plugins/fetch-csr.mjs

@@ -34,6 +34,7 @@ export function describe() {
   return {
     name: "fetch-csr",
     summary: "Fetches pinned subject versions from a Confluent Schema Registry into the tree, with a lock beside each, so a schema published to a registry can be read without talking to it.",
+    category: "sources",
     phases: ["extract"],
     options: optionsSchema,
   };

package/scripts/host-plugins/fetch-git.mjs

@@ -44,10 +44,23 @@ export const LOCK_NAME = "git.lock.json";
 export const PIN_NAME = "git.repo.json";
 export const OFFLINE_ENV = "PORTOLAN_OFFLINE";
 
+const SNIFF_BYTES = 8 * 1024;
+const BINARY_SUFFIXES = [
+  ".tar.gz", ".tar.bz2", ".tar.xz",
+  ".7z", ".a", ".apk", ".avif", ".bin", ".bmp", ".bz2", ".class",
+  ".deb", ".dll", ".dmg", ".doc", ".docx", ".eot", ".exe", ".gif",
+  ".gz", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lib", ".mmdb",
+  ".mov", ".mp3", ".mp4", ".o", ".otf", ".pdf", ".png", ".pyc",
+  ".rar", ".so", ".tar", ".tgz", ".tif", ".tiff", ".ttf", ".war",
+  ".wasm", ".webm", ".webp", ".woff", ".woff2", ".xls", ".xlsx",
+  ".xz", ".zip",
+];
+
 export function describe() {
   return {
     name: "fetch-git",
     summary: "Fetches pinned directories of another git repository into the tree, with a lock, so extractors can read a service that lives elsewhere.",
+    category: "sources",
     phases: ["extract"],
     options: optionsSchema,
   };
@@ -105,7 +118,7 @@ export function run(request, { env = process.env } = {}) {
       }
       continue;
     }
-    emitFetched(out, dir, want, fetched.commit, fetched.files, generatedAt);
+    emitFetched(out, dir, want, fetched.commit, fetched.files, fetched.skipped, generatedAt);
   }
 
   return out.response();
@@ -172,9 +185,11 @@ export function pin(repo, commit, generatedAt = "") {
 /** The lock, written the way every generated file here is written. */
 export function encodeLock(entry) {
   const files = [...entry.files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  const skipped = [...(entry.skipped ?? [])].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
   const record = { repo: entry.repo, commit: entry.commit };
   if (entry.paths?.length) record.paths = [...entry.paths].sort();
   record.files = files.map((file) => ({ path: file.path, sha256: file.sha256, size: file.size }));
+  if (skipped.length) record.skipped = skipped.map((file) => ({ path: file.path, size: file.size, reason: file.reason }));
   return `${JSON.stringify({ repos: [record] }, null, 2)}\n`;
 }
 
@@ -191,13 +206,15 @@ function live(url, want, out, env) {
     commit = resolve(url, want.ref, env);
     out.warn(want.repo, `is not pinned; "${want.ref || "HEAD"}" resolved to ${commit}. Pin it in portolan.json or every run is a lottery.`);
   }
-  const files = download(url, commit, want.paths ?? [], env);
-  if (files.size === 0) throw new Error(`${want.repo} at ${commit} holds no files under ${(want.paths ?? []).join(", ")}`);
-  return { commit, files };
+  const fetched = download(url, commit, want.paths ?? [], env);
+  if (fetched.files.size === 0 && fetched.skipped.length === 0) {
+    throw new Error(`${want.repo} at ${commit} holds no files under ${(want.paths ?? []).join(", ")}`);
+  }
+  return { commit, ...fetched };
 }
 
-function emitFetched(out, dir, want, commit, files, generatedAt) {
-  const entry = { repo: want.repo, commit, paths: want.paths ?? [], files: [] };
+function emitFetched(out, dir, want, commit, files, skipped, generatedAt) {
+  const entry = { repo: want.repo, commit, paths: want.paths ?? [], files: [], skipped };
   for (const path of [...files.keys()].sort()) {
     const contents = files.get(path);
     out.file(posix.join(dir, path), contents);
@@ -205,6 +222,7 @@ function emitFetched(out, dir, want, commit, files, generatedAt) {
   }
   out.file(posix.join(dir, LOCK_NAME), encodeLock(entry));
   out.file(posix.join(dir, PIN_NAME), pin(want.repo, commit, generatedAt));
+  warnSkipped(out, want.repo, skipped);
 }
 
 /**
@@ -225,6 +243,13 @@ function emitCached(out, dir, at, want, why, generatedAt = "") {
   // what is actually on disk rather than what was asked for.
   out.file(posix.join(dir, PIN_NAME), pin(want.repo, held.lock.commit, generatedAt));
   out.warn(want.repo, `not fetched (${why}); the copy committed in this repository is used unchanged`);
+  warnSkipped(out, want.repo, held.lock.skipped);
+}
+
+function warnSkipped(out, repo, skipped = []) {
+  if (skipped.length === 0) return;
+  const bytes = skipped.reduce((total, file) => total + file.size, 0);
+  out.warn(repo, `skipped ${skipped.length} binary ${skipped.length === 1 ? "file" : "files"} (${bytes} bytes); metadata is recorded in ${LOCK_NAME}`);
 }
 
 /**
@@ -252,7 +277,14 @@ function replay(dir) {
   if (repos.length !== 1) throw new Error(`${lockPath} names ${repos.length} repositories; expected exactly one`);
   const entry = repos[0];
   const files = new Map();
+  const kept = [];
+  const skipped = [...(Array.isArray(entry.skipped) ? entry.skipped : [])];
   for (const want of entry.files ?? []) {
+    const known = knownBinaryReason(want.path);
+    if (known) {
+      skipped.push({ path: want.path, size: want.size, reason: known });
+      continue;
+    }
     let contents;
     try {
       contents = readFileSync(join(dir, ...String(want.path).split("/")));
@@ -261,9 +293,15 @@ function replay(dir) {
       throw cause;
     }
     if (digestOf(contents) !== want.sha256) throw new Error(`${want.path} does not match its digest; the vendored copy was edited by hand`);
+    const sniffed = sniffedBinaryReason(contents);
+    if (sniffed) {
+      skipped.push({ path: want.path, size: contents.length, reason: sniffed });
+      continue;
+    }
     files.set(want.path, contents);
+    kept.push(want);
   }
-  return { lock: { repo: entry.repo, commit: entry.commit, paths: entry.paths ?? [], files: entry.files ?? [] }, files };
+  return { lock: { repo: entry.repo, commit: entry.commit, paths: entry.paths ?? [], files: kept, skipped }, files };
 }
 
 // --- git --------------------------------------------------------------------
@@ -305,7 +343,7 @@ function resolve(url, ref, env) {
  * its branches where it does not, which is the case for a plain file://
  * remote; either way what is read is verified to be that commit.
  *
- * @returns {Map<string, Buffer>} path → contents
+ * @returns {{files: Map<string, Buffer>, skipped: {path: string, size: number, reason: string}[]}}
  */
 function download(url, commit, paths, env) {
   const tmp = mkdtempSync(join(tmpdir(), "portolan-fetch-git-"));
@@ -326,19 +364,24 @@ function download(url, commit, paths, env) {
       throw new Error(`${commit} is not a commit ${url} has, or not one reachable from a branch`);
     }
 
-    const listing = git(tmp, ["ls-tree", "-r", "-z", "--format=%(objectname) %(objecttype) %(path)", commit, "--", ...paths], env);
+    const listing = git(tmp, ["ls-tree", "-r", "-z", "--format=%(objectname) %(objecttype) %(objectsize) %(path)", commit, "--", ...paths], env);
     const blobs = [];
+    const skipped = [];
     for (const line of listing.split("\0")) {
       if (!line) continue;
-      const [sha, type, ...rest] = line.split(" ");
+      const [sha, type, rawSize, ...rest] = line.split(" ");
       if (type !== "blob") continue;
-      blobs.push({ sha, path: posix.normalize(rest.join(" ")) });
+      const path = posix.normalize(rest.join(" "));
+      const size = Number(rawSize);
+      const reason = knownBinaryReason(path);
+      if (reason) skipped.push({ path, size, reason });
+      else blobs.push({ sha, path, size });
     }
     const files = new Map();
-    if (blobs.length === 0) return files;
+    if (blobs.length === 0) return { files, skipped };
 
-    // One process for every blob: `cat-file --batch` answers each sha with a
-    // header line and the bytes.
+    // One `cat-file --batch` process answers every remaining sha with a header
+    // line and the bytes. Known binaries never enter this batch.
     const batch = execFileSync("git", ["cat-file", "--batch"], {
       cwd: tmp,
       env: { ...env, GIT_TERMINAL_PROMPT: "0" },
@@ -352,15 +395,30 @@ function download(url, commit, paths, env) {
       if (header[1] === "missing") throw new Error(`${blob.path} is missing from ${commit}`);
       const size = Number(header[2]);
       const start = newline + 1;
-      files.set(blob.path, Buffer.from(batch.subarray(start, start + size)));
+      const contents = batch.subarray(start, start + size);
+      const reason = sniffedBinaryReason(contents);
+      if (reason) skipped.push({ path: blob.path, size, reason });
+      else files.set(blob.path, Buffer.from(contents));
       offset = start + size + 1;
     }
-    return files;
+    return { files, skipped };
   } finally {
     rmSync(tmp, { recursive: true, force: true });
   }
 }
 
+function knownBinaryReason(path) {
+  const lower = String(path).toLowerCase();
+  const suffix = BINARY_SUFFIXES.find((candidate) => lower.endsWith(candidate));
+  return suffix ? `known binary extension ${suffix}` : "";
+}
+
+function sniffedBinaryReason(contents) {
+  if (contents.subarray(0, SNIFF_BYTES).includes(0)) return `NUL byte in first ${SNIFF_BYTES} bytes`;
+  const text = contents.toString("utf8");
+  return Buffer.from(text, "utf8").equals(contents) ? "" : "invalid UTF-8";
+}
+
 /** Accumulates a response the way plugin.Builder does. */
 class Builder {
   files = [];
@@ -372,11 +430,10 @@ class Builder {
       return;
     }
     const text = contents.toString("utf8");
-    this.files.push(
-      Buffer.from(text, "utf8").equals(contents)
-        ? { name, contents: text }
-        : { name, contents: contents.toString("base64"), encoding: "base64" },
-    );
+    if (!Buffer.from(text, "utf8").equals(contents) || contents.subarray(0, SNIFF_BYTES).includes(0)) {
+      throw new Error(`${name} is binary and cannot be emitted by fetch-git`);
+    }
+    this.files.push({ name, contents: text });
   }
 
   warn(ref, message) {

package/scripts/host-plugins/fetch-git.test.mjs

@@ -1,11 +1,12 @@
 import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
 import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { basename, dirname, join } from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
 
 import { runPlugin } from "../plugin-host.mjs";
-import { OFFLINE_ENV, encodeLock, pin, run, splitRepo, webRepo } from "./fetch-git.mjs";
+import { LOCK_NAME, OFFLINE_ENV, encodeLock, pin, run, splitRepo, webRepo } from "./fetch-git.mjs";
 
 const created = [];
 afterEach(() => {
@@ -105,18 +106,71 @@ describe("fetch-git", () => {
     expect(inCi.warnings).toHaveLength(1);
   });
 
-  it("preserves binary blobs online and from the offline cache", () => {
+  it("skips binary blobs online and from the offline cache", () => {
     const remote = repository();
     const cacheDir = cache();
     const first = fetch(options(remote, cacheDir, ["docs/example.png"]), online);
     const name = `${remote.copyDir}/docs/example.png`;
-    const image = first.files.find((file) => file.name === name);
-    expect(image?.encoding).toBe("base64");
+    expect(first.files.find((file) => file.name === name)).toBeUndefined();
+    expect(first.files.every((file) => file.encoding === undefined)).toBe(true);
+    const lockName = `${remote.copyDir}/${LOCK_NAME}`;
+    expect(JSON.parse(contentsOf(first, lockName)).repos[0].skipped).toEqual([
+      { path: "docs/example.png", size: 10, reason: "known binary extension .png" },
+    ]);
+    expect(first.warnings[0].message).toContain("skipped 1 binary file (10 bytes)");
     write(cacheDir, first);
-    expect(readFileSync(join(cacheDir, name))).toEqual(readFileSync(join(remote.dir, "docs/example.png")));
+    expect(() => readFileSync(join(cacheDir, name))).toThrow();
 
     const replayed = fetch(options(remote, cacheDir, ["docs/example.png"]), offline);
-    expect(replayed.files.find((file) => file.name === name)).toEqual(image);
+    expect(replayed.files.find((file) => file.name === name)).toBeUndefined();
+    expect(JSON.parse(contentsOf(replayed, lockName)).repos[0].skipped).toEqual([
+      { path: "docs/example.png", size: 10, reason: "known binary extension .png" },
+    ]);
+  });
+
+  it("does not read a large known binary into the plugin response and sniffs unknown binary files", () => {
+    const remote = repository();
+    const mmdb = "tests/integration/resources/GeoLite2-Country.mmdb";
+    const unknown = "tests/integration/resources/blob.data";
+    mkdirSync(dirname(join(remote.dir, mmdb)), { recursive: true });
+    writeFileSync(join(remote.dir, mmdb), Buffer.alloc(10 * 1024 * 1024, 0xab));
+    writeFileSync(join(remote.dir, unknown), Buffer.from([1, 0, 2]));
+    remote.git(["add", "."]);
+    remote.git(["commit", "--quiet", "-m", "binary fixtures"]);
+    remote.commit = remote.git(["rev-parse", "HEAD"]);
+
+    const response = fetch(options(remote, cache(), [mmdb, unknown]), online);
+    expect(response.files.every((file) => file.encoding === undefined)).toBe(true);
+    expect(JSON.stringify(response).length).toBeLessThan(20_000);
+    const lock = JSON.parse(contentsOf(response, `${remote.copyDir}/${LOCK_NAME}`)).repos[0];
+    expect(lock.files).toEqual([]);
+    expect(lock.skipped).toEqual([
+      { path: mmdb, size: 10 * 1024 * 1024, reason: "known binary extension .mmdb" },
+      { path: unknown, size: 3, reason: "NUL byte in first 8192 bytes" },
+    ]);
+  });
+
+  it("drops binary files from a cache written by an older Portolan", () => {
+    const remote = repository();
+    const cacheDir = cache();
+    const at = join(cacheDir, remote.copyDir);
+    const path = "docs/example.png";
+    const contents = readFileSync(join(remote.dir, path));
+    mkdirSync(dirname(join(at, path)), { recursive: true });
+    writeFileSync(join(at, path), contents);
+    writeFileSync(join(at, LOCK_NAME), encodeLock({
+      repo: remote.url,
+      commit: remote.commit,
+      paths: [path],
+      files: [{ path, size: contents.length, sha256: createHash("sha256").update(contents).digest("hex") }],
+    }));
+
+    const replayed = fetch(options(remote, cacheDir, [path]), offline);
+    expect(replayed.files.some((file) => file.name.endsWith(path))).toBe(false);
+    expect(JSON.parse(contentsOf(replayed, `${remote.copyDir}/${LOCK_NAME}`)).repos[0]).toMatchObject({
+      files: [],
+      skipped: [{ path, size: contents.length, reason: "known binary extension .png" }],
+    });
   });
 
   it("names the edited file when a vendored copy no longer matches its lock", () => {
@@ -223,8 +277,8 @@ describe("names", () => {
       repos: [{ repo: "github.com/acme/shop", commit: "c1d2e3f" }],
     });
 
-    const lock = encodeLock({ repo: "github.com/acme/shop", commit: "abc", paths: ["services/oms", "proto"], files: [{ path: "b", sha256: "2", size: 1 }, { path: "a", sha256: "1", size: 1 }] });
-    expect(JSON.parse(lock)).toEqual({ repos: [{ repo: "github.com/acme/shop", commit: "abc", paths: ["proto", "services/oms"], files: [{ path: "a", sha256: "1", size: 1 }, { path: "b", sha256: "2", size: 1 }] }] });
+    const lock = encodeLock({ repo: "github.com/acme/shop", commit: "abc", paths: ["services/oms", "proto"], files: [{ path: "b", sha256: "2", size: 1 }, { path: "a", sha256: "1", size: 1 }], skipped: [{ path: "z.png", size: 2, reason: "known binary extension .png" }] });
+    expect(JSON.parse(lock)).toEqual({ repos: [{ repo: "github.com/acme/shop", commit: "abc", paths: ["proto", "services/oms"], files: [{ path: "a", sha256: "1", size: 1 }, { path: "b", sha256: "2", size: 1 }], skipped: [{ path: "z.png", size: 2, reason: "known binary extension .png" }] }] });
     expect(JSON.parse(encodeLock({ repo: "r", commit: "c", paths: [], files: [] })).repos[0]).not.toHaveProperty("paths");
   });
 });