@synovitec/trellis 0.0.0-stage → 0.6.0-rc.2

package/src/verify/deps.mjs

@@ -0,0 +1,382 @@
+// The dependencies a change adds or moves, checked at their exact version: the package exists on
+// its registry (models invent package names), that version is published, and it has no known
+// vulnerability (OSV). Exact versions come from what the change itself records: npm's lockfile,
+// pip pins, go.mod. A dependency without one is reported as unpinned, never passed silently, and
+// a registry or OSV that cannot be reached makes the check unavailable, never clean.
+import { spawnSync } from "node:child_process";
+import { addedDependencies } from "../agents/manifests.mjs";
+import { GitError } from "../core/diff.mjs";
+
+/** @typedef {{ ecosystem: string, name: string, version: string | null, source?: string | null, integrity?: string | null }} Dependency  source: where it comes from when not its registry; integrity: the lockfile's hash of it */
+/** @typedef {{ ecosystem: string, name: string, version: string | null, problem: string }} DependencyFinding */
+/**
+ * @typedef {object} DependencyCheck
+ * @property {"clean" | "findings" | "unavailable"} status
+ * @property {DependencyFinding[]} findings
+ * @property {string[]} unpinned    exist, but with no exact version their vulnerabilities are unknown
+ * @property {string[]} unchecked   what no registry vouched for: a package from git, a URL or another
+ *                                  registry, or one of an ecosystem Trellis does not query yet
+ * @property {string} [detail]
+ */
+
+/** A blob's text, or null for no blob. @param {string} root @param {string | null} blob */
+function blobText(root, blob) {
+  if (!blob) return null;
+  const r = spawnSync("git", ["cat-file", "blob", blob], { cwd: root, encoding: "utf8", maxBuffer: 256 * 1024 * 1024, windowsHide: true });
+  if (r.status !== 0) throw new GitError(["cat-file", "blob", blob], (r.stderr || `exit ${r.status}`).trim());
+  return r.stdout;
+}
+
+/** @param {string} path */
+const fileOf = (path) => path.slice(path.lastIndexOf("/") + 1);
+/** @param {string} path */
+const dirOf = (path) => (path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "");
+
+const NPM_REGISTRY = "https://registry.npmjs.org/";
+
+/**
+ * Where a lockfile entry was fetched from, when it is not this package's own tarball on the npm
+ * registry: git, a URL, another registry, or another package's tarball on npm (an entry named
+ * lodash that downloads something else).
+ * @param {unknown} resolved @param {string} name @param {string} version
+ */
+const sourceOf = (resolved, name, version) => {
+  if (typeof resolved !== "string") return null;
+  if (resolved.startsWith(NPM_REGISTRY)) return resolved === `${NPM_REGISTRY}${name}/-/${name.split("/").pop()}-${version}.tgz` ? null : resolved;
+  return resolved.match(/^[^:]+:\/\/[^/#]+/)?.[0] ?? resolved.split(/[#?]/)[0];
+};
+
+/**
+ * One lockfile entry as the dependency it installs. An alias (`"lodash": "npm:other@1.0.0"`)
+ * installs `other`, by `name` (v2/v3) or by its version (v1).
+ * @param {string} folderName @param {any} entry
+ * @returns {Dependency | null}
+ */
+function entryOf(folderName, entry) {
+  if (entry?.link) return null;
+  // No version, but a place it is fetched from: what is installed is whatever that URL serves.
+  if (typeof entry?.version !== "string") return typeof entry?.resolved === "string" ? { ecosystem: "npm", name: typeof entry.name === "string" ? entry.name : folderName, version: null, source: entry.resolved } : null;
+  let name = typeof entry.name === "string" ? entry.name : folderName;
+  let version = entry.version;
+  const alias = version.match(/^npm:((?:@[^/]+\/)?[^@]+)@(.+)$/);
+  if (alias) [, name, version] = alias;
+  return { ecosystem: "npm", name, version, source: sourceOf(entry.resolved, name, version), integrity: typeof entry.integrity === "string" ? entry.integrity : null };
+}
+
+/**
+ * Every installed package of an npm lockfile, by its place in it: v2/v3 `packages`, and v1's
+ * nested `dependencies`.
+ * @param {string | null} text
+ * @returns {Map<string, Dependency>}
+ */
+function lockEntries(text) {
+  /** @type {Map<string, Dependency>} */
+  const out = new Map();
+  if (!text) return out;
+  let lock;
+  try {
+    lock = JSON.parse(text.replace(/^\uFEFF/, ""));
+  } catch {
+    return out;
+  }
+  for (const [key, entry] of Object.entries(lock?.packages ?? {})) {
+    const i = key.lastIndexOf("node_modules/");
+    const d = i === -1 ? null : entryOf(key.slice(i + "node_modules/".length), entry);
+    if (d) out.set(key, d);
+  }
+  if (!lock?.packages) {
+    const walk = (/** @type {any} */ deps, /** @type {string} */ at) => {
+      for (const [name, entry] of Object.entries(deps ?? {})) {
+        const key = `${at}node_modules/${name}`;
+        const d = entryOf(name, entry);
+        if (d) out.set(key, d);
+        walk(/** @type {any} */ (entry)?.dependencies, `${key}/`);
+      }
+    };
+    walk(lock?.dependencies, "");
+  }
+  return out;
+}
+
+/** @param {Dependency} d */
+const idOf = (d) => `${d.ecosystem}:${d.name}@${d.version}${d.source ? ` ${d.source}` : ""}${d.integrity ? ` ${d.integrity}` : ""}`;
+
+/** Whether a JSON file parses, past a byte-order mark. @param {string | null} text */
+const parses = (text) => {
+  try {
+    JSON.parse((text ?? "").replace(/^\uFEFF/, ""));
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/** The dependency specs of a package.json, by name, across its dependency fields. @param {string | null} text */
+function npmSpecs(text) {
+  /** @type {Map<string, string>} */
+  const out = new Map();
+  let doc;
+  try {
+    doc = JSON.parse((text ?? "null").replace(/^\uFEFF/, ""));
+  } catch {
+    return out;
+  }
+  for (const field of ["dependencies", "devDependencies", "optionalDependencies", "peerDependencies"]) {
+    for (const [name, spec] of Object.entries(doc?.[field] ?? {})) if (typeof spec === "string") out.set(name, spec);
+  }
+  return out;
+}
+
+/** A spec that does not name a version on the npm registry: git, a URL, a file, an alias. @param {string} spec */
+const nonRegistrySpec = (spec) => (/^(git\+|git:|github:|gitlab:|bitbucket:|https?:|file:|link:|npm:)|^[\w.-]+\/[\w.-]+(#.*)?$/.test(spec) ? spec : null);
+
+/**
+ * What a requirements file installs: `name==version` pins and bare names, and what PyPI does not
+ * vouch for: another index, a VCS or URL requirement, `name @ url`.
+ * @param {string | null} text
+ * @returns {Map<string, { version: string | null, source: string | null }>}
+ */
+function pins(text) {
+  /** @type {Map<string, { version: string | null, source: string | null }>} */
+  const out = new Map();
+  for (const raw of (text ?? "").split(/\r?\n/)) {
+    const line = raw.replace(/(^|\s)#.*/, "").trim();
+    if (!line) continue;
+    const index = line.match(/^(--index-url|--extra-index-url|-i|--find-links|-f)[\s=]+(\S+)/);
+    if (index) {
+      out.set(index[1], { version: null, source: index[2] });
+      continue;
+    }
+    const vcs = line.match(/^(?:-e\s+|--editable[\s=]+)?((?:git\+|hg\+|svn\+|bzr\+|https?:|file:)\S+)/);
+    if (vcs) {
+      out.set(vcs[1].match(/#egg=([\w.-]+)/)?.[1]?.toLowerCase() ?? vcs[1], { version: null, source: vcs[1] });
+      continue;
+    }
+    // Another file's requirements: read only when it is itself named requirements*.txt.
+    const include = line.match(/^(-r|-c|--requirement|--constraint)[\s=]+(\S+)/);
+    if (include) {
+      out.set(`${include[1]} ${include[2]}`, { version: null, source: "an included requirements file" });
+      continue;
+    }
+    if (line.startsWith("-")) continue;
+    const name = line.split(/[\s\[<>=!~;@]/)[0].toLowerCase();
+    const url = line.match(/^[\w.\-\[\], ]+@\s*(\S+)/)?.[1] ?? null;
+    const exact = line.match(/==\s*([^\s;,]+)/);
+    if (name) out.set(name, { version: exact ? exact[1] : null, source: url });
+  }
+  return out;
+}
+
+/** `replace` directives of a go.mod: the module, and what it is replaced with. @param {string | null} text */
+function goReplaces(text) {
+  /** @type {Map<string, string>} */
+  const out = new Map();
+  let block = false;
+  for (const raw of (text ?? "").split(/\r?\n/)) {
+    const line = raw.replace(/\/\/.*/, "").trim();
+    if (/^replace\s*\($/.test(line)) block = true;
+    else if (block && line === ")") block = false;
+    else {
+      const m = (block ? line : line.replace(/^replace\s+/, "")).match(/^(\S+)(?:\s+\S+)?\s+=>\s+(.+)$/);
+      if (m && (block || /^replace\s/.test(line))) out.set(m[1], m[2].trim());
+    }
+  }
+  return out;
+}
+
+/** Settings that change where packages come from: a registry, a proxy, a scope's registry. */
+const REGISTRY_SETTINGS = new Set([".npmrc", ".yarnrc", ".yarnrc.yml", "pip.conf", "pip.ini", ".pypirc"]);
+
+/** What overrides the versions a package.json's dependencies resolve to. @param {string | null} text */
+const overridesOf = (text) => {
+  let doc;
+  try {
+    doc = JSON.parse((text ?? "null").replace(/^\uFEFF/, ""));
+  } catch {
+    return "";
+  }
+  return JSON.stringify([doc?.overrides ?? null, doc?.resolutions ?? null, doc?.pnpm?.overrides ?? null]);
+};
+
+/** Manifests Trellis does not read yet: a change to one is named, never passed silently. */
+const UNREAD_MANIFESTS = /^(setup\.py|setup\.cfg|Pipfile|environment\.ya?ml|conda\.ya?ml|pom\.xml|build\.gradle(\.kts)?|settings\.gradle(\.kts)?|gradle\.properties|.+\.csproj|.+\.fsproj|.+\.vbproj|packages\.config|Directory\.Packages\.props|pubspec\.ya?ml|deno\.jsonc?|import_map\.json|mix\.exs|Package\.swift|Podfile|build\.sbt|project\.clj|deps\.edn|.+\.cabal|stack\.yaml|rebar\.config|vcpkg\.json|conanfile\.(txt|py))$/;
+
+/** Lockfiles Trellis does not read yet: a change to one is named, never passed silently. */
+const UNREAD_LOCKS = new Set(["pnpm-lock.yaml", "yarn.lock", "bun.lock", "bun.lockb", "npm-shrinkwrap.json", "poetry.lock", "uv.lock", "Pipfile.lock", "Cargo.lock", "Gemfile.lock", "composer.lock", "go.sum"]);
+
+/** `module version` requirements of a go.mod. @param {string | null} text */
+function goRequires(text) {
+  /** @type {Map<string, string>} */
+  const out = new Map();
+  let block = false;
+  for (const raw of (text ?? "").split(/\r?\n/)) {
+    const line = raw.replace(/\/\/.*/, "").trim();
+    if (/^require\s*\($/.test(line)) block = true;
+    else if (block && line === ")") block = false;
+    else {
+      const m = (block ? line : line.replace(/^require\s+/, "")).match(/^(\S+)\s+(v\S+)$/);
+      if (m && (block || /^require\s/.test(line))) out.set(m[1], m[2]);
+    }
+  }
+  return out;
+}
+
+const OTHER_ECOSYSTEMS = { "pyproject.toml": "PyPI", "Cargo.toml": "crates.io", "Gemfile": "RubyGems", "composer.json": "Packagist" };
+
+/**
+ * The dependencies a change adds, or moves to another version, read from the change's own blobs.
+ * @param {string} root
+ * @param {import("../core/diff.mjs").Change[]} changes
+ * @returns {Dependency[]}
+ */
+export function dependencyChanges(root, changes) {
+  /** @type {Map<string, Dependency>} */
+  const found = new Map();
+  const add = (/** @type {Dependency} */ d) => found.set(idOf(d), d);
+  // The lockfiles the change writes, by folder: what they install covers the manifest beside them.
+  const locks = new Map(changes.filter((c) => c.status !== "D" && fileOf(c.path) === "package-lock.json").map((c) => [dirOf(c.path), lockEntries(blobText(root, c.newBlob))]));
+  for (const c of changes) {
+    if (c.status === "D") continue;
+    const file = fileOf(c.path);
+    // A manifest or lockfile that does not parse holds dependencies Trellis cannot see: the check
+    // is unavailable, never clean.
+    if ((file === "package-lock.json" || file === "package.json") && !parses(blobText(root, c.newBlob))) {
+      add({ ecosystem: "unreadable", name: c.path, version: null, source: "does not parse as JSON" });
+      continue;
+    }
+    if (file === "package-lock.json") {
+      const before = new Set([...lockEntries(blobText(root, c.oldBlob)).values()].map(idOf));
+      for (const d of /** @type {Map<string, Dependency>} */ (locks.get(dirOf(c.path))).values()) if (!before.has(idOf(d))) add(d);
+    } else if (file === "package.json") {
+      // Each spec added or changed. One that names no registry version (git, a URL, an alias) is
+      // named; a name the lockfile beside it installs is checked from there, at its exact
+      // version; any other is checked here, by name.
+      const lock = locks.get(dirOf(c.path));
+      const before = npmSpecs(blobText(root, c.oldBlob));
+      if (overridesOf(blobText(root, c.oldBlob)) !== overridesOf(blobText(root, c.newBlob))) add({ ecosystem: "npm", name: `${c.path} overrides`, version: null, source: "overrides or resolutions Trellis does not resolve" });
+      for (const [name, spec] of npmSpecs(blobText(root, c.newBlob))) {
+        if (before.get(name) === spec) continue;
+        const elsewhere = nonRegistrySpec(spec);
+        if (elsewhere) add({ ecosystem: "npm", name, version: null, source: elsewhere });
+        else if (!lock?.has(`node_modules/${name}`)) add({ ecosystem: "npm", name, version: null });
+      }
+    } else if (/^requirements.*\.txt$/.test(file) || /(^|\/)requirements\/[^/]+\.txt$/.test(c.path)) {
+      const before = pins(blobText(root, c.oldBlob));
+      for (const [name, pin] of pins(blobText(root, c.newBlob))) {
+        const had = before.get(name);
+        if (had && had.version === pin.version && had.source === pin.source) continue;
+        add({ ecosystem: "PyPI", name, version: pin.version, source: pin.source });
+      }
+    } else if (file === "go.mod") {
+      const before = goRequires(blobText(root, c.oldBlob));
+      for (const [name, version] of goRequires(blobText(root, c.newBlob))) if (before.get(name) !== version) add({ ecosystem: "Go", name, version });
+      const replacedBefore = goReplaces(blobText(root, c.oldBlob));
+      for (const [name, by] of goReplaces(blobText(root, c.newBlob))) if (replacedBefore.get(name) !== by) add({ ecosystem: "Go", name, version: null, source: `replace => ${by}` });
+    } else if (UNREAD_MANIFESTS.test(file)) {
+      add({ ecosystem: "manifest", name: c.path, version: null, source: "a manifest Trellis does not read yet" });
+    } else if (REGISTRY_SETTINGS.has(file)) {
+      add({ ecosystem: "settings", name: c.path, version: null, source: "registry settings: where packages come from" });
+    } else if (UNREAD_LOCKS.has(file)) {
+      add({ ecosystem: "lockfile", name: c.path, version: null, source: "a lockfile Trellis does not read yet" });
+    } else if (file in OTHER_ECOSYSTEMS) {
+      const ecosystem = OTHER_ECOSYSTEMS[/** @type {keyof typeof OTHER_ECOSYSTEMS} */ (file)];
+      for (const name of addedDependencies(c.path, blobText(root, c.oldBlob), blobText(root, c.newBlob))) add({ ecosystem, name, version: null });
+    }
+  }
+  return [...found.values()];
+}
+
+/** A Go module path as the module proxy wants it: each capital as `!` and its lower case. @param {string} path */
+export const goEscape = (path) => path.replace(/[A-Z]/g, (c) => `!${c.toLowerCase()}`);
+
+/** @typedef {(url: string, init?: RequestInit) => Promise<{ status: number, json: () => Promise<any> }>} Http */
+
+/** @type {Http} */
+const defaultHttp = (url, init) => fetch(url, { ...init, signal: AbortSignal.timeout(15_000) });
+
+/** Thrown when a registry or OSV did not answer: the check is unavailable, not passed. */
+class Unreachable extends Error {}
+
+/** @param {Http} http @param {string} url @param {RequestInit} [init] */
+async function get(http, url, init) {
+  let r;
+  try {
+    r = await http(url, init);
+  } catch (e) {
+    throw new Unreachable(`${new URL(url).host} could not be reached (${/** @type {Error} */ (e).message})`);
+  }
+  if (r.status === 404 || r.status === 410) return null;
+  if (r.status !== 200) throw new Unreachable(`${new URL(url).host} answered ${r.status}`);
+  return r.json();
+}
+
+/**
+ * Whether the package exists and the version is published, for the registries Trellis reads.
+ * @param {Http} http
+ * @param {Dependency} d
+ * @returns {Promise<string | null | undefined>}  the problem, null when fine, undefined when not checked
+ */
+async function registryProblem(http, d) {
+  if (d.ecosystem === "npm") {
+    const doc = await get(http, `https://registry.npmjs.org/${encodeURIComponent(d.name).replace(/^%40/, "@")}`, { headers: { accept: "application/vnd.npm.install-v1+json" } });
+    if (!doc) return "does not exist on npm";
+    if (d.version && !(d.version in (doc.versions ?? {}))) return "version not published";
+    // The lockfile's hash must be npm's own for that version, or the tarball is not npm's package.
+    const published = d.version ? doc.versions?.[d.version]?.dist?.integrity : null;
+    return d.integrity && published && d.integrity !== published ? "its lockfile integrity is not the registry's: the tarball installed is not this package" : null;
+  }
+  if (d.ecosystem === "PyPI") {
+    const doc = await get(http, `https://pypi.org/pypi/${encodeURIComponent(d.name)}/json`);
+    if (!doc) return "does not exist on PyPI";
+    return d.version && !(doc.releases?.[d.version]?.length) ? "version not published" : null;
+  }
+  if (d.ecosystem === "Go" && d.version) {
+    const info = await get(http, `https://proxy.golang.org/${goEscape(d.name)}/@v/${encodeURIComponent(d.version)}.info`);
+    return info ? null : "module version not found on the Go module proxy";
+  }
+  return undefined;
+}
+
+/**
+ * Checks dependencies: existence and published version on their registry, and known
+ * vulnerabilities of exact versions in one OSV batch.
+ * @param {Dependency[]} deps
+ * @param {{ http?: Http }} [opts]
+ * @returns {Promise<DependencyCheck>}
+ */
+export async function checkDependencies(deps, opts = {}) {
+  const http = opts.http ?? defaultHttp;
+  /** @type {DependencyFinding[]} */
+  const findings = [];
+  /** @type {Set<string>} */
+  const unchecked = new Set();
+  const unreadable = deps.filter((d) => d.ecosystem === "unreadable");
+  if (unreadable.length) return { status: "unavailable", findings: [], unpinned: [], unchecked: [], detail: `${unreadable.map((d) => d.name).join(", ")} could not be read (not valid JSON), so the dependencies it holds were not checked` };
+  // A package fetched from elsewhere is not the registry's package of that name: neither is looked up.
+  for (const d of deps) if (d.source) unchecked.add(`${d.name} (from ${d.source})`);
+  const fromRegistry = deps.filter((d) => !d.source);
+  try {
+    const problems = await Promise.all(fromRegistry.map((d) => registryProblem(http, d)));
+    problems.forEach((p, i) => {
+      if (p === undefined) unchecked.add(`${fromRegistry[i].name} (${fromRegistry[i].ecosystem})`);
+      if (p) findings.push({ ...fromRegistry[i], problem: p });
+    });
+    const missing = new Set(findings.map((f) => `${f.ecosystem}:${f.name}`));
+    const exact = fromRegistry.filter((d) => d.version && !missing.has(`${d.ecosystem}:${d.name}`));
+    if (exact.length) {
+      const queries = exact.map((d) => ({ package: { name: d.name, ecosystem: d.ecosystem }, version: d.version }));
+      const res = await get(http, "https://api.osv.dev/v1/querybatch", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ queries }) });
+      if (!res || !Array.isArray(res.results) || res.results.length !== exact.length) throw new Unreachable("OSV's answer could not be read");
+      res.results.forEach((/** @type {any} */ r, /** @type {number} */ i) => {
+        const ids = (r?.vulns ?? []).map((/** @type {any} */ v) => v.id);
+        if (ids.length) findings.push({ ...exact[i], problem: `vulnerable (${ids.join(", ")})` });
+      });
+    }
+  } catch (e) {
+    if (e instanceof Unreachable) return { status: "unavailable", findings, unpinned: [], unchecked: [...unchecked], detail: e.message };
+    throw e;
+  }
+  // What comes from elsewhere is named under unchecked, not as merely unpinned.
+  const unpinned = deps.filter((d) => d.version == null && !d.source && !findings.some((f) => f.name === d.name)).map((d) => d.name);
+  return { status: findings.length ? "findings" : "clean", findings, unpinned, unchecked: [...unchecked] };
+}