@better-internet/oss-verify 0.1.0-draft

package/dist/src/checks/reuse.js

@@ -0,0 +1,78 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { lsFiles } from "../git.js";
+// Files that don't need a license header.
+//   - License files themselves (the license declaration itself)
+//   - Common config files that are factually un-copyrightable
+//   - Generated / lock files
+const SKIP_PATTERNS = [
+    /^LICENSE(\..+)?$/i,
+    /^LICENCE(\..+)?$/i,
+    /^COPYING(\..+)?$/i,
+    /^NOTICE(\..+)?$/i,
+    /^\.gitignore$/,
+    /^\.gitattributes$/,
+    /^\.editorconfig$/,
+    /^\.npmrc$/,
+    /^\.nvmrc$/,
+    /^\.tool-versions$/,
+    /(^|\/)pnpm-lock\.yaml$/,
+    /(^|\/)package-lock\.json$/,
+    /(^|\/)yarn\.lock$/,
+    /(^|\/)bun\.lock(b)?$/,
+    /(^|\/)Cargo\.lock$/,
+    /(^|\/)go\.sum$/,
+    /(^|\/)poetry\.lock$/,
+    /\.terraform\.lock\.hcl$/,
+    // JSON has no comment syntax — cannot carry an inline SPDX header. Real
+    // REUSE-compliant projects register these via .reuse/dep5 or REUSE.toml;
+    // MVP CLI doesn't yet parse those, so we skip JSON to avoid false negatives.
+    /\.json$/,
+    /\.jsonc$/,
+];
+const SPDX_HEADER_RE = /SPDX-License-Identifier:\s*([A-Za-z0-9.+\-\s()]+)/;
+const looksBinary = (buf) => {
+    const max = Math.min(buf.length, 4096);
+    for (let i = 0; i < max; i++)
+        if (buf[i] === 0)
+            return true;
+    return false;
+};
+const skip = (path) => SKIP_PATTERNS.some((re) => re.test(path));
+export function checkReuse(ctx) {
+    const files = lsFiles(ctx.repoRoot);
+    const missing = [];
+    let checked = 0;
+    for (const rel of files) {
+        if (skip(rel))
+            continue;
+        const abs = join(ctx.repoRoot, rel);
+        let buf;
+        try {
+            buf = readFileSync(abs);
+        }
+        catch {
+            continue; // symlink to non-file, etc.
+        }
+        if (looksBinary(buf))
+            continue;
+        checked++;
+        // Check the first ~30 lines (or 8 KB) for an SPDX header.
+        const head = buf.subarray(0, 8192).toString("utf8");
+        if (!SPDX_HEADER_RE.test(head)) {
+            missing.push(rel);
+        }
+    }
+    if (missing.length === 0) {
+        return {
+            pass: true,
+            details: `${checked} text files all carry SPDX-License-Identifier headers`,
+        };
+    }
+    const sample = missing.slice(0, 10);
+    const more = missing.length > sample.length ? ` (+${missing.length - sample.length} more)` : "";
+    return {
+        pass: false,
+        details: `${missing.length} of ${checked} text files missing SPDX-License-Identifier:\n  - ${sample.join("\n  - ")}${more}\n\nNote: this MVP doesn't yet honor .reuse/dep5 or REUSE.toml exemptions; that's a known limitation.`,
+    };
+}

package/dist/src/checks/sbom/cargo.js

@@ -0,0 +1,124 @@
+// Cargo (Rust) ecosystem detector.
+//
+// Cargo.lock is the source of truth for the resolved dependency graph. It's
+// a TOML file with `[[package]]` entries. We parse the minimal subset we
+// need (name + version + dependencies) without pulling in a TOML library.
+//
+// Per-crate license metadata isn't in Cargo.lock — we have to look it up
+// from the crates.io API:
+//   GET https://crates.io/api/v1/crates/<name>/<version>
+//   -> { "version": { "license": "MIT OR Apache-2.0", ... } }
+//
+// Lookups are concurrency-limited (CRATESIO_CONCURRENCY) and memoised by
+// (name, version) so re-runs are fast and crates.io stays happy.
+//
+// Scope: includes ALL packages from Cargo.lock — Rust's dev-dependency
+// distinction lives in Cargo.toml not Cargo.lock, and parsing the dep tree
+// to filter would require a real resolver. SPEC §3.3 says runtime deps only,
+// so this is conservative: we may include a few dev/build deps in the
+// audited set. Fixing it requires shipping cargo-metadata output alongside,
+// which is out of scope for the MVP detector.
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+const CRATESIO = "https://crates.io/api/v1/crates";
+const CRATESIO_CONCURRENCY = 4;
+const USER_AGENT = "oss-verify/0.1 (https://github.com/better-internet-org/oss-verify)";
+export async function detect(ctx) {
+    const lockPath = join(ctx.repoRoot, "Cargo.lock");
+    if (!existsSync(lockPath))
+        return null;
+    let lockText;
+    try {
+        lockText = readFileSync(lockPath, "utf8");
+    }
+    catch (e) {
+        return {
+            ecosystem: "cargo",
+            components: [],
+            missing: [`Cargo.lock read failed: ${e.message}`],
+        };
+    }
+    const packages = parseCargoLock(lockText);
+    if (packages.length === 0) {
+        return { ecosystem: "cargo", components: [], missing: [], details: "no Cargo.lock packages" };
+    }
+    // Filter out the root package — Cargo.lock includes the project itself.
+    // Best signal: entries that have no `source` line are local (the workspace
+    // root or path-deps). Drop them so the SBOM is about EXTERNAL deps only.
+    const external = packages.filter((p) => p.source !== null);
+    const components = [];
+    const missing = [];
+    // Fetch licenses with a small concurrency limit.
+    const queue = [...external];
+    const workers = Array.from({ length: CRATESIO_CONCURRENCY }, async () => {
+        while (queue.length > 0) {
+            const p = queue.shift();
+            if (!p)
+                break;
+            const license = await fetchCrateLicense(p.name, p.version);
+            if (license === undefined) {
+                missing.push(`${p.name}@${p.version}`);
+                continue;
+            }
+            components.push({
+                name: p.name,
+                version: p.version,
+                license,
+                purl: `pkg:cargo/${p.name}@${p.version}`,
+            });
+        }
+    });
+    await Promise.all(workers);
+    components.sort((a, b) => a.name === b.name ? (a.version < b.version ? -1 : 1) : a.name < b.name ? -1 : 1);
+    return { ecosystem: "cargo", components, missing };
+}
+/**
+ * Minimal Cargo.lock parser. The format is TOML v3 with one top-level
+ * `[[package]]` array. We only need name/version/source. Lock files are
+ * machine-generated and the format is stable, so we lean on a simple
+ * regex rather than depending on a TOML library.
+ */
+function parseCargoLock(text) {
+    const packages = [];
+    const blocks = text.split(/^\[\[package\]\]\s*$/m);
+    for (let i = 1; i < blocks.length; i++) {
+        const block = blocks[i];
+        const name = matchField(block, "name");
+        const version = matchField(block, "version");
+        const source = matchField(block, "source");
+        if (name && version)
+            packages.push({ name, version, source });
+    }
+    return packages;
+}
+function matchField(block, field) {
+    const re = new RegExp(`^${field}\\s*=\\s*"([^"]+)"`, "m");
+    const m = block.match(re);
+    return m ? m[1] : null;
+}
+const licenseCache = new Map();
+async function fetchCrateLicense(name, version) {
+    const key = `${name}@${version}`;
+    if (licenseCache.has(key))
+        return licenseCache.get(key);
+    try {
+        const url = `${CRATESIO}/${encodeURIComponent(name)}/${encodeURIComponent(version)}`;
+        const res = await fetch(url, {
+            headers: { accept: "application/json", "user-agent": USER_AGENT },
+        });
+        if (!res.ok) {
+            licenseCache.set(key, undefined);
+            return undefined;
+        }
+        const data = (await res.json());
+        const license = data.version?.license?.trim();
+        const result = license && license.length > 0 ? license : null;
+        // crates.io returns a SPDX expression in `version.license` (since 2020).
+        licenseCache.set(key, result ?? undefined);
+        return result ?? undefined;
+    }
+    catch {
+        licenseCache.set(key, undefined);
+        return undefined;
+    }
+}

package/dist/src/checks/sbom/go.js

@@ -0,0 +1,137 @@
+// Go modules ecosystem detector.
+//
+// Strategy:
+//   1. Read go.mod to find the project's own `module` path (so we don't
+//      flag self-references as missing).
+//   2. Parse go.sum to enumerate every (module, version) pair Go's
+//      resolver pinned. go.sum lists each dep twice (content hash and
+//      go.mod hash); we dedupe.
+//   3. Look up the license for each via deps.dev v3:
+//        GET https://api.deps.dev/v3/systems/GO/packages/<urlencoded-name>/versions/<version>
+//        -> { "licenses": ["MIT", "Apache-2.0"], ... }
+//
+// Lookups are concurrency-limited (DEPSDEV_CONCURRENCY) and memoised by
+// (name, version) so re-runs are fast.
+//
+// Scope note: go.sum doesn't distinguish runtime vs test/build deps. The
+// SPEC §3.3 requirement is runtime-only, but accurate filtering requires
+// `go list -m -json all` or `go mod why` output that's not in the lockfile.
+// We err conservative: audit all modules in go.sum. If a test-only dep
+// has a non-OSI license, the criterion fails. The remedy for affected
+// projects is to remove or replace the offending dep — strictly correct
+// per the SPEC; mildly inconvenient in edge cases.
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+const DEPSDEV = "https://api.deps.dev/v3/systems/GO/packages";
+const DEPSDEV_CONCURRENCY = 4;
+const USER_AGENT = "oss-verify/0.1 (https://github.com/better-internet-org/oss-verify)";
+export async function detect(ctx) {
+    const sumPath = join(ctx.repoRoot, "go.sum");
+    const modPath = join(ctx.repoRoot, "go.mod");
+    if (!existsSync(modPath))
+        return null;
+    // Empty go.sum is legal if the project has no deps at all.
+    if (!existsSync(sumPath)) {
+        return { ecosystem: "go", components: [], missing: [], details: "no go.sum (no deps)" };
+    }
+    const selfModule = readGoMod(modPath);
+    const sumText = readFileSync(sumPath, "utf8");
+    const pairs = parseGoSum(sumText, selfModule);
+    if (pairs.length === 0) {
+        return { ecosystem: "go", components: [], missing: [], details: "no go.sum entries" };
+    }
+    const components = [];
+    const missing = [];
+    const queue = [...pairs];
+    const workers = Array.from({ length: DEPSDEV_CONCURRENCY }, async () => {
+        while (queue.length > 0) {
+            const p = queue.shift();
+            if (!p)
+                break;
+            const license = await fetchGoLicense(p.name, p.version);
+            if (license === undefined) {
+                missing.push(`${p.name}@${p.version}`);
+                continue;
+            }
+            components.push({
+                name: p.name,
+                version: p.version,
+                license,
+                purl: `pkg:golang/${p.name.replace(/^v\d+$/, "")}@${p.version}`,
+            });
+        }
+    });
+    await Promise.all(workers);
+    components.sort((a, b) => a.name === b.name ? (a.version < b.version ? -1 : 1) : a.name < b.name ? -1 : 1);
+    return { ecosystem: "go", components, missing };
+}
+function readGoMod(modPath) {
+    try {
+        const text = readFileSync(modPath, "utf8");
+        const m = text.match(/^\s*module\s+(\S+)/m);
+        return m ? m[1] : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Parse go.sum into a deduplicated (name, version) list. Format:
+ *   <module>  <version>      h1:<base64-hash>
+ *   <module>  <version>/go.mod  h1:<base64-hash>
+ * Pre-release versions have suffixes like `-pre.0`, pseudo-versions look
+ * like `v0.0.0-<timestamp>-<sha>`. We don't try to normalise — deps.dev
+ * accepts the raw string back.
+ */
+function parseGoSum(text, selfModule) {
+    const seen = new Set();
+    const out = [];
+    for (const line of text.split("\n")) {
+        const parts = line.trim().split(/\s+/);
+        if (parts.length < 3)
+            continue;
+        const name = parts[0];
+        // Strip `/go.mod` suffix on version lines so we dedupe.
+        const version = parts[1].endsWith("/go.mod") ? parts[1].slice(0, -7) : parts[1];
+        if (selfModule && (name === selfModule || name.startsWith(`${selfModule}/`)))
+            continue;
+        const key = `${name}@${version}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push({ name, version });
+    }
+    return out;
+}
+const licenseCache = new Map();
+async function fetchGoLicense(name, version) {
+    const key = `${name}@${version}`;
+    if (licenseCache.has(key))
+        return licenseCache.get(key);
+    try {
+        const url = `${DEPSDEV}/${encodeURIComponent(name)}/versions/${encodeURIComponent(version)}`;
+        const res = await fetch(url, {
+            headers: { accept: "application/json", "user-agent": USER_AGENT },
+        });
+        if (!res.ok) {
+            licenseCache.set(key, undefined);
+            return undefined;
+        }
+        const data = (await res.json());
+        const licenses = (data.licenses ?? []).filter((s) => typeof s === "string" && s.length > 0);
+        if (licenses.length === 0) {
+            licenseCache.set(key, undefined);
+            return undefined;
+        }
+        // Convert deps.dev's array to a SPDX expression. Most Go modules
+        // declare a single license; multi-license entries get joined with OR
+        // since Go has no metadata to express AND-style co-licensing.
+        const result = licenses.length === 1 ? licenses[0] : `(${licenses.join(" OR ")})`;
+        licenseCache.set(key, result);
+        return result;
+    }
+    catch {
+        licenseCache.set(key, undefined);
+        return undefined;
+    }
+}

package/dist/src/checks/sbom/javascript.js

@@ -0,0 +1,125 @@
+// JavaScript / Node ecosystem detector.
+//
+// Walks node_modules from the root package.json's `dependencies` (runtime
+// only) using Node's resolution algorithm. Pulls licenses out of each
+// dep's installed package.json so no network calls or registry lookups
+// are needed.
+//
+// Requires `pnpm install` / `npm install` to have run in the consumer
+// project before the CLI runs — direct + transitive deps must be on disk.
+import { existsSync, readFileSync, realpathSync } from "node:fs";
+import { dirname, join } from "node:path";
+export async function detect(ctx) {
+    const rootPkgPath = join(ctx.repoRoot, "package.json");
+    if (!existsSync(rootPkgPath))
+        return null;
+    let rootPkg;
+    try {
+        rootPkg = JSON.parse(readFileSync(rootPkgPath, "utf8"));
+    }
+    catch (e) {
+        return {
+            ecosystem: "javascript",
+            components: [],
+            missing: [`package.json parse failed: ${e.message}`],
+        };
+    }
+    const directDeps = Object.keys(rootPkg.dependencies ?? {});
+    if (directDeps.length === 0) {
+        return { ecosystem: "javascript", components: [], missing: [], details: "no runtime deps" };
+    }
+    const visited = new Map();
+    const missing = [];
+    const queue = directDeps.map((name) => ({ name, requestedFrom: ctx.repoRoot }));
+    while (queue.length > 0) {
+        const entry = queue.shift();
+        if (entry === undefined)
+            break;
+        const { name, requestedFrom } = entry;
+        const pkgJsonPath = resolvePackageJson(requestedFrom, name);
+        if (!pkgJsonPath) {
+            if (!missing.includes(name))
+                missing.push(name);
+            continue;
+        }
+        let pkg;
+        try {
+            pkg = JSON.parse(readFileSync(pkgJsonPath, "utf8"));
+        }
+        catch {
+            if (!missing.includes(name))
+                missing.push(name);
+            continue;
+        }
+        const version = pkg.version ?? "0.0.0";
+        const key = `${name}@${version}`;
+        if (visited.has(key))
+            continue;
+        visited.set(key, {
+            name,
+            version,
+            license: normaliseLicense(pkg.license ?? pkg.licenses),
+            purl: `pkg:npm/${name.replace(/^@/, "%40")}@${version}`,
+        });
+        const ownDir = dirname(pkgJsonPath);
+        for (const dep of Object.keys(pkg.dependencies ?? {})) {
+            queue.push({ name: dep, requestedFrom: ownDir });
+        }
+    }
+    return {
+        ecosystem: "javascript",
+        components: [...visited.values()].sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0),
+        missing,
+    };
+}
+/**
+ * Walk up from `fromDir` looking for `node_modules/<name>/package.json`,
+ * matching Node's own resolution algorithm. Handles pnpm strict mode
+ * (where each package only sees its declared deps via local node_modules)
+ * and npm's hoisted flat tree.
+ */
+function resolvePackageJson(fromDir, name) {
+    let dir = fromDir;
+    while (true) {
+        const candidate = join(dir, "node_modules", name, "package.json");
+        if (existsSync(candidate)) {
+            try {
+                return realpathSync(candidate);
+            }
+            catch {
+                return candidate;
+            }
+        }
+        const parent = dirname(dir);
+        if (parent === dir)
+            return null;
+        dir = parent;
+    }
+}
+const MAX_LICENSE_LEN = 256;
+function normaliseLicense(raw) {
+    if (typeof raw === "string") {
+        const s = raw.trim();
+        if (!s || s.length > MAX_LICENSE_LEN)
+            return null;
+        if (s.toUpperCase() === "UNLICENSED")
+            return null;
+        if (s.toUpperCase().startsWith("SEE LICENSE IN"))
+            return null;
+        return s;
+    }
+    if (raw && typeof raw === "object" && !Array.isArray(raw)) {
+        const t = raw.type;
+        if (typeof t === "string")
+            return normaliseLicense(t);
+    }
+    if (Array.isArray(raw)) {
+        const parts = raw
+            .map((r) => normaliseLicense(r.type))
+            .filter((v) => Boolean(v));
+        if (parts.length === 0)
+            return null;
+        return parts.length === 1 ? parts[0] : `(${parts.join(" OR ")})`;
+    }
+    return null;
+}