@better-internet/oss-verify 0.1.1 → 0.1.3

package/dist/src/checks/license-text.js

@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: MIT
+//
+// Heuristic license detection from LICENSE / LICENCE / COPYING body text.
+// Used as a fallback when a project declares its license via the file content
+// alone (no SPDX-License-Identifier header, no package.json license field).
+//
+// Order matters: more specific patterns must come first (AGPL contains the
+// GPL phrase, BSD-3-Clause includes the BSD-2 boilerplate, etc.).
+//
+// This isn't a replacement for full license-scanning tooling (licensee,
+// ScanCode). It catches the ~10 most common OSI licenses with high precision
+// — enough to keep checkOsiLicense and checkReuse from emitting false
+// negatives against the majority of real-world OSS repos.
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+const LICENSE_FILES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "LICENCE", "COPYING"];
+const DETECTORS = [
+    // AGPL must beat the GPL detector below (AGPL text contains "GNU GENERAL
+    // PUBLIC LICENSE" as a referenced phrase).
+    {
+        spdx: "AGPL-3.0-only",
+        matches: (b) => /GNU AFFERO GENERAL PUBLIC LICENSE/i.test(b) && /Version 3/i.test(b),
+    },
+    {
+        spdx: "GPL-3.0-only",
+        matches: (b) => /GNU GENERAL PUBLIC LICENSE/i.test(b) &&
+            /Version 3/i.test(b) &&
+            !/AFFERO/i.test(b) &&
+            !/LESSER/i.test(b),
+    },
+    {
+        spdx: "GPL-2.0-only",
+        matches: (b) => /GNU GENERAL PUBLIC LICENSE/i.test(b) && /Version 2/i.test(b) && !/LESSER/i.test(b),
+    },
+    {
+        spdx: "LGPL-3.0-only",
+        matches: (b) => /GNU LESSER GENERAL PUBLIC LICENSE/i.test(b) && /Version 3/i.test(b),
+    },
+    {
+        spdx: "LGPL-2.1-only",
+        matches: (b) => /GNU LESSER GENERAL PUBLIC LICENSE/i.test(b) && /Version 2\.1/i.test(b),
+    },
+    {
+        spdx: "Apache-2.0",
+        matches: (b) => /Apache License/i.test(b) && /Version 2\.0/i.test(b),
+    },
+    {
+        spdx: "MPL-2.0",
+        matches: (b) => /Mozilla Public License/i.test(b) && /Version 2\.0/i.test(b),
+    },
+    {
+        spdx: "BSD-3-Clause",
+        matches: (b) => /Redistribution and use/i.test(b) && /Neither the name/i.test(b),
+    },
+    {
+        spdx: "BSD-2-Clause",
+        matches: (b) => /Redistribution and use/i.test(b) && !/Neither the name/i.test(b),
+    },
+    {
+        spdx: "MIT",
+        matches: (b) => /Permission is hereby granted, free of charge/i.test(b) && /MERCHANTABILITY/i.test(b),
+    },
+    {
+        spdx: "ISC",
+        matches: (b) => /Permission to use, copy, modify, and\/or distribute/i.test(b),
+    },
+    {
+        spdx: "Unlicense",
+        matches: (b) => /This is free and unencumbered software released into the public domain/i.test(b),
+    },
+];
+/**
+ * Returns the SPDX identifier of the first detector that matches the LICENSE
+ * body, or null when nothing recognisable is present.
+ */
+export function detectLicenseFromText(body) {
+    for (const d of DETECTORS) {
+        if (d.matches(body))
+            return d.spdx;
+    }
+    return null;
+}
+/**
+ * Returns the SPDX identifier detected from the repo's root LICENSE file (or
+ * its common variants), or null if no LICENSE file exists or its content
+ * doesn't match any known license.
+ */
+export function detectRootLicense(repoRoot) {
+    for (const name of LICENSE_FILES) {
+        const p = join(repoRoot, name);
+        if (!existsSync(p))
+            continue;
+        try {
+            const body = readFileSync(p, "utf8").slice(0, 16384);
+            const id = detectLicenseFromText(body);
+            if (id)
+                return id;
+        }
+        catch { }
+    }
+    return null;
+}
+/**
+ * True iff the repo has *any* license declaration our checks can recognise:
+ * a root LICENSE we can text-detect, an SPDX-License-Identifier in a root
+ * license file, or a package.json license field. Used by checkReuse to
+ * decide whether a missing per-file SPDX header is a real problem.
+ */
+export function hasAnyLicenseDeclaration(repoRoot) {
+    if (detectRootLicense(repoRoot))
+        return true;
+    // SPDX header in a root LICENSE file
+    for (const name of LICENSE_FILES) {
+        const p = join(repoRoot, name);
+        if (!existsSync(p))
+            continue;
+        try {
+            const head = readFileSync(p, "utf8").slice(0, 8192);
+            if (/SPDX-License-Identifier:/i.test(head))
+                return true;
+        }
+        catch { }
+    }
+    // package.json license field
+    const pkgPath = join(repoRoot, "package.json");
+    if (existsSync(pkgPath)) {
+        try {
+            const pkg = JSON.parse(readFileSync(pkgPath, "utf8"));
+            if (pkg.license && pkg.license !== "UNLICENSED")
+                return true;
+        }
+        catch { }
+    }
+    return false;
+}

package/dist/src/checks/llm-audit.js

@@ -62,6 +62,15 @@ export async function runLlmAudit(ctx, opts) {
     // SPEC §7.4: three independent calls at temperature=0; majority verdict wins.
     // "Block" must be a strict majority — a 1:1:1 outcome (one of each + an error
     // or unparseable) defaults to block, since the audit is a veto layer.
+    //
+    // The three calls run *sequentially* rather than in parallel. Parallel
+    // firing creates a 3x burst within ~1s against Anthropic's per-minute
+    // token rate limit (30k tpm on default tier). Posthog-sized envelopes
+    // (~10k input tokens × 3) reliably tripped that ceiling. Sequential adds
+    // ~20s of wallclock per project — invisible for an offline watchlist —
+    // and lets the per-minute window relax between passes. callAnthropic
+    // also retries with backoff on 429, so even sequential bursts that
+    // exceed the limit recover instead of failing.
     const apiKey = opts.apiKey;
     const callOnce = () => callAnthropic({
         modelId: opts.modelId,
@@ -70,7 +79,9 @@ export async function runLlmAudit(ctx, opts) {
         system: SYSTEM_PROMPT,
         envelope: envelope.text,
     });
-    const verdicts = await Promise.all([callOnce(), callOnce(), callOnce()]);
+    const verdicts = [];
+    for (let i = 0; i < 3; i++)
+        verdicts.push(await callOnce());
     const verdict = majorityVerdict(verdicts);
     return { verdict, promptHash, modelId: opts.modelId };
 }
@@ -141,47 +152,79 @@ function containsNul(buf, max) {
             return true;
     return false;
 }
+// Retry policy for transient Anthropic failures. 429 (rate-limit) and 5xx
+// (gateway / server) are retryable; 4xx-except-429 (auth, bad request, etc.)
+// are not. Honors `retry-after` (seconds) and `retry-after-ms` headers when
+// present; otherwise exponential backoff capped at 30s. Max 4 attempts.
+const MAX_RETRIES = 3;
+const BASE_DELAY_MS = 1000;
+const MAX_DELAY_MS = 30_000;
+function isRetryable(status) {
+    return status === 429 || (status >= 500 && status < 600);
+}
+function backoffDelay(attempt, res) {
+    const ms = res.headers.get("retry-after-ms");
+    if (ms && /^\d+$/.test(ms))
+        return Math.min(Number(ms), MAX_DELAY_MS);
+    const sec = res.headers.get("retry-after");
+    if (sec && /^\d+$/.test(sec))
+        return Math.min(Number(sec) * 1000, MAX_DELAY_MS);
+    // Exponential with full jitter — 1s, 2s, 4s, 8s capped at 30s.
+    const exp = Math.min(BASE_DELAY_MS * 2 ** attempt, MAX_DELAY_MS);
+    return Math.floor(Math.random() * exp);
+}
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 async function callAnthropic(args) {
-    const res = await fetch(args.endpoint, {
-        method: "POST",
-        headers: {
-            "x-api-key": args.apiKey,
-            "anthropic-version": "2023-06-01",
-            "content-type": "application/json",
-        },
-        body: JSON.stringify({
-            model: args.modelId,
-            max_tokens: 256,
-            temperature: 0,
-            system: args.system,
-            messages: [{ role: "user", content: args.envelope }],
-        }),
+    const body = JSON.stringify({
+        model: args.modelId,
+        max_tokens: 256,
+        temperature: 0,
+        system: args.system,
+        messages: [{ role: "user", content: args.envelope }],
     });
-    if (!res.ok) {
-        const body = await res.text();
-        // Network/auth failure must BLOCK — we have no opinion if the audit
-        // didn't actually run, and the predicate must not be emitted on a
-        // silent fallback.
-        return {
-            verdict: "block",
-            rationale: `Anthropic API call failed (${res.status}): ${body.slice(0, 200)}`,
-            passes: 0,
-        };
-    }
-    const data = (await res.json());
-    const text = data.content?.find((b) => b.type === "text")?.text?.trim() ?? "";
-    const parsed = parseModelVerdict(text);
-    // Belt-and-braces: if the response.model field doesn't match what we
-    // asked for, block. Vendors can route to fallback models; we need the
-    // exact one for predicate integrity.
-    if (data.model && data.model !== args.modelId) {
-        return {
-            verdict: "block",
-            rationale: `response.model '${data.model}' != requested '${args.modelId}'`,
-            passes: 1,
-        };
+    let lastStatus = 0;
+    let lastBody = "";
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+        const res = await fetch(args.endpoint, {
+            method: "POST",
+            headers: {
+                "x-api-key": args.apiKey,
+                "anthropic-version": "2023-06-01",
+                "content-type": "application/json",
+            },
+            body,
+        });
+        if (res.ok) {
+            const data = (await res.json());
+            const text = data.content?.find((b) => b.type === "text")?.text?.trim() ?? "";
+            const parsed = parseModelVerdict(text);
+            // Belt-and-braces: if the response.model field doesn't match what we
+            // asked for, block. Vendors can route to fallback models; we need
+            // the exact one for predicate integrity.
+            if (data.model && data.model !== args.modelId) {
+                return {
+                    verdict: "block",
+                    rationale: `response.model '${data.model}' != requested '${args.modelId}'`,
+                    passes: 1,
+                };
+            }
+            return { ...parsed, passes: 1 };
+        }
+        lastStatus = res.status;
+        lastBody = await res.text();
+        if (!isRetryable(res.status) || attempt === MAX_RETRIES)
+            break;
+        await sleep(backoffDelay(attempt, res));
     }
-    return { ...parsed, passes: 1 };
+    // Network/auth failure must BLOCK — we have no opinion if the audit
+    // didn't actually run, and the predicate must not be emitted on a
+    // silent fallback. The "retried N times" suffix flags this as the
+    // outcome of a giving-up retry vs a single-shot failure.
+    return {
+        verdict: "block",
+        rationale: `Anthropic API call failed (${lastStatus}) after ${MAX_RETRIES + 1} attempts: ${lastBody.slice(0, 200)}`,
+        passes: 0,
+    };
 }
 function parseModelVerdict(text) {
     // Per the system prompt, the model returns one JSON object. Be forgiving

package/dist/src/checks/osi-license.js

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import parseSpdx from "spdx-expression-parse";
 import licenseIds from "spdx-license-ids" with { type: "json" };
 import { sha256Hex } from "../hash.js";
+import { detectRootLicense } from "./license-text.js";
 // OSI used to publish a JSON API at api.opensource.org/licenses; that's been
 // deprecated. SPDX maintains the canonical list of licenses with an
 // isOsiApproved field, refreshed when OSI approves new ones. Source of truth.
@@ -31,8 +32,9 @@ function readDeclaredLicense(repoRoot) {
     if (existsSync(pkgPath)) {
         try {
             const pkg = JSON.parse(readFileSync(pkgPath, "utf8"));
-            if (pkg.license && pkg.license !== "UNLICENSED")
-                return pkg.license;
+            if (pkg.license && pkg.license !== "UNLICENSED") {
+                return { spdx: pkg.license, source: "package.json" };
+            }
         }
         catch { }
     }
@@ -43,9 +45,16 @@ function readDeclaredLicense(repoRoot) {
             const head = readFileSync(p, "utf8").slice(0, 8192);
             const m = head.match(/SPDX-License-Identifier:\s*([A-Za-z0-9.+\-\s()]+)/);
             if (m)
-                return m[1].trim();
+                return { spdx: m[1].trim(), source: "spdx-header" };
         }
     }
+    // 3. Fall back to text-pattern detection from the LICENSE body. Many older
+    // OSS repos declare their license via the file content alone, with no
+    // SPDX header (e.g. GPL/AGPL/Apache/BSD preambles). Less precise than an
+    // explicit header but covers the long tail of real repos.
+    const detected = detectRootLicense(repoRoot);
+    if (detected)
+        return { spdx: detected, source: "text-match" };
     return null;
 }
 export function leafIdentifiers(expr) {
@@ -55,26 +64,31 @@ export function leafIdentifiers(expr) {
         return [...leafIdentifiers(expr.left), ...leafIdentifiers(expr.right)];
     return [];
 }
+const SOURCE_LABEL = {
+    "package.json": "package.json `license` field",
+    "spdx-header": "SPDX-License-Identifier header",
+    "text-match": "LICENSE text match",
+};
 export async function checkOsiLicense(ctx) {
     const declared = readDeclaredLicense(ctx.repoRoot);
     if (!declared) {
         return {
             result: {
                 pass: false,
-                details: "No declared license found. Looked at package.json `license` field and SPDX-License-Identifier headers in LICENSE/LICENCE/COPYING.",
+                details: "No declared license found. Looked at package.json `license` field, SPDX-License-Identifier headers in LICENSE/LICENCE/COPYING, and text-pattern detection against the LICENSE body.",
             },
             osiResponseHash: "",
         };
     }
     let parsed;
     try {
-        parsed = parseSpdx(declared);
+        parsed = parseSpdx(declared.spdx);
     }
     catch (e) {
         return {
             result: {
                 pass: false,
-                details: `Declared license '${declared}' is not a valid SPDX expression: ${e.message}`,
+                details: `Declared license '${declared.spdx}' is not a valid SPDX expression: ${e.message}`,
             },
             osiResponseHash: "",
         };
@@ -82,7 +96,10 @@ export async function checkOsiLicense(ctx) {
     const leaves = leafIdentifiers(parsed);
     if (leaves.length === 0) {
         return {
-            result: { pass: false, details: `Could not extract any SPDX identifiers from '${declared}'` },
+            result: {
+                pass: false,
+                details: `Could not extract any SPDX identifiers from '${declared.spdx}'`,
+            },
             osiResponseHash: "",
         };
     }
@@ -101,14 +118,15 @@ export async function checkOsiLicense(ctx) {
     const unknownSpdx = leaves.filter((id) => !licenseIds.includes(id));
     if (nonOsi.length > 0) {
         const reason = unknownSpdx.length === leaves.length
-            ? `'${declared}' contains identifiers not in the SPDX license list: ${unknownSpdx.join(", ")}`
-            : `'${declared}' contains non-OSI-approved identifiers: ${nonOsi.join(", ")}`;
+            ? `'${declared.spdx}' contains identifiers not in the SPDX license list: ${unknownSpdx.join(", ")}`
+            : `'${declared.spdx}' contains non-OSI-approved identifiers: ${nonOsi.join(", ")}`;
         return { result: { pass: false, details: reason }, osiResponseHash: osi.hash };
     }
+    const sourceNote = declared.source === "text-match" ? ` (detected via ${SOURCE_LABEL[declared.source]})` : "";
     return {
         result: {
             pass: true,
-            details: `Declared '${declared}' resolves to OSI-approved leaves: ${leaves.join(", ")}`,
+            details: `Declared '${declared.spdx}'${sourceNote} resolves to OSI-approved leaves: ${leaves.join(", ")}`,
         },
         osiResponseHash: osi.hash,
     };

package/dist/src/checks/reuse.js

@@ -1,6 +1,7 @@
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { lsFiles } from "../git.js";
+import { hasAnyLicenseDeclaration } from "./license-text.js";
 // Files that don't need a license header.
 //   - License files themselves (the license declaration itself)
 //   - Common config files that are factually un-copyrightable
@@ -39,7 +40,36 @@ const looksBinary = (buf) => {
     return false;
 };
 const skip = (path) => SKIP_PATTERNS.some((re) => re.test(path));
+/**
+ * SPEC §3.1 "REUSE compliance". The REUSE standard itself accepts three valid
+ * declaration patterns; we recognise all three:
+ *
+ *   1. Per-file SPDX-License-Identifier headers across every source file
+ *      (strict REUSE).
+ *   2. A repo-level .reuse/dep5 or REUSE.toml file (REUSE's own blanket-
+ *      declaration mechanism). We don't parse the file's content — its
+ *      presence indicates the maintainer has opted into REUSE format.
+ *   3. A root LICENSE / LICENCE / COPYING file with a recognisable license
+ *      declaration, in the absence of a REUSE-format file. This is the
+ *      common case for projects that declare one license repo-wide without
+ *      using REUSE-style per-file headers.
+ *
+ * Only patterns (1) and (2) are strictly "REUSE-compliant" per the spec;
+ * pattern (3) is the pragmatic recognition that a project with a single
+ * top-level license has made an unambiguous declaration without going
+ * through REUSE's per-file ceremony. Treating (3) as a soft fail (or
+ * blanket pass with a note) avoids 100% false-positive rates on the
+ * majority of real OSS repos.
+ */
 export function checkReuse(ctx) {
+    const hasReuseFormat = existsSync(join(ctx.repoRoot, ".reuse", "dep5")) ||
+        existsSync(join(ctx.repoRoot, "REUSE.toml"));
+    if (hasReuseFormat) {
+        return {
+            pass: true,
+            details: "Project uses REUSE-format declarations (.reuse/dep5 or REUSE.toml). Per-file SPDX headers not required.",
+        };
+    }
     const files = lsFiles(ctx.repoRoot);
     const missing = [];
     let checked = 0;
@@ -69,10 +99,19 @@ export function checkReuse(ctx) {
             details: `${checked} text files all carry SPDX-License-Identifier headers`,
         };
     }
+    // No per-file SPDX headers, no REUSE-format file. Fall back to "is there
+    // a recognisable repo-level declaration?" If yes, accept it as a blanket
+    // declaration; if no, this is a real REUSE gap.
+    if (hasAnyLicenseDeclaration(ctx.repoRoot)) {
+        return {
+            pass: true,
+            details: `${missing.length} of ${checked} source files lack per-file SPDX headers, but a repo-level license declaration (LICENSE file or package.json) is present. Accepted as a blanket declaration.`,
+        };
+    }
     const sample = missing.slice(0, 10);
     const more = missing.length > sample.length ? ` (+${missing.length - sample.length} more)` : "";
     return {
         pass: false,
-        details: `${missing.length} of ${checked} text files missing SPDX-License-Identifier:\n  - ${sample.join("\n  - ")}${more}\n\nNote: this MVP doesn't yet honor .reuse/dep5 or REUSE.toml exemptions; that's a known limitation.`,
+        details: `${missing.length} of ${checked} source files missing SPDX-License-Identifier and no repo-level license declaration was found:\n  - ${sample.join("\n  - ")}${more}`,
     };
 }

package/dist/src/checks/sbom.js

@@ -47,10 +47,20 @@ export async function checkSbom(ctx) {
     const sbom = buildCycloneDx(ctx, meta, allComponents);
     const sbomHash = sha256Hex(canonicalJson(sbom));
     if (allMissing.length > 0) {
+        // "Unresolved" = the registry/index lookup failed for this dependency
+        // (e.g. unpublished Go module on deps.dev, custom forked package, or
+        // transient network failure). Distinct from "found a non-OSI license":
+        // these may well be OSI-licensed but we can't confirm. SPEC §3.3
+        // requires us to be able to verify *every* dependency's license, so
+        // this still fails the check — but the details now make it clear this
+        // is a resolution gap, not a confirmed violation, and re-running may
+        // succeed (registry mirror, package republished, etc.).
+        const detName = (s) => s.split("@")[0];
+        const ecosystems = detections.map((d) => d.ecosystem).join("+");
         return {
             result: {
                 pass: false,
-                details: `${allMissing.length} unresolved entr${allMissing.length === 1 ? "y" : "ies"}: ${allMissing.slice(0, 5).join(" | ")}${allMissing.length > 5 ? `, +${allMissing.length - 5} more` : ""}`,
+                details: `${allMissing.length} dependenc${allMissing.length === 1 ? "y" : "ies"} (${ecosystems}) had no resolvable license — registry lookup failed (retry-eligible; these may be OSI-licensed but we can't confirm):\n  - ${allMissing.slice(0, 10).map(detName).join("\n  - ")}${allMissing.length > 10 ? `\n  +${allMissing.length - 10} more` : ""}`,
             },
             sbomHash,
             sbomFormat: "cyclonedx-1.5",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@better-internet/oss-verify",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "private": false,
   "publishConfig": {
     "access": "public"