@better-internet/oss-verify 0.1.0-draft → 0.1.2

package/dist/src/checks/license-text.js

@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: MIT
+//
+// Heuristic license detection from LICENSE / LICENCE / COPYING body text.
+// Used as a fallback when a project declares its license via the file content
+// alone (no SPDX-License-Identifier header, no package.json license field).
+//
+// Order matters: more specific patterns must come first (AGPL contains the
+// GPL phrase, BSD-3-Clause includes the BSD-2 boilerplate, etc.).
+//
+// This isn't a replacement for full license-scanning tooling (licensee,
+// ScanCode). It catches the ~10 most common OSI licenses with high precision
+// — enough to keep checkOsiLicense and checkReuse from emitting false
+// negatives against the majority of real-world OSS repos.
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+const LICENSE_FILES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "LICENCE", "COPYING"];
+const DETECTORS = [
+    // AGPL must beat the GPL detector below (AGPL text contains "GNU GENERAL
+    // PUBLIC LICENSE" as a referenced phrase).
+    {
+        spdx: "AGPL-3.0-only",
+        matches: (b) => /GNU AFFERO GENERAL PUBLIC LICENSE/i.test(b) && /Version 3/i.test(b),
+    },
+    {
+        spdx: "GPL-3.0-only",
+        matches: (b) => /GNU GENERAL PUBLIC LICENSE/i.test(b) &&
+            /Version 3/i.test(b) &&
+            !/AFFERO/i.test(b) &&
+            !/LESSER/i.test(b),
+    },
+    {
+        spdx: "GPL-2.0-only",
+        matches: (b) => /GNU GENERAL PUBLIC LICENSE/i.test(b) && /Version 2/i.test(b) && !/LESSER/i.test(b),
+    },
+    {
+        spdx: "LGPL-3.0-only",
+        matches: (b) => /GNU LESSER GENERAL PUBLIC LICENSE/i.test(b) && /Version 3/i.test(b),
+    },
+    {
+        spdx: "LGPL-2.1-only",
+        matches: (b) => /GNU LESSER GENERAL PUBLIC LICENSE/i.test(b) && /Version 2\.1/i.test(b),
+    },
+    {
+        spdx: "Apache-2.0",
+        matches: (b) => /Apache License/i.test(b) && /Version 2\.0/i.test(b),
+    },
+    {
+        spdx: "MPL-2.0",
+        matches: (b) => /Mozilla Public License/i.test(b) && /Version 2\.0/i.test(b),
+    },
+    {
+        spdx: "BSD-3-Clause",
+        matches: (b) => /Redistribution and use/i.test(b) && /Neither the name/i.test(b),
+    },
+    {
+        spdx: "BSD-2-Clause",
+        matches: (b) => /Redistribution and use/i.test(b) && !/Neither the name/i.test(b),
+    },
+    {
+        spdx: "MIT",
+        matches: (b) => /Permission is hereby granted, free of charge/i.test(b) && /MERCHANTABILITY/i.test(b),
+    },
+    {
+        spdx: "ISC",
+        matches: (b) => /Permission to use, copy, modify, and\/or distribute/i.test(b),
+    },
+    {
+        spdx: "Unlicense",
+        matches: (b) => /This is free and unencumbered software released into the public domain/i.test(b),
+    },
+];
+/**
+ * Returns the SPDX identifier of the first detector that matches the LICENSE
+ * body, or null when nothing recognisable is present.
+ */
+export function detectLicenseFromText(body) {
+    for (const d of DETECTORS) {
+        if (d.matches(body))
+            return d.spdx;
+    }
+    return null;
+}
+/**
+ * Returns the SPDX identifier detected from the repo's root LICENSE file (or
+ * its common variants), or null if no LICENSE file exists or its content
+ * doesn't match any known license.
+ */
+export function detectRootLicense(repoRoot) {
+    for (const name of LICENSE_FILES) {
+        const p = join(repoRoot, name);
+        if (!existsSync(p))
+            continue;
+        try {
+            const body = readFileSync(p, "utf8").slice(0, 16384);
+            const id = detectLicenseFromText(body);
+            if (id)
+                return id;
+        }
+        catch { }
+    }
+    return null;
+}
+/**
+ * True iff the repo has *any* license declaration our checks can recognise:
+ * a root LICENSE we can text-detect, an SPDX-License-Identifier in a root
+ * license file, or a package.json license field. Used by checkReuse to
+ * decide whether a missing per-file SPDX header is a real problem.
+ */
+export function hasAnyLicenseDeclaration(repoRoot) {
+    if (detectRootLicense(repoRoot))
+        return true;
+    // SPDX header in a root LICENSE file
+    for (const name of LICENSE_FILES) {
+        const p = join(repoRoot, name);
+        if (!existsSync(p))
+            continue;
+        try {
+            const head = readFileSync(p, "utf8").slice(0, 8192);
+            if (/SPDX-License-Identifier:/i.test(head))
+                return true;
+        }
+        catch { }
+    }
+    // package.json license field
+    const pkgPath = join(repoRoot, "package.json");
+    if (existsSync(pkgPath)) {
+        try {
+            const pkg = JSON.parse(readFileSync(pkgPath, "utf8"));
+            if (pkg.license && pkg.license !== "UNLICENSED")
+                return true;
+        }
+        catch { }
+    }
+    return false;
+}

package/dist/src/checks/osi-license.js

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import parseSpdx from "spdx-expression-parse";
 import licenseIds from "spdx-license-ids" with { type: "json" };
 import { sha256Hex } from "../hash.js";
+import { detectRootLicense } from "./license-text.js";
 // OSI used to publish a JSON API at api.opensource.org/licenses; that's been
 // deprecated. SPDX maintains the canonical list of licenses with an
 // isOsiApproved field, refreshed when OSI approves new ones. Source of truth.
@@ -31,8 +32,9 @@ function readDeclaredLicense(repoRoot) {
     if (existsSync(pkgPath)) {
         try {
             const pkg = JSON.parse(readFileSync(pkgPath, "utf8"));
-            if (pkg.license && pkg.license !== "UNLICENSED")
-                return pkg.license;
+            if (pkg.license && pkg.license !== "UNLICENSED") {
+                return { spdx: pkg.license, source: "package.json" };
+            }
         }
         catch { }
     }
@@ -43,9 +45,16 @@ function readDeclaredLicense(repoRoot) {
             const head = readFileSync(p, "utf8").slice(0, 8192);
             const m = head.match(/SPDX-License-Identifier:\s*([A-Za-z0-9.+\-\s()]+)/);
             if (m)
-                return m[1].trim();
+                return { spdx: m[1].trim(), source: "spdx-header" };
         }
     }
+    // 3. Fall back to text-pattern detection from the LICENSE body. Many older
+    // OSS repos declare their license via the file content alone, with no
+    // SPDX header (e.g. GPL/AGPL/Apache/BSD preambles). Less precise than an
+    // explicit header but covers the long tail of real repos.
+    const detected = detectRootLicense(repoRoot);
+    if (detected)
+        return { spdx: detected, source: "text-match" };
     return null;
 }
 export function leafIdentifiers(expr) {
@@ -55,26 +64,31 @@ export function leafIdentifiers(expr) {
         return [...leafIdentifiers(expr.left), ...leafIdentifiers(expr.right)];
     return [];
 }
+const SOURCE_LABEL = {
+    "package.json": "package.json `license` field",
+    "spdx-header": "SPDX-License-Identifier header",
+    "text-match": "LICENSE text match",
+};
 export async function checkOsiLicense(ctx) {
     const declared = readDeclaredLicense(ctx.repoRoot);
     if (!declared) {
         return {
             result: {
                 pass: false,
-                details: "No declared license found. Looked at package.json `license` field and SPDX-License-Identifier headers in LICENSE/LICENCE/COPYING.",
+                details: "No declared license found. Looked at package.json `license` field, SPDX-License-Identifier headers in LICENSE/LICENCE/COPYING, and text-pattern detection against the LICENSE body.",
             },
             osiResponseHash: "",
         };
     }
     let parsed;
     try {
-        parsed = parseSpdx(declared);
+        parsed = parseSpdx(declared.spdx);
     }
     catch (e) {
         return {
             result: {
                 pass: false,
-                details: `Declared license '${declared}' is not a valid SPDX expression: ${e.message}`,
+                details: `Declared license '${declared.spdx}' is not a valid SPDX expression: ${e.message}`,
             },
             osiResponseHash: "",
         };
@@ -82,7 +96,10 @@ export async function checkOsiLicense(ctx) {
     const leaves = leafIdentifiers(parsed);
     if (leaves.length === 0) {
         return {
-            result: { pass: false, details: `Could not extract any SPDX identifiers from '${declared}'` },
+            result: {
+                pass: false,
+                details: `Could not extract any SPDX identifiers from '${declared.spdx}'`,
+            },
             osiResponseHash: "",
         };
     }
@@ -101,14 +118,15 @@ export async function checkOsiLicense(ctx) {
     const unknownSpdx = leaves.filter((id) => !licenseIds.includes(id));
     if (nonOsi.length > 0) {
         const reason = unknownSpdx.length === leaves.length
-            ? `'${declared}' contains identifiers not in the SPDX license list: ${unknownSpdx.join(", ")}`
-            : `'${declared}' contains non-OSI-approved identifiers: ${nonOsi.join(", ")}`;
+            ? `'${declared.spdx}' contains identifiers not in the SPDX license list: ${unknownSpdx.join(", ")}`
+            : `'${declared.spdx}' contains non-OSI-approved identifiers: ${nonOsi.join(", ")}`;
         return { result: { pass: false, details: reason }, osiResponseHash: osi.hash };
     }
+    const sourceNote = declared.source === "text-match" ? ` (detected via ${SOURCE_LABEL[declared.source]})` : "";
     return {
         result: {
             pass: true,
-            details: `Declared '${declared}' resolves to OSI-approved leaves: ${leaves.join(", ")}`,
+            details: `Declared '${declared.spdx}'${sourceNote} resolves to OSI-approved leaves: ${leaves.join(", ")}`,
         },
         osiResponseHash: osi.hash,
     };

package/dist/src/checks/reuse.js

@@ -1,6 +1,7 @@
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { lsFiles } from "../git.js";
+import { hasAnyLicenseDeclaration } from "./license-text.js";
 // Files that don't need a license header.
 //   - License files themselves (the license declaration itself)
 //   - Common config files that are factually un-copyrightable
@@ -39,7 +40,36 @@ const looksBinary = (buf) => {
     return false;
 };
 const skip = (path) => SKIP_PATTERNS.some((re) => re.test(path));
+/**
+ * SPEC §3.1 "REUSE compliance". The REUSE standard itself accepts three valid
+ * declaration patterns; we recognise all three:
+ *
+ *   1. Per-file SPDX-License-Identifier headers across every source file
+ *      (strict REUSE).
+ *   2. A repo-level .reuse/dep5 or REUSE.toml file (REUSE's own blanket-
+ *      declaration mechanism). We don't parse the file's content — its
+ *      presence indicates the maintainer has opted into REUSE format.
+ *   3. A root LICENSE / LICENCE / COPYING file with a recognisable license
+ *      declaration, in the absence of a REUSE-format file. This is the
+ *      common case for projects that declare one license repo-wide without
+ *      using REUSE-style per-file headers.
+ *
+ * Only patterns (1) and (2) are strictly "REUSE-compliant" per the spec;
+ * pattern (3) is the pragmatic recognition that a project with a single
+ * top-level license has made an unambiguous declaration without going
+ * through REUSE's per-file ceremony. Treating (3) as a soft fail (or
+ * blanket pass with a note) avoids 100% false-positive rates on the
+ * majority of real OSS repos.
+ */
 export function checkReuse(ctx) {
+    const hasReuseFormat = existsSync(join(ctx.repoRoot, ".reuse", "dep5")) ||
+        existsSync(join(ctx.repoRoot, "REUSE.toml"));
+    if (hasReuseFormat) {
+        return {
+            pass: true,
+            details: "Project uses REUSE-format declarations (.reuse/dep5 or REUSE.toml). Per-file SPDX headers not required.",
+        };
+    }
     const files = lsFiles(ctx.repoRoot);
     const missing = [];
     let checked = 0;
@@ -69,10 +99,19 @@ export function checkReuse(ctx) {
             details: `${checked} text files all carry SPDX-License-Identifier headers`,
         };
     }
+    // No per-file SPDX headers, no REUSE-format file. Fall back to "is there
+    // a recognisable repo-level declaration?" If yes, accept it as a blanket
+    // declaration; if no, this is a real REUSE gap.
+    if (hasAnyLicenseDeclaration(ctx.repoRoot)) {
+        return {
+            pass: true,
+            details: `${missing.length} of ${checked} source files lack per-file SPDX headers, but a repo-level license declaration (LICENSE file or package.json) is present. Accepted as a blanket declaration.`,
+        };
+    }
     const sample = missing.slice(0, 10);
     const more = missing.length > sample.length ? ` (+${missing.length - sample.length} more)` : "";
     return {
         pass: false,
-        details: `${missing.length} of ${checked} text files missing SPDX-License-Identifier:\n  - ${sample.join("\n  - ")}${more}\n\nNote: this MVP doesn't yet honor .reuse/dep5 or REUSE.toml exemptions; that's a known limitation.`,
+        details: `${missing.length} of ${checked} source files missing SPDX-License-Identifier and no repo-level license declaration was found:\n  - ${sample.join("\n  - ")}${more}`,
     };
 }

package/dist/src/checks/sbom.js

@@ -47,10 +47,20 @@ export async function checkSbom(ctx) {
     const sbom = buildCycloneDx(ctx, meta, allComponents);
     const sbomHash = sha256Hex(canonicalJson(sbom));
     if (allMissing.length > 0) {
+        // "Unresolved" = the registry/index lookup failed for this dependency
+        // (e.g. unpublished Go module on deps.dev, custom forked package, or
+        // transient network failure). Distinct from "found a non-OSI license":
+        // these may well be OSI-licensed but we can't confirm. SPEC §3.3
+        // requires us to be able to verify *every* dependency's license, so
+        // this still fails the check — but the details now make it clear this
+        // is a resolution gap, not a confirmed violation, and re-running may
+        // succeed (registry mirror, package republished, etc.).
+        const detName = (s) => s.split("@")[0];
+        const ecosystems = detections.map((d) => d.ecosystem).join("+");
         return {
             result: {
                 pass: false,
-                details: `${allMissing.length} unresolved entr${allMissing.length === 1 ? "y" : "ies"}: ${allMissing.slice(0, 5).join(" | ")}${allMissing.length > 5 ? `, +${allMissing.length - 5} more` : ""}`,
+                details: `${allMissing.length} dependenc${allMissing.length === 1 ? "y" : "ies"} (${ecosystems}) had no resolvable license — registry lookup failed (retry-eligible; these may be OSI-licensed but we can't confirm):\n  - ${allMissing.slice(0, 10).map(detName).join("\n  - ")}${allMissing.length > 10 ? `\n  +${allMissing.length - 10} more` : ""}`,
             },
             sbomHash,
             sbomFormat: "cyclonedx-1.5",

package/dist/src/git.js

@@ -1,5 +1,14 @@
 import { execSync } from "node:child_process";
-const exec = (cmd, cwd) => execSync(cmd, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim();
+// 64 MiB stdout — `git ls-files` on a 21k-file repo (posthog) exceeds the
+// 1 MiB default and throws ENOBUFS before we can do anything with it; the
+// other helpers here are small but inherit the same setting for consistency.
+const MAX_BUFFER = 64 * 1024 * 1024;
+const exec = (cmd, cwd) => execSync(cmd, {
+    cwd,
+    encoding: "utf8",
+    stdio: ["ignore", "pipe", "pipe"],
+    maxBuffer: MAX_BUFFER,
+}).trim();
 export function commitSha(repoRoot) {
     return exec("git rev-parse HEAD", repoRoot);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@better-internet/oss-verify",
-  "version": "0.1.0-draft",
+  "version": "0.1.2",
   "private": false,
   "publishConfig": {
     "access": "public"