fad-checker 2.3.1 → 2.4.0

package/lib/provenance.js

@@ -0,0 +1,135 @@
+/**
+ * lib/provenance.js — scan-provenance / reproducibility manifest.
+ *
+ * A professional audit must be defensible and reproducible: a finding has to be
+ * explainable from the exact inputs that produced it, and a second auditor should be
+ * able to reproduce it. This module builds a manifest of WHAT fad-checker ran with —
+ * tool version, runtime, run mode (offline/online), the run configuration (only the
+ * flags that change WHAT is found), and the freshness of every data source (read from
+ * its own cache under ~/.fad-checker/).
+ *
+ * The manifest reports the CACHE STATE of each source, not "online vs offline this
+ * run" — the cache state is the reproducible truth and is what an --offline re-run
+ * reads from. Pure given a cacheDir (injected for tests).
+ *
+ * @author: N.BRAUN
+ * @email: pp9ping@gmail.com
+ */
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+
+const DEFAULT_CACHE_DIR = path.join(os.homedir(), ".fad-checker");
+
+function readJson(file) {
+	try { return JSON.parse(fs.readFileSync(file, "utf8")); } catch { return null; }
+}
+
+function iso(ms) {
+	if (!ms || !Number.isFinite(Number(ms))) return null;
+	try { return new Date(Number(ms)).toISOString(); } catch { return null; }
+}
+
+// Each data source declares how to read its freshness from its cache file/dir.
+// kind:
+//   "cve-meta"   cve-data/meta.json  → { builtAt, cveCount }
+//   "meta"       <file>.json         → { meta: { fetchedAt }, entries }
+//   "kev"        kev-cache.json       → { _fetchedAt, body: { catalogVersion, dateReleased } }
+//   "dir"        <dir>/               → file count + newest mtime
+const SOURCE_DEFS = [
+	{ id: "cve", label: "CVEProject (Maven index)", rel: "cve-data/meta.json", kind: "cve-meta" },
+	{ id: "osv", label: "OSV.dev", rel: "osv-cache", kind: "dir", flag: "osv" },
+	{ id: "osvDb", label: "OSV local DB (Maven)", rel: "osv-db/maven-index.json", kind: "mtime" },
+	{ id: "nvd", label: "NIST NVD", rel: "nvd-cache", kind: "dir", flag: "nvd" },
+	{ id: "epss", label: "EPSS (FIRST.org)", rel: "epss-cache.json", kind: "meta", flag: "epss" },
+	{ id: "kev", label: "CISA KEV", rel: "kev-cache.json", kind: "kev", flag: "kev" },
+	{ id: "eol", label: "endoflife.date", rel: "eol-cache.json", kind: "meta" },
+	{ id: "mavenCentral", label: "Maven Central (latest)", rel: "version-cache.json", kind: "meta" },
+	{ id: "npm", label: "npm registry", rel: "npm-registry-cache.json", kind: "meta" },
+	{ id: "nuget", label: "NuGet registry", rel: "nuget-cache.json", kind: "meta" },
+	{ id: "packagist", label: "Packagist", rel: "packagist-cache.json", kind: "meta" },
+	{ id: "go", label: "Go module proxy", rel: "go-proxy-cache.json", kind: "meta" },
+	{ id: "rubygems", label: "RubyGems", rel: "rubygems-cache.json", kind: "meta" },
+];
+
+// Read one source's freshness → { status, asOf, detail }.
+function readSource(def, cacheDir) {
+	const target = path.join(cacheDir, def.rel);
+	if (def.kind === "dir") {
+		let files; try { files = fs.readdirSync(target).filter(f => f.endsWith(".json")); } catch { return { status: "missing", asOf: null, detail: null }; }
+		if (!files.length) return { status: "missing", asOf: null, detail: null };
+		let newest = 0;
+		for (const f of files) { try { const m = fs.statSync(path.join(target, f)).mtimeMs; if (m > newest) newest = m; } catch { /* skip */ } }
+		return { status: "cached", asOf: iso(newest), detail: `${files.length} cached` };
+	}
+	if (def.kind === "mtime") {
+		let st; try { st = fs.statSync(target); } catch { return { status: "missing", asOf: null, detail: null }; }
+		return { status: "cached", asOf: iso(st.mtimeMs), detail: null };
+	}
+	const data = readJson(target);
+	if (!data) return { status: "missing", asOf: null, detail: null };
+	if (def.kind === "cve-meta") {
+		return { status: "cached", asOf: data.builtAt || null, detail: data.cveCount != null ? `${data.cveCount} CVEs` : null };
+	}
+	if (def.kind === "kev") {
+		const body = data.body || {};
+		const ver = body.catalogVersion || body.dateReleased || null;
+		return { status: "cached", asOf: iso(data._fetchedAt), detail: ver ? `catalog ${ver}` : null };
+	}
+	// "meta": { meta: { fetchedAt }, entries }
+	const n = data.entries && typeof data.entries === "object" ? Object.keys(data.entries).length : null;
+	return { status: "cached", asOf: iso(data.meta && data.meta.fetchedAt), detail: n != null ? `${n} entries` : null };
+}
+
+/**
+ * Inspect the cache directory → an array of data-source descriptors:
+ *   { id, label, status: "disabled"|"missing"|"cached", asOf, detail }
+ * A source whose feature flag is turned off this run is "disabled".
+ */
+function collectDataSources({ cacheDir = DEFAULT_CACHE_DIR, options = {} } = {}) {
+	return SOURCE_DEFS.map(def => {
+		if (def.flag && options[def.flag] === false) {
+			return { id: def.id, label: def.label, status: "disabled", asOf: null, detail: null };
+		}
+		const r = readSource(def, cacheDir);
+		return { id: def.id, label: def.label, ...r };
+	});
+}
+
+// Only the options that change WHAT is found belong in the reproducibility record.
+function runConfiguration(options = {}) {
+	return {
+		ecosystems: options.ecosystem || "auto",
+		transitive: options.transitive !== false,
+		transitiveDepth: options.transitiveDepth != null ? String(options.transitiveDepth) : null,
+		osv: options.osv !== false,
+		nvd: options.nvd !== false,
+		epss: options.epss !== false,
+		kev: options.kev !== false,
+		allLibs: options.allLibs !== false,
+		licenses: !!options.licenses,
+		typosquat: !!options.typosquat,
+		failOn: options.failOn || "none",
+		ignoreFile: options.ignore || null,
+		vexFile: options.vex || null,
+		excludePath: options.excludePath || [],
+		defaultExcludes: options.defaultExcludes !== false,
+	};
+}
+
+/**
+ * Assemble the full scan-provenance manifest. Pure given cacheDir + runtime.
+ */
+function buildScanProvenance({ toolVersion, generatedAt, options = {}, cacheDir = DEFAULT_CACHE_DIR, runtime } = {}) {
+	const rt = runtime || { node: process.version, platform: process.platform, arch: process.arch };
+	return {
+		tool: { name: "fad-checker", version: String(toolVersion || "0") },
+		generatedAt: generatedAt || null,
+		mode: options.offline ? "offline" : "online",
+		runtime: { node: rt.node, platform: rt.platform, arch: rt.arch },
+		configuration: runConfiguration(options),
+		dataSources: collectDataSources({ cacheDir, options }),
+	};
+}
+
+module.exports = { buildScanProvenance, collectDataSources, runConfiguration, DEFAULT_CACHE_DIR };

package/lib/registries.js

@@ -12,7 +12,7 @@
  * @author: N.BRAUN
  * @email: pp9ping@gmail.com
  */
-const SUPPORTED = ["maven", "npm", "pypi", "ruby", "go"];
+const SUPPORTED = ["maven", "npm", "pypi", "ruby", "go", "nuget", "composer"];
 
 const PUBLIC_BASES = {
 	maven: "https://repo1.maven.org/maven2/",
@@ -20,6 +20,11 @@ const PUBLIC_BASES = {
 	pypi: "https://pypi.org/pypi/",
 	ruby: "https://rubygems.org/api/v1/gems/",
 	go: "https://proxy.golang.org/",
+	// NuGet: the public v3 registration base (each fetch appends "<lowerid>/index.json").
+	nuget: "https://api.nuget.org/v3/registration5-gz-semver2/",
+	// Composer: the Packagist root (the public path is "packages/<name>.json"; custom
+	// feeds are queried via the v2 "p2/<name>.json" metadata API — see composer/registry.js).
+	composer: "https://packagist.org/",
 };
 
 function normalise(url) {

package/lib/report-integrity.js

@@ -0,0 +1,52 @@
+/**
+ * lib/report-integrity.js — tamper-evidence for the report deliverable.
+ *
+ * An audit handed to a client should be verifiable as unaltered. This writes a
+ * standard `sha256sum`-format manifest (`<hex>␠␠<relative-path>`) over every artifact
+ * produced by a run, verifiable offline with `sha256sum -c SHA256SUMS`. Checksums
+ * (not signatures) keep the CLI zero-config; signing the manifest with the auditor's
+ * own key is then a trivial external step.
+ *
+ * @author: N.BRAUN
+ * @email: pp9ping@gmail.com
+ */
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+
+/** SHA-256 of a file's bytes, as lowercase hex. */
+function sha256OfFile(filePath) {
+	const buf = fs.readFileSync(filePath);
+	return crypto.createHash("sha256").update(buf).digest("hex");
+}
+
+function relName(file, baseDir) {
+	if (!baseDir) return path.basename(file);
+	const rel = path.relative(baseDir, file);
+	return rel && !rel.startsWith("..") ? rel : path.basename(file);
+}
+
+/**
+ * Build a sha256sum-format manifest over `files` (existing ones only), sorted by
+ * relative name for a deterministic, diffable output.
+ */
+function buildChecksumManifest(files, { baseDir } = {}) {
+	const rows = [];
+	for (const f of files || []) {
+		if (!f) continue;
+		let hex;
+		try { hex = sha256OfFile(f); } catch { continue; } // missing/unreadable → skip
+		rows.push({ name: relName(f, baseDir), hex });
+	}
+	rows.sort((a, b) => a.name.localeCompare(b.name));
+	return rows.map(r => `${r.hex}  ${r.name}`).join("\n") + (rows.length ? "\n" : "");
+}
+
+/** Write the manifest to `manifestPath`; returns the manifest text. */
+function writeChecksums(files, manifestPath, { baseDir } = {}) {
+	const text = buildChecksumManifest(files, { baseDir: baseDir || path.dirname(manifestPath) });
+	fs.writeFileSync(manifestPath, text);
+	return text;
+}
+
+module.exports = { sha256OfFile, buildChecksumManifest, writeChecksums };

package/lib/retire.js

@@ -142,9 +142,58 @@ async function ensureSignatures({ verbose, offline } = {}) {
 	return !!r.ok;
 }
 
-// Pure: build the retire CLI argv. `jsRepo` (a local signature file) is added
-// when available so retire loads signatures from disk instead of the network.
-function buildRetireArgs({ srcDir, outPath, ignoredDirs, jsRepo }) {
+// Directories retire never needs to walk. Mirrors the per-walker default SKIP sets
+// the codecs use (lib/path-filter.js / npm parse.js): package stores, build output,
+// IDE/VCS metadata. Matched by BASENAME at any depth (see buildRetireIgnorePatterns).
+const DEFAULT_RETIRE_SKIP_DIRS = [
+	"node_modules", "bower_components", "jspm_packages",
+	".git", ".idea", ".vscode", ".gradle", ".mvn",
+	"target", "dist", "build", "build-output", "out", "coverage", ".next", ".nuxt",
+];
+
+/**
+ * Build the lines for a retire ignorefile, mirroring the rest of the scan's
+ * exclude policy (lib/path-filter.js#makeDirFilter) so retire skips the SAME dirs
+ * every codec walker prunes.
+ *
+ * Why an ignorefile and not `--ignore`: retire `path.resolve()`s every `--ignore`
+ * entry against ITS OWN cwd (cli.js), so a bare "node_modules" becomes
+ * `<cwd>/node_modules` and silently misses `<srcDir>/…/node_modules` whenever fad
+ * runs from a different directory than `-s` — and it can never express "this dir
+ * at ANY depth". An ignorefile line prefixed with `@` is kept VERBATIM (no resolve)
+ * and used as a substring regex, which is the only retire mechanism that can.
+ *
+ * Two layers, exactly like makeDirFilter:
+ *   - default skip dirs → `@/<name>/` : segment-bounded substring → matches the dir
+ *     at ANY depth (the bug: a deeply-nested node_modules), and the surrounding
+ *     slashes keep lookalike files (`node_modules_shim.js`) from being clipped.
+ *     Dropped entirely when defaultExcludes === false (--no-default-excludes).
+ *   - user --exclude-path globs → `@<absSrcDir>/<glob>/` : ANCHORED to the scan
+ *     root (not any-depth), retire expands `*`→[^/]* and `**`→.* itself.
+ *
+ * retire walks with POSIX-style paths (its own matchers use `/`), so we emit `/`.
+ */
+function buildRetireIgnorePatterns({ srcDir, excludePath = [], defaultExcludes = true } = {}) {
+	const lines = [];
+	if (defaultExcludes !== false) {
+		for (const d of DEFAULT_RETIRE_SKIP_DIRS) lines.push(`@/${d}/`);
+	}
+	const root = srcDir ? path.resolve(srcDir).split(path.sep).join("/").replace(/\/+$/, "") : "";
+	for (const g of excludePath || []) {
+		// Normalise like compileGlobs: strip a leading ./ or /, a trailing /, and a
+		// trailing /** (the dir's whole subtree is already covered by the segment).
+		const base = String(g || "").trim().replace(/^\.?\/+/, "").replace(/\/+$/, "").replace(/\/\*\*$/, "");
+		if (!base) continue;
+		lines.push(root ? `@${root}/${base}/` : `@/${base}/`);
+	}
+	return lines;
+}
+
+// Pure: build the retire CLI argv. `ignoreFile` (a path-anchored ignorefile, see
+// buildRetireIgnorePatterns) is passed when there's anything to exclude. `jsRepo`
+// (a local signature file) is added when available so retire loads signatures from
+// disk instead of the network.
+function buildRetireArgs({ srcDir, outPath, ignoreFile, jsRepo }) {
 	const args = [
 		// --verbose makes retire list EVERY identified library, not just the
 		// vulnerable ones (its own wording: "by default only vulnerable files are
@@ -154,8 +203,8 @@ function buildRetireArgs({ srcDir, outPath, ignoredDirs, jsRepo }) {
 		"--outputformat", "json",
 		"--outputpath", outPath,
 		"--jspath", srcDir,
-		"--ignore", ignoredDirs,
 	];
+	if (ignoreFile) { args.push("--ignorefile", ignoreFile); }
 	if (jsRepo) { args.push("--jsrepo", jsRepo); }
 	return args;
 }
@@ -222,11 +271,13 @@ function extractVendoredInventory(raw, srcDir) {
  * Run retire.js against `srcDir`. Returns the parsed JSON output (an array
  * of file-level findings) or null if retire is unavailable.
  *
- * `--outputformat json` is the structured form; `--ignore` takes a list of
- * dirs to skip; we add the standard suspects to keep runtime bounded.
+ * `--outputformat json` is the structured form; a generated `--ignorefile`
+ * prunes the same dirs the rest of the scan skips (default SKIP set at any depth
+ * + the user's --exclude-path globs, anchored to the scan root) so retire never
+ * walks node_modules/target/… or anything the auditor excluded.
  */
 async function runRetire(srcDir, opts = {}) {
-	const { verbose, force, offline } = opts;
+	const { verbose, force, offline, excludePath, defaultExcludes } = opts;
 	// Optional diagnostics collector: callers (scanWithRetireFull) read diag.error to
 	// surface a genuine SCAN FAILURE (retire crashed / produced no parseable output)
 	// as a report warning — instead of letting it masquerade as "no vendored JS found".
@@ -252,16 +303,25 @@ async function runRetire(srcDir, opts = {}) {
 	}
 
 	const launcher = findRetireLauncher();
-	const ignoredDirs = [
-		"node_modules", "bower_components", "jspm_packages",
-		".git", ".idea", ".vscode", ".gradle", ".mvn",
-		"target", "dist", "build", "build-output", "out", "coverage", ".next", ".nuxt",
-	].join(",");
 
 	// retire.js refuses to write to /dev/stdout, so we use a real temp file
 	// and read it back. Falls back to stdout if --outputpath is rejected.
 	const tmpOut = path.join(os.tmpdir(), `fad-checker-retire-${process.pid}-${Date.now()}.json`);
-	const args = buildRetireArgs({ srcDir, outPath: tmpOut, ignoredDirs, jsRepo: haveSig ? RETIRE_SIG_FILE : null });
+
+	// Pruning policy, written to a temp ignorefile so retire honors it for the tree
+	// it ACTUALLY walks (anchored to srcDir, not retire's cwd — the nested
+	// node_modules bug) and respects the same --exclude-path the codecs do.
+	const ignorePatterns = buildRetireIgnorePatterns({ srcDir, excludePath, defaultExcludes });
+	// retire treats a `.json` ignorefile as the structured descriptor form; keep a
+	// plain-text extension so the `@`-prefixed regex lines are read as such.
+	const tmpIgnore = ignorePatterns.length ? path.join(os.tmpdir(), `fad-checker-retire-ignore-${process.pid}-${Date.now()}.txt`) : null;
+	if (tmpIgnore) fs.writeFileSync(tmpIgnore, ignorePatterns.join("\n") + "\n");
+	const cleanup = () => {
+		try { fs.unlinkSync(tmpOut); } catch { /* best effort */ }
+		if (tmpIgnore) { try { fs.unlinkSync(tmpIgnore); } catch { /* best effort */ } }
+	};
+
+	const args = buildRetireArgs({ srcDir, outPath: tmpOut, ignoreFile: tmpIgnore, jsRepo: haveSig ? RETIRE_SIG_FILE : null });
 	if (verbose) console.log(`retire: scanning ${srcDir}…${haveSig ? " (local signatures)" : ""}`);
 
 	let execErr = null;
@@ -284,7 +344,7 @@ async function runRetire(srcDir, opts = {}) {
 			const reason = retireFailureReason(err.stderr, err.message);
 			diag.error = `retire.js scan failed: ${reason}`;
 			if (verbose) console.warn(`retire: failed to run — ${reason}`);
-			try { fs.unlinkSync(tmpOut); } catch { /* best effort */ }
+			cleanup();
 			return null;
 		}
 	}
@@ -299,7 +359,7 @@ async function runRetire(srcDir, opts = {}) {
 		if (verbose) console.warn(`retire: could not parse output — ${err.message}`);
 		return null;
 	} finally {
-		try { fs.unlinkSync(tmpOut); } catch { /* best effort */ }
+		cleanup();
 	}
 	writeCache(srcDir, parsed);
 	return parsed;
@@ -417,6 +477,8 @@ module.exports = {
 	warmRetireSignatures,
 	ensureSignatures,
 	buildRetireArgs,
+	buildRetireIgnorePatterns,
+	DEFAULT_RETIRE_SKIP_DIRS,
 	retireFailureReason,
 	readCache,
 	writeCache,

package/lib/snyk.js

@@ -28,13 +28,46 @@ async function runSnykTest(outputDir, opts = {}) {
 		});
 		return parseSnykStdout(stdout);
 	} catch (err) {
-		// snyk exits 1 when vulns are found — stdout still contains the JSON
-		if (err.stdout) return parseSnykStdout(err.stdout);
 		if (err.code === "ENOENT") throw new Error("snyk CLI not found on PATH — install snyk separately");
-		throw err;
+		if (err.killed) throw new Error(`snyk timed out after ${Math.round(timeoutMs / 1000)}s (raise opts.timeoutMs)`);
+		// snyk exits 1 when vulnerabilities are found — stdout holds the real findings
+		// JSON, so that's NOT a failure. BUT exit 2 (command error, e.g. auth) and
+		// exit 3 (no supported projects) ALSO write JSON to stdout, shaped
+		// { ok:false, error:"…" }. Returning that as-is makes parseSnykResults read it
+		// as "0 vulnerabilities" → a silent success that hides a real snyk failure.
+		if (err.stdout) {
+			const parsed = parseSnykStdout(err.stdout);
+			const snykErr = snykOutputError(parsed);
+			if (snykErr) throw new Error(`snyk failed (exit ${err.code}): ${snykErr}`);
+			return parsed;
+		}
+		// No JSON on stdout at all → surface stderr (the human-readable reason) rather
+		// than execFile's generic "Command failed: snyk test…".
+		const detail = (err.stderr && String(err.stderr).trim()) || err.message;
+		throw new Error(`snyk failed (exit ${err.code}): ${detail}`);
 	}
 }
 
+/**
+ * Inspect parsed `snyk --json` output and return a human-readable error string when
+ * snyk actually FAILED — i.e. every project is an error stub ({ ok:false, error }) and
+ * none carries usable results — or `null` when the output holds real findings.
+ *
+ * Snyk emits this error-shaped JSON on stdout for exit 2 (command error, e.g. auth) and
+ * exit 3 (no supported projects). Without this check those are read as "0 vulnerabilities",
+ * silently hiding the failure behind a green "Snyk: 0 findings merged".
+ */
+function snykOutputError(parsed) {
+	const projects = Array.isArray(parsed) ? parsed : (parsed != null ? [parsed] : []);
+	if (!projects.length) return null;
+	// A usable project either lists vulnerabilities (exit 1) or reports ok:true (clean).
+	const usable = projects.some(p => p && (Array.isArray(p.vulnerabilities) || p.ok === true));
+	if (usable) return null;
+	const errored = projects.filter(p => p && p.ok === false && p.error);
+	if (!errored.length) return null;
+	return [...new Set(errored.map(p => String(p.error).trim()).filter(Boolean))].join("; ");
+}
+
 function parseSnykStdout(stdout) {
 	if (!stdout) return [];
 	const str = String(stdout).trim();
@@ -126,5 +159,6 @@ module.exports = {
 	runSnykTest,
 	parseSnykResults,
 	parseSnykStdout,
+	snykOutputError,
 	mergeWithFadResults,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fad-checker",
-  "version": "2.3.1",
+  "version": "2.4.0",
   "description": "Scan ALL Maven, Gradle, npm, Yarn, Composer, Python, C#/.NET, Go & Ruby dependencies — plus embedded JARs (fat-jars/war/ear) — in a source tree ONE SHOT without mvn/python/etc — CVE (EPSS/KEV-prioritised), EOL, obsolete, outdated & licenses, with SBOM/CSAF/SARIF/JSON exports, CI gating and fix recos",
   "keywords": [
     "sca",