fad-checker 2.1.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +100 -32
- package/fad-checker-report/cve-report.doc +643 -105
- package/fad-checker-report/cve-report.html +653 -108
- package/fad-checker.js +265 -55
- package/lib/codecs/binary/scan.js +71 -0
- package/lib/codecs/binary/sniff.js +37 -0
- package/lib/codecs/binary.codec.js +54 -0
- package/lib/codecs/composer.codec.js +7 -3
- package/lib/codecs/go/registry.js +19 -11
- package/lib/codecs/go.codec.js +7 -3
- package/lib/codecs/index.js +73 -4
- package/lib/codecs/maven/jar-scan.js +179 -28
- package/lib/codecs/maven.codec.js +4 -2
- package/lib/codecs/npm/collect.js +3 -1
- package/lib/codecs/npm/parse.js +29 -1
- package/lib/codecs/npm/registry.js +29 -18
- package/lib/codecs/npm.codec.js +10 -4
- package/lib/codecs/nuget.codec.js +7 -3
- package/lib/codecs/pypi/registry.js +16 -10
- package/lib/codecs/pypi.codec.js +7 -3
- package/lib/codecs/recipes.js +9 -1
- package/lib/codecs/ruby/registry.js +16 -10
- package/lib/codecs/ruby.codec.js +7 -3
- package/lib/config.js +34 -23
- package/lib/core.js +22 -3
- package/lib/cve-match.js +3 -2
- package/lib/cve-report.js +59 -6
- package/lib/dep-record.js +12 -9
- package/lib/hash-id.js +78 -0
- package/lib/json-export.js +8 -1
- package/lib/license-policy.js +3 -1
- package/lib/options-env.js +113 -0
- package/lib/outdated.js +3 -0
- package/lib/parallel-walk.js +50 -0
- package/lib/path-filter.js +58 -0
- package/lib/registries.js +112 -0
- package/lib/retire.js +66 -0
- package/lib/scan-completeness.js +7 -1
- package/lib/unmanaged.js +82 -0
- package/package.json +1 -1
package/lib/codecs/npm/parse.js
CHANGED
|
@@ -362,6 +362,7 @@ const DEFAULT_JS_SKIP_DIRS = new Set([
|
|
|
362
362
|
|
|
363
363
|
function findJsManifests(rootDir, opts = {}) {
|
|
364
364
|
const { skipDirs = DEFAULT_JS_SKIP_DIRS } = opts;
|
|
365
|
+
const skipDir = opts.skipDir || ((child, name) => skipDirs.has(name));
|
|
365
366
|
const found = [];
|
|
366
367
|
const stack = [rootDir];
|
|
367
368
|
while (stack.length) {
|
|
@@ -374,7 +375,7 @@ function findJsManifests(rootDir, opts = {}) {
|
|
|
374
375
|
for (const e of entries) {
|
|
375
376
|
const p = path.join(cur, e.name);
|
|
376
377
|
if (e.isDirectory()) {
|
|
377
|
-
if (
|
|
378
|
+
if (skipDir(p, e.name)) continue;
|
|
378
379
|
stack.push(p);
|
|
379
380
|
} else if (e.isFile()) {
|
|
380
381
|
if (e.name === "package.json") here.packageJson = p;
|
|
@@ -388,6 +389,31 @@ function findJsManifests(rootDir, opts = {}) {
|
|
|
388
389
|
return found;
|
|
389
390
|
}
|
|
390
391
|
|
|
392
|
+
// Parallel equivalent of findJsManifests — concurrent readdir so the walk isn't
|
|
393
|
+
// serialized one round-trip at a time on a high-latency filesystem.
|
|
394
|
+
async function findJsManifestsAsync(rootDir, opts = {}) {
|
|
395
|
+
const { skipDirs = DEFAULT_JS_SKIP_DIRS } = opts;
|
|
396
|
+
const skipDir = opts.skipDir || ((child, name) => skipDirs.has(name));
|
|
397
|
+
const { walkDirs } = require("../../parallel-walk");
|
|
398
|
+
const found = [];
|
|
399
|
+
await walkDirs(rootDir, {
|
|
400
|
+
skipDir,
|
|
401
|
+
onDir: (cur, entries) => {
|
|
402
|
+
const here = { dir: cur, packageJson: null, packageLock: null, yarnLock: null, pnpmLock: null };
|
|
403
|
+
for (const e of entries) {
|
|
404
|
+
if (!e.isFile()) continue;
|
|
405
|
+
const p = path.join(cur, e.name);
|
|
406
|
+
if (e.name === "package.json") here.packageJson = p;
|
|
407
|
+
else if (e.name === "package-lock.json") here.packageLock = p;
|
|
408
|
+
else if (e.name === "yarn.lock") here.yarnLock = p;
|
|
409
|
+
else if (e.name === "pnpm-lock.yaml") here.pnpmLock = p;
|
|
410
|
+
}
|
|
411
|
+
if (here.packageJson || here.packageLock || here.yarnLock || here.pnpmLock) found.push(here);
|
|
412
|
+
},
|
|
413
|
+
});
|
|
414
|
+
return found;
|
|
415
|
+
}
|
|
416
|
+
|
|
391
417
|
module.exports = {
|
|
392
418
|
parsePackageJson,
|
|
393
419
|
parsePackageLock,
|
|
@@ -395,4 +421,6 @@ module.exports = {
|
|
|
395
421
|
parseYarnBerry,
|
|
396
422
|
parsePnpmLock,
|
|
397
423
|
findJsManifests,
|
|
424
|
+
findJsManifestsAsync,
|
|
425
|
+
DEFAULT_JS_SKIP_DIRS,
|
|
398
426
|
};
|
|
@@ -21,11 +21,11 @@ const path = require("path");
|
|
|
21
21
|
const os = require("os");
|
|
22
22
|
const pLimit = require("p-limit");
|
|
23
23
|
const { semverCompare, webjarToNpm } = require("./collect");
|
|
24
|
+
const { withPublic, authHeaderFor, normalise } = require("../../registries");
|
|
24
25
|
|
|
25
26
|
const CACHE_DIR = path.join(os.homedir(), ".fad-checker");
|
|
26
27
|
const CACHE_PATH = path.join(CACHE_DIR, "npm-registry-cache.json");
|
|
27
28
|
const CACHE_MAX_AGE_MS = 24 * 3600 * 1000; // 1 day, aligned with Maven Central
|
|
28
|
-
const REGISTRY = "https://registry.npmjs.org";
|
|
29
29
|
|
|
30
30
|
function loadCache() {
|
|
31
31
|
try { return JSON.parse(fs.readFileSync(CACHE_PATH, "utf8")); }
|
|
@@ -89,29 +89,40 @@ function packumentToFindings(packument, dep) {
|
|
|
89
89
|
return out;
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
-
/** registry
|
|
93
|
-
function
|
|
94
|
-
return name.startsWith("@")
|
|
95
|
-
? `${REGISTRY}/${name.replace("/", "%2F")}`
|
|
96
|
-
: `${REGISTRY}/${encodeURIComponent(name)}`;
|
|
92
|
+
/** registry path-encodes a scoped name's slash. */
|
|
93
|
+
function encodeName(name) {
|
|
94
|
+
return name.startsWith("@") ? name.replace("/", "%2F") : encodeURIComponent(name);
|
|
97
95
|
}
|
|
98
96
|
|
|
97
|
+
/**
|
|
98
|
+
* Fetch a packument, trying each custom registry first (with per-registry auth),
|
|
99
|
+
* then the public registry.npmjs.org as the final fallback. Returns the parsed
|
|
100
|
+
* packument, or { error } if every base failed.
|
|
101
|
+
*/
|
|
99
102
|
async function fetchPackument(name, opts = {}) {
|
|
100
103
|
if (opts.offline) return null;
|
|
101
104
|
const timeoutMs = opts.timeoutMs || 15000;
|
|
102
|
-
|
|
105
|
+
const fetcher = opts.fetcher || globalThis.fetch;
|
|
106
|
+
const bases = withPublic("npm", opts.registries || []);
|
|
107
|
+
const enc = encodeName(name);
|
|
108
|
+
let lastErr = null;
|
|
109
|
+
for (const base of bases) {
|
|
103
110
|
// Per-request timeout: a single stalled connection must never hang the
|
|
104
111
|
// whole run (one slow package would otherwise occupy a concurrency slot
|
|
105
112
|
// forever and starve the pool).
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
113
|
+
const url = normalise(base.url) + enc;
|
|
114
|
+
const headers = { "User-Agent": "fad-checker-npm-registry", Accept: "application/json" };
|
|
115
|
+
const ah = authHeaderFor(base);
|
|
116
|
+
if (ah) headers.Authorization = ah;
|
|
117
|
+
try {
|
|
118
|
+
const res = await fetcher(url, { headers, signal: AbortSignal.timeout(timeoutMs) });
|
|
119
|
+
if (res.ok) return await res.json();
|
|
120
|
+
lastErr = `HTTP ${res.status}`;
|
|
121
|
+
} catch (err) {
|
|
122
|
+
lastErr = err.name === "TimeoutError" ? `timeout after ${timeoutMs}ms` : err.message;
|
|
123
|
+
}
|
|
114
124
|
}
|
|
125
|
+
return { error: lastErr || "no data" };
|
|
115
126
|
}
|
|
116
127
|
|
|
117
128
|
/**
|
|
@@ -124,7 +135,7 @@ async function fetchPackument(name, opts = {}) {
|
|
|
124
135
|
* opts: { verbose, offline, allLibs, concurrency = 8 }
|
|
125
136
|
*/
|
|
126
137
|
async function checkNpmRegistryDeps(resolvedDeps, opts = {}) {
|
|
127
|
-
const { verbose, offline, allLibs = true, concurrency = 8, onProgress } = opts;
|
|
138
|
+
const { verbose, offline, allLibs = true, concurrency = 8, onProgress, registries = [] } = opts;
|
|
128
139
|
// Targets = npm deps (queried by their own name) + WebJars (queried by their
|
|
129
140
|
// derived npm-equivalent name; version matches upstream). The original dep
|
|
130
141
|
// is kept for display/results so the report shows e.g. org.webjars:angularjs.
|
|
@@ -168,7 +179,7 @@ async function checkNpmRegistryDeps(resolvedDeps, opts = {}) {
|
|
|
168
179
|
const key = cacheKey(t);
|
|
169
180
|
let extracted = cache.entries[key];
|
|
170
181
|
if (!extracted) {
|
|
171
|
-
const packument = await fetchPackument(npmName, { offline });
|
|
182
|
+
const packument = await fetchPackument(npmName, { offline, registries });
|
|
172
183
|
if (packument && !packument.error) {
|
|
173
184
|
const f = packumentToFindings(packument, { version });
|
|
174
185
|
extracted = {
|
|
@@ -212,4 +223,4 @@ async function checkNpmRegistryDeps(resolvedDeps, opts = {}) {
|
|
|
212
223
|
return result;
|
|
213
224
|
}
|
|
214
225
|
|
|
215
|
-
module.exports = { packumentToFindings, checkNpmRegistryDeps, replacementFromMessage };
|
|
226
|
+
module.exports = { packumentToFindings, checkNpmRegistryDeps, replacementFromMessage, fetchPackument };
|
package/lib/codecs/npm.codec.js
CHANGED
|
@@ -20,9 +20,9 @@ const retireScanner = {
|
|
|
20
20
|
id: "retire",
|
|
21
21
|
kind: "vendored", // résultats → chapitre vendored-JS (séparé des CVE)
|
|
22
22
|
async scan(_deps, opts = {}) {
|
|
23
|
-
const {
|
|
24
|
-
const matches = await
|
|
25
|
-
return { matches, meta: {} };
|
|
23
|
+
const { scanWithRetireFull } = require("../retire");
|
|
24
|
+
const { matches, inventory } = await scanWithRetireFull(opts.src, { verbose: opts.verbose, force: !!opts.retireRefresh, offline: !!opts.offline });
|
|
25
|
+
return { matches, meta: { inventory } };
|
|
26
26
|
},
|
|
27
27
|
};
|
|
28
28
|
|
|
@@ -49,7 +49,13 @@ module.exports = {
|
|
|
49
49
|
recipe: require("./recipes").npm,
|
|
50
50
|
// collect ramasse TOUTES les deps JS (package-lock + yarn.lock).
|
|
51
51
|
async collect(dir, opts = {}) {
|
|
52
|
-
|
|
52
|
+
// Parallel manifest discovery (concurrent readdir) — much faster than the
|
|
53
|
+
// serial walk on a high-latency filesystem; parsing then runs synchronously.
|
|
54
|
+
const { findJsManifestsAsync, DEFAULT_JS_SKIP_DIRS } = require("./npm/parse");
|
|
55
|
+
const { makeDirFilter } = require("../path-filter");
|
|
56
|
+
const skipDir = makeDirFilter({ srcRoot: opts.srcRoot || dir, defaultSkip: DEFAULT_JS_SKIP_DIRS, excludePath: opts.excludePath, useDefaults: opts.defaultExcludes !== false });
|
|
57
|
+
const manifestGroups = await findJsManifestsAsync(dir, { skipDir });
|
|
58
|
+
const deps = collectNpmDeps(dir, { ...opts, manifestGroups });
|
|
53
59
|
return { deps, warnings: deps.warnings || [] };
|
|
54
60
|
},
|
|
55
61
|
};
|
|
@@ -19,7 +19,7 @@ const SKIP = new Set([".git", ".idea", ".vscode", "node_modules", "dist", "build
|
|
|
19
19
|
// MSBuild project files share the same <PackageReference> schema — C#, F#, VB.
|
|
20
20
|
const PROJ_RE = /\.(csproj|fsproj|vbproj)$/i;
|
|
21
21
|
|
|
22
|
-
function findNugetDirs(dir) {
|
|
22
|
+
function findNugetDirs(dir, skipDir = (child, name) => SKIP.has(name)) {
|
|
23
23
|
const groups = [];
|
|
24
24
|
const stack = [dir];
|
|
25
25
|
while (stack.length) {
|
|
@@ -30,11 +30,15 @@ function findNugetDirs(dir) {
|
|
|
30
30
|
if (files.includes("packages.lock.json") || files.includes("packages.config") || csprojs.length) {
|
|
31
31
|
groups.push({ dir: cur, files, csprojs });
|
|
32
32
|
}
|
|
33
|
-
for (const e of entries) if (e.isDirectory() && !
|
|
33
|
+
for (const e of entries) if (e.isDirectory() && !skipDir(path.join(cur, e.name), e.name)) stack.push(path.join(cur, e.name));
|
|
34
34
|
}
|
|
35
35
|
return groups;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
function dirFilter(dir, opts) {
|
|
39
|
+
return require("../path-filter").makeDirFilter({ srcRoot: opts.srcRoot || dir, defaultSkip: SKIP, excludePath: opts.excludePath, useDefaults: opts.defaultExcludes !== false });
|
|
40
|
+
}
|
|
41
|
+
|
|
38
42
|
module.exports = {
|
|
39
43
|
id: "nuget",
|
|
40
44
|
label: "NuGet",
|
|
@@ -51,7 +55,7 @@ module.exports = {
|
|
|
51
55
|
if (deps2Exclude && deps2Exclude.test(d.name)) return;
|
|
52
56
|
out.set(coordKeyFor("nuget", "", d.name), makeDepRecord({ ecosystem: "nuget", namespace: "", name: d.name, version: d.version, manifestPath, scope: d.scope, isDev: d.isDev }));
|
|
53
57
|
};
|
|
54
|
-
for (const g of findNugetDirs(dir)) {
|
|
58
|
+
for (const g of findNugetDirs(dir, dirFilter(dir, opts))) {
|
|
55
59
|
if (g.files.includes("packages.lock.json")) {
|
|
56
60
|
const fp = path.join(g.dir, "packages.lock.json");
|
|
57
61
|
try { const { deps } = await N.parsePackagesLockJson(fp); for (const d of deps) add(d, fp); }
|
|
@@ -14,11 +14,11 @@ const fs = require("fs");
|
|
|
14
14
|
const path = require("path");
|
|
15
15
|
const os = require("os");
|
|
16
16
|
let pLimit; try { pLimit = require("p-limit"); } catch { pLimit = () => (fn) => fn(); }
|
|
17
|
+
const { withPublic, authHeaderFor, normalise } = require("../../registries");
|
|
17
18
|
|
|
18
19
|
const CACHE_DIR = path.join(os.homedir(), ".fad-checker");
|
|
19
20
|
const CACHE_PATH = path.join(CACHE_DIR, "pypi-cache.json");
|
|
20
21
|
const CACHE_MAX_AGE_MS = 24 * 3600 * 1000;
|
|
21
|
-
const API = "https://pypi.org/pypi";
|
|
22
22
|
|
|
23
23
|
function loadCache() { try { return JSON.parse(fs.readFileSync(CACHE_PATH, "utf8")); } catch { return { entries: {}, meta: {} }; } }
|
|
24
24
|
function saveCache(d) { try { fs.mkdirSync(CACHE_DIR, { recursive: true }); fs.writeFileSync(CACHE_PATH, JSON.stringify(d)); } catch { /* ignore */ } }
|
|
@@ -56,18 +56,24 @@ function pypiToFindings(data, { version }) {
|
|
|
56
56
|
return out;
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
// Custom registries first (same JSON API: <base>/<name>/json), then pypi.org.
|
|
60
|
+
async function fetchProject(name, { offline, registries = [], fetcher = globalThis.fetch } = {}) {
|
|
60
61
|
if (offline) return null;
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
62
|
+
const bases = withPublic("pypi", registries);
|
|
63
|
+
let lastErr = null;
|
|
64
|
+
for (const base of bases) {
|
|
65
|
+
const url = normalise(base.url) + `${name}/json`;
|
|
66
|
+
const headers = { "User-Agent": "fad-checker-pypi", Accept: "application/json" };
|
|
67
|
+
const ah = authHeaderFor(base); if (ah) headers.Authorization = ah;
|
|
68
|
+
try { const res = await fetcher(url, { headers }); if (res.ok) return await res.json(); lastErr = `HTTP ${res.status}`; }
|
|
69
|
+
catch (e) { lastErr = e.message; }
|
|
70
|
+
}
|
|
71
|
+
return { error: lastErr || "no data" };
|
|
66
72
|
}
|
|
67
73
|
|
|
68
74
|
// Mirror of checkNpmRegistryDeps: returns { deprecated:[], outdated:[] }.
|
|
69
75
|
async function checkPypiRegistryDeps(deps, opts = {}) {
|
|
70
|
-
const { verbose, offline, allLibs = true, concurrency = 8 } = opts;
|
|
76
|
+
const { verbose, offline, allLibs = true, concurrency = 8, registries = [] } = opts;
|
|
71
77
|
const targets = [...deps.values()].filter(d => d.ecosystem === "pypi" && d.version);
|
|
72
78
|
const result = { deprecated: [], outdated: [], licensed: [] };
|
|
73
79
|
if (!targets.length) return result;
|
|
@@ -79,7 +85,7 @@ async function checkPypiRegistryDeps(deps, opts = {}) {
|
|
|
79
85
|
const key = `${t.name}@${t.version}`;
|
|
80
86
|
let ex = cache.entries[key];
|
|
81
87
|
if (!ex) {
|
|
82
|
-
const data = await fetchProject(t.name, { offline });
|
|
88
|
+
const data = await fetchProject(t.name, { offline, registries });
|
|
83
89
|
if (data && !data.error) {
|
|
84
90
|
const f = pypiToFindings(data, { version: t.version });
|
|
85
91
|
ex = { yanked: f.yanked, inactive: f.inactive, latest: f.outdated?.latest || null, license: f.license || null };
|
|
@@ -105,4 +111,4 @@ async function checkPypiRegistryDeps(deps, opts = {}) {
|
|
|
105
111
|
return result;
|
|
106
112
|
}
|
|
107
113
|
|
|
108
|
-
module.exports = { pypiToFindings, pypiLicense, checkPypiRegistryDeps };
|
|
114
|
+
module.exports = { pypiToFindings, pypiLicense, checkPypiRegistryDeps, fetchProject };
|
package/lib/codecs/pypi.codec.js
CHANGED
|
@@ -24,7 +24,7 @@ const LOCKS = [
|
|
|
24
24
|
|
|
25
25
|
const FALLBACK_MANIFESTS = ["pyproject.toml", "requirements.txt"];
|
|
26
26
|
|
|
27
|
-
function findPyDirs(dir) {
|
|
27
|
+
function findPyDirs(dir, skipDir = (child, name) => SKIP.has(name)) {
|
|
28
28
|
const groups = [];
|
|
29
29
|
const stack = [dir];
|
|
30
30
|
while (stack.length) {
|
|
@@ -32,11 +32,15 @@ function findPyDirs(dir) {
|
|
|
32
32
|
let entries; try { entries = fs.readdirSync(cur, { withFileTypes: true }); } catch { continue; }
|
|
33
33
|
const names = new Set(entries.filter(e => e.isFile()).map(e => e.name));
|
|
34
34
|
if ([...LOCKS.map(l => l[0]), ...FALLBACK_MANIFESTS].some(n => names.has(n))) groups.push({ dir: cur, names });
|
|
35
|
-
for (const e of entries) if (e.isDirectory() && !
|
|
35
|
+
for (const e of entries) if (e.isDirectory() && !skipDir(path.join(cur, e.name), e.name)) stack.push(path.join(cur, e.name));
|
|
36
36
|
}
|
|
37
37
|
return groups;
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
function pyDirFilter(dir, opts) {
|
|
41
|
+
return require("../path-filter").makeDirFilter({ srcRoot: opts.srcRoot || dir, defaultSkip: SKIP, excludePath: opts.excludePath, useDefaults: opts.defaultExcludes !== false });
|
|
42
|
+
}
|
|
43
|
+
|
|
40
44
|
module.exports = {
|
|
41
45
|
id: "pypi",
|
|
42
46
|
label: "PyPI",
|
|
@@ -54,7 +58,7 @@ module.exports = {
|
|
|
54
58
|
if (deps2Exclude && deps2Exclude.test(d.name)) return;
|
|
55
59
|
out.set(coordKeyFor("pypi", "", d.name), makeDepRecord({ ecosystem: "pypi", namespace: "", name: d.name, version: d.version, manifestPath, scope: d.scope, isDev: d.isDev }));
|
|
56
60
|
};
|
|
57
|
-
for (const g of findPyDirs(dir)) {
|
|
61
|
+
for (const g of findPyDirs(dir, pyDirFilter(dir, opts))) {
|
|
58
62
|
const lock = LOCKS.find(([n]) => g.names.has(n));
|
|
59
63
|
if (lock) {
|
|
60
64
|
const fp = path.join(g.dir, lock[0]);
|
package/lib/codecs/recipes.js
CHANGED
|
@@ -132,4 +132,12 @@ const ruby = {
|
|
|
132
132
|
directSection: "B. Or pin them in the Gemfile and run bundle update",
|
|
133
133
|
};
|
|
134
134
|
|
|
135
|
-
|
|
135
|
+
const binary = {
|
|
136
|
+
label: "Binaries",
|
|
137
|
+
pinSection: "A. Replace or remove the vendored binary",
|
|
138
|
+
pinIntro: cnt => `Replace ${cnt} committed binar${cnt > 1 ? "ies" : "y"} with a managed dependency, or verify their provenance/checksum:`,
|
|
139
|
+
snippet: (items) => ((items || []).map(it => `# ${it.artifactId || "binary"}: replace with a managed dependency, or record its origin and verify its checksum`).join("\n")) || "# verify the provenance/checksum of each vendored binary, or replace it with a managed dependency",
|
|
140
|
+
directSection: "B. Prefer declaring these through a package manager (Maven/npm/NuGet/…)",
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
module.exports = { maven, npm, yarn, composer, pypi, nuget, go, ruby, binary, dependencyManagementSnippet, npmOverridesSnippet, yarnResolutionsSnippet, composerRequireSnippet, pipInstallSnippet, dotnetAddSnippet, goGetSnippet, bundleUpdateSnippet };
|
|
@@ -11,11 +11,11 @@ const fs = require("fs");
|
|
|
11
11
|
const path = require("path");
|
|
12
12
|
const os = require("os");
|
|
13
13
|
let pLimit; try { pLimit = require("p-limit"); } catch { pLimit = () => (fn) => fn(); }
|
|
14
|
+
const { withPublic, authHeaderFor, normalise } = require("../../registries");
|
|
14
15
|
|
|
15
16
|
const CACHE_DIR = path.join(os.homedir(), ".fad-checker");
|
|
16
17
|
const CACHE_PATH = path.join(CACHE_DIR, "rubygems-cache.json");
|
|
17
18
|
const CACHE_MAX_AGE_MS = 24 * 3600 * 1000;
|
|
18
|
-
const API = "https://rubygems.org/api/v1/gems";
|
|
19
19
|
|
|
20
20
|
function loadCache() { try { return JSON.parse(fs.readFileSync(CACHE_PATH, "utf8")); } catch { return { entries: {}, meta: {} }; } }
|
|
21
21
|
function saveCache(d) { try { fs.mkdirSync(CACHE_DIR, { recursive: true }); fs.writeFileSync(CACHE_PATH, JSON.stringify(d)); } catch { /* ignore */ } }
|
|
@@ -36,17 +36,23 @@ function gemToFindings(data) {
|
|
|
36
36
|
return out;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
// Custom gem servers first (same JSON API: <base>/<gem>.json), then rubygems.org.
|
|
40
|
+
async function fetchGem(name, { offline, registries = [], fetcher = globalThis.fetch } = {}) {
|
|
40
41
|
if (offline) return null;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
42
|
+
const bases = withPublic("ruby", registries);
|
|
43
|
+
let lastErr = null;
|
|
44
|
+
for (const base of bases) {
|
|
45
|
+
const url = normalise(base.url) + `${encodeURIComponent(name)}.json`;
|
|
46
|
+
const headers = { "User-Agent": "fad-checker-rubygems", Accept: "application/json" };
|
|
47
|
+
const ah = authHeaderFor(base); if (ah) headers.Authorization = ah;
|
|
48
|
+
try { const res = await fetcher(url, { headers }); if (res.ok) return await res.json(); lastErr = `HTTP ${res.status}`; }
|
|
49
|
+
catch (e) { lastErr = e.message; }
|
|
50
|
+
}
|
|
51
|
+
return { error: lastErr || "no data" };
|
|
46
52
|
}
|
|
47
53
|
|
|
48
54
|
async function checkRubyRegistryDeps(deps, opts = {}) {
|
|
49
|
-
const { verbose, offline, allLibs = true, concurrency = 8 } = opts;
|
|
55
|
+
const { verbose, offline, allLibs = true, concurrency = 8, registries = [] } = opts;
|
|
50
56
|
const targets = [...deps.values()].filter(d => d.ecosystem === "ruby" && d.version);
|
|
51
57
|
const result = { deprecated: [], outdated: [], licensed: [] };
|
|
52
58
|
if (!targets.length) return result;
|
|
@@ -58,7 +64,7 @@ async function checkRubyRegistryDeps(deps, opts = {}) {
|
|
|
58
64
|
const key = `${t.name}@${t.version}`;
|
|
59
65
|
let ex = cache.entries[key];
|
|
60
66
|
if (!ex) {
|
|
61
|
-
const data = await fetchGem(t.name, { offline });
|
|
67
|
+
const data = await fetchGem(t.name, { offline, registries });
|
|
62
68
|
ex = (data && !data.error) ? gemToFindings(data) : { latest: null, license: null };
|
|
63
69
|
if (data && !data.error) cache.entries[key] = ex;
|
|
64
70
|
}
|
|
@@ -73,4 +79,4 @@ async function checkRubyRegistryDeps(deps, opts = {}) {
|
|
|
73
79
|
return result;
|
|
74
80
|
}
|
|
75
81
|
|
|
76
|
-
module.exports = { gemToFindings, checkRubyRegistryDeps };
|
|
82
|
+
module.exports = { gemToFindings, checkRubyRegistryDeps, fetchGem };
|
package/lib/codecs/ruby.codec.js
CHANGED
|
@@ -14,7 +14,7 @@ const R = require("./ruby/parse");
|
|
|
14
14
|
|
|
15
15
|
const SKIP = new Set([".git", ".idea", ".vscode", "node_modules", "dist", "build", "out", "target", "vendor", "tmp"]);
|
|
16
16
|
|
|
17
|
-
function findGemfileLocks(dir) {
|
|
17
|
+
function findGemfileLocks(dir, skipDir = (child, name) => SKIP.has(name)) {
|
|
18
18
|
const found = [];
|
|
19
19
|
const stack = [dir];
|
|
20
20
|
while (stack.length) {
|
|
@@ -22,12 +22,16 @@ function findGemfileLocks(dir) {
|
|
|
22
22
|
let entries; try { entries = fs.readdirSync(cur, { withFileTypes: true }); } catch { continue; }
|
|
23
23
|
for (const e of entries) {
|
|
24
24
|
if (e.isFile() && e.name === "Gemfile.lock") found.push(path.join(cur, e.name));
|
|
25
|
-
else if (e.isDirectory() && !
|
|
25
|
+
else if (e.isDirectory() && !skipDir(path.join(cur, e.name), e.name)) stack.push(path.join(cur, e.name));
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
28
|
return found;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
function dirFilter(dir, opts) {
|
|
32
|
+
return require("../path-filter").makeDirFilter({ srcRoot: opts.srcRoot || dir, defaultSkip: SKIP, excludePath: opts.excludePath, useDefaults: opts.defaultExcludes !== false });
|
|
33
|
+
}
|
|
34
|
+
|
|
31
35
|
module.exports = {
|
|
32
36
|
id: "ruby",
|
|
33
37
|
label: "Ruby",
|
|
@@ -40,7 +44,7 @@ module.exports = {
|
|
|
40
44
|
const { deps2Exclude } = opts;
|
|
41
45
|
const out = new Map();
|
|
42
46
|
const warnings = [];
|
|
43
|
-
for (const fp of findGemfileLocks(dir)) {
|
|
47
|
+
for (const fp of findGemfileLocks(dir, dirFilter(dir, opts))) {
|
|
44
48
|
try {
|
|
45
49
|
const { deps } = R.parseGemfileLockFile(fp);
|
|
46
50
|
for (const d of deps) {
|
package/lib/config.js
CHANGED
|
@@ -44,35 +44,45 @@ function getNvdApiKey() {
|
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
/**
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
47
|
+
* Per-ecosystem custom registries (Nexus/Artifactory/JBoss, Verdaccio, devpi,
|
|
48
|
+
* Gemfury, Athens, …) the user has configured. Stored under config key
|
|
49
|
+
* `registries`: { <ecosystem>: [{ name, url, auth?, token? }] } where `auth` is
|
|
50
|
+
* "user:pass" (→ Basic) and `token` is a Bearer token.
|
|
50
51
|
*
|
|
51
|
-
* Maven Central
|
|
52
|
-
* the final fallback
|
|
53
|
-
*
|
|
52
|
+
* Public registries (Maven Central, registry.npmjs.org, …) are intentionally
|
|
53
|
+
* NOT stored here — callers append them as the final fallback so the user's
|
|
54
|
+
* registries stay in priority order while the public one is always a safety net.
|
|
54
55
|
*/
|
|
55
|
-
function
|
|
56
|
-
const
|
|
56
|
+
function getRegistryMap() {
|
|
57
|
+
const m = get("registries");
|
|
58
|
+
return (m && typeof m === "object" && !Array.isArray(m)) ? m : {};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function getRegistries(ecosystem) {
|
|
62
|
+
const list = getRegistryMap()[ecosystem];
|
|
57
63
|
return Array.isArray(list) ? list : [];
|
|
58
64
|
}
|
|
59
65
|
|
|
60
|
-
function
|
|
61
|
-
return set("
|
|
66
|
+
function setRegistryMap(map) {
|
|
67
|
+
return set("registries", map && Object.keys(map).length ? map : null);
|
|
62
68
|
}
|
|
63
69
|
|
|
64
|
-
function
|
|
65
|
-
const
|
|
66
|
-
list
|
|
67
|
-
|
|
70
|
+
function addRegistry(ecosystem, name, url, { auth = null, token = null } = {}) {
|
|
71
|
+
const map = getRegistryMap();
|
|
72
|
+
const list = (map[ecosystem] || []).filter(r => r.name !== name);
|
|
73
|
+
list.push({ name, url, ...(auth ? { auth } : {}), ...(token ? { token } : {}) });
|
|
74
|
+
map[ecosystem] = list;
|
|
75
|
+
setRegistryMap(map);
|
|
68
76
|
return list;
|
|
69
77
|
}
|
|
70
78
|
|
|
71
|
-
function
|
|
72
|
-
const
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
function removeRegistry(ecosystem, name) {
|
|
80
|
+
const map = getRegistryMap();
|
|
81
|
+
const before = (map[ecosystem] || []).length;
|
|
82
|
+
map[ecosystem] = (map[ecosystem] || []).filter(r => r.name !== name);
|
|
83
|
+
if (!map[ecosystem].length) delete map[ecosystem];
|
|
84
|
+
setRegistryMap(map);
|
|
85
|
+
return before !== (map[ecosystem]?.length || 0);
|
|
76
86
|
}
|
|
77
87
|
|
|
78
88
|
module.exports = {
|
|
@@ -83,8 +93,9 @@ module.exports = {
|
|
|
83
93
|
set,
|
|
84
94
|
get,
|
|
85
95
|
getNvdApiKey,
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
96
|
+
getRegistryMap,
|
|
97
|
+
getRegistries,
|
|
98
|
+
setRegistryMap,
|
|
99
|
+
addRegistry,
|
|
100
|
+
removeRegistry,
|
|
90
101
|
};
|
package/lib/core.js
CHANGED
|
@@ -19,7 +19,9 @@ const SKIP_DIRS = new Set([
|
|
|
19
19
|
|
|
20
20
|
const coord = v => (v == null ? null : String(v).trim() || null);
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
const defaultPomSkip = child => SKIP_DIRS.has(path.basename(child));
|
|
23
|
+
|
|
24
|
+
function findPomFiles(dir, skipDir = defaultPomSkip) {
|
|
23
25
|
const out = [];
|
|
24
26
|
const stack = [dir];
|
|
25
27
|
while (stack.length) {
|
|
@@ -29,8 +31,9 @@ function findPomFiles(dir) {
|
|
|
29
31
|
catch { continue; }
|
|
30
32
|
for (const e of entries) {
|
|
31
33
|
if (e.isDirectory()) {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
+
const child = path.join(cur, e.name);
|
|
35
|
+
if (skipDir(child, e.name)) continue;
|
|
36
|
+
stack.push(child);
|
|
34
37
|
} else if (e.name === "pom.xml") {
|
|
35
38
|
out.push(path.join(cur, e.name));
|
|
36
39
|
}
|
|
@@ -39,6 +42,20 @@ function findPomFiles(dir) {
|
|
|
39
42
|
return out;
|
|
40
43
|
}
|
|
41
44
|
|
|
45
|
+
// Parallel equivalent of findPomFiles — same result, but readdir runs concurrently
|
|
46
|
+
// so the walk isn't serialized one round-trip at a time on a high-latency filesystem.
|
|
47
|
+
async function findPomFilesAsync(dir, skipDir = defaultPomSkip) {
|
|
48
|
+
const { walkDirs } = require("./parallel-walk");
|
|
49
|
+
const out = [];
|
|
50
|
+
await walkDirs(dir, {
|
|
51
|
+
skipDir,
|
|
52
|
+
onDir: (cur, entries) => {
|
|
53
|
+
for (const e of entries) if (e.isFile() && e.name === "pom.xml") out.push(path.join(cur, e.name));
|
|
54
|
+
},
|
|
55
|
+
});
|
|
56
|
+
return out;
|
|
57
|
+
}
|
|
58
|
+
|
|
42
59
|
function newMetadataStore() {
|
|
43
60
|
return {
|
|
44
61
|
byPath: {},
|
|
@@ -345,7 +362,9 @@ async function rewritePoms(pomPath, allPomMetadata, allPropsByPom, opts) {
|
|
|
345
362
|
|
|
346
363
|
module.exports = {
|
|
347
364
|
coord,
|
|
365
|
+
SKIP_DIRS,
|
|
348
366
|
findPomFiles,
|
|
367
|
+
findPomFilesAsync,
|
|
349
368
|
newMetadataStore,
|
|
350
369
|
parsePom,
|
|
351
370
|
resolveParentPath,
|
package/lib/cve-match.js
CHANGED
|
@@ -125,14 +125,14 @@ async function expandWithTransitives(resolvedDeps, opts = {}) {
|
|
|
125
125
|
// Embedded-binary coords don't participate in Maven Central resolution (a
|
|
126
126
|
// fat-jar already ships its deps, discovered by recursion) and must not act as
|
|
127
127
|
// a depMgmt override for the declared tree.
|
|
128
|
-
if (dep.provenance === "embedded") continue;
|
|
128
|
+
if (dep.provenance === "embedded" || dep.provenance === "binary") continue;
|
|
129
129
|
if (dep.version && !/\$\{/.test(dep.version)) {
|
|
130
130
|
rootDepMgmt.set(`${dep.groupId}:${dep.artifactId}`, { version: dep.version });
|
|
131
131
|
}
|
|
132
132
|
}
|
|
133
133
|
|
|
134
134
|
const directs = [...resolvedDeps.values()]
|
|
135
|
-
.filter(d => d.provenance !== "embedded")
|
|
135
|
+
.filter(d => d.provenance !== "embedded" && d.provenance !== "binary")
|
|
136
136
|
.filter(d => d.version && !/\$\{/.test(d.version))
|
|
137
137
|
.filter(d => d.scope !== "test" || opts.includeTestDeps)
|
|
138
138
|
.filter(d => d.scope !== "parent");
|
|
@@ -227,6 +227,7 @@ function matchDepsAgainstCves(resolvedDeps, cveIndex, opts = {}) {
|
|
|
227
227
|
// They are caught by the OSV pipeline (which is multi-ecosystem) and
|
|
228
228
|
// the CPE refinement step (post-NVD).
|
|
229
229
|
if (dep.ecosystem === "npm") continue;
|
|
230
|
+
if (dep.provenance === "binary") continue; // no resolved coordinate yet (Plan 2 identifies it)
|
|
230
231
|
const key = `${dep.groupId}:${dep.artifactId}`.toLowerCase();
|
|
231
232
|
// Tier 1: exact packageName match
|
|
232
233
|
const t1 = matchOne(dep, byPackage[key], "exact");
|