Package not found. Please check the package name and try again.
@cyclonedx/cdxgen 9.9.0 → 9.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -21
- package/bin/cdxgen.js +132 -28
- package/bin/evinse.js +1 -18
- package/data/frameworks-list.json +29 -11
- package/data/lic-mapping.json +44 -5
- package/data/pypi-pkg-aliases.json +6 -0
- package/data/wrapdb-releases.json +503 -206
- package/evinser.js +145 -45
- package/index.js +74 -19
- package/package.json +2 -2
- package/server.js +35 -17
- package/utils.js +452 -112
- package/utils.test.js +24 -4
package/utils.js
CHANGED
|
@@ -51,6 +51,13 @@ if (!url.startsWith("file://")) {
|
|
|
51
51
|
}
|
|
52
52
|
const dirNameStr = import.meta ? dirname(fileURLToPath(url)) : __dirname;
|
|
53
53
|
const isWin = platform() === "win32";
|
|
54
|
+
const isMac = platform() === "darwin";
|
|
55
|
+
export let ATOM_DB = join(homedir(), ".local", "share", ".atomdb");
|
|
56
|
+
if (isWin) {
|
|
57
|
+
ATOM_DB = join(homedir(), "AppData", "Local", ".atomdb");
|
|
58
|
+
} else if (isMac) {
|
|
59
|
+
ATOM_DB = join(homedir(), "Library", "Application Support", ".atomdb");
|
|
60
|
+
}
|
|
54
61
|
|
|
55
62
|
const licenseMapping = JSON.parse(
|
|
56
63
|
readFileSync(join(dirNameStr, "data", "lic-mapping.json"))
|
|
@@ -116,6 +123,8 @@ const MAX_LICENSE_ID_LENGTH = 100;
|
|
|
116
123
|
let PYTHON_CMD = "python";
|
|
117
124
|
if (process.env.PYTHON_CMD) {
|
|
118
125
|
PYTHON_CMD = process.env.PYTHON_CMD;
|
|
126
|
+
} else if (process.env.CONDA_PYTHON_EXE) {
|
|
127
|
+
PYTHON_CMD = process.env.CONDA_PYTHON_EXE;
|
|
119
128
|
}
|
|
120
129
|
|
|
121
130
|
// Custom user-agent for cdxgen
|
|
@@ -330,8 +339,12 @@ export const getNpmMetadata = async function (pkgList) {
|
|
|
330
339
|
body = res.body;
|
|
331
340
|
metadata_cache[key] = body;
|
|
332
341
|
}
|
|
333
|
-
p.description =
|
|
334
|
-
|
|
342
|
+
p.description =
|
|
343
|
+
body.versions?.[p.version]?.description || body.description;
|
|
344
|
+
p.license =
|
|
345
|
+
body.versions?.[p.version]?.license ||
|
|
346
|
+
body.license ||
|
|
347
|
+
(await getRepoLicense(body.repository?.url, undefined));
|
|
335
348
|
if (body.repository && body.repository.url) {
|
|
336
349
|
p.repository = { url: body.repository.url };
|
|
337
350
|
}
|
|
@@ -2282,46 +2295,24 @@ export const getMvnMetadata = async function (pkgList, jarNSMapping = {}) {
|
|
|
2282
2295
|
if (group.indexOf("android") !== -1) {
|
|
2283
2296
|
urlPrefix = ANDROID_MAVEN;
|
|
2284
2297
|
}
|
|
2285
|
-
const groupPart = group.replace(/\./g, "/");
|
|
2286
2298
|
// Querying maven requires a valid group name
|
|
2287
|
-
if (!
|
|
2299
|
+
if (!group || group === "") {
|
|
2288
2300
|
cdepList.push(p);
|
|
2289
2301
|
continue;
|
|
2290
2302
|
}
|
|
2291
|
-
const
|
|
2292
|
-
urlPrefix
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
p.
|
|
2296
|
-
|
|
2297
|
-
p.version +
|
|
2298
|
-
"/" +
|
|
2299
|
-
p.name +
|
|
2300
|
-
"-" +
|
|
2301
|
-
p.version +
|
|
2302
|
-
".pom";
|
|
2303
|
+
const pomMetadata = {
|
|
2304
|
+
urlPrefix: urlPrefix,
|
|
2305
|
+
group: group,
|
|
2306
|
+
name: p.name,
|
|
2307
|
+
version: p.version
|
|
2308
|
+
};
|
|
2303
2309
|
try {
|
|
2304
2310
|
if (DEBUG_MODE) {
|
|
2305
|
-
console.log(
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
const bodyJson = xml2js(res.body, {
|
|
2309
|
-
compact: true,
|
|
2310
|
-
spaces: 4,
|
|
2311
|
-
textKey: "_",
|
|
2312
|
-
attributesKey: "$",
|
|
2313
|
-
commentKey: "value"
|
|
2314
|
-
}).project;
|
|
2315
|
-
if (bodyJson && bodyJson.licenses && bodyJson.licenses.license) {
|
|
2316
|
-
if (Array.isArray(bodyJson.licenses.license)) {
|
|
2317
|
-
p.license = bodyJson.licenses.license.map((l) => {
|
|
2318
|
-
return findLicenseId(l.name._);
|
|
2319
|
-
});
|
|
2320
|
-
} else if (Object.keys(bodyJson.licenses.license).length) {
|
|
2321
|
-
const l = bodyJson.licenses.license;
|
|
2322
|
-
p.license = [findLicenseId(l.name._)];
|
|
2323
|
-
}
|
|
2311
|
+
console.log(
|
|
2312
|
+
`Querying ${pomMetadata} from ${composePomXmlUrl(pomMetadata)}`
|
|
2313
|
+
);
|
|
2324
2314
|
}
|
|
2315
|
+
const bodyJson = await fetchPomXmlAsJson(pomMetadata);
|
|
2325
2316
|
p.publisher =
|
|
2326
2317
|
bodyJson.organization && bodyJson.organization.name
|
|
2327
2318
|
? bodyJson.organization.name._
|
|
@@ -2330,23 +2321,151 @@ export const getMvnMetadata = async function (pkgList, jarNSMapping = {}) {
|
|
|
2330
2321
|
if (bodyJson.scm && bodyJson.scm.url) {
|
|
2331
2322
|
p.repository = { url: bodyJson.scm.url._ };
|
|
2332
2323
|
}
|
|
2333
|
-
|
|
2324
|
+
p.license =
|
|
2325
|
+
parseLicenseEntryOrArrayFromPomXml(bodyJson?.licenses?.license) ||
|
|
2326
|
+
(await extractLicenseCommentFromPomXml(pomMetadata)) ||
|
|
2327
|
+
(await getRepoLicense(p.repository?.url, undefined));
|
|
2334
2328
|
} catch (err) {
|
|
2335
2329
|
if (DEBUG_MODE) {
|
|
2336
2330
|
console.log(
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
p.name,
|
|
2340
|
-
p.version,
|
|
2341
|
-
fullUrl
|
|
2331
|
+
`An error occurred when trying to fetch metadata ${pomMetadata}`,
|
|
2332
|
+
err
|
|
2342
2333
|
);
|
|
2343
2334
|
}
|
|
2335
|
+
} finally {
|
|
2344
2336
|
cdepList.push(p);
|
|
2345
2337
|
}
|
|
2346
2338
|
}
|
|
2347
2339
|
return cdepList;
|
|
2348
2340
|
};
|
|
2349
2341
|
|
|
2342
|
+
/**
|
|
2343
|
+
* Method to compose URL of pom.xml
|
|
2344
|
+
*
|
|
2345
|
+
* @param {String} urlPrefix
|
|
2346
|
+
* @param {String} group
|
|
2347
|
+
* @param {String} name
|
|
2348
|
+
* @param {String} version
|
|
2349
|
+
*
|
|
2350
|
+
* @return {String} fullUrl
|
|
2351
|
+
*/
|
|
2352
|
+
export const composePomXmlUrl = function ({ urlPrefix, group, name, version }) {
|
|
2353
|
+
const groupPart = group.replace(/\./g, "/");
|
|
2354
|
+
const fullUrl =
|
|
2355
|
+
urlPrefix +
|
|
2356
|
+
groupPart +
|
|
2357
|
+
"/" +
|
|
2358
|
+
name +
|
|
2359
|
+
"/" +
|
|
2360
|
+
version +
|
|
2361
|
+
"/" +
|
|
2362
|
+
name +
|
|
2363
|
+
"-" +
|
|
2364
|
+
version +
|
|
2365
|
+
".pom";
|
|
2366
|
+
return fullUrl;
|
|
2367
|
+
};
|
|
2368
|
+
|
|
2369
|
+
/**
|
|
2370
|
+
* Method to fetch pom.xml data and parse it to JSON
|
|
2371
|
+
*
|
|
2372
|
+
* @param {String} urlPrefix
|
|
2373
|
+
* @param {String} group
|
|
2374
|
+
* @param {String} name
|
|
2375
|
+
* @param {String} version
|
|
2376
|
+
*
|
|
2377
|
+
* @return {Object|undefined}
|
|
2378
|
+
*/
|
|
2379
|
+
export const fetchPomXmlAsJson = async function ({
|
|
2380
|
+
urlPrefix,
|
|
2381
|
+
group,
|
|
2382
|
+
name,
|
|
2383
|
+
version
|
|
2384
|
+
}) {
|
|
2385
|
+
const pomXml = await fetchPomXml({ urlPrefix, group, name, version });
|
|
2386
|
+
const options = {
|
|
2387
|
+
compact: true,
|
|
2388
|
+
spaces: 4,
|
|
2389
|
+
textKey: "_",
|
|
2390
|
+
attributesKey: "$",
|
|
2391
|
+
commentKey: "value"
|
|
2392
|
+
};
|
|
2393
|
+
const pomJson = xml2js(pomXml, options).project;
|
|
2394
|
+
if (pomJson?.parent) {
|
|
2395
|
+
const parentXml = await fetchPomXml({
|
|
2396
|
+
urlPrefix,
|
|
2397
|
+
group: pomJson.parent.groupId?._,
|
|
2398
|
+
name: pomJson.parent.artifactId?._,
|
|
2399
|
+
version: pomJson.parent.version?._
|
|
2400
|
+
});
|
|
2401
|
+
const parentJson = xml2js(parentXml, options).project;
|
|
2402
|
+
const result = { ...parentJson, ...pomJson };
|
|
2403
|
+
return result;
|
|
2404
|
+
}
|
|
2405
|
+
return pomJson;
|
|
2406
|
+
};
|
|
2407
|
+
|
|
2408
|
+
/**
|
|
2409
|
+
* Method to fetch pom.xml data
|
|
2410
|
+
*
|
|
2411
|
+
* @param {String} urlPrefix
|
|
2412
|
+
* @param {String} group
|
|
2413
|
+
* @param {String} name
|
|
2414
|
+
* @param {String} version
|
|
2415
|
+
*
|
|
2416
|
+
* @return {String}
|
|
2417
|
+
*/
|
|
2418
|
+
export const fetchPomXml = async function ({
|
|
2419
|
+
urlPrefix,
|
|
2420
|
+
group,
|
|
2421
|
+
name,
|
|
2422
|
+
version
|
|
2423
|
+
}) {
|
|
2424
|
+
let fullUrl = composePomXmlUrl({ urlPrefix, group, name, version });
|
|
2425
|
+
const res = await cdxgenAgent.get(fullUrl);
|
|
2426
|
+
return res.body;
|
|
2427
|
+
};
|
|
2428
|
+
|
|
2429
|
+
/**
|
|
2430
|
+
* Method extract single or multiple license entries that might appear in pom.xml
|
|
2431
|
+
*
|
|
2432
|
+
* @param {Object|Array} license
|
|
2433
|
+
*/
|
|
2434
|
+
export const parseLicenseEntryOrArrayFromPomXml = function (license) {
|
|
2435
|
+
if (!license) return;
|
|
2436
|
+
if (Array.isArray(license)) {
|
|
2437
|
+
return license.map((l) => {
|
|
2438
|
+
return findLicenseId(l.name._);
|
|
2439
|
+
});
|
|
2440
|
+
} else if (Object.keys(license).length) {
|
|
2441
|
+
return [findLicenseId(license.name._)];
|
|
2442
|
+
}
|
|
2443
|
+
};
|
|
2444
|
+
|
|
2445
|
+
/**
|
|
2446
|
+
* Method to parse pom.xml in search of a comment containing license text
|
|
2447
|
+
*
|
|
2448
|
+
* @param {String} urlPrefix
|
|
2449
|
+
* @param {String} group
|
|
2450
|
+
* @param {String} name
|
|
2451
|
+
* @param {String} version
|
|
2452
|
+
*
|
|
2453
|
+
* @return {String} License ID
|
|
2454
|
+
*/
|
|
2455
|
+
export const extractLicenseCommentFromPomXml = async function ({
|
|
2456
|
+
urlPrefix,
|
|
2457
|
+
group,
|
|
2458
|
+
name,
|
|
2459
|
+
version
|
|
2460
|
+
}) {
|
|
2461
|
+
const pom_xml = await fetchPomXml({ urlPrefix, group, name, version });
|
|
2462
|
+
const licenseRegex = /<!--([\s\S]*?)-->[\s\n]*<project/m;
|
|
2463
|
+
const match = licenseRegex.exec(pom_xml);
|
|
2464
|
+
if (match && match[1]) {
|
|
2465
|
+
return findLicenseId(match[1].trim());
|
|
2466
|
+
}
|
|
2467
|
+
};
|
|
2468
|
+
|
|
2350
2469
|
/**
|
|
2351
2470
|
* Method to parse python requires_dist attribute found in pypi setup.py
|
|
2352
2471
|
*
|
|
@@ -2422,13 +2541,23 @@ export const getPyMetadata = async function (pkgList, fetchDepsInfo) {
|
|
|
2422
2541
|
cdepList.push(p);
|
|
2423
2542
|
continue;
|
|
2424
2543
|
}
|
|
2544
|
+
const origName = p.name;
|
|
2425
2545
|
// Some packages support extra modules
|
|
2426
2546
|
if (p.name.includes("[")) {
|
|
2427
2547
|
p.name = p.name.split("[")[0];
|
|
2428
2548
|
}
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2549
|
+
let res = undefined;
|
|
2550
|
+
try {
|
|
2551
|
+
res = await cdxgenAgent.get(PYPI_URL + p.name + "/json", {
|
|
2552
|
+
responseType: "json"
|
|
2553
|
+
});
|
|
2554
|
+
} catch (err) {
|
|
2555
|
+
// retry by prefixing django- to the package name
|
|
2556
|
+
res = await cdxgenAgent.get(PYPI_URL + "django-" + p.name + "/json", {
|
|
2557
|
+
responseType: "json"
|
|
2558
|
+
});
|
|
2559
|
+
p.name = "django-" + p.name;
|
|
2560
|
+
}
|
|
2432
2561
|
const body = res.body;
|
|
2433
2562
|
if (body.info.author && body.info.author.trim() !== "") {
|
|
2434
2563
|
if (body.info.author_email && body.info.author_email.trim() !== "") {
|
|
@@ -2443,7 +2572,24 @@ export const getPyMetadata = async function (pkgList, fetchDepsInfo) {
|
|
|
2443
2572
|
p.author = body.info.author_email.trim();
|
|
2444
2573
|
}
|
|
2445
2574
|
p.description = body.info.summary;
|
|
2446
|
-
p.license =
|
|
2575
|
+
p.license = [];
|
|
2576
|
+
if (body.info.classifiers) {
|
|
2577
|
+
for (const c of body.info.classifiers) {
|
|
2578
|
+
if (c.startsWith("License :: ")) {
|
|
2579
|
+
let licenseName = c.split("::").slice(-1)[0].trim();
|
|
2580
|
+
let licenseId = findLicenseId(licenseName);
|
|
2581
|
+
if (licenseId && !p.license.includes(licenseId)) {
|
|
2582
|
+
p.license.push(licenseId);
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
}
|
|
2587
|
+
if (body.info.license) {
|
|
2588
|
+
let licenseId = findLicenseId(body.info.license);
|
|
2589
|
+
if (licenseId && !p.license.includes(licenseId)) {
|
|
2590
|
+
p.license.push(licenseId);
|
|
2591
|
+
}
|
|
2592
|
+
}
|
|
2447
2593
|
if (body.info.home_page) {
|
|
2448
2594
|
if (body.info.home_page.includes("git")) {
|
|
2449
2595
|
p.repository = { url: body.info.home_page };
|
|
@@ -2517,6 +2663,10 @@ export const getPyMetadata = async function (pkgList, fetchDepsInfo) {
|
|
|
2517
2663
|
name: "cdx:pypi:latest_version",
|
|
2518
2664
|
value: body.info.version
|
|
2519
2665
|
});
|
|
2666
|
+
p.properties.push({
|
|
2667
|
+
name: "cdx:pypi:resolved_from",
|
|
2668
|
+
value: origName
|
|
2669
|
+
});
|
|
2520
2670
|
}
|
|
2521
2671
|
if (
|
|
2522
2672
|
body.releases &&
|
|
@@ -2530,12 +2680,22 @@ export const getPyMetadata = async function (pkgList, fetchDepsInfo) {
|
|
|
2530
2680
|
p._integrity = "md5-" + digest["md5"];
|
|
2531
2681
|
}
|
|
2532
2682
|
}
|
|
2683
|
+
const purlString = new PackageURL(
|
|
2684
|
+
"pypi",
|
|
2685
|
+
"",
|
|
2686
|
+
p.name,
|
|
2687
|
+
p.version,
|
|
2688
|
+
null,
|
|
2689
|
+
null
|
|
2690
|
+
).toString();
|
|
2691
|
+
p.purl = purlString;
|
|
2692
|
+
p["bom-ref"] = decodeURIComponent(purlString);
|
|
2533
2693
|
cdepList.push(p);
|
|
2534
2694
|
} catch (err) {
|
|
2535
2695
|
if (DEBUG_MODE) {
|
|
2536
2696
|
console.error(p.name, "is not found on PyPI.");
|
|
2537
2697
|
console.log(
|
|
2538
|
-
"If this package is available from PyPI or a registry, its name might be different
|
|
2698
|
+
"If this package is available from PyPI or a registry, its name might be different from the module name. Raise a ticket at https://github.com/CycloneDX/cdxgen/issues so that this can be added to the mapping file pypi-pkg-aliases.json"
|
|
2539
2699
|
);
|
|
2540
2700
|
console.log(
|
|
2541
2701
|
"Alternatively, if this is a package that gets installed directly in your environment and offers a python binding, then track such packages manually."
|
|
@@ -2563,6 +2723,16 @@ export const getPyMetadata = async function (pkgList, fetchDepsInfo) {
|
|
|
2563
2723
|
}
|
|
2564
2724
|
};
|
|
2565
2725
|
}
|
|
2726
|
+
const purlString = new PackageURL(
|
|
2727
|
+
"pypi",
|
|
2728
|
+
"",
|
|
2729
|
+
p.name,
|
|
2730
|
+
p.version,
|
|
2731
|
+
null,
|
|
2732
|
+
null
|
|
2733
|
+
).toString();
|
|
2734
|
+
p.purl = purlString;
|
|
2735
|
+
p["bom-ref"] = decodeURIComponent(purlString);
|
|
2566
2736
|
cdepList.push(p);
|
|
2567
2737
|
}
|
|
2568
2738
|
}
|
|
@@ -2952,35 +3122,29 @@ export const getPyModules = async (src, epkgList, options) => {
|
|
|
2952
3122
|
const allImports = {};
|
|
2953
3123
|
const dependenciesList = [];
|
|
2954
3124
|
let modList = [];
|
|
3125
|
+
const slicesFile = resolve(
|
|
3126
|
+
options.depsSlicesFile || options.usagesSlicesFile
|
|
3127
|
+
);
|
|
2955
3128
|
// Issue: 615 fix. Reuse existing slices file
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
if (options.usagesSlicesFile && existsSync(options.usagesSlicesFile)) {
|
|
2959
|
-
const slicesData = JSON.parse(
|
|
2960
|
-
readFileSync(options.usagesSlicesFile, "utf-8")
|
|
2961
|
-
);
|
|
3129
|
+
if (slicesFile && existsSync(slicesFile)) {
|
|
3130
|
+
const slicesData = JSON.parse(readFileSync(slicesFile, "utf-8"));
|
|
2962
3131
|
if (slicesData && Object.keys(slicesData) && slicesData.modules) {
|
|
2963
3132
|
modList = slicesData.modules;
|
|
2964
3133
|
} else {
|
|
2965
3134
|
modList = slicesData;
|
|
2966
3135
|
}
|
|
2967
3136
|
} else {
|
|
2968
|
-
modList = findAppModules(
|
|
2969
|
-
src,
|
|
2970
|
-
"python",
|
|
2971
|
-
"parsedeps",
|
|
2972
|
-
options.usagesSlicesFile
|
|
2973
|
-
);
|
|
3137
|
+
modList = findAppModules(src, "python", "parsedeps", slicesFile);
|
|
2974
3138
|
}
|
|
2975
3139
|
const pyDefaultModules = new Set(PYTHON_STD_MODULES);
|
|
2976
|
-
|
|
3140
|
+
modList = modList.filter(
|
|
2977
3141
|
(x) =>
|
|
2978
3142
|
!pyDefaultModules.has(x.name.toLowerCase()) &&
|
|
2979
3143
|
!x.name.startsWith("_") &&
|
|
2980
3144
|
!x.name.startsWith(".")
|
|
2981
3145
|
);
|
|
2982
|
-
let pkgList =
|
|
2983
|
-
|
|
3146
|
+
let pkgList = modList.map((p) => {
|
|
3147
|
+
const apkg = {
|
|
2984
3148
|
name:
|
|
2985
3149
|
PYPI_MODULE_PACKAGE_MAPPING[p.name.toLowerCase()] ||
|
|
2986
3150
|
PYPI_MODULE_PACKAGE_MAPPING[p.name.replace(/_/g, "-").toLowerCase()] ||
|
|
@@ -2994,6 +3158,13 @@ export const getPyModules = async (src, epkgList, options) => {
|
|
|
2994
3158
|
}
|
|
2995
3159
|
]
|
|
2996
3160
|
};
|
|
3161
|
+
if (p.importedSymbols) {
|
|
3162
|
+
apkg.properties.push({
|
|
3163
|
+
name: "ImportedModules",
|
|
3164
|
+
value: p.importedSymbols
|
|
3165
|
+
});
|
|
3166
|
+
}
|
|
3167
|
+
return apkg;
|
|
2997
3168
|
});
|
|
2998
3169
|
pkgList = pkgList.filter(
|
|
2999
3170
|
(obj, index) => pkgList.findIndex((i) => i.name === obj.name) === index
|
|
@@ -3017,7 +3188,7 @@ export const getPyModules = async (src, epkgList, options) => {
|
|
|
3017
3188
|
});
|
|
3018
3189
|
}
|
|
3019
3190
|
}
|
|
3020
|
-
return { allImports, pkgList, dependenciesList };
|
|
3191
|
+
return { allImports, pkgList, dependenciesList, modList };
|
|
3021
3192
|
};
|
|
3022
3193
|
|
|
3023
3194
|
/**
|
|
@@ -3049,14 +3220,15 @@ export const parseSetupPyFile = async function (setupPyData) {
|
|
|
3049
3220
|
};
|
|
3050
3221
|
|
|
3051
3222
|
/**
|
|
3052
|
-
* Method to construct a
|
|
3223
|
+
* Method to construct a GitHub API url for the given repo metadata
|
|
3053
3224
|
* @param {Object} repoMetadata Repo metadata with group and name
|
|
3225
|
+
* @return {String|undefined} github api url (or undefined - if not enough data)
|
|
3054
3226
|
*/
|
|
3055
|
-
export const
|
|
3227
|
+
export const repoMetadataToGitHubApiUrl = function (repoMetadata) {
|
|
3056
3228
|
if (repoMetadata) {
|
|
3057
3229
|
const group = repoMetadata.group;
|
|
3058
3230
|
const name = repoMetadata.name;
|
|
3059
|
-
let ghUrl = "https://github.com";
|
|
3231
|
+
let ghUrl = "https://api.github.com/repos";
|
|
3060
3232
|
if (group && group !== "." && group != "") {
|
|
3061
3233
|
ghUrl = ghUrl + "/" + group.replace("github.com/", "");
|
|
3062
3234
|
}
|
|
@@ -3067,6 +3239,32 @@ export const toGitHubUrl = function (repoMetadata) {
|
|
|
3067
3239
|
}
|
|
3068
3240
|
};
|
|
3069
3241
|
|
|
3242
|
+
/**
|
|
3243
|
+
* Method to construct GitHub api url from repo metadata or one of multiple formats of repo URLs
|
|
3244
|
+
* @param {String} repoUrl Repository url
|
|
3245
|
+
* @param {Object} repoMetadata Object containing group and package name strings
|
|
3246
|
+
* @return {String|undefined} github api url (or undefined - if not a GitHub repo)
|
|
3247
|
+
*/
|
|
3248
|
+
export const toGitHubApiUrl = function (repoUrl, repoMetadata) {
|
|
3249
|
+
if (!repoUrl || !repoUrl.includes("://github.com/")) {
|
|
3250
|
+
return repoMetadataToGitHubApiUrl(repoMetadata);
|
|
3251
|
+
}
|
|
3252
|
+
if (repoUrl.toLowerCase().endsWith(".git")) {
|
|
3253
|
+
repoUrl = repoUrl.slice(0, -4);
|
|
3254
|
+
}
|
|
3255
|
+
repoUrl.replace(/\/$/, "");
|
|
3256
|
+
const parts = repoUrl.split("/");
|
|
3257
|
+
|
|
3258
|
+
if (parts.length < 5 || parts[2] !== "github.com") {
|
|
3259
|
+
return undefined; // Not a valid GitHub repo URL
|
|
3260
|
+
} else {
|
|
3261
|
+
return repoMetadataToGitHubApiUrl({
|
|
3262
|
+
group: parts[3],
|
|
3263
|
+
name: parts[4]
|
|
3264
|
+
});
|
|
3265
|
+
}
|
|
3266
|
+
};
|
|
3267
|
+
|
|
3070
3268
|
/**
|
|
3071
3269
|
* Method to retrieve repo license by querying github api
|
|
3072
3270
|
*
|
|
@@ -3075,22 +3273,16 @@ export const toGitHubUrl = function (repoMetadata) {
|
|
|
3075
3273
|
* @return {String} SPDX license id
|
|
3076
3274
|
*/
|
|
3077
3275
|
export const getRepoLicense = async function (repoUrl, repoMetadata) {
|
|
3078
|
-
|
|
3079
|
-
repoUrl = toGitHubUrl(repoMetadata);
|
|
3080
|
-
}
|
|
3276
|
+
let apiUrl = toGitHubApiUrl(repoUrl, repoMetadata);
|
|
3081
3277
|
// Perform github lookups
|
|
3082
|
-
if (
|
|
3083
|
-
let
|
|
3084
|
-
"https://github.com",
|
|
3085
|
-
"https://api.github.com/repos"
|
|
3086
|
-
);
|
|
3087
|
-
apiUrl += "/license";
|
|
3278
|
+
if (apiUrl) {
|
|
3279
|
+
let licenseUrl = apiUrl + "/license";
|
|
3088
3280
|
const headers = {};
|
|
3089
3281
|
if (process.env.GITHUB_TOKEN) {
|
|
3090
3282
|
headers["Authorization"] = "Bearer " + process.env.GITHUB_TOKEN;
|
|
3091
3283
|
}
|
|
3092
3284
|
try {
|
|
3093
|
-
const res = await cdxgenAgent.get(
|
|
3285
|
+
const res = await cdxgenAgent.get(licenseUrl, {
|
|
3094
3286
|
responseType: "json",
|
|
3095
3287
|
headers: headers
|
|
3096
3288
|
});
|
|
@@ -4555,7 +4747,26 @@ export const parseConanLockData = function (conanLockData) {
|
|
|
4555
4747
|
if (nodes[nk].ref) {
|
|
4556
4748
|
const tmpA = nodes[nk].ref.split("/");
|
|
4557
4749
|
if (tmpA.length === 2) {
|
|
4558
|
-
|
|
4750
|
+
let version = tmpA[1] || "latest";
|
|
4751
|
+
if (tmpA[1].includes("@")) {
|
|
4752
|
+
version = version.split("@")[0];
|
|
4753
|
+
} else if (tmpA[1].includes("#")) {
|
|
4754
|
+
version = version.split("#")[0];
|
|
4755
|
+
}
|
|
4756
|
+
const purlString = new PackageURL(
|
|
4757
|
+
"conan",
|
|
4758
|
+
"",
|
|
4759
|
+
tmpA[0],
|
|
4760
|
+
version,
|
|
4761
|
+
null,
|
|
4762
|
+
null
|
|
4763
|
+
).toString();
|
|
4764
|
+
pkgList.push({
|
|
4765
|
+
name: tmpA[0],
|
|
4766
|
+
version,
|
|
4767
|
+
purl: purlString,
|
|
4768
|
+
"bom-ref": decodeURIComponent(purlString)
|
|
4769
|
+
});
|
|
4559
4770
|
}
|
|
4560
4771
|
}
|
|
4561
4772
|
}
|
|
@@ -4567,15 +4778,50 @@ export const parseConanData = function (conanData) {
|
|
|
4567
4778
|
if (!conanData) {
|
|
4568
4779
|
return pkgList;
|
|
4569
4780
|
}
|
|
4781
|
+
let scope = "required";
|
|
4570
4782
|
conanData.split("\n").forEach((l) => {
|
|
4571
4783
|
l = l.replace("\r", "");
|
|
4784
|
+
if (l.includes("[build_requires]")) {
|
|
4785
|
+
scope = "optional";
|
|
4786
|
+
}
|
|
4787
|
+
if (l.includes("[requires]")) {
|
|
4788
|
+
scope = "required";
|
|
4789
|
+
}
|
|
4572
4790
|
if (!l.includes("/")) {
|
|
4573
4791
|
return;
|
|
4574
4792
|
}
|
|
4575
4793
|
if (l.includes("/")) {
|
|
4576
4794
|
const tmpA = l.trim().split("#")[0].split("/");
|
|
4577
|
-
if (tmpA.length
|
|
4578
|
-
|
|
4795
|
+
if (tmpA.length >= 2 && /^\d+/.test(tmpA[1])) {
|
|
4796
|
+
let version = tmpA[1] || "latest";
|
|
4797
|
+
let qualifiers = undefined;
|
|
4798
|
+
if (tmpA[1].includes("#")) {
|
|
4799
|
+
const tmpB = version.split("#");
|
|
4800
|
+
version = tmpB[0];
|
|
4801
|
+
qualifiers = { revision: tmpB[1] };
|
|
4802
|
+
}
|
|
4803
|
+
if (l.includes("#")) {
|
|
4804
|
+
const tmpB = l.split("#");
|
|
4805
|
+
qualifiers = { revision: tmpB[1] };
|
|
4806
|
+
}
|
|
4807
|
+
if (tmpA[1].includes("@")) {
|
|
4808
|
+
version = version.split("@")[0];
|
|
4809
|
+
}
|
|
4810
|
+
const purlString = new PackageURL(
|
|
4811
|
+
"conan",
|
|
4812
|
+
"",
|
|
4813
|
+
tmpA[0],
|
|
4814
|
+
version,
|
|
4815
|
+
qualifiers,
|
|
4816
|
+
null
|
|
4817
|
+
).toString();
|
|
4818
|
+
pkgList.push({
|
|
4819
|
+
name: tmpA[0],
|
|
4820
|
+
version,
|
|
4821
|
+
purl: purlString,
|
|
4822
|
+
"bom-ref": decodeURIComponent(purlString),
|
|
4823
|
+
scope
|
|
4824
|
+
});
|
|
4579
4825
|
}
|
|
4580
4826
|
}
|
|
4581
4827
|
});
|
|
@@ -5911,7 +6157,7 @@ export const collectGradleDependencies = (
|
|
|
5911
6157
|
* Method to collect class names from all jars in a directory
|
|
5912
6158
|
*
|
|
5913
6159
|
* @param {string} jarPath Path containing jars
|
|
5914
|
-
* @param {object} pomPathMap Map containing jar to pom names. Required to
|
|
6160
|
+
* @param {object} pomPathMap Map containing jar to pom names. Required to successfully parse gradle cache.
|
|
5915
6161
|
*
|
|
5916
6162
|
* @return object containing jar name and class list
|
|
5917
6163
|
*/
|
|
@@ -6838,9 +7084,19 @@ const flattenDeps = (dependenciesMap, pkgList, reqOrSetupFile, t) => {
|
|
|
6838
7084
|
if (!dependenciesMap[pkgRef]) {
|
|
6839
7085
|
dependenciesMap[pkgRef] = [];
|
|
6840
7086
|
}
|
|
7087
|
+
const purlString = new PackageURL(
|
|
7088
|
+
"pypi",
|
|
7089
|
+
"",
|
|
7090
|
+
d.name,
|
|
7091
|
+
d.version,
|
|
7092
|
+
null,
|
|
7093
|
+
null
|
|
7094
|
+
).toString();
|
|
6841
7095
|
pkgList.push({
|
|
6842
7096
|
name: d.name,
|
|
6843
7097
|
version: d.version,
|
|
7098
|
+
purl: purlString,
|
|
7099
|
+
"bom-ref": decodeURIComponent(purlString),
|
|
6844
7100
|
properties: [
|
|
6845
7101
|
{
|
|
6846
7102
|
name: "SrcFile",
|
|
@@ -6850,11 +7106,11 @@ const flattenDeps = (dependenciesMap, pkgList, reqOrSetupFile, t) => {
|
|
|
6850
7106
|
evidence: {
|
|
6851
7107
|
identity: {
|
|
6852
7108
|
field: "purl",
|
|
6853
|
-
confidence:
|
|
7109
|
+
confidence: 0.8,
|
|
6854
7110
|
methods: [
|
|
6855
7111
|
{
|
|
6856
7112
|
technique: "manifest-analysis",
|
|
6857
|
-
confidence:
|
|
7113
|
+
confidence: 0.8,
|
|
6858
7114
|
value: reqOrSetupFile
|
|
6859
7115
|
}
|
|
6860
7116
|
]
|
|
@@ -6894,6 +7150,7 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => {
|
|
|
6894
7150
|
*/
|
|
6895
7151
|
if (
|
|
6896
7152
|
!process.env.VIRTUAL_ENV &&
|
|
7153
|
+
!process.env.CONDA_PREFIX &&
|
|
6897
7154
|
reqOrSetupFile &&
|
|
6898
7155
|
!reqOrSetupFile.endsWith("poetry.lock")
|
|
6899
7156
|
) {
|
|
@@ -6904,7 +7161,10 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => {
|
|
|
6904
7161
|
if (result.status !== 0 || result.error) {
|
|
6905
7162
|
if (DEBUG_MODE) {
|
|
6906
7163
|
console.log("Virtual env creation has failed");
|
|
6907
|
-
if (
|
|
7164
|
+
if (
|
|
7165
|
+
result.stderr &&
|
|
7166
|
+
result.stderr.includes("spawnSync python ENOENT")
|
|
7167
|
+
) {
|
|
6908
7168
|
console.log(
|
|
6909
7169
|
"Install suitable version of python or set the environment variable PYTHON_CMD."
|
|
6910
7170
|
);
|
|
@@ -7082,7 +7342,10 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => {
|
|
|
7082
7342
|
}
|
|
7083
7343
|
}
|
|
7084
7344
|
// Bug #375. Attempt pip freeze on existing and new virtual environments
|
|
7085
|
-
if (
|
|
7345
|
+
if (
|
|
7346
|
+
(env.VIRTUAL_ENV && env.VIRTUAL_ENV.length) ||
|
|
7347
|
+
(env.CONDA_PREFIX && env.CONDA_PREFIX.length)
|
|
7348
|
+
) {
|
|
7086
7349
|
/**
|
|
7087
7350
|
* At this point, the previous attempt to do a pip install might have failed and we might have an unclean virtual environment with an incomplete list
|
|
7088
7351
|
* The position taken by cdxgen is "Some SBOM is better than no SBOM", so we proceed to collecting the dependencies that got installed with pip freeze
|
|
@@ -7112,9 +7375,19 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => {
|
|
|
7112
7375
|
const version = t.version;
|
|
7113
7376
|
let exclude = ["pip", "setuptools", "wheel"];
|
|
7114
7377
|
if (!exclude.includes(name)) {
|
|
7378
|
+
const purlString = new PackageURL(
|
|
7379
|
+
"pypi",
|
|
7380
|
+
"",
|
|
7381
|
+
name,
|
|
7382
|
+
version,
|
|
7383
|
+
null,
|
|
7384
|
+
null
|
|
7385
|
+
).toString();
|
|
7115
7386
|
pkgList.push({
|
|
7116
7387
|
name,
|
|
7117
7388
|
version,
|
|
7389
|
+
purl: purlString,
|
|
7390
|
+
"bom-ref": decodeURIComponent(purlString),
|
|
7118
7391
|
evidence: {
|
|
7119
7392
|
identity: {
|
|
7120
7393
|
field: "purl",
|
|
@@ -7123,7 +7396,7 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => {
|
|
|
7123
7396
|
{
|
|
7124
7397
|
technique: "instrumentation",
|
|
7125
7398
|
confidence: 1,
|
|
7126
|
-
value: env.VIRTUAL_ENV
|
|
7399
|
+
value: env.VIRTUAL_ENV || env.CONDA_PREFIX
|
|
7127
7400
|
}
|
|
7128
7401
|
]
|
|
7129
7402
|
}
|
|
@@ -7139,12 +7412,6 @@ export const getPipFrozenTree = (basePath, reqOrSetupFile, tempVenvDir) => {
|
|
|
7139
7412
|
for (const k of Object.keys(dependenciesMap)) {
|
|
7140
7413
|
dependenciesList.push({ ref: k, dependsOn: dependenciesMap[k] });
|
|
7141
7414
|
}
|
|
7142
|
-
} else {
|
|
7143
|
-
if (DEBUG_MODE) {
|
|
7144
|
-
console.log(
|
|
7145
|
-
"NOTE: Setup and activate a python virtual environment for this project prior to invoking cdxgen to improve SBOM accuracy."
|
|
7146
|
-
);
|
|
7147
|
-
}
|
|
7148
7415
|
}
|
|
7149
7416
|
return {
|
|
7150
7417
|
pkgList,
|
|
@@ -7198,11 +7465,16 @@ export const addEvidenceForImports = (pkgList, allImports) => {
|
|
|
7198
7465
|
? [name, `${group}/${name}`, `@${group}/${name}`]
|
|
7199
7466
|
: [name];
|
|
7200
7467
|
for (const alias of aliases) {
|
|
7201
|
-
|
|
7202
|
-
|
|
7203
|
-
|
|
7204
|
-
|
|
7205
|
-
|
|
7468
|
+
const all_includes = impPkgs.filter(
|
|
7469
|
+
(find_pkg) =>
|
|
7470
|
+
find_pkg.startsWith(alias) &&
|
|
7471
|
+
(find_pkg.length === alias.length || find_pkg[alias.length] === "/")
|
|
7472
|
+
);
|
|
7473
|
+
if (impPkgs.includes(alias) || all_includes.length) {
|
|
7474
|
+
let importedModules = new Set();
|
|
7475
|
+
pkg.scope = "required";
|
|
7476
|
+
for (const subevidence of all_includes) {
|
|
7477
|
+
const evidences = allImports[subevidence];
|
|
7206
7478
|
for (const evidence of evidences) {
|
|
7207
7479
|
if (evidence && Object.keys(evidence).length && evidence.fileName) {
|
|
7208
7480
|
pkg.evidence = pkg.evidence || {};
|
|
@@ -7223,14 +7495,14 @@ export const addEvidenceForImports = (pkgList, allImports) => {
|
|
|
7223
7495
|
}
|
|
7224
7496
|
}
|
|
7225
7497
|
}
|
|
7226
|
-
|
|
7227
|
-
|
|
7228
|
-
|
|
7229
|
-
|
|
7230
|
-
|
|
7231
|
-
|
|
7232
|
-
|
|
7233
|
-
}
|
|
7498
|
+
}
|
|
7499
|
+
importedModules = Array.from(importedModules);
|
|
7500
|
+
if (importedModules.length) {
|
|
7501
|
+
pkg.properties = pkg.properties || [];
|
|
7502
|
+
pkg.properties.push({
|
|
7503
|
+
name: "ImportedModules",
|
|
7504
|
+
value: importedModules.join(",")
|
|
7505
|
+
});
|
|
7234
7506
|
}
|
|
7235
7507
|
break;
|
|
7236
7508
|
}
|
|
@@ -7532,6 +7804,7 @@ export const parseCmakeLikeFile = (cmakeListFile, pkgType, options = {}) => {
|
|
|
7532
7804
|
!n.startsWith("@")
|
|
7533
7805
|
) {
|
|
7534
7806
|
n = n.replace(/"/g, "");
|
|
7807
|
+
// Can this be replaced with a db lookup?
|
|
7535
7808
|
for (const wrapkey of Object.keys(mesonWrapDB)) {
|
|
7536
7809
|
const awrap = mesonWrapDB[wrapkey];
|
|
7537
7810
|
if (
|
|
@@ -7645,6 +7918,44 @@ export const getCppModules = (src, options, osPkgsList, epkgList) => {
|
|
|
7645
7918
|
const pkgAddedMap = {};
|
|
7646
7919
|
let sliceData = {};
|
|
7647
7920
|
const epkgMap = {};
|
|
7921
|
+
let parentComponent = undefined;
|
|
7922
|
+
const dependsOn = [];
|
|
7923
|
+
// Let's look for any vcpkg.json file to tell us about the directory we're scanning
|
|
7924
|
+
// users can use this file to give us a clue even if they do not use vcpkg library manager
|
|
7925
|
+
if (existsSync(join(src, "vcpkg.json"))) {
|
|
7926
|
+
const vcPkgData = JSON.parse(join(src, "vcpkg.json"));
|
|
7927
|
+
if (
|
|
7928
|
+
vcPkgData &&
|
|
7929
|
+
Object.keys(vcPkgData).length &&
|
|
7930
|
+
vcPkgData.name &&
|
|
7931
|
+
vcPkgData.version
|
|
7932
|
+
) {
|
|
7933
|
+
const parentPurl = new PackageURL(
|
|
7934
|
+
pkgType,
|
|
7935
|
+
"",
|
|
7936
|
+
vcPkgData.name,
|
|
7937
|
+
vcPkgData.version,
|
|
7938
|
+
null,
|
|
7939
|
+
null
|
|
7940
|
+
).toString();
|
|
7941
|
+
parentComponent = {
|
|
7942
|
+
name: vcPkgData.name,
|
|
7943
|
+
version: vcPkgData.version,
|
|
7944
|
+
description: vcPkgData.description,
|
|
7945
|
+
license: vcPkgData.license,
|
|
7946
|
+
purl: parentPurl,
|
|
7947
|
+
"bom-ref": decodeURIComponent(parentPurl)
|
|
7948
|
+
};
|
|
7949
|
+
if (vcPkgData.homepage) {
|
|
7950
|
+
parentComponent.homepage = { url: vcPkgData.homepage };
|
|
7951
|
+
}
|
|
7952
|
+
}
|
|
7953
|
+
} else if (existsSync(join(src, "CMakeLists.txt"))) {
|
|
7954
|
+
const retMap = parseCmakeLikeFile(join(src, "CMakeLists.txt"), pkgType);
|
|
7955
|
+
if (retMap.parentComponent && Object.keys(retMap.parentComponent).length) {
|
|
7956
|
+
parentComponent = retMap.parentComponent;
|
|
7957
|
+
}
|
|
7958
|
+
}
|
|
7648
7959
|
(epkgList || []).forEach((p) => {
|
|
7649
7960
|
epkgMap[p.name] = p;
|
|
7650
7961
|
});
|
|
@@ -7717,15 +8028,14 @@ export const getCppModules = (src, options, osPkgsList, epkgList) => {
|
|
|
7717
8028
|
}
|
|
7718
8029
|
if (usageData[afile]) {
|
|
7719
8030
|
const usymbols = Array.from(usageData[afile]).filter(
|
|
7720
|
-
(v) =>
|
|
7721
|
-
!v.startsWith("<operator") &&
|
|
7722
|
-
!v.startsWith("<global") &&
|
|
7723
|
-
!v.startsWith("<empty")
|
|
8031
|
+
(v) => !v.startsWith("<") && !v.startsWith("__")
|
|
7724
8032
|
);
|
|
7725
|
-
if (!apkg["properties"]) {
|
|
8033
|
+
if (!apkg["properties"] && usymbols.length) {
|
|
7726
8034
|
apkg["properties"] = [
|
|
7727
8035
|
{ name: "ImportedSymbols", value: usymbols.join(", ") }
|
|
7728
8036
|
];
|
|
8037
|
+
} else {
|
|
8038
|
+
apkg["properties"] = [];
|
|
7729
8039
|
}
|
|
7730
8040
|
const newProps = [];
|
|
7731
8041
|
let symbolsPropertyFound = false;
|
|
@@ -7738,7 +8048,7 @@ export const getCppModules = (src, options, osPkgsList, epkgList) => {
|
|
|
7738
8048
|
}
|
|
7739
8049
|
newProps.push(prop);
|
|
7740
8050
|
}
|
|
7741
|
-
if (!symbolsPropertyFound) {
|
|
8051
|
+
if (!symbolsPropertyFound && usymbols.length) {
|
|
7742
8052
|
apkg["properties"].push({
|
|
7743
8053
|
name: "ImportedSymbols",
|
|
7744
8054
|
value: usymbols.join(", ")
|
|
@@ -7746,10 +8056,40 @@ export const getCppModules = (src, options, osPkgsList, epkgList) => {
|
|
|
7746
8056
|
}
|
|
7747
8057
|
apkg["properties"] = newProps;
|
|
7748
8058
|
}
|
|
7749
|
-
|
|
7750
|
-
|
|
8059
|
+
// At this point, we have a package but we don't know what it's called
|
|
8060
|
+
// So let's try to locate this generic package using some heuristics
|
|
8061
|
+
apkg = locateGenericPackage(apkg);
|
|
8062
|
+
if (!pkgAddedMap[name]) {
|
|
8063
|
+
pkgList.push(apkg);
|
|
8064
|
+
dependsOn.push(apkg["bom-ref"]);
|
|
8065
|
+
pkgAddedMap[name] = true;
|
|
8066
|
+
}
|
|
7751
8067
|
}
|
|
7752
|
-
|
|
8068
|
+
const dependenciesList =
|
|
8069
|
+
dependsOn.length && parentComponent
|
|
8070
|
+
? [
|
|
8071
|
+
{
|
|
8072
|
+
ref: parentComponent["bom-ref"],
|
|
8073
|
+
dependsOn
|
|
8074
|
+
}
|
|
8075
|
+
]
|
|
8076
|
+
: [];
|
|
8077
|
+
return {
|
|
8078
|
+
parentComponent,
|
|
8079
|
+
pkgList,
|
|
8080
|
+
dependenciesList
|
|
8081
|
+
};
|
|
8082
|
+
};
|
|
8083
|
+
|
|
8084
|
+
/**
|
|
8085
|
+
* NOT IMPLEMENTED YET.
|
|
8086
|
+
* A future method to locate a generic package given some name and properties
|
|
8087
|
+
*
|
|
8088
|
+
* @param {object} apkg Package to locate
|
|
8089
|
+
* @returns Located project with precise purl or the original unmodified input.
|
|
8090
|
+
*/
|
|
8091
|
+
export const locateGenericPackage = (apkg) => {
|
|
8092
|
+
return apkg;
|
|
7753
8093
|
};
|
|
7754
8094
|
|
|
7755
8095
|
export const parseCUsageSlice = (sliceData) => {
|