@trustify-da/trustify-da-javascript-client 0.3.0-ea.e12bc82 → 0.3.0-ea.e348975
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +193 -13
- package/dist/package.json +43 -27
- package/dist/src/analysis.d.ts +28 -5
- package/dist/src/analysis.js +99 -89
- package/dist/src/batch_opts.d.ts +24 -0
- package/dist/src/batch_opts.js +35 -0
- package/dist/src/cli.js +346 -9
- package/dist/src/cyclone_dx_sbom.d.ts +17 -3
- package/dist/src/cyclone_dx_sbom.js +56 -8
- package/dist/src/index.d.ts +202 -12
- package/dist/src/index.js +361 -9
- package/dist/src/license/index.d.ts +28 -0
- package/dist/src/license/index.js +103 -0
- package/dist/src/license/license_utils.d.ts +40 -0
- package/dist/src/license/license_utils.js +137 -0
- package/dist/src/license/licenses_api.d.ts +34 -0
- package/dist/src/license/licenses_api.js +98 -0
- package/dist/src/license/project_license.d.ts +20 -0
- package/dist/src/license/project_license.js +62 -0
- package/dist/src/oci_image/images.d.ts +4 -5
- package/dist/src/oci_image/images.js +15 -1
- package/dist/src/oci_image/utils.d.ts +7 -7
- package/dist/src/oci_image/utils.js +11 -2
- package/dist/src/provider.d.ts +18 -5
- package/dist/src/provider.js +33 -5
- package/dist/src/providers/base_java.d.ts +15 -15
- package/dist/src/providers/base_java.js +18 -44
- package/dist/src/providers/base_javascript.d.ts +45 -12
- package/dist/src/providers/base_javascript.js +138 -24
- package/dist/src/providers/base_pyproject.d.ts +158 -0
- package/dist/src/providers/base_pyproject.js +322 -0
- package/dist/src/providers/containerfile_parser.d.ts +5 -0
- package/dist/src/providers/containerfile_parser.js +20 -0
- package/dist/src/providers/golang_gomodules.d.ts +30 -13
- package/dist/src/providers/golang_gomodules.js +176 -121
- package/dist/src/providers/gomod_parser.d.ts +4 -0
- package/dist/src/providers/gomod_parser.js +16 -0
- package/dist/src/providers/java_gradle.d.ts +28 -3
- package/dist/src/providers/java_gradle.js +128 -4
- package/dist/src/providers/java_gradle_groovy.d.ts +1 -1
- package/dist/src/providers/java_gradle_kotlin.d.ts +1 -1
- package/dist/src/providers/java_maven.d.ts +20 -5
- package/dist/src/providers/java_maven.js +198 -6
- package/dist/src/providers/javascript_bun.d.ts +10 -0
- package/dist/src/providers/javascript_bun.js +100 -0
- package/dist/src/providers/javascript_npm.d.ts +6 -0
- package/dist/src/providers/javascript_npm.js +21 -0
- package/dist/src/providers/javascript_pnpm.d.ts +1 -1
- package/dist/src/providers/javascript_pnpm.js +8 -4
- package/dist/src/providers/manifest.d.ts +2 -0
- package/dist/src/providers/manifest.js +22 -4
- package/dist/src/providers/marker_evaluator.d.ts +14 -0
- package/dist/src/providers/marker_evaluator.js +191 -0
- package/dist/src/providers/oci_dockerfile.d.ts +51 -0
- package/dist/src/providers/oci_dockerfile.js +177 -0
- package/dist/src/providers/processors/yarn_berry_processor.d.ts +2 -2
- package/dist/src/providers/processors/yarn_berry_processor.js +88 -5
- package/dist/src/providers/processors/yarn_classic_processor.d.ts +2 -2
- package/dist/src/providers/python_controller.d.ts +10 -3
- package/dist/src/providers/python_controller.js +122 -63
- package/dist/src/providers/python_pip.d.ts +16 -4
- package/dist/src/providers/python_pip.js +51 -58
- package/dist/src/providers/python_pip_pyproject.d.ts +61 -0
- package/dist/src/providers/python_pip_pyproject.js +146 -0
- package/dist/src/providers/python_poetry.d.ts +98 -0
- package/dist/src/providers/python_poetry.js +286 -0
- package/dist/src/providers/python_uv.d.ts +46 -0
- package/dist/src/providers/python_uv.js +269 -0
- package/dist/src/providers/requirements_parser.d.ts +6 -0
- package/dist/src/providers/requirements_parser.js +24 -0
- package/dist/src/providers/rust_cargo.d.ts +56 -0
- package/dist/src/providers/rust_cargo.js +641 -0
- package/dist/src/providers/tree-sitter-containerfile.wasm +0 -0
- package/dist/src/providers/tree-sitter-gomod.wasm +0 -0
- package/dist/src/providers/tree-sitter-requirements.wasm +0 -0
- package/dist/src/remediate.d.ts +21 -0
- package/dist/src/remediate.js +149 -0
- package/dist/src/remediation.d.ts +57 -0
- package/dist/src/remediation.js +436 -0
- package/dist/src/remediation_report.d.ts +40 -0
- package/dist/src/remediation_report.js +159 -0
- package/dist/src/sbom.d.ts +17 -2
- package/dist/src/sbom.js +16 -4
- package/dist/src/tools.d.ts +48 -6
- package/dist/src/tools.js +114 -1
- package/dist/src/updaters/maven_updater.d.ts +26 -0
- package/dist/src/updaters/maven_updater.js +322 -0
- package/dist/src/updaters/toml_updater.d.ts +33 -0
- package/dist/src/updaters/toml_updater.js +244 -0
- package/dist/src/workspace.d.ts +70 -0
- package/dist/src/workspace.js +256 -0
- package/package.json +44 -28
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/** @typedef {{name: string, version: string, dependencies: DependencyEntry[]}} DependencyEntry */
|
|
1
|
+
/** @typedef {{name: string, version: string, dependencies: DependencyEntry[], hashes?: Array<{alg: string, content: string}>}} DependencyEntry */
|
|
2
2
|
export default class Python_controller {
|
|
3
3
|
/**
|
|
4
4
|
* Constructor to create new python controller instance to interact with pip package manager
|
|
@@ -15,17 +15,24 @@ export default class Python_controller {
|
|
|
15
15
|
realEnvironment: boolean;
|
|
16
16
|
pathToRequirements: string;
|
|
17
17
|
options: {};
|
|
18
|
+
parser: Promise<import("web-tree-sitter").Parser>;
|
|
19
|
+
requirementsQuery: Promise<import("web-tree-sitter").Query>;
|
|
20
|
+
pinnedVersionQuery: Promise<import("web-tree-sitter").Query>;
|
|
18
21
|
prepareEnvironment(): void;
|
|
19
22
|
/**
|
|
20
23
|
*
|
|
21
24
|
* @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not
|
|
22
|
-
* @return {[DependencyEntry]}
|
|
25
|
+
* @return {Promise<[DependencyEntry]>}
|
|
23
26
|
*/
|
|
24
|
-
getDependencies(includeTransitive: boolean): [DependencyEntry]
|
|
27
|
+
getDependencies(includeTransitive: boolean): Promise<[DependencyEntry]>;
|
|
25
28
|
#private;
|
|
26
29
|
}
|
|
27
30
|
export type DependencyEntry = {
|
|
28
31
|
name: string;
|
|
29
32
|
version: string;
|
|
30
33
|
dependencies: DependencyEntry[];
|
|
34
|
+
hashes?: Array<{
|
|
35
|
+
alg: string;
|
|
36
|
+
content: string;
|
|
37
|
+
}>;
|
|
31
38
|
};
|
|
@@ -2,6 +2,7 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import os, { EOL } from "os";
|
|
4
4
|
import { environmentVariableIsPopulated, getCustom, invokeCommand } from "../tools.js";
|
|
5
|
+
import { getParser, getRequirementQuery, getPinnedVersionQuery } from './requirements_parser.js';
|
|
5
6
|
function getPipFreezeOutput() {
|
|
6
7
|
try {
|
|
7
8
|
return environmentVariableIsPopulated("TRUSTIFY_DA_PIP_FREEZE") ? new Buffer.from(process.env["TRUSTIFY_DA_PIP_FREEZE"], 'base64').toString('ascii') : invokeCommand(this.pathToPipBin, ['freeze', '--all']).toString();
|
|
@@ -18,7 +19,54 @@ function getPipShowOutput(depNames) {
|
|
|
18
19
|
throw new Error('fail invoking \'pip show\' to fetch metadata for all installed packages in environment', { cause: error });
|
|
19
20
|
}
|
|
20
21
|
}
|
|
21
|
-
/**
|
|
22
|
+
/**
|
|
23
|
+
* Get pip inspect output from env var override or by invoking pip inspect.
|
|
24
|
+
* Returns null if pip inspect is unavailable (pip < 22.2) or if freeze/show
|
|
25
|
+
* env vars are set without an inspect override (to avoid inconsistent data).
|
|
26
|
+
* @return {string|null} pip inspect JSON output, or null on failure
|
|
27
|
+
*/
|
|
28
|
+
function getPipInspectOutput() {
|
|
29
|
+
if (environmentVariableIsPopulated("TRUSTIFY_DA_PIP_INSPECT")) {
|
|
30
|
+
return new Buffer.from(process.env["TRUSTIFY_DA_PIP_INSPECT"], 'base64').toString('ascii');
|
|
31
|
+
}
|
|
32
|
+
if (environmentVariableIsPopulated("TRUSTIFY_DA_PIP_FREEZE") || environmentVariableIsPopulated("TRUSTIFY_DA_PIP_SHOW")) {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
return invokeCommand(this.pathToPipBin, ['inspect']).toString();
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
console.warn('pip inspect is not available (requires pip 22.2+), SBOM will be generated without hashes');
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Parse pip inspect JSON output and build a hash lookup map.
|
|
45
|
+
* @param {string|null} inspectOutput - raw pip inspect JSON string
|
|
46
|
+
* @return {Map<string, Array<{alg: string, content: string}>>} map of lowercase package name to hashes
|
|
47
|
+
*/
|
|
48
|
+
function parsePipInspectHashes(inspectOutput) {
|
|
49
|
+
if (!inspectOutput) {
|
|
50
|
+
return new Map();
|
|
51
|
+
}
|
|
52
|
+
try {
|
|
53
|
+
let inspectData = JSON.parse(inspectOutput);
|
|
54
|
+
let hashMap = new Map();
|
|
55
|
+
for (let pkg of (inspectData.installed || [])) {
|
|
56
|
+
let name = pkg.metadata?.name;
|
|
57
|
+
let sha256 = pkg.download_info?.archive_info?.hashes?.sha256;
|
|
58
|
+
if (name && sha256) {
|
|
59
|
+
hashMap.set(name.toLowerCase(), [{ alg: "SHA-256", content: sha256 }]);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return hashMap;
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
console.warn('Failed to parse pip inspect output, SBOM will be generated without hashes');
|
|
66
|
+
return new Map();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** @typedef {{name: string, version: string, dependencies: DependencyEntry[], hashes?: Array<{alg: string, content: string}>}} DependencyEntry */
|
|
22
70
|
export default class Python_controller {
|
|
23
71
|
pythonEnvDir;
|
|
24
72
|
pathToPipBin;
|
|
@@ -26,6 +74,9 @@ export default class Python_controller {
|
|
|
26
74
|
realEnvironment;
|
|
27
75
|
pathToRequirements;
|
|
28
76
|
options;
|
|
77
|
+
parser;
|
|
78
|
+
requirementsQuery;
|
|
79
|
+
pinnedVersionQuery;
|
|
29
80
|
/**
|
|
30
81
|
* Constructor to create new python controller instance to interact with pip package manager
|
|
31
82
|
* @param {boolean} realEnvironment - whether to use real environment supplied by client or to create virtual environment
|
|
@@ -41,6 +92,9 @@ export default class Python_controller {
|
|
|
41
92
|
this.prepareEnvironment();
|
|
42
93
|
this.pathToRequirements = pathToRequirements;
|
|
43
94
|
this.options = options;
|
|
95
|
+
this.parser = getParser();
|
|
96
|
+
this.requirementsQuery = getRequirementQuery();
|
|
97
|
+
this.pinnedVersionQuery = getPinnedVersionQuery();
|
|
44
98
|
}
|
|
45
99
|
prepareEnvironment() {
|
|
46
100
|
if (!this.realEnvironment) {
|
|
@@ -86,6 +140,24 @@ export default class Python_controller {
|
|
|
86
140
|
}
|
|
87
141
|
}
|
|
88
142
|
}
|
|
143
|
+
/**
|
|
144
|
+
* Parse the requirements.txt file using tree-sitter and return structured requirement data.
|
|
145
|
+
* @return {Promise<{name: string, version: string|null, hasMarker: boolean}[]>}
|
|
146
|
+
*/
|
|
147
|
+
async #parseRequirements() {
|
|
148
|
+
const content = fs.readFileSync(this.pathToRequirements).toString();
|
|
149
|
+
const tree = (await this.parser).parse(content);
|
|
150
|
+
return Promise.all((await this.requirementsQuery).matches(tree.rootNode).map(async (match) => {
|
|
151
|
+
const reqNode = match.captures.find(c => c.name === 'req').node;
|
|
152
|
+
const name = match.captures.find(c => c.name === 'name').node.text;
|
|
153
|
+
const versionMatches = (await this.pinnedVersionQuery).matches(reqNode);
|
|
154
|
+
const version = versionMatches.length > 0
|
|
155
|
+
? versionMatches[0].captures.find(c => c.name === 'version').node.text
|
|
156
|
+
: null;
|
|
157
|
+
const hasMarker = reqNode.children.some(c => c.type === 'marker_spec');
|
|
158
|
+
return { name, version, hasMarker };
|
|
159
|
+
}));
|
|
160
|
+
}
|
|
89
161
|
#decideIfWindowsOrLinuxPath(fileName) {
|
|
90
162
|
if (os.platform() === "win32") {
|
|
91
163
|
return fileName + ".exe";
|
|
@@ -97,17 +169,20 @@ export default class Python_controller {
|
|
|
97
169
|
/**
|
|
98
170
|
*
|
|
99
171
|
* @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not
|
|
100
|
-
* @return {[DependencyEntry]}
|
|
172
|
+
* @return {Promise<[DependencyEntry]>}
|
|
101
173
|
*/
|
|
102
|
-
getDependencies(includeTransitive) {
|
|
174
|
+
async getDependencies(includeTransitive) {
|
|
103
175
|
let startingTime;
|
|
104
176
|
let endingTime;
|
|
105
177
|
if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
|
|
106
178
|
startingTime = new Date();
|
|
107
179
|
console.log("Starting time to get requirements.txt dependency tree = " + startingTime);
|
|
108
180
|
}
|
|
181
|
+
let installBestEfforts = getCustom("TRUSTIFY_DA_PYTHON_INSTALL_BEST_EFFORTS", "false", this.options);
|
|
182
|
+
if (installBestEfforts === "true" && this.realEnvironment) {
|
|
183
|
+
throw new Error("Conflicting settings, TRUSTIFY_DA_PYTHON_INSTALL_BEST_EFFORTS=true requires TRUSTIFY_DA_PYTHON_VIRTUAL_ENV=true");
|
|
184
|
+
}
|
|
109
185
|
if (!this.realEnvironment) {
|
|
110
|
-
let installBestEfforts = getCustom("TRUSTIFY_DA_PYTHON_INSTALL_BEST_EFFORTS", "false", this.options);
|
|
111
186
|
if (installBestEfforts === "false") {
|
|
112
187
|
try {
|
|
113
188
|
invokeCommand(this.pathToPipBin, ['install', '-r', this.pathToRequirements]);
|
|
@@ -124,10 +199,10 @@ export default class Python_controller {
|
|
|
124
199
|
if (matchManifestVersions === "true") {
|
|
125
200
|
throw new Error("Conflicting settings, TRUSTIFY_DA_PYTHON_INSTALL_BEST_EFFORTS=true can only work with MATCH_MANIFEST_VERSIONS=false");
|
|
126
201
|
}
|
|
127
|
-
this.#installingRequirementsOneByOne();
|
|
202
|
+
await this.#installingRequirementsOneByOne();
|
|
128
203
|
}
|
|
129
204
|
}
|
|
130
|
-
let dependencies = this.#getDependenciesImpl(includeTransitive);
|
|
205
|
+
let dependencies = await this.#getDependenciesImpl(includeTransitive);
|
|
131
206
|
this.#cleanEnvironment();
|
|
132
207
|
if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
|
|
133
208
|
endingTime = new Date();
|
|
@@ -137,16 +212,14 @@ export default class Python_controller {
|
|
|
137
212
|
}
|
|
138
213
|
return dependencies;
|
|
139
214
|
}
|
|
140
|
-
#installingRequirementsOneByOne() {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
requirementsRows.filter((line) => !line.trim().startsWith("#")).filter((line) => line.trim() !== "").forEach((dependency) => {
|
|
144
|
-
let dependencyName = getDependencyName(dependency);
|
|
215
|
+
async #installingRequirementsOneByOne() {
|
|
216
|
+
const requirements = await this.#parseRequirements();
|
|
217
|
+
requirements.forEach(({ name }) => {
|
|
145
218
|
try {
|
|
146
|
-
invokeCommand(this.pathToPipBin, ['install',
|
|
219
|
+
invokeCommand(this.pathToPipBin, ['install', name]);
|
|
147
220
|
}
|
|
148
221
|
catch (error) {
|
|
149
|
-
throw new Error(`Failed in best-effort installing ${
|
|
222
|
+
throw new Error(`Failed in best-effort installing ${name} in virtual python environment`, { cause: error });
|
|
150
223
|
}
|
|
151
224
|
});
|
|
152
225
|
}
|
|
@@ -163,34 +236,26 @@ export default class Python_controller {
|
|
|
163
236
|
}
|
|
164
237
|
}
|
|
165
238
|
}
|
|
166
|
-
#getDependenciesImpl(includeTransitive) {
|
|
167
|
-
let dependencies =
|
|
239
|
+
async #getDependenciesImpl(includeTransitive) {
|
|
240
|
+
let dependencies = [];
|
|
168
241
|
let usePipDepTree = getCustom("TRUSTIFY_DA_PIP_USE_DEP_TREE", "false", this.options);
|
|
169
|
-
let freezeOutput;
|
|
170
|
-
let lines;
|
|
171
|
-
let depNames;
|
|
172
|
-
let pipShowOutput;
|
|
173
242
|
let allPipShowDeps;
|
|
174
243
|
let pipDepTreeJsonArrayOutput;
|
|
175
244
|
if (usePipDepTree !== "true") {
|
|
176
|
-
freezeOutput = getPipFreezeOutput.call(this);
|
|
177
|
-
lines = freezeOutput.split(EOL);
|
|
178
|
-
depNames = lines.map(line => getDependencyName(line));
|
|
245
|
+
const freezeOutput = getPipFreezeOutput.call(this);
|
|
246
|
+
const lines = freezeOutput.split(EOL);
|
|
247
|
+
const depNames = lines.map(line => getDependencyName(line));
|
|
248
|
+
const pipShowOutput = getPipShowOutput.call(this, depNames);
|
|
249
|
+
allPipShowDeps = pipShowOutput.split(EOL + "---" + EOL);
|
|
179
250
|
}
|
|
180
251
|
else {
|
|
181
252
|
pipDepTreeJsonArrayOutput = getDependencyTreeJsonFromPipDepTree(this.pathToPipBin, this.pathToPythonBin);
|
|
182
253
|
}
|
|
183
|
-
if (usePipDepTree !== "true") {
|
|
184
|
-
pipShowOutput = getPipShowOutput.call(this, depNames);
|
|
185
|
-
allPipShowDeps = pipShowOutput.split(EOL + "---" + EOL);
|
|
186
|
-
}
|
|
187
|
-
//debug
|
|
188
|
-
// pipShowOutput = "alternative pip show output goes here for debugging"
|
|
189
254
|
let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS", "true", this.options);
|
|
190
|
-
let
|
|
255
|
+
let parsedRequirements = await this.#parseRequirements();
|
|
191
256
|
let CachedEnvironmentDeps = {};
|
|
192
257
|
if (usePipDepTree !== "true") {
|
|
193
|
-
allPipShowDeps.forEach(
|
|
258
|
+
allPipShowDeps.forEach(record => {
|
|
194
259
|
let dependencyName = getDependencyNameShow(record).toLowerCase();
|
|
195
260
|
CachedEnvironmentDeps[dependencyName] = record;
|
|
196
261
|
CachedEnvironmentDeps[dependencyName.replace("-", "_")] = record;
|
|
@@ -210,42 +275,31 @@ export default class Python_controller {
|
|
|
210
275
|
CachedEnvironmentDeps[packageName.replace("_", "-")] = pipDepTreeEntryForCache;
|
|
211
276
|
});
|
|
212
277
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
278
|
+
const inspectOutput = getPipInspectOutput.call(this);
|
|
279
|
+
const hashMap = parsePipInspectHashes(inspectOutput);
|
|
280
|
+
parsedRequirements.forEach(({ name: depName, version: manifestVersion, hasMarker }) => {
|
|
281
|
+
if (hasMarker && CachedEnvironmentDeps[depName.toLowerCase()] === undefined) {
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
if (matchManifestVersions === "true" && manifestVersion != null) {
|
|
218
285
|
let installedVersion;
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim();
|
|
223
|
-
if (manifestVersion.includes("#")) {
|
|
224
|
-
let hashCharIndex = manifestVersion.indexOf("#");
|
|
225
|
-
manifestVersion = manifestVersion.substring(0, hashCharIndex);
|
|
286
|
+
if (CachedEnvironmentDeps[depName.toLowerCase()] !== undefined) {
|
|
287
|
+
if (usePipDepTree !== "true") {
|
|
288
|
+
installedVersion = getDependencyVersion(CachedEnvironmentDeps[depName.toLowerCase()]);
|
|
226
289
|
}
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
if (CachedEnvironmentDeps[dependencyName.toLowerCase()] !== undefined) {
|
|
230
|
-
if (usePipDepTree !== "true") {
|
|
231
|
-
installedVersion = getDependencyVersion(CachedEnvironmentDeps[dependencyName.toLowerCase()]);
|
|
232
|
-
}
|
|
233
|
-
else {
|
|
234
|
-
installedVersion = CachedEnvironmentDeps[dependencyName.toLowerCase()].version;
|
|
235
|
-
}
|
|
290
|
+
else {
|
|
291
|
+
installedVersion = CachedEnvironmentDeps[depName.toLowerCase()].version;
|
|
236
292
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
}
|
|
293
|
+
}
|
|
294
|
+
if (installedVersion) {
|
|
295
|
+
if (manifestVersion.trim() !== installedVersion.trim()) {
|
|
296
|
+
throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${depName} (manifest version=${manifestVersion}, installed version=${installedVersion}).If you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting MATCH_MANIFEST_VERSIONS=false`);
|
|
241
297
|
}
|
|
242
298
|
}
|
|
243
299
|
}
|
|
244
|
-
let path =
|
|
245
|
-
let depName = getDependencyName(dep);
|
|
246
|
-
//array to track a path for each branch in the dependency tree
|
|
300
|
+
let path = [];
|
|
247
301
|
path.push(depName.toLowerCase());
|
|
248
|
-
bringAllDependencies(dependencies, depName, CachedEnvironmentDeps, includeTransitive, path, usePipDepTree);
|
|
302
|
+
bringAllDependencies(dependencies, depName, CachedEnvironmentDeps, includeTransitive, path, usePipDepTree, hashMap);
|
|
249
303
|
});
|
|
250
304
|
dependencies.sort((dep1, dep2) => {
|
|
251
305
|
const DEP1 = dep1.name.toLowerCase();
|
|
@@ -325,8 +379,9 @@ function getDepsList(record) {
|
|
|
325
379
|
* @param includeTransitive
|
|
326
380
|
* @param usePipDepTree
|
|
327
381
|
* @param {[string]}path array representing the path of the current branch in dependency tree, starting with a root dependency - that is - a given dependency in requirements.txt
|
|
382
|
+
* @param {Map<string, Array<{alg: string, content: string}>>} hashMap - map of lowercase package name to hashes
|
|
328
383
|
*/
|
|
329
|
-
function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDeps, includeTransitive, path, usePipDepTree) {
|
|
384
|
+
function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDeps, includeTransitive, path, usePipDepTree, hashMap) {
|
|
330
385
|
if (dependencyName?.trim() === "") {
|
|
331
386
|
return;
|
|
332
387
|
}
|
|
@@ -347,18 +402,22 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
|
|
|
347
402
|
version = record.version;
|
|
348
403
|
directDeps = record.dependencies;
|
|
349
404
|
}
|
|
350
|
-
let targetDeps =
|
|
405
|
+
let targetDeps = [];
|
|
406
|
+
let hashes = hashMap?.get(depName.toLowerCase());
|
|
351
407
|
let entry = { "name": depName, "version": version, "dependencies": [] };
|
|
408
|
+
if (hashes) {
|
|
409
|
+
entry.hashes = hashes;
|
|
410
|
+
}
|
|
352
411
|
dependencies.push(entry);
|
|
353
412
|
directDeps.forEach((dep) => {
|
|
354
|
-
let depArray =
|
|
413
|
+
let depArray = [];
|
|
355
414
|
// to avoid infinite loop, check if the dependency not already on current path, before going recursively resolving its dependencies.
|
|
356
415
|
if (!path.includes(dep.toLowerCase())) {
|
|
357
416
|
// send to recurrsion the path + the current dep
|
|
358
417
|
depArray.push(dep.toLowerCase());
|
|
359
418
|
if (includeTransitive) {
|
|
360
419
|
// send to recurrsion the array of all deps in path + the current dependency name which is not on the path.
|
|
361
|
-
bringAllDependencies(targetDeps, dep, cachedEnvironmentDeps, includeTransitive, path.concat(depArray), usePipDepTree);
|
|
420
|
+
bringAllDependencies(targetDeps, dep, cachedEnvironmentDeps, includeTransitive, path.concat(depArray), usePipDepTree, hashMap);
|
|
362
421
|
}
|
|
363
422
|
}
|
|
364
423
|
// sort ra
|
|
@@ -3,12 +3,18 @@ declare namespace _default {
|
|
|
3
3
|
export { validateLockFile };
|
|
4
4
|
export { provideComponent };
|
|
5
5
|
export { provideStack };
|
|
6
|
+
export { readLicenseFromManifest };
|
|
7
|
+
export function packageManagerName(): string;
|
|
6
8
|
}
|
|
7
9
|
export default _default;
|
|
8
10
|
export type DependencyEntry = {
|
|
9
11
|
name: string;
|
|
10
12
|
version: string;
|
|
11
13
|
dependencies: DependencyEntry[];
|
|
14
|
+
hashes?: Array<{
|
|
15
|
+
alg: string;
|
|
16
|
+
content: string;
|
|
17
|
+
}>;
|
|
12
18
|
};
|
|
13
19
|
/**
|
|
14
20
|
* @param {string} manifestName - the subject manifest name-type
|
|
@@ -23,13 +29,19 @@ declare function validateLockFile(): boolean;
|
|
|
23
29
|
* Provide content and content type for python-pip component analysis.
|
|
24
30
|
* @param {string} manifest - path to requirements.txt for component report
|
|
25
31
|
* @param {{}} [opts={}] - optional various options to pass along the application
|
|
26
|
-
* @returns {Provided}
|
|
32
|
+
* @returns {Promise<Provided>}
|
|
27
33
|
*/
|
|
28
|
-
declare function provideComponent(manifest: string, opts?: {}
|
|
34
|
+
declare function provideComponent(manifest: string, opts?: {}): Promise<Provided>;
|
|
29
35
|
/**
|
|
30
36
|
* Provide content and content type for python-pip stack analysis.
|
|
31
37
|
* @param {string} manifest - the manifest path or name
|
|
32
38
|
* @param {{}} [opts={}] - optional various options to pass along the application
|
|
33
|
-
* @returns {Provided}
|
|
39
|
+
* @returns {Promise<Provided>}
|
|
34
40
|
*/
|
|
35
|
-
declare function provideStack(manifest: string, opts?: {}
|
|
41
|
+
declare function provideStack(manifest: string, opts?: {}): Promise<Provided>;
|
|
42
|
+
/**
|
|
43
|
+
* Python requirements.txt has no standard license field
|
|
44
|
+
* @param {string} manifestPath - path to requirements.txt
|
|
45
|
+
* @returns {string|null}
|
|
46
|
+
*/
|
|
47
|
+
declare function readLicenseFromManifest(manifestPath: string): string | null;
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
import fs from 'node:fs';
|
|
2
|
-
import { EOL } from 'os';
|
|
3
2
|
import { PackageURL } from 'packageurl-js';
|
|
3
|
+
import { readLicenseFile } from '../license/license_utils.js';
|
|
4
4
|
import Sbom from '../sbom.js';
|
|
5
5
|
import { environmentVariableIsPopulated, getCustom, getCustomPath, invokeCommand } from "../tools.js";
|
|
6
6
|
import Python_controller from './python_controller.js';
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
import { getParser, getIgnoreQuery, getPinnedVersionQuery } from './requirements_parser.js';
|
|
8
|
+
export default { isSupported, validateLockFile, provideComponent, provideStack, readLicenseFromManifest, packageManagerName() { return 'pip'; } };
|
|
9
|
+
/** @typedef {{name: string, version: string, dependencies: DependencyEntry[], hashes?: Array<{alg: string, content: string}>}} DependencyEntry */
|
|
9
10
|
/**
|
|
10
11
|
* @type {string} ecosystem for python-pip is 'pip'
|
|
11
12
|
* @private
|
|
12
13
|
*/
|
|
13
14
|
const ecosystem = 'pip';
|
|
15
|
+
const NO_SCOPE = undefined;
|
|
14
16
|
/**
|
|
15
17
|
* @param {string} manifestName - the subject manifest name-type
|
|
16
18
|
* @returns {boolean} - return true if `requirements.txt` is the manifest name-type
|
|
@@ -18,6 +20,13 @@ const ecosystem = 'pip';
|
|
|
18
20
|
function isSupported(manifestName) {
|
|
19
21
|
return 'requirements.txt' === manifestName;
|
|
20
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* Python requirements.txt has no standard license field
|
|
25
|
+
* @param {string} manifestPath - path to requirements.txt
|
|
26
|
+
* @returns {string|null}
|
|
27
|
+
*/
|
|
28
|
+
// eslint-disable-next-line no-unused-vars
|
|
29
|
+
function readLicenseFromManifest(manifestPath) { return readLicenseFile(manifestPath); }
|
|
21
30
|
/**
|
|
22
31
|
* @param {string} manifestDir - the directory where the manifest lies
|
|
23
32
|
*/
|
|
@@ -26,12 +35,12 @@ function validateLockFile() { return true; }
|
|
|
26
35
|
* Provide content and content type for python-pip stack analysis.
|
|
27
36
|
* @param {string} manifest - the manifest path or name
|
|
28
37
|
* @param {{}} [opts={}] - optional various options to pass along the application
|
|
29
|
-
* @returns {Provided}
|
|
38
|
+
* @returns {Promise<Provided>}
|
|
30
39
|
*/
|
|
31
|
-
function provideStack(manifest, opts = {}) {
|
|
40
|
+
async function provideStack(manifest, opts = {}) {
|
|
32
41
|
return {
|
|
33
42
|
ecosystem,
|
|
34
|
-
content: createSbomStackAnalysis(manifest, opts),
|
|
43
|
+
content: await createSbomStackAnalysis(manifest, opts),
|
|
35
44
|
contentType: 'application/vnd.cyclonedx+json'
|
|
36
45
|
};
|
|
37
46
|
}
|
|
@@ -39,16 +48,15 @@ function provideStack(manifest, opts = {}) {
|
|
|
39
48
|
* Provide content and content type for python-pip component analysis.
|
|
40
49
|
* @param {string} manifest - path to requirements.txt for component report
|
|
41
50
|
* @param {{}} [opts={}] - optional various options to pass along the application
|
|
42
|
-
* @returns {Provided}
|
|
51
|
+
* @returns {Promise<Provided>}
|
|
43
52
|
*/
|
|
44
|
-
function provideComponent(manifest, opts = {}) {
|
|
53
|
+
async function provideComponent(manifest, opts = {}) {
|
|
45
54
|
return {
|
|
46
55
|
ecosystem,
|
|
47
|
-
content: getSbomForComponentAnalysis(manifest, opts),
|
|
56
|
+
content: await getSbomForComponentAnalysis(manifest, opts),
|
|
48
57
|
contentType: 'application/vnd.cyclonedx+json'
|
|
49
58
|
};
|
|
50
59
|
}
|
|
51
|
-
/** @typedef {{name: string, , version: string, dependencies: DependencyEntry[]}} DependencyEntry */
|
|
52
60
|
/**
|
|
53
61
|
*
|
|
54
62
|
* @param {PackageURL}source
|
|
@@ -58,7 +66,7 @@ function provideComponent(manifest, opts = {}) {
|
|
|
58
66
|
*/
|
|
59
67
|
function addAllDependencies(source, dep, sbom) {
|
|
60
68
|
let targetPurl = toPurl(dep["name"], dep["version"]);
|
|
61
|
-
sbom.addDependency(source, targetPurl);
|
|
69
|
+
sbom.addDependency(source, targetPurl, NO_SCOPE, dep["hashes"]);
|
|
62
70
|
let directDeps = dep["dependencies"];
|
|
63
71
|
if (directDeps !== undefined && directDeps.length > 0) {
|
|
64
72
|
directDeps.forEach((dependency) => { addAllDependencies(toPurl(dep["name"], dep["version"]), dependency, sbom); });
|
|
@@ -66,49 +74,34 @@ function addAllDependencies(source, dep, sbom) {
|
|
|
66
74
|
}
|
|
67
75
|
/**
|
|
68
76
|
*
|
|
69
|
-
* @param
|
|
70
|
-
* @return {
|
|
71
|
-
*/
|
|
72
|
-
function splitToNameVersion(nameVersion) {
|
|
73
|
-
let result = [];
|
|
74
|
-
if (nameVersion.includes("==")) {
|
|
75
|
-
return nameVersion.split("==");
|
|
76
|
-
}
|
|
77
|
-
const regex = /[^\w\s-_]/g;
|
|
78
|
-
let endIndex = nameVersion.search(regex);
|
|
79
|
-
if (endIndex === -1) {
|
|
80
|
-
return [nameVersion.trim()];
|
|
81
|
-
}
|
|
82
|
-
result.push(nameVersion.substring(0, endIndex).trim());
|
|
83
|
-
return result;
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
*
|
|
87
|
-
* @param {string} requirementTxtContent
|
|
88
|
-
* @return {PackageURL []}
|
|
77
|
+
* @param {string} manifest - path to requirements.txt
|
|
78
|
+
* @return {Promise<PackageURL[]>}
|
|
89
79
|
*/
|
|
90
|
-
function getIgnoredDependencies(
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
80
|
+
async function getIgnoredDependencies(manifest) {
|
|
81
|
+
const [parser, ignoreQuery, pinnedVersionQuery] = await Promise.all([
|
|
82
|
+
getParser(), getIgnoreQuery(), getPinnedVersionQuery()
|
|
83
|
+
]);
|
|
84
|
+
const content = fs.readFileSync(manifest).toString();
|
|
85
|
+
const tree = parser.parse(content);
|
|
86
|
+
return ignoreQuery.matches(tree.rootNode).map(match => {
|
|
87
|
+
const reqNode = match.captures.find(c => c.name === 'req').node;
|
|
88
|
+
const name = match.captures.find(c => c.name === 'name').node.text;
|
|
89
|
+
const versionMatches = pinnedVersionQuery.matches(reqNode);
|
|
90
|
+
const version = versionMatches.length > 0
|
|
91
|
+
? versionMatches[0].captures.find(c => c.name === 'version').node.text
|
|
92
|
+
: undefined;
|
|
93
|
+
return toPurl(name, version);
|
|
101
94
|
});
|
|
102
95
|
}
|
|
103
96
|
/**
|
|
104
97
|
*
|
|
105
|
-
* @param {string}
|
|
98
|
+
* @param {string} manifest - path to requirements.txt
|
|
106
99
|
* @param {Sbom} sbom object to filter out from it exhortignore dependencies.
|
|
107
100
|
* @param {{Object}} opts - various options and settings for the application
|
|
108
101
|
* @private
|
|
109
102
|
*/
|
|
110
|
-
function handleIgnoredDependencies(
|
|
111
|
-
let ignoredDeps = getIgnoredDependencies(
|
|
103
|
+
async function handleIgnoredDependencies(manifest, sbom, opts = {}) {
|
|
104
|
+
let ignoredDeps = await getIgnoredDependencies(manifest);
|
|
112
105
|
let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS", "true", opts);
|
|
113
106
|
if (matchManifestVersions === "true") {
|
|
114
107
|
const ignoredDepsVersion = ignoredDeps.filter(dep => dep.version !== undefined);
|
|
@@ -172,22 +165,22 @@ const DEFAULT_PIP_ROOT_COMPONENT_VERSION = "0.0.0";
|
|
|
172
165
|
* Create sbom json string out of a manifest path for stack analysis.
|
|
173
166
|
* @param {string} manifest - path for requirements.txt
|
|
174
167
|
* @param {{}} [opts={}] - optional various options to pass along the application
|
|
175
|
-
* @returns {string} the sbom json string content
|
|
168
|
+
* @returns {Promise<string>} the sbom json string content
|
|
176
169
|
* @private
|
|
177
170
|
*/
|
|
178
|
-
function createSbomStackAnalysis(manifest, opts = {}) {
|
|
171
|
+
async function createSbomStackAnalysis(manifest, opts = {}) {
|
|
179
172
|
let binaries = {};
|
|
180
173
|
let createVirtualPythonEnv = handlePythonEnvironment(binaries, opts);
|
|
181
174
|
let pythonController = new Python_controller(createVirtualPythonEnv === "false", binaries.pip, binaries.python, manifest, opts);
|
|
182
|
-
let dependencies = pythonController.getDependencies(true);
|
|
175
|
+
let dependencies = await pythonController.getDependencies(true);
|
|
183
176
|
let sbom = new Sbom();
|
|
184
177
|
const rootPurl = toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION);
|
|
185
|
-
|
|
178
|
+
const license = readLicenseFromManifest(manifest);
|
|
179
|
+
sbom.addRoot(rootPurl, license);
|
|
186
180
|
dependencies.forEach(dep => {
|
|
187
181
|
addAllDependencies(rootPurl, dep, sbom);
|
|
188
182
|
});
|
|
189
|
-
|
|
190
|
-
handleIgnoredDependencies(requirementTxtContent, sbom, opts);
|
|
183
|
+
await handleIgnoredDependencies(manifest, sbom, opts);
|
|
191
184
|
// In python there is no root component, then we must remove the dummy root we added, so the sbom json will be accepted by the DA backend
|
|
192
185
|
// sbom.removeRootComponent()
|
|
193
186
|
return sbom.getAsJsonString(opts);
|
|
@@ -196,22 +189,22 @@ function createSbomStackAnalysis(manifest, opts = {}) {
|
|
|
196
189
|
* Create a sbom json string out of a manifest content for component analysis
|
|
197
190
|
* @param {string} manifest - path to requirements.txt
|
|
198
191
|
* @param {{}} [opts={}] - optional various options to pass along the application
|
|
199
|
-
* @returns {string} the sbom json string content
|
|
192
|
+
* @returns {Promise<string>} the sbom json string content
|
|
200
193
|
* @private
|
|
201
194
|
*/
|
|
202
|
-
function getSbomForComponentAnalysis(manifest, opts = {}) {
|
|
195
|
+
async function getSbomForComponentAnalysis(manifest, opts = {}) {
|
|
203
196
|
let binaries = {};
|
|
204
197
|
let createVirtualPythonEnv = handlePythonEnvironment(binaries, opts);
|
|
205
198
|
let pythonController = new Python_controller(createVirtualPythonEnv === "false", binaries.pip, binaries.python, manifest, opts);
|
|
206
|
-
let dependencies = pythonController.getDependencies(false);
|
|
199
|
+
let dependencies = await pythonController.getDependencies(false);
|
|
207
200
|
let sbom = new Sbom();
|
|
208
201
|
const rootPurl = toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION);
|
|
209
|
-
|
|
202
|
+
const license = readLicenseFromManifest(manifest);
|
|
203
|
+
sbom.addRoot(rootPurl, license);
|
|
210
204
|
dependencies.forEach(dep => {
|
|
211
|
-
sbom.addDependency(rootPurl, toPurl(dep.name, dep.version));
|
|
205
|
+
sbom.addDependency(rootPurl, toPurl(dep.name, dep.version), NO_SCOPE, dep.hashes);
|
|
212
206
|
});
|
|
213
|
-
|
|
214
|
-
handleIgnoredDependencies(requirementTxtContent, sbom, opts);
|
|
207
|
+
await handleIgnoredDependencies(manifest, sbom, opts);
|
|
215
208
|
// In python there is no root component, then we must remove the dummy root we added, so the sbom json will be accepted by the DA backend
|
|
216
209
|
// sbom.removeRootComponent()
|
|
217
210
|
return sbom.getAsJsonString(opts);
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Python provider for pyproject.toml files using PEP 621 format without a lock file.
|
|
3
|
+
* Uses `pip install --dry-run --ignore-installed --report` to resolve the full dependency tree.
|
|
4
|
+
* Acts as the fallback provider when no lock file (uv.lock/poetry.lock) is found.
|
|
5
|
+
*/
|
|
6
|
+
export default class Python_pip_pyproject extends Base_pyproject {
|
|
7
|
+
/**
|
|
8
|
+
* Always returns true — pip provider is the fallback when no lock file is found.
|
|
9
|
+
* @param {string} manifestDir
|
|
10
|
+
* @param {{}} [opts={}]
|
|
11
|
+
* @returns {boolean}
|
|
12
|
+
*/
|
|
13
|
+
validateLockFile(manifestDir: string, opts?: {}): boolean;
|
|
14
|
+
/**
|
|
15
|
+
* Get pip report output from env var override or by running pip.
|
|
16
|
+
* @param {string} manifestDir - directory containing pyproject.toml
|
|
17
|
+
* @param {{}} [opts={}]
|
|
18
|
+
* @returns {string} pip report JSON string
|
|
19
|
+
*/
|
|
20
|
+
_getPipReportOutput(manifestDir: string, opts?: {}): string;
|
|
21
|
+
/**
|
|
22
|
+
* Parse pip report JSON and build dependency graph.
|
|
23
|
+
* @param {string} reportJson - pip report JSON string
|
|
24
|
+
* @returns {{directDeps: string[], graph: Map<string, {name: string, version: string, children: string[]}>}}
|
|
25
|
+
*/
|
|
26
|
+
_parsePipReport(reportJson: string): {
|
|
27
|
+
directDeps: string[];
|
|
28
|
+
graph: Map<string, {
|
|
29
|
+
name: string;
|
|
30
|
+
version: string;
|
|
31
|
+
children: string[];
|
|
32
|
+
}>;
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Check if a requires_dist entry is an extras-only dependency.
|
|
36
|
+
* @param {string} req - e.g. "PySocks!=1.5.7,>=1.5.6; extra == \"socks\""
|
|
37
|
+
* @returns {boolean}
|
|
38
|
+
*/
|
|
39
|
+
_hasExtraMarker(req: string): boolean;
|
|
40
|
+
/**
|
|
41
|
+
* Extract package name from a requires_dist entry.
|
|
42
|
+
* @param {string} req - e.g. "charset_normalizer<4,>=2"
|
|
43
|
+
* @returns {string|null}
|
|
44
|
+
*/
|
|
45
|
+
_extractDepName(req: string): string | null;
|
|
46
|
+
/**
|
|
47
|
+
* Resolve dependencies using pip install --dry-run --report.
|
|
48
|
+
* @param {string} manifestDir
|
|
49
|
+
* @param {string} _workspaceDir - unused (pip resolves from manifest directory)
|
|
50
|
+
* @param {object} parsed - parsed pyproject.toml
|
|
51
|
+
* @param {{}} [opts={}]
|
|
52
|
+
* @returns {Promise<{directDeps: string[], graph: Map}>}
|
|
53
|
+
*/
|
|
54
|
+
_getDependencyData(manifestDir: string, _workspaceDir: string, parsed: object, opts?: {}): Promise<{
|
|
55
|
+
directDeps: string[];
|
|
56
|
+
graph: Map<any, any>;
|
|
57
|
+
}>;
|
|
58
|
+
_findEggInfoDirs(dir: any): string[];
|
|
59
|
+
_cleanupEggInfo(dir: any, existing: any): void;
|
|
60
|
+
}
|
|
61
|
+
import Base_pyproject from './base_pyproject.js';
|