@trustify-da/trustify-da-javascript-client 0.3.0-ea.8adb67b → 0.3.0-ea.8eab29b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +191 -11
- package/dist/package.json +19 -6
- package/dist/src/analysis.d.ts +16 -0
- package/dist/src/analysis.js +74 -80
- package/dist/src/batch_opts.d.ts +24 -0
- package/dist/src/batch_opts.js +35 -0
- package/dist/src/cli.js +241 -8
- package/dist/src/cyclone_dx_sbom.d.ts +17 -2
- package/dist/src/cyclone_dx_sbom.js +56 -8
- package/dist/src/index.d.ts +138 -4
- package/dist/src/index.js +356 -8
- package/dist/src/license/index.d.ts +28 -0
- package/dist/src/license/index.js +100 -0
- package/dist/src/license/license_utils.d.ts +40 -0
- package/dist/src/license/license_utils.js +134 -0
- package/dist/src/license/licenses_api.d.ts +34 -0
- package/dist/src/license/licenses_api.js +98 -0
- package/dist/src/license/project_license.d.ts +20 -0
- package/dist/src/license/project_license.js +62 -0
- package/dist/src/oci_image/utils.js +11 -2
- package/dist/src/provider.d.ts +18 -5
- package/dist/src/provider.js +31 -5
- package/dist/src/providers/base_java.d.ts +5 -9
- package/dist/src/providers/base_java.js +9 -38
- package/dist/src/providers/base_javascript.d.ts +40 -7
- package/dist/src/providers/base_javascript.js +138 -24
- package/dist/src/providers/base_pyproject.d.ts +158 -0
- package/dist/src/providers/base_pyproject.js +322 -0
- package/dist/src/providers/golang_gomodules.d.ts +29 -12
- package/dist/src/providers/golang_gomodules.js +176 -121
- package/dist/src/providers/gomod_parser.d.ts +4 -0
- package/dist/src/providers/gomod_parser.js +16 -0
- package/dist/src/providers/java_gradle.d.ts +25 -0
- package/dist/src/providers/java_gradle.js +128 -4
- package/dist/src/providers/java_maven.d.ts +16 -1
- package/dist/src/providers/java_maven.js +125 -5
- package/dist/src/providers/javascript_bun.d.ts +10 -0
- package/dist/src/providers/javascript_bun.js +100 -0
- package/dist/src/providers/javascript_npm.d.ts +1 -0
- package/dist/src/providers/javascript_npm.js +21 -0
- package/dist/src/providers/javascript_pnpm.d.ts +1 -1
- package/dist/src/providers/javascript_pnpm.js +8 -4
- package/dist/src/providers/manifest.d.ts +2 -0
- package/dist/src/providers/manifest.js +22 -4
- package/dist/src/providers/marker_evaluator.d.ts +14 -0
- package/dist/src/providers/marker_evaluator.js +191 -0
- package/dist/src/providers/processors/yarn_berry_processor.js +88 -5
- package/dist/src/providers/python_controller.d.ts +10 -3
- package/dist/src/providers/python_controller.js +118 -62
- package/dist/src/providers/python_pip.d.ts +16 -4
- package/dist/src/providers/python_pip.js +51 -58
- package/dist/src/providers/python_pip_pyproject.d.ts +61 -0
- package/dist/src/providers/python_pip_pyproject.js +146 -0
- package/dist/src/providers/python_poetry.d.ts +75 -0
- package/dist/src/providers/python_poetry.js +238 -0
- package/dist/src/providers/python_uv.d.ts +55 -0
- package/dist/src/providers/python_uv.js +227 -0
- package/dist/src/providers/requirements_parser.d.ts +6 -0
- package/dist/src/providers/requirements_parser.js +24 -0
- package/dist/src/providers/rust_cargo.d.ts +56 -0
- package/dist/src/providers/rust_cargo.js +641 -0
- package/dist/src/providers/tree-sitter-gomod.wasm +0 -0
- package/dist/src/providers/tree-sitter-requirements.wasm +0 -0
- package/dist/src/sbom.d.ts +17 -2
- package/dist/src/sbom.js +16 -4
- package/dist/src/tools.d.ts +44 -0
- package/dist/src/tools.js +114 -1
- package/dist/src/workspace.d.ts +70 -0
- package/dist/src/workspace.js +256 -0
- package/package.json +20 -7
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// PEP 508 environment marker evaluator.
|
|
2
|
+
// Filters Python dependencies by platform/version markers so that e.g.
|
|
3
|
+
// "pywin32 ; sys_platform == 'win32'" is excluded on Linux/macOS.
|
|
4
|
+
// See https://peps.python.org/pep-0508/#environment-markers
|
|
5
|
+
import os from 'node:os';
|
|
6
|
+
import { getCustomPath, invokeCommand } from '../tools.js';
|
|
7
|
+
let cachedPythonVersions = undefined;
|
|
8
|
+
function getPythonVersions() {
|
|
9
|
+
if (cachedPythonVersions !== undefined) {
|
|
10
|
+
return cachedPythonVersions;
|
|
11
|
+
}
|
|
12
|
+
try {
|
|
13
|
+
let python = getCustomPath('python3');
|
|
14
|
+
let out = invokeCommand(python, ['-c', "import sys; v=sys.version_info; print(f'{v.major}.{v.minor} {v.major}.{v.minor}.{v.micro}')"], { timeout: 5000 }).toString().trim();
|
|
15
|
+
let [short, full] = out.split(' ');
|
|
16
|
+
cachedPythonVersions = { short, full };
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
cachedPythonVersions = null;
|
|
20
|
+
}
|
|
21
|
+
return cachedPythonVersions;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Maps Node.js/OS values to PEP 508 marker variables.
|
|
25
|
+
* Example: on Linux, sys_platform='linux', platform_system='Linux', os_name='posix'
|
|
26
|
+
* @returns {Record<string, string>}
|
|
27
|
+
*/
|
|
28
|
+
export function getEnvironmentMarkers() {
|
|
29
|
+
let platform = process.platform;
|
|
30
|
+
let systemMap = { win32: 'Windows', linux: 'Linux', darwin: 'Darwin' };
|
|
31
|
+
let machine = typeof os.machine === 'function' ? os.machine() : process.arch;
|
|
32
|
+
let pyVer = getPythonVersions();
|
|
33
|
+
return {
|
|
34
|
+
sys_platform: platform,
|
|
35
|
+
platform_system: systemMap[platform] || platform,
|
|
36
|
+
os_name: platform === 'win32' ? 'nt' : 'posix',
|
|
37
|
+
platform_machine: machine,
|
|
38
|
+
platform_release: os.release(),
|
|
39
|
+
platform_version: os.version?.() || '',
|
|
40
|
+
python_version: pyVer?.short || '',
|
|
41
|
+
python_full_version: pyVer?.full || '',
|
|
42
|
+
implementation_name: 'cpython',
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function compareVersions(left, right) {
|
|
46
|
+
let lParts = left.split('.').map(Number);
|
|
47
|
+
let rParts = right.split('.').map(Number);
|
|
48
|
+
let len = Math.max(lParts.length, rParts.length);
|
|
49
|
+
for (let i = 0; i < len; i++) {
|
|
50
|
+
let l = lParts[i] || 0;
|
|
51
|
+
let r = rParts[i] || 0;
|
|
52
|
+
if (l < r) {
|
|
53
|
+
return -1;
|
|
54
|
+
}
|
|
55
|
+
if (l > r) {
|
|
56
|
+
return 1;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return 0;
|
|
60
|
+
}
|
|
61
|
+
// Evaluates a single comparison like sys_platform == 'win32' or python_version >= '3.8'.
|
|
62
|
+
// Version-bearing variables (python_version, python_full_version) use numeric comparison;
|
|
63
|
+
// all others use string equality. Returns false when the env value is missing.
|
|
64
|
+
function evaluateComparison(variable, op, value, env) {
|
|
65
|
+
let envVal = env[variable];
|
|
66
|
+
if (envVal === undefined || envVal === '') {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
let isVersion = variable.includes('version');
|
|
70
|
+
if (isVersion) {
|
|
71
|
+
let cmp = compareVersions(envVal, value);
|
|
72
|
+
switch (op) {
|
|
73
|
+
case '==': return cmp === 0;
|
|
74
|
+
case '!=': return cmp !== 0;
|
|
75
|
+
case '>=': return cmp >= 0;
|
|
76
|
+
case '<=': return cmp <= 0;
|
|
77
|
+
case '>': return cmp > 0;
|
|
78
|
+
case '<': return cmp < 0;
|
|
79
|
+
case '~=': {
|
|
80
|
+
let parts = value.split('.');
|
|
81
|
+
parts.pop();
|
|
82
|
+
let prefix = parts.join('.');
|
|
83
|
+
return envVal.startsWith(prefix) && cmp >= 0;
|
|
84
|
+
}
|
|
85
|
+
default: return true;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
switch (op) {
|
|
89
|
+
case '==': return envVal === value;
|
|
90
|
+
case '!=': return envVal !== value;
|
|
91
|
+
case 'in': return value.includes(envVal);
|
|
92
|
+
case 'not in': return !value.includes(envVal);
|
|
93
|
+
default: return envVal === value;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// Parses a single marker comparison into {variable, op, value}.
|
|
97
|
+
// Handles both normal and reversed forms:
|
|
98
|
+
// "sys_platform == 'linux'" → { variable: 'sys_platform', op: '==', value: 'linux' }
|
|
99
|
+
// "'linux' == sys_platform" → { variable: 'sys_platform', op: '==', value: 'linux' }
|
|
100
|
+
function parseAtom(expr) {
|
|
101
|
+
// Normal form: variable op 'value'
|
|
102
|
+
let m = expr.match(/^\s*([\w.]+)\s*(~=|!=|==|>=|<=|>|<|not\s+in|in)\s*["']([^"']*)["']\s*$/);
|
|
103
|
+
if (m) {
|
|
104
|
+
return { variable: m[1], op: m[2].replace(/\s+/g, ' '), value: m[3] };
|
|
105
|
+
}
|
|
106
|
+
// Reversed form: 'value' op variable — reverse directional operators
|
|
107
|
+
let mReverse = expr.match(/^\s*["']([^"']*)['"]\s*(~=|!=|==|>=|<=|>|<|not\s+in|in)\s*([\w.]+)\s*$/);
|
|
108
|
+
if (mReverse) {
|
|
109
|
+
let reverseOp = { '<': '>', '>': '<', '<=': '>=', '>=': '<=' };
|
|
110
|
+
let op = mReverse[2].replace(/\s+/g, ' ');
|
|
111
|
+
return { variable: mReverse[3], op: reverseOp[op] || op, value: mReverse[1] };
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Evaluates a full PEP 508 marker expression against the current platform.
|
|
117
|
+
* Example: "sys_platform == 'win32' and python_version >= '3.8'" → false on Linux
|
|
118
|
+
* Empty/missing markers return true (unconditional dependency).
|
|
119
|
+
* @param {string} markerExpr
|
|
120
|
+
* @returns {boolean}
|
|
121
|
+
*/
|
|
122
|
+
export function evaluateMarker(markerExpr) {
|
|
123
|
+
if (!markerExpr || !markerExpr.trim()) {
|
|
124
|
+
return true;
|
|
125
|
+
}
|
|
126
|
+
let env = getEnvironmentMarkers();
|
|
127
|
+
return evaluateExpr(markerExpr.trim(), env);
|
|
128
|
+
}
|
|
129
|
+
function evaluateExpr(expr, env) {
|
|
130
|
+
let orParts = splitLogical(expr, ' or ');
|
|
131
|
+
if (orParts.length > 1) {
|
|
132
|
+
return orParts.some(part => evaluateExpr(part, env));
|
|
133
|
+
}
|
|
134
|
+
let andParts = splitLogical(expr, ' and ');
|
|
135
|
+
if (andParts.length > 1) {
|
|
136
|
+
return andParts.every(part => evaluateExpr(part, env));
|
|
137
|
+
}
|
|
138
|
+
let trimmed = expr.trim();
|
|
139
|
+
if (trimmed.startsWith('(') && trimmed.endsWith(')')) {
|
|
140
|
+
return evaluateExpr(trimmed.slice(1, -1), env);
|
|
141
|
+
}
|
|
142
|
+
let atom = parseAtom(trimmed);
|
|
143
|
+
if (!atom) {
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
return evaluateComparison(atom.variable, atom.op, atom.value, env);
|
|
147
|
+
}
|
|
148
|
+
// Splits an expression by " and " or " or " at the top level, skipping
|
|
149
|
+
// separators inside parentheses or quoted strings.
|
|
150
|
+
// Example: splitLogical("a == 'x' and (b == 'y' or c == 'z')", " and ")
|
|
151
|
+
// → ["a == 'x'", "(b == 'y' or c == 'z')"]
|
|
152
|
+
function splitLogical(expr, sep) {
|
|
153
|
+
let parts = [];
|
|
154
|
+
let depth = 0;
|
|
155
|
+
let current = '';
|
|
156
|
+
let i = 0;
|
|
157
|
+
let quoteChar = null;
|
|
158
|
+
while (i < expr.length) {
|
|
159
|
+
let ch = expr[i];
|
|
160
|
+
if (quoteChar) {
|
|
161
|
+
if (ch === quoteChar) {
|
|
162
|
+
quoteChar = null;
|
|
163
|
+
}
|
|
164
|
+
current += ch;
|
|
165
|
+
i++;
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
if (ch === '"' || ch === "'") {
|
|
169
|
+
quoteChar = ch;
|
|
170
|
+
current += ch;
|
|
171
|
+
i++;
|
|
172
|
+
continue;
|
|
173
|
+
}
|
|
174
|
+
if (ch === '(') {
|
|
175
|
+
depth++;
|
|
176
|
+
}
|
|
177
|
+
if (ch === ')') {
|
|
178
|
+
depth--;
|
|
179
|
+
}
|
|
180
|
+
if (depth === 0 && expr.substring(i, i + sep.length) === sep) {
|
|
181
|
+
parts.push(current);
|
|
182
|
+
current = '';
|
|
183
|
+
i += sep.length;
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
current += ch;
|
|
187
|
+
i++;
|
|
188
|
+
}
|
|
189
|
+
parts.push(current);
|
|
190
|
+
return parts.filter(p => p.trim());
|
|
191
|
+
}
|
|
@@ -15,7 +15,10 @@ export default class Yarn_berry_processor extends Yarn_processor {
|
|
|
15
15
|
* @returns {string[]} Command arguments for listing dependencies
|
|
16
16
|
*/
|
|
17
17
|
listCmdArgs(includeTransitive) {
|
|
18
|
-
|
|
18
|
+
// --all is needed to include workspace members in the output
|
|
19
|
+
return includeTransitive
|
|
20
|
+
? ['info', '--recursive', '--all', '--json']
|
|
21
|
+
: ['info', '--all', '--json'];
|
|
19
22
|
}
|
|
20
23
|
/**
|
|
21
24
|
* Returns the command arguments for updating the lock file
|
|
@@ -48,13 +51,15 @@ export default class Yarn_berry_processor extends Yarn_processor {
|
|
|
48
51
|
if (!depTree) {
|
|
49
52
|
return new Map();
|
|
50
53
|
}
|
|
51
|
-
return new Map(depTree.filter(dep => !this.#isRoot(dep.value))
|
|
54
|
+
return new Map(depTree.filter(dep => !this.#isRoot(dep.value))
|
|
55
|
+
.map(dep => {
|
|
52
56
|
const depName = dep.value;
|
|
53
57
|
const idx = depName.lastIndexOf('@');
|
|
54
58
|
const name = depName.substring(0, idx);
|
|
55
59
|
const version = dep.children.Version;
|
|
56
60
|
return [name, toPurl(purlType, name, version)];
|
|
57
|
-
})
|
|
61
|
+
})
|
|
62
|
+
.filter(([name]) => this._manifest.dependencies.includes(name)));
|
|
58
63
|
}
|
|
59
64
|
/**
|
|
60
65
|
* Checks if a dependency is the root package
|
|
@@ -66,7 +71,8 @@ export default class Yarn_berry_processor extends Yarn_processor {
|
|
|
66
71
|
if (!name) {
|
|
67
72
|
return false;
|
|
68
73
|
}
|
|
69
|
-
|
|
74
|
+
// Workspace members use paths like "member-a@workspace:packages/member-a", not just "@workspace:."
|
|
75
|
+
return name.startsWith(`${this._manifest.name}@workspace:`);
|
|
70
76
|
}
|
|
71
77
|
/**
|
|
72
78
|
* Adds dependencies to the SBOM
|
|
@@ -77,14 +83,58 @@ export default class Yarn_berry_processor extends Yarn_processor {
|
|
|
77
83
|
if (!depTree) {
|
|
78
84
|
return;
|
|
79
85
|
}
|
|
86
|
+
// Build index of nodes by their value for quick lookup
|
|
87
|
+
const nodeIndex = new Map();
|
|
88
|
+
depTree.forEach(n => nodeIndex.set(n.value, n));
|
|
89
|
+
// Determine the set of node values reachable from root via production deps
|
|
90
|
+
const prodDeps = new Set(this._manifest.dependencies);
|
|
91
|
+
const reachable = new Set();
|
|
92
|
+
const queue = [];
|
|
93
|
+
// Seed with root's production dependencies
|
|
94
|
+
const rootNode = depTree.find(n => this.#isRoot(n.value));
|
|
95
|
+
if (rootNode?.children?.Dependencies) {
|
|
96
|
+
for (const d of rootNode.children.Dependencies) {
|
|
97
|
+
const to = this.#purlFromLocator(d.locator);
|
|
98
|
+
if (to) {
|
|
99
|
+
const fullName = to.namespace ? `${to.namespace}/${to.name}` : to.name;
|
|
100
|
+
if (prodDeps.has(fullName)) {
|
|
101
|
+
queue.push(d.locator);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
// BFS to find all transitively reachable packages
|
|
107
|
+
while (queue.length > 0) {
|
|
108
|
+
const locator = queue.shift();
|
|
109
|
+
if (reachable.has(locator)) {
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
reachable.add(locator);
|
|
113
|
+
const node = nodeIndex.get(this.#nodeValueFromLocator(locator));
|
|
114
|
+
if (node?.children?.Dependencies) {
|
|
115
|
+
for (const d of node.children.Dependencies) {
|
|
116
|
+
if (!reachable.has(d.locator)) {
|
|
117
|
+
queue.push(d.locator);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
// Only emit edges for root and reachable nodes
|
|
80
123
|
depTree.forEach(n => {
|
|
81
124
|
const depName = n.value;
|
|
82
|
-
const
|
|
125
|
+
const isRoot = this.#isRoot(depName);
|
|
126
|
+
if (!isRoot && !this.#isReachableNode(depName, reachable)) {
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
const from = isRoot ? toPurlFromString(sbom.getRoot().purl) : this.#purlFromNode(depName, n);
|
|
83
130
|
const deps = n.children?.Dependencies;
|
|
84
131
|
if (!deps) {
|
|
85
132
|
return;
|
|
86
133
|
}
|
|
87
134
|
deps.forEach(d => {
|
|
135
|
+
if (!reachable.has(d.locator)) {
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
88
138
|
const to = this.#purlFromLocator(d.locator);
|
|
89
139
|
if (to) {
|
|
90
140
|
sbom.addDependency(from, to);
|
|
@@ -92,6 +142,39 @@ export default class Yarn_berry_processor extends Yarn_processor {
|
|
|
92
142
|
});
|
|
93
143
|
});
|
|
94
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* Converts a locator to the node value format used in yarn info output
|
|
147
|
+
* @param {string} locator - e.g. "express@npm:4.17.1"
|
|
148
|
+
* @returns {string} The node value, same as locator for non-virtual
|
|
149
|
+
* @private
|
|
150
|
+
*/
|
|
151
|
+
#nodeValueFromLocator(locator) {
|
|
152
|
+
// Virtual locators: "@scope/name@virtual:hash#npm:version" → "@scope/name@npm:version"
|
|
153
|
+
const virtualMatch = Yarn_berry_processor.VIRTUAL_LOCATOR_PATTERN.exec(locator);
|
|
154
|
+
if (virtualMatch) {
|
|
155
|
+
return `${virtualMatch[1]}@npm:${virtualMatch[2]}`;
|
|
156
|
+
}
|
|
157
|
+
return locator;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Checks if a node is in the reachable set by matching its value against reachable locators
|
|
161
|
+
* @param {string} depName - The node value (e.g. "express@npm:4.17.1")
|
|
162
|
+
* @param {Set<string>} reachable - Set of reachable locators
|
|
163
|
+
* @returns {boolean}
|
|
164
|
+
* @private
|
|
165
|
+
*/
|
|
166
|
+
#isReachableNode(depName, reachable) {
|
|
167
|
+
if (reachable.has(depName)) {
|
|
168
|
+
return true;
|
|
169
|
+
}
|
|
170
|
+
// Check if any reachable locator resolves to this node value
|
|
171
|
+
for (const locator of reachable) {
|
|
172
|
+
if (this.#nodeValueFromLocator(locator) === depName) {
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
95
178
|
/**
|
|
96
179
|
* Creates a PackageURL from a dependency locator
|
|
97
180
|
* @param {string} locator - The dependency locator
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
/** @typedef {{name: string, version: string, dependencies: DependencyEntry[]}} DependencyEntry */
|
|
1
|
+
/** @typedef {{name: string, version: string, dependencies: DependencyEntry[], hashes?: Array<{alg: string, content: string}>}} DependencyEntry */
|
|
2
2
|
export default class Python_controller {
|
|
3
3
|
/**
|
|
4
4
|
* Constructor to create new python controller instance to interact with pip package manager
|
|
@@ -15,17 +15,24 @@ export default class Python_controller {
|
|
|
15
15
|
realEnvironment: boolean;
|
|
16
16
|
pathToRequirements: string;
|
|
17
17
|
options: {};
|
|
18
|
+
parser: Promise<import("web-tree-sitter").Parser>;
|
|
19
|
+
requirementsQuery: Promise<import("web-tree-sitter").Query>;
|
|
20
|
+
pinnedVersionQuery: Promise<import("web-tree-sitter").Query>;
|
|
18
21
|
prepareEnvironment(): void;
|
|
19
22
|
/**
|
|
20
23
|
*
|
|
21
24
|
* @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not
|
|
22
|
-
* @return {[DependencyEntry]}
|
|
25
|
+
* @return {Promise<[DependencyEntry]>}
|
|
23
26
|
*/
|
|
24
|
-
getDependencies(includeTransitive: boolean): [DependencyEntry]
|
|
27
|
+
getDependencies(includeTransitive: boolean): Promise<[DependencyEntry]>;
|
|
25
28
|
#private;
|
|
26
29
|
}
|
|
27
30
|
export type DependencyEntry = {
|
|
28
31
|
name: string;
|
|
29
32
|
version: string;
|
|
30
33
|
dependencies: DependencyEntry[];
|
|
34
|
+
hashes?: Array<{
|
|
35
|
+
alg: string;
|
|
36
|
+
content: string;
|
|
37
|
+
}>;
|
|
31
38
|
};
|
|
@@ -2,6 +2,7 @@ import fs from "node:fs";
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import os, { EOL } from "os";
|
|
4
4
|
import { environmentVariableIsPopulated, getCustom, invokeCommand } from "../tools.js";
|
|
5
|
+
import { getParser, getRequirementQuery, getPinnedVersionQuery } from './requirements_parser.js';
|
|
5
6
|
function getPipFreezeOutput() {
|
|
6
7
|
try {
|
|
7
8
|
return environmentVariableIsPopulated("TRUSTIFY_DA_PIP_FREEZE") ? new Buffer.from(process.env["TRUSTIFY_DA_PIP_FREEZE"], 'base64').toString('ascii') : invokeCommand(this.pathToPipBin, ['freeze', '--all']).toString();
|
|
@@ -18,7 +19,54 @@ function getPipShowOutput(depNames) {
|
|
|
18
19
|
throw new Error('fail invoking \'pip show\' to fetch metadata for all installed packages in environment', { cause: error });
|
|
19
20
|
}
|
|
20
21
|
}
|
|
21
|
-
/**
|
|
22
|
+
/**
|
|
23
|
+
* Get pip inspect output from env var override or by invoking pip inspect.
|
|
24
|
+
* Returns null if pip inspect is unavailable (pip < 22.2) or if freeze/show
|
|
25
|
+
* env vars are set without an inspect override (to avoid inconsistent data).
|
|
26
|
+
* @return {string|null} pip inspect JSON output, or null on failure
|
|
27
|
+
*/
|
|
28
|
+
function getPipInspectOutput() {
|
|
29
|
+
if (environmentVariableIsPopulated("TRUSTIFY_DA_PIP_INSPECT")) {
|
|
30
|
+
return new Buffer.from(process.env["TRUSTIFY_DA_PIP_INSPECT"], 'base64').toString('ascii');
|
|
31
|
+
}
|
|
32
|
+
if (environmentVariableIsPopulated("TRUSTIFY_DA_PIP_FREEZE") || environmentVariableIsPopulated("TRUSTIFY_DA_PIP_SHOW")) {
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
return invokeCommand(this.pathToPipBin, ['inspect']).toString();
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
console.warn('pip inspect is not available (requires pip 22.2+), SBOM will be generated without hashes');
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Parse pip inspect JSON output and build a hash lookup map.
|
|
45
|
+
* @param {string|null} inspectOutput - raw pip inspect JSON string
|
|
46
|
+
* @return {Map<string, Array<{alg: string, content: string}>>} map of lowercase package name to hashes
|
|
47
|
+
*/
|
|
48
|
+
function parsePipInspectHashes(inspectOutput) {
|
|
49
|
+
if (!inspectOutput) {
|
|
50
|
+
return new Map();
|
|
51
|
+
}
|
|
52
|
+
try {
|
|
53
|
+
let inspectData = JSON.parse(inspectOutput);
|
|
54
|
+
let hashMap = new Map();
|
|
55
|
+
for (let pkg of (inspectData.installed || [])) {
|
|
56
|
+
let name = pkg.metadata?.name;
|
|
57
|
+
let sha256 = pkg.download_info?.archive_info?.hashes?.sha256;
|
|
58
|
+
if (name && sha256) {
|
|
59
|
+
hashMap.set(name.toLowerCase(), [{ alg: "SHA-256", content: sha256 }]);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return hashMap;
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
console.warn('Failed to parse pip inspect output, SBOM will be generated without hashes');
|
|
66
|
+
return new Map();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/** @typedef {{name: string, version: string, dependencies: DependencyEntry[], hashes?: Array<{alg: string, content: string}>}} DependencyEntry */
|
|
22
70
|
export default class Python_controller {
|
|
23
71
|
pythonEnvDir;
|
|
24
72
|
pathToPipBin;
|
|
@@ -26,6 +74,9 @@ export default class Python_controller {
|
|
|
26
74
|
realEnvironment;
|
|
27
75
|
pathToRequirements;
|
|
28
76
|
options;
|
|
77
|
+
parser;
|
|
78
|
+
requirementsQuery;
|
|
79
|
+
pinnedVersionQuery;
|
|
29
80
|
/**
|
|
30
81
|
* Constructor to create new python controller instance to interact with pip package manager
|
|
31
82
|
* @param {boolean} realEnvironment - whether to use real environment supplied by client or to create virtual environment
|
|
@@ -41,6 +92,9 @@ export default class Python_controller {
|
|
|
41
92
|
this.prepareEnvironment();
|
|
42
93
|
this.pathToRequirements = pathToRequirements;
|
|
43
94
|
this.options = options;
|
|
95
|
+
this.parser = getParser();
|
|
96
|
+
this.requirementsQuery = getRequirementQuery();
|
|
97
|
+
this.pinnedVersionQuery = getPinnedVersionQuery();
|
|
44
98
|
}
|
|
45
99
|
prepareEnvironment() {
|
|
46
100
|
if (!this.realEnvironment) {
|
|
@@ -86,6 +140,24 @@ export default class Python_controller {
|
|
|
86
140
|
}
|
|
87
141
|
}
|
|
88
142
|
}
|
|
143
|
+
/**
|
|
144
|
+
* Parse the requirements.txt file using tree-sitter and return structured requirement data.
|
|
145
|
+
* @return {Promise<{name: string, version: string|null, hasMarker: boolean}[]>}
|
|
146
|
+
*/
|
|
147
|
+
async #parseRequirements() {
|
|
148
|
+
const content = fs.readFileSync(this.pathToRequirements).toString();
|
|
149
|
+
const tree = (await this.parser).parse(content);
|
|
150
|
+
return Promise.all((await this.requirementsQuery).matches(tree.rootNode).map(async (match) => {
|
|
151
|
+
const reqNode = match.captures.find(c => c.name === 'req').node;
|
|
152
|
+
const name = match.captures.find(c => c.name === 'name').node.text;
|
|
153
|
+
const versionMatches = (await this.pinnedVersionQuery).matches(reqNode);
|
|
154
|
+
const version = versionMatches.length > 0
|
|
155
|
+
? versionMatches[0].captures.find(c => c.name === 'version').node.text
|
|
156
|
+
: null;
|
|
157
|
+
const hasMarker = reqNode.children.some(c => c.type === 'marker_spec');
|
|
158
|
+
return { name, version, hasMarker };
|
|
159
|
+
}));
|
|
160
|
+
}
|
|
89
161
|
#decideIfWindowsOrLinuxPath(fileName) {
|
|
90
162
|
if (os.platform() === "win32") {
|
|
91
163
|
return fileName + ".exe";
|
|
@@ -97,9 +169,9 @@ export default class Python_controller {
|
|
|
97
169
|
/**
|
|
98
170
|
*
|
|
99
171
|
* @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not
|
|
100
|
-
* @return {[DependencyEntry]}
|
|
172
|
+
* @return {Promise<[DependencyEntry]>}
|
|
101
173
|
*/
|
|
102
|
-
getDependencies(includeTransitive) {
|
|
174
|
+
async getDependencies(includeTransitive) {
|
|
103
175
|
let startingTime;
|
|
104
176
|
let endingTime;
|
|
105
177
|
if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
|
|
@@ -124,10 +196,10 @@ export default class Python_controller {
|
|
|
124
196
|
if (matchManifestVersions === "true") {
|
|
125
197
|
throw new Error("Conflicting settings, TRUSTIFY_DA_PYTHON_INSTALL_BEST_EFFORTS=true can only work with MATCH_MANIFEST_VERSIONS=false");
|
|
126
198
|
}
|
|
127
|
-
this.#installingRequirementsOneByOne();
|
|
199
|
+
await this.#installingRequirementsOneByOne();
|
|
128
200
|
}
|
|
129
201
|
}
|
|
130
|
-
let dependencies = this.#getDependenciesImpl(includeTransitive);
|
|
202
|
+
let dependencies = await this.#getDependenciesImpl(includeTransitive);
|
|
131
203
|
this.#cleanEnvironment();
|
|
132
204
|
if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
|
|
133
205
|
endingTime = new Date();
|
|
@@ -137,16 +209,14 @@ export default class Python_controller {
|
|
|
137
209
|
}
|
|
138
210
|
return dependencies;
|
|
139
211
|
}
|
|
140
|
-
#installingRequirementsOneByOne() {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
requirementsRows.filter((line) => !line.trim().startsWith("#")).filter((line) => line.trim() !== "").forEach((dependency) => {
|
|
144
|
-
let dependencyName = getDependencyName(dependency);
|
|
212
|
+
async #installingRequirementsOneByOne() {
|
|
213
|
+
const requirements = await this.#parseRequirements();
|
|
214
|
+
requirements.forEach(({ name }) => {
|
|
145
215
|
try {
|
|
146
|
-
invokeCommand(this.pathToPipBin, ['install',
|
|
216
|
+
invokeCommand(this.pathToPipBin, ['install', name]);
|
|
147
217
|
}
|
|
148
218
|
catch (error) {
|
|
149
|
-
throw new Error(`Failed in best-effort installing ${
|
|
219
|
+
throw new Error(`Failed in best-effort installing ${name} in virtual python environment`, { cause: error });
|
|
150
220
|
}
|
|
151
221
|
});
|
|
152
222
|
}
|
|
@@ -163,34 +233,26 @@ export default class Python_controller {
|
|
|
163
233
|
}
|
|
164
234
|
}
|
|
165
235
|
}
|
|
166
|
-
#getDependenciesImpl(includeTransitive) {
|
|
167
|
-
let dependencies =
|
|
236
|
+
async #getDependenciesImpl(includeTransitive) {
|
|
237
|
+
let dependencies = [];
|
|
168
238
|
let usePipDepTree = getCustom("TRUSTIFY_DA_PIP_USE_DEP_TREE", "false", this.options);
|
|
169
|
-
let freezeOutput;
|
|
170
|
-
let lines;
|
|
171
|
-
let depNames;
|
|
172
|
-
let pipShowOutput;
|
|
173
239
|
let allPipShowDeps;
|
|
174
240
|
let pipDepTreeJsonArrayOutput;
|
|
175
241
|
if (usePipDepTree !== "true") {
|
|
176
|
-
freezeOutput = getPipFreezeOutput.call(this);
|
|
177
|
-
lines = freezeOutput.split(EOL);
|
|
178
|
-
depNames = lines.map(line => getDependencyName(line));
|
|
242
|
+
const freezeOutput = getPipFreezeOutput.call(this);
|
|
243
|
+
const lines = freezeOutput.split(EOL);
|
|
244
|
+
const depNames = lines.map(line => getDependencyName(line));
|
|
245
|
+
const pipShowOutput = getPipShowOutput.call(this, depNames);
|
|
246
|
+
allPipShowDeps = pipShowOutput.split(EOL + "---" + EOL);
|
|
179
247
|
}
|
|
180
248
|
else {
|
|
181
249
|
pipDepTreeJsonArrayOutput = getDependencyTreeJsonFromPipDepTree(this.pathToPipBin, this.pathToPythonBin);
|
|
182
250
|
}
|
|
183
|
-
if (usePipDepTree !== "true") {
|
|
184
|
-
pipShowOutput = getPipShowOutput.call(this, depNames);
|
|
185
|
-
allPipShowDeps = pipShowOutput.split(EOL + "---" + EOL);
|
|
186
|
-
}
|
|
187
|
-
//debug
|
|
188
|
-
// pipShowOutput = "alternative pip show output goes here for debugging"
|
|
189
251
|
let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS", "true", this.options);
|
|
190
|
-
let
|
|
252
|
+
let parsedRequirements = await this.#parseRequirements();
|
|
191
253
|
let CachedEnvironmentDeps = {};
|
|
192
254
|
if (usePipDepTree !== "true") {
|
|
193
|
-
allPipShowDeps.forEach(
|
|
255
|
+
allPipShowDeps.forEach(record => {
|
|
194
256
|
let dependencyName = getDependencyNameShow(record).toLowerCase();
|
|
195
257
|
CachedEnvironmentDeps[dependencyName] = record;
|
|
196
258
|
CachedEnvironmentDeps[dependencyName.replace("-", "_")] = record;
|
|
@@ -210,42 +272,31 @@ export default class Python_controller {
|
|
|
210
272
|
CachedEnvironmentDeps[packageName.replace("_", "-")] = pipDepTreeEntryForCache;
|
|
211
273
|
});
|
|
212
274
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
275
|
+
const inspectOutput = getPipInspectOutput.call(this);
|
|
276
|
+
const hashMap = parsePipInspectHashes(inspectOutput);
|
|
277
|
+
parsedRequirements.forEach(({ name: depName, version: manifestVersion, hasMarker }) => {
|
|
278
|
+
if (hasMarker && CachedEnvironmentDeps[depName.toLowerCase()] === undefined) {
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
if (matchManifestVersions === "true" && manifestVersion != null) {
|
|
218
282
|
let installedVersion;
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim();
|
|
223
|
-
if (manifestVersion.includes("#")) {
|
|
224
|
-
let hashCharIndex = manifestVersion.indexOf("#");
|
|
225
|
-
manifestVersion = manifestVersion.substring(0, hashCharIndex);
|
|
283
|
+
if (CachedEnvironmentDeps[depName.toLowerCase()] !== undefined) {
|
|
284
|
+
if (usePipDepTree !== "true") {
|
|
285
|
+
installedVersion = getDependencyVersion(CachedEnvironmentDeps[depName.toLowerCase()]);
|
|
226
286
|
}
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
if (CachedEnvironmentDeps[dependencyName.toLowerCase()] !== undefined) {
|
|
230
|
-
if (usePipDepTree !== "true") {
|
|
231
|
-
installedVersion = getDependencyVersion(CachedEnvironmentDeps[dependencyName.toLowerCase()]);
|
|
232
|
-
}
|
|
233
|
-
else {
|
|
234
|
-
installedVersion = CachedEnvironmentDeps[dependencyName.toLowerCase()].version;
|
|
235
|
-
}
|
|
287
|
+
else {
|
|
288
|
+
installedVersion = CachedEnvironmentDeps[depName.toLowerCase()].version;
|
|
236
289
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
}
|
|
290
|
+
}
|
|
291
|
+
if (installedVersion) {
|
|
292
|
+
if (manifestVersion.trim() !== installedVersion.trim()) {
|
|
293
|
+
throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${depName} (manifest version=${manifestVersion}, installed version=${installedVersion}).If you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting MATCH_MANIFEST_VERSIONS=false`);
|
|
241
294
|
}
|
|
242
295
|
}
|
|
243
296
|
}
|
|
244
|
-
let path =
|
|
245
|
-
let depName = getDependencyName(dep);
|
|
246
|
-
//array to track a path for each branch in the dependency tree
|
|
297
|
+
let path = [];
|
|
247
298
|
path.push(depName.toLowerCase());
|
|
248
|
-
bringAllDependencies(dependencies, depName, CachedEnvironmentDeps, includeTransitive, path, usePipDepTree);
|
|
299
|
+
bringAllDependencies(dependencies, depName, CachedEnvironmentDeps, includeTransitive, path, usePipDepTree, hashMap);
|
|
249
300
|
});
|
|
250
301
|
dependencies.sort((dep1, dep2) => {
|
|
251
302
|
const DEP1 = dep1.name.toLowerCase();
|
|
@@ -325,8 +376,9 @@ function getDepsList(record) {
|
|
|
325
376
|
* @param includeTransitive
|
|
326
377
|
* @param usePipDepTree
|
|
327
378
|
* @param {[string]}path array representing the path of the current branch in dependency tree, starting with a root dependency - that is - a given dependency in requirements.txt
|
|
379
|
+
* @param {Map<string, Array<{alg: string, content: string}>>} hashMap - map of lowercase package name to hashes
|
|
328
380
|
*/
|
|
329
|
-
function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDeps, includeTransitive, path, usePipDepTree) {
|
|
381
|
+
function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDeps, includeTransitive, path, usePipDepTree, hashMap) {
|
|
330
382
|
if (dependencyName?.trim() === "") {
|
|
331
383
|
return;
|
|
332
384
|
}
|
|
@@ -347,18 +399,22 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
|
|
|
347
399
|
version = record.version;
|
|
348
400
|
directDeps = record.dependencies;
|
|
349
401
|
}
|
|
350
|
-
let targetDeps =
|
|
402
|
+
let targetDeps = [];
|
|
403
|
+
let hashes = hashMap?.get(depName.toLowerCase());
|
|
351
404
|
let entry = { "name": depName, "version": version, "dependencies": [] };
|
|
405
|
+
if (hashes) {
|
|
406
|
+
entry.hashes = hashes;
|
|
407
|
+
}
|
|
352
408
|
dependencies.push(entry);
|
|
353
409
|
directDeps.forEach((dep) => {
|
|
354
|
-
let depArray =
|
|
410
|
+
let depArray = [];
|
|
355
411
|
// to avoid infinite loop, check if the dependency not already on current path, before going recursively resolving its dependencies.
|
|
356
412
|
if (!path.includes(dep.toLowerCase())) {
|
|
357
413
|
// send to recurrsion the path + the current dep
|
|
358
414
|
depArray.push(dep.toLowerCase());
|
|
359
415
|
if (includeTransitive) {
|
|
360
416
|
// send to recurrsion the array of all deps in path + the current dependency name which is not on the path.
|
|
361
|
-
bringAllDependencies(targetDeps, dep, cachedEnvironmentDeps, includeTransitive, path.concat(depArray), usePipDepTree);
|
|
417
|
+
bringAllDependencies(targetDeps, dep, cachedEnvironmentDeps, includeTransitive, path.concat(depArray), usePipDepTree, hashMap);
|
|
362
418
|
}
|
|
363
419
|
}
|
|
364
420
|
// sort ra
|