@trustify-da/trustify-da-javascript-client 0.3.0-ea.e12bc82 → 0.3.0-ea.e5bb86c

package/dist/src/providers/java_maven.js

@@ -3,6 +3,7 @@ import os from 'node:os';
 import path from 'node:path';
 import { EOL } from 'os';
 import { XMLParser } from 'fast-xml-parser';
+import { getLicense } from '../license/license_utils.js';
 import Sbom from '../sbom.js';
 import { getCustom } from '../tools.js';
 import Base_java, { ecosystem_maven } from "./base_java.js";
@@ -51,6 +52,30 @@ export default class Java_maven extends Base_java {
             contentType: 'application/vnd.cyclonedx+json'
         };
     }
+    /**
+     * Read license from pom.xml manifest, with fallback to LICENSE file
+     * @param {string} manifestPath - path to pom.xml
+     * @returns {string|null}
+     */
+    readLicenseFromManifest(manifestPath) {
+        let fromPom = null;
+        try {
+            const xml = fs.readFileSync(manifestPath, 'utf-8');
+            const parser = new XMLParser({ ignoreAttributes: false });
+            const obj = parser.parse(xml);
+            const project = obj?.project;
+            if (project?.licenses?.license) {
+                const license = Array.isArray(project.licenses.license)
+                    ? project.licenses.license[0]
+                    : project.licenses.license;
+                fromPom = (license?.name && license.name.trim()) || null;
+            }
+        }
+        catch {
+            // leave fromPom as null
+        }
+        return getLicense(fromPom, manifestPath);
+    }
     /**
      * Create a Dot Graph dependency tree for a manifest path.
      * @param {string} manifest - path for pom.xml
@@ -105,7 +130,7 @@ export default class Java_maven extends Base_java {
         if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
             console.error("Dependency tree that will be used as input for creating the BOM =>" + EOL + EOL + content.toString());
         }
-        let sbom = this.createSbomFileFromTextFormat(content.toString(), ignoredDeps, opts);
+        let sbom = this.createSbomFileFromTextFormat(content.toString(), ignoredDeps, opts, manifest);
         // delete temp file and directory
         fs.rmSync(tmpDir, { recursive: true, force: true });
         // return dependency graph as string
@@ -115,15 +140,17 @@ export default class Java_maven extends Base_java {
      *
      * @param {String} textGraphList Text graph String of the manifest
      * @param {[String]} ignoredDeps List of ignored dependencies to be omitted from sbom
+     * @param {String} manifestPath Path to the pom.xml manifest
      * @return {String} formatted sbom Json String with all dependencies
      */
-    createSbomFileFromTextFormat(textGraphList, ignoredDeps, opts) {
+    createSbomFileFromTextFormat(textGraphList, ignoredDeps, opts, manifestPath) {
         let lines = textGraphList.split(EOL);
         // get root component
         let root = lines[0];
         let rootPurl = this.parseDep(root);
+        const license = this.readLicenseFromManifest(manifestPath);
         let sbom = new Sbom();
-        sbom.addRoot(rootPurl);
+        sbom.addRoot(rootPurl, license);
         this.parseDependencyTree(root, 0, lines.slice(1), sbom);
         return sbom.filterIgnoredDeps(ignoredDeps).getAsJsonString(opts);
     }
@@ -156,7 +183,8 @@ export default class Java_maven extends Base_java {
         let sbom = new Sbom();
         let rootDependency = this.#getRootFromPom(tmpEffectivePom, manifestPath);
         let purlRoot = this.toPurl(rootDependency.groupId, rootDependency.artifactId, rootDependency.version);
-        sbom.addRoot(purlRoot);
+        const license = this.readLicenseFromManifest(manifestPath);
+        sbom.addRoot(purlRoot, license);
         dependencies.forEach(dep => {
             let currentPurl = this.toPurl(dep.groupId, dep.artifactId, dep.version);
             sbom.addDependency(purlRoot, currentPurl);
@@ -209,7 +237,7 @@ export default class Java_maven extends Base_java {
         let ignored = [];
         // build xml parser with options
         let parser = new XMLParser({
-            commentPropName: '#comment',
+            commentPropName: '#comment', // mark comments with #comment
             isArray: (_, jpath) => 'project.dependencies.dependency' === jpath,
             parseTagValue: false
         });

package/dist/src/providers/javascript_pnpm.d.ts

@@ -1,5 +1,5 @@
 export default class Javascript_pnpm extends Base_javascript {
     _listCmdArgs(includeTransitive: any): string[];
-    _buildDependencyTree(includeTransitive: any, manifest: any): any;
+    _buildDependencyTree(includeTransitive: any, opts?: {}): any;
 }
 import Base_javascript from './base_javascript.js';

package/dist/src/providers/javascript_pnpm.js

@@ -12,8 +12,8 @@ export default class Javascript_pnpm extends Base_javascript {
     _updateLockFileCmdArgs() {
         return ['install', '--frozen-lockfile'];
     }
-    _buildDependencyTree(includeTransitive, manifest) {
-        const tree = super._buildDependencyTree(includeTransitive, manifest);
+    _buildDependencyTree(includeTransitive, opts = {}) {
+        const tree = super._buildDependencyTree(includeTransitive, opts);
         if (Array.isArray(tree) && tree.length > 0) {
             return tree[0];
         }

package/dist/src/providers/manifest.d.ts

@@ -2,6 +2,8 @@ export default class Manifest {
     constructor(manifestPath: any);
     manifestPath: any;
     dependencies: any[];
+    peerDependencies: any;
+    optionalDependencies: any;
     name: any;
     version: any;
     ignored: any[];

package/dist/src/providers/manifest.js

@@ -9,6 +9,8 @@ export default class Manifest {
         this.manifestPath = manifestPath;
         const content = this.loadManifest();
         this.dependencies = this.loadDependencies(content);
+        this.peerDependencies = content.peerDependencies || {};
+        this.optionalDependencies = content.optionalDependencies || {};
         this.name = content.name;
         this.version = content.version || DEFAULT_VERSION;
         this.ignored = this.loadIgnored(content);
@@ -27,11 +29,27 @@ export default class Manifest {
     }
     loadDependencies(content) {
         let deps = [];
-        if (!content.dependencies) {
-            return deps;
+        const depSources = [
+            content.dependencies,
+            content.peerDependencies,
+            content.optionalDependencies,
+        ];
+        for (const source of depSources) {
+            if (source) {
+                for (let dep in source) {
+                    if (!deps.includes(dep)) {
+                        deps.push(dep);
+                    }
+                }
+            }
         }
-        for (let dep in content.dependencies) {
-            deps.push(dep);
+        // bundledDependencies is an array of package names (subset of dependencies)
+        if (Array.isArray(content.bundledDependencies)) {
+            for (const dep of content.bundledDependencies) {
+                if (!deps.includes(dep)) {
+                    deps.push(dep);
+                }
+            }
         }
         return deps;
     }

package/dist/src/providers/processors/yarn_berry_processor.js

@@ -48,13 +48,15 @@ export default class Yarn_berry_processor extends Yarn_processor {
         if (!depTree) {
             return new Map();
         }
-        return new Map(depTree.filter(dep => !this.#isRoot(dep.value)).map(dep => {
+        return new Map(depTree.filter(dep => !this.#isRoot(dep.value))
+            .map(dep => {
             const depName = dep.value;
             const idx = depName.lastIndexOf('@');
             const name = depName.substring(0, idx);
             const version = dep.children.Version;
             return [name, toPurl(purlType, name, version)];
-        }));
+        })
+            .filter(([name]) => this._manifest.dependencies.includes(name)));
     }
     /**
    * Checks if a dependency is the root package
@@ -77,14 +79,58 @@ export default class Yarn_berry_processor extends Yarn_processor {
         if (!depTree) {
             return;
         }
+        // Build index of nodes by their value for quick lookup
+        const nodeIndex = new Map();
+        depTree.forEach(n => nodeIndex.set(n.value, n));
+        // Determine the set of node values reachable from root via production deps
+        const prodDeps = new Set(this._manifest.dependencies);
+        const reachable = new Set();
+        const queue = [];
+        // Seed with root's production dependencies
+        const rootNode = depTree.find(n => this.#isRoot(n.value));
+        if (rootNode?.children?.Dependencies) {
+            for (const d of rootNode.children.Dependencies) {
+                const to = this.#purlFromLocator(d.locator);
+                if (to) {
+                    const fullName = to.namespace ? `${to.namespace}/${to.name}` : to.name;
+                    if (prodDeps.has(fullName)) {
+                        queue.push(d.locator);
+                    }
+                }
+            }
+        }
+        // BFS to find all transitively reachable packages
+        while (queue.length > 0) {
+            const locator = queue.shift();
+            if (reachable.has(locator)) {
+                continue;
+            }
+            reachable.add(locator);
+            const node = nodeIndex.get(this.#nodeValueFromLocator(locator));
+            if (node?.children?.Dependencies) {
+                for (const d of node.children.Dependencies) {
+                    if (!reachable.has(d.locator)) {
+                        queue.push(d.locator);
+                    }
+                }
+            }
+        }
+        // Only emit edges for root and reachable nodes
         depTree.forEach(n => {
             const depName = n.value;
-            const from = this.#isRoot(depName) ? toPurlFromString(sbom.getRoot().purl) : this.#purlFromNode(depName, n);
+            const isRoot = this.#isRoot(depName);
+            if (!isRoot && !this.#isReachableNode(depName, reachable)) {
+                return;
+            }
+            const from = isRoot ? toPurlFromString(sbom.getRoot().purl) : this.#purlFromNode(depName, n);
             const deps = n.children?.Dependencies;
             if (!deps) {
                 return;
             }
             deps.forEach(d => {
+                if (!reachable.has(d.locator)) {
+                    return;
+                }
                 const to = this.#purlFromLocator(d.locator);
                 if (to) {
                     sbom.addDependency(from, to);
@@ -92,6 +138,39 @@ export default class Yarn_berry_processor extends Yarn_processor {
             });
         });
     }
+    /**
+     * Converts a locator to the node value format used in yarn info output
+     * @param {string} locator - e.g. "express@npm:4.17.1"
+     * @returns {string} The node value, same as locator for non-virtual
+     * @private
+     */
+    #nodeValueFromLocator(locator) {
+        // Virtual locators: "@scope/name@virtual:hash#npm:version" → "@scope/name@npm:version"
+        const virtualMatch = Yarn_berry_processor.VIRTUAL_LOCATOR_PATTERN.exec(locator);
+        if (virtualMatch) {
+            return `${virtualMatch[1]}@npm:${virtualMatch[2]}`;
+        }
+        return locator;
+    }
+    /**
+     * Checks if a node is in the reachable set by matching its value against reachable locators
+     * @param {string} depName - The node value (e.g. "express@npm:4.17.1")
+     * @param {Set<string>} reachable - Set of reachable locators
+     * @returns {boolean}
+     * @private
+     */
+    #isReachableNode(depName, reachable) {
+        if (reachable.has(depName)) {
+            return true;
+        }
+        // Check if any reachable locator resolves to this node value
+        for (const locator of reachable) {
+            if (this.#nodeValueFromLocator(locator) === depName) {
+                return true;
+            }
+        }
+        return false;
+    }
     /**
    * Creates a PackageURL from a dependency locator
    * @param {string} locator - The dependency locator

package/dist/src/providers/python_controller.d.ts

@@ -15,13 +15,16 @@ export default class Python_controller {
     realEnvironment: boolean;
     pathToRequirements: string;
     options: {};
+    parser: Promise<import("web-tree-sitter").Parser>;
+    requirementsQuery: Promise<import("web-tree-sitter").Query>;
+    pinnedVersionQuery: Promise<import("web-tree-sitter").Query>;
     prepareEnvironment(): void;
     /**
      *
      * @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not
-     * @return {[DependencyEntry]}
+     * @return {Promise<[DependencyEntry]>}
      */
-    getDependencies(includeTransitive: boolean): [DependencyEntry];
+    getDependencies(includeTransitive: boolean): Promise<[DependencyEntry]>;
     #private;
 }
 export type DependencyEntry = {

package/dist/src/providers/python_controller.js

@@ -2,6 +2,7 @@ import fs from "node:fs";
 import path from 'node:path';
 import os, { EOL } from "os";
 import { environmentVariableIsPopulated, getCustom, invokeCommand } from "../tools.js";
+import { getParser, getRequirementQuery, getPinnedVersionQuery } from './requirements_parser.js';
 function getPipFreezeOutput() {
     try {
         return environmentVariableIsPopulated("TRUSTIFY_DA_PIP_FREEZE") ? new Buffer.from(process.env["TRUSTIFY_DA_PIP_FREEZE"], 'base64').toString('ascii') : invokeCommand(this.pathToPipBin, ['freeze', '--all']).toString();
@@ -26,6 +27,9 @@ export default class Python_controller {
     realEnvironment;
     pathToRequirements;
     options;
+    parser;
+    requirementsQuery;
+    pinnedVersionQuery;
     /**
      * Constructor to create new python controller instance to interact with pip package manager
      * @param {boolean} realEnvironment - whether to use real environment supplied by client or to create virtual environment
@@ -41,6 +45,9 @@ export default class Python_controller {
         this.prepareEnvironment();
         this.pathToRequirements = pathToRequirements;
         this.options = options;
+        this.parser = getParser();
+        this.requirementsQuery = getRequirementQuery();
+        this.pinnedVersionQuery = getPinnedVersionQuery();
     }
     prepareEnvironment() {
         if (!this.realEnvironment) {
@@ -86,6 +93,23 @@ export default class Python_controller {
             }
         }
     }
+    /**
+     * Parse the requirements.txt file using tree-sitter and return structured requirement data.
+     * @return {Promise<{name: string, version: string|null}[]>}
+     */
+    async #parseRequirements() {
+        const content = fs.readFileSync(this.pathToRequirements).toString();
+        const tree = (await this.parser).parse(content);
+        return Promise.all((await this.requirementsQuery).matches(tree.rootNode).map(async (match) => {
+            const reqNode = match.captures.find(c => c.name === 'req').node;
+            const name = match.captures.find(c => c.name === 'name').node.text;
+            const versionMatches = (await this.pinnedVersionQuery).matches(reqNode);
+            const version = versionMatches.length > 0
+                ? versionMatches[0].captures.find(c => c.name === 'version').node.text
+                : null;
+            return { name, version };
+        }));
+    }
     #decideIfWindowsOrLinuxPath(fileName) {
         if (os.platform() === "win32") {
             return fileName + ".exe";
@@ -97,9 +121,9 @@ export default class Python_controller {
     /**
      *
      * @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not
-     * @return {[DependencyEntry]}
+     * @return {Promise<[DependencyEntry]>}
      */
-    getDependencies(includeTransitive) {
+    async getDependencies(includeTransitive) {
         let startingTime;
         let endingTime;
         if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
@@ -124,10 +148,10 @@ export default class Python_controller {
                 if (matchManifestVersions === "true") {
                     throw new Error("Conflicting settings, TRUSTIFY_DA_PYTHON_INSTALL_BEST_EFFORTS=true can only work with MATCH_MANIFEST_VERSIONS=false");
                 }
-                this.#installingRequirementsOneByOne();
+                await this.#installingRequirementsOneByOne();
             }
         }
-        let dependencies = this.#getDependenciesImpl(includeTransitive);
+        let dependencies = await this.#getDependenciesImpl(includeTransitive);
         this.#cleanEnvironment();
         if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
             endingTime = new Date();
@@ -137,16 +161,14 @@ export default class Python_controller {
         }
         return dependencies;
     }
-    #installingRequirementsOneByOne() {
-        let requirementsContent = fs.readFileSync(this.pathToRequirements);
-        let requirementsRows = requirementsContent.toString().split(EOL);
-        requirementsRows.filter((line) => !line.trim().startsWith("#")).filter((line) => line.trim() !== "").forEach((dependency) => {
-            let dependencyName = getDependencyName(dependency);
+    async #installingRequirementsOneByOne() {
+        const requirements = await this.#parseRequirements();
+        requirements.forEach(({ name }) => {
             try {
-                invokeCommand(this.pathToPipBin, ['install', dependencyName]);
+                invokeCommand(this.pathToPipBin, ['install', name]);
             }
             catch (error) {
-                throw new Error(`Failed in best-effort installing ${dependencyName} in virtual python environment`, { cause: error });
+                throw new Error(`Failed in best-effort installing ${name} in virtual python environment`, { cause: error });
             }
         });
     }
@@ -163,34 +185,26 @@ export default class Python_controller {
             }
         }
     }
-    #getDependenciesImpl(includeTransitive) {
-        let dependencies = new Array();
+    async #getDependenciesImpl(includeTransitive) {
+        let dependencies = [];
         let usePipDepTree = getCustom("TRUSTIFY_DA_PIP_USE_DEP_TREE", "false", this.options);
-        let freezeOutput;
-        let lines;
-        let depNames;
-        let pipShowOutput;
         let allPipShowDeps;
         let pipDepTreeJsonArrayOutput;
         if (usePipDepTree !== "true") {
-            freezeOutput = getPipFreezeOutput.call(this);
-            lines = freezeOutput.split(EOL);
-            depNames = lines.map(line => getDependencyName(line));
+            const freezeOutput = getPipFreezeOutput.call(this);
+            const lines = freezeOutput.split(EOL);
+            const depNames = lines.map(line => getDependencyName(line));
+            const pipShowOutput = getPipShowOutput.call(this, depNames);
+            allPipShowDeps = pipShowOutput.split(EOL + "---" + EOL);
         }
         else {
             pipDepTreeJsonArrayOutput = getDependencyTreeJsonFromPipDepTree(this.pathToPipBin, this.pathToPythonBin);
         }
-        if (usePipDepTree !== "true") {
-            pipShowOutput = getPipShowOutput.call(this, depNames);
-            allPipShowDeps = pipShowOutput.split(EOL + "---" + EOL);
-        }
-        //debug
-        // pipShowOutput = "alternative pip show output goes here for debugging"
         let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS", "true", this.options);
-        let linesOfRequirements = fs.readFileSync(this.pathToRequirements).toString().split(EOL).filter((line) => !line.trim().startsWith("#")).map(line => line.trim());
+        let parsedRequirements = await this.#parseRequirements();
         let CachedEnvironmentDeps = {};
         if (usePipDepTree !== "true") {
-            allPipShowDeps.forEach((record) => {
+            allPipShowDeps.forEach(record => {
                 let dependencyName = getDependencyNameShow(record).toLowerCase();
                 CachedEnvironmentDeps[dependencyName] = record;
                 CachedEnvironmentDeps[dependencyName.replace("-", "_")] = record;
@@ -210,40 +224,24 @@ export default class Python_controller {
                 CachedEnvironmentDeps[packageName.replace("_", "-")] = pipDepTreeEntryForCache;
             });
         }
-        linesOfRequirements.forEach((dep) => {
-            // if matchManifestVersions setting is turned on , then
-            if (matchManifestVersions === "true") {
-                let dependencyName;
-                let manifestVersion;
+        parsedRequirements.forEach(({ name: depName, version: manifestVersion }) => {
+            if (matchManifestVersions === "true" && manifestVersion != null) {
                 let installedVersion;
-                let doubleEqualSignPosition;
-                if (dep.includes("==")) {
-                    doubleEqualSignPosition = dep.indexOf("==");
-                    manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim();
-                    if (manifestVersion.includes("#")) {
-                        let hashCharIndex = manifestVersion.indexOf("#");
-                        manifestVersion = manifestVersion.substring(0, hashCharIndex);
+                if (CachedEnvironmentDeps[depName.toLowerCase()] !== undefined) {
+                    if (usePipDepTree !== "true") {
+                        installedVersion = getDependencyVersion(CachedEnvironmentDeps[depName.toLowerCase()]);
                     }
-                    dependencyName = getDependencyName(dep);
-                    // only compare between declared version in manifest to installed version , if the package is installed.
-                    if (CachedEnvironmentDeps[dependencyName.toLowerCase()] !== undefined) {
-                        if (usePipDepTree !== "true") {
-                            installedVersion = getDependencyVersion(CachedEnvironmentDeps[dependencyName.toLowerCase()]);
-                        }
-                        else {
-                            installedVersion = CachedEnvironmentDeps[dependencyName.toLowerCase()].version;
-                        }
+                    else {
+                        installedVersion = CachedEnvironmentDeps[depName.toLowerCase()].version;
                     }
-                    if (installedVersion) {
-                        if (manifestVersion.trim() !== installedVersion.trim()) {
-                            throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${dependencyName} (manifest version=${manifestVersion}, installed version=${installedVersion}).If you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting MATCH_MANIFEST_VERSIONS=false`);
-                        }
+                }
+                if (installedVersion) {
+                    if (manifestVersion.trim() !== installedVersion.trim()) {
+                        throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${depName} (manifest version=${manifestVersion}, installed version=${installedVersion}).If you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting MATCH_MANIFEST_VERSIONS=false`);
                     }
                 }
             }
-            let path = new Array();
-            let depName = getDependencyName(dep);
-            //array to track a path for each branch in the dependency tree
+            let path = [];
             path.push(depName.toLowerCase());
             bringAllDependencies(dependencies, depName, CachedEnvironmentDeps, includeTransitive, path, usePipDepTree);
         });
@@ -347,11 +345,11 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
         version = record.version;
         directDeps = record.dependencies;
     }
-    let targetDeps = new Array();
+    let targetDeps = [];
     let entry = { "name": depName, "version": version, "dependencies": [] };
     dependencies.push(entry);
     directDeps.forEach((dep) => {
-        let depArray = new Array();
+        let depArray = [];
         // to avoid infinite loop, check if the dependency not already on current path, before going recursively resolving its dependencies.
         if (!path.includes(dep.toLowerCase())) {
             // send to recurrsion the path + the current dep

package/dist/src/providers/python_pip.d.ts

@@ -3,6 +3,7 @@ declare namespace _default {
     export { validateLockFile };
     export { provideComponent };
     export { provideStack };
+    export { readLicenseFromManifest };
 }
 export default _default;
 export type DependencyEntry = {
@@ -23,13 +24,19 @@ declare function validateLockFile(): boolean;
  * Provide content and content type for python-pip component analysis.
  * @param {string} manifest - path to requirements.txt for component report
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {Provided}
+ * @returns {Promise<Provided>}
  */
-declare function provideComponent(manifest: string, opts?: {} | undefined): Provided;
+declare function provideComponent(manifest: string, opts?: {}): Promise<Provided>;
 /**
  * Provide content and content type for python-pip stack analysis.
  * @param {string} manifest - the manifest path or name
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {Provided}
+ * @returns {Promise<Provided>}
  */
-declare function provideStack(manifest: string, opts?: {} | undefined): Provided;
+declare function provideStack(manifest: string, opts?: {}): Promise<Provided>;
+/**
+ * Python requirements.txt has no standard license field
+ * @param {string} manifestPath - path to requirements.txt
+ * @returns {string|null}
+ */
+declare function readLicenseFromManifest(manifestPath: string): string | null;