@trustify-da/trustify-da-javascript-client 0.3.0-ea.62f6bc7 → 0.3.0-ea.63ae5c2

package/dist/package.json

@@ -46,13 +46,16 @@
     "dependencies": {
         "@babel/core": "^7.23.2",
         "@cyclonedx/cyclonedx-library": "^6.13.0",
+        "eslint-import-resolver-typescript": "^4.4.4",
         "fast-toml": "^0.5.4",
         "fast-xml-parser": "^5.3.4",
         "help": "^3.0.2",
         "https-proxy-agent": "^7.0.6",
         "node-fetch": "^3.3.2",
         "packageurl-js": "~1.0.2",
-        "yargs": "^17.7.2"
+        "tree-sitter-requirements": "github:Strum355/tree-sitter-requirements#d0261ee76b84253997fe70d7d397e78c006c3801",
+        "web-tree-sitter": "^0.26.6",
+        "yargs": "^18.0.0"
     },
     "devDependencies": {
         "@babel/core": "^7.23.2",
@@ -60,9 +63,10 @@
         "@types/node": "^20.17.30",
         "@types/which": "^3.0.4",
         "babel-plugin-rewire": "^1.2.0",
-        "c8": "^10.1.3",
+        "c8": "^11.0.0",
         "chai": "^4.3.7",
         "eslint": "^8.42.0",
+        "eslint-import-resolver-typescript": "^4.4.4",
         "eslint-plugin-editorconfig": "^4.0.3",
         "eslint-plugin-import": "^2.29.1",
         "esmock": "^2.6.2",

package/dist/src/analysis.d.ts

@@ -1,3 +1,9 @@
+/**
+ * Utility function for fetching vendor tokens
+ * @param {import("index.js").Options} [opts={}] - optional various options to pass along the application
+ * @returns {{}}
+ */
+export function getTokenHeaders(opts?: import("index.js").Options): {};
 declare namespace _default {
     export { requestComponent };
     export { requestStack };

package/dist/src/analysis.js

@@ -5,11 +5,11 @@ import { HttpsProxyAgent } from "https-proxy-agent";
 import { generateImageSBOM, parseImageRef } from "./oci_image/utils.js";
 import { RegexNotToBeLogged, getCustom } from "./tools.js";
 export default { requestComponent, requestStack, requestImages, validateToken };
-const rhdaTokenHeader = "rhda-token";
-const rhdaTelemetryId = "rhda-telemetry-id";
-const rhdaSourceHeader = "rhda-source";
-const rhdaOperationTypeHeader = "rhda-operation-type";
-const rhdaPackageManagerHeader = "rhda-pkg-manager";
+const rhdaTokenHeader = "trust-da-token";
+const rhdaTelemetryId = "telemetry-anonymous-id";
+const rhdaSourceHeader = "trust-da-source";
+const rhdaOperationTypeHeader = "trust-da-operation-type";
+const rhdaPackageManagerHeader = "trust-da-pkg-manager";
 /**
  * Adds proxy agent configuration to fetch options if a proxy URL is specified
  * @param {RequestInit} options - The base fetch options
@@ -35,7 +35,7 @@ function addProxyAgent(options, opts) {
 async function requestStack(provider, manifest, url, html = false, opts = {}) {
     opts["source-manifest"] = Buffer.from(fs.readFileSync(manifest).toString()).toString('base64');
     opts["manifest-type"] = path.parse(manifest).base;
-    let provided = provider.provideStack(manifest, opts); // throws error if content providing failed
+    let provided = await provider.provideStack(manifest, opts); // throws error if content providing failed
     opts["source-manifest"] = "";
     opts[rhdaOperationTypeHeader.toUpperCase().replaceAll("-", "_")] = "stack-analysis";
     let startTime = new Date();
@@ -94,7 +94,7 @@ async function requestStack(provider, manifest, url, html = false, opts = {}) {
  */
 async function requestComponent(provider, manifest, url, opts = {}) {
     opts["source-manifest"] = Buffer.from(fs.readFileSync(manifest).toString()).toString('base64');
-    let provided = provider.provideComponent(manifest, opts); // throws error if content providing failed
+    let provided = await provider.provideComponent(manifest, opts); // throws error if content providing failed
     opts["source-manifest"] = "";
     opts[rhdaOperationTypeHeader.toUpperCase().replaceAll("-", "_")] = "component-analysis";
     if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
@@ -208,11 +208,12 @@ async function validateToken(url, opts = {}) {
  *
  * @param {string} headerName - the header name to populate in request
  * @param headers
- * @param {import("index.js").Options} [opts={}] - optional various options to pass along the application
+ * @param {string} optsKey - key in the options object to use the value for
+ * @param {import("index.js").Options} [opts={}] - options input object to fetch header values from
  * @private
  */
-function setRhdaHeader(headerName, headers, opts) {
-    let rhdaHeaderValue = getCustom(headerName.toUpperCase().replaceAll("-", "_"), null, opts);
+function setRhdaHeader(headerName, headers, optsKey, opts) {
+    let rhdaHeaderValue = getCustom(optsKey, null, opts);
     if (rhdaHeaderValue) {
         headers[headerName] = rhdaHeaderValue;
     }
@@ -222,25 +223,14 @@ function setRhdaHeader(headerName, headers, opts) {
  * @param {import("index.js").Options} [opts={}] - optional various options to pass along the application
  * @returns {{}}
  */
-function getTokenHeaders(opts = {}) {
-    let supportedTokens = ['snyk', 'oss-index'];
+export function getTokenHeaders(opts = {}) {
     let headers = {};
-    supportedTokens.forEach(vendor => {
-        let token = getCustom(`TRUSTIFY_DA_${vendor.replace("-", "_").toUpperCase()}_TOKEN`, null, opts);
-        if (token) {
-            headers[`ex-${vendor}-token`] = token;
-        }
-        let user = getCustom(`TRUSTIFY_DA_${vendor.replace("-", "_").toUpperCase()}_USER`, null, opts);
-        if (user) {
-            headers[`ex-${vendor}-user`] = user;
-        }
-    });
-    setRhdaHeader(rhdaTokenHeader, headers, opts);
-    setRhdaHeader(rhdaSourceHeader, headers, opts);
-    setRhdaHeader(rhdaOperationTypeHeader, headers, opts);
-    setRhdaHeader(rhdaPackageManagerHeader, headers, opts);
-    setRhdaHeader(rhdaTelemetryId, headers, opts);
-    if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
+    setRhdaHeader(rhdaTokenHeader, headers, 'TRUSTIFY_DA_TOKEN', opts);
+    setRhdaHeader(rhdaSourceHeader, headers, 'TRUSTIFY_DA_SOURCE', opts);
+    setRhdaHeader(rhdaOperationTypeHeader, headers, rhdaOperationTypeHeader.toUpperCase().replaceAll("-", "_"), opts);
+    setRhdaHeader(rhdaPackageManagerHeader, headers, rhdaPackageManagerHeader.toUpperCase().replaceAll("-", "_"), opts);
+    setRhdaHeader(rhdaTelemetryId, headers, 'TRUSTIFY_DA_TELEMETRY_ID', opts);
+    if (getCustom("TRUSTIFY_DA_DEBUG", null, opts) === "true") {
         console.log("Headers Values to be sent to Trustify DA backend:" + EOL);
         for (const headerKey in headers) {
             if (!headerKey.match(RegexNotToBeLogged)) {

package/dist/src/index.d.ts

@@ -48,9 +48,9 @@ export type Options = {
     TRUSTIFY_DA_SYFT_PATH?: string | undefined;
     TRUSTIFY_DA_YARN_PATH?: string | undefined;
     MATCH_MANIFEST_VERSIONS?: string | undefined;
-    RHDA_SOURCE?: string | undefined;
-    RHDA_TOKEN?: string | undefined;
-    RHDA_TELEMETRY_ID?: string | undefined;
+    TRUSTIFY_DA_SOURCE?: string | undefined;
+    TRUSTIFY_DA_TOKEN?: string | undefined;
+    TRUSTIFY_DA_TELEMETRY_ID?: string | undefined;
     [key: string]: string | undefined;
 };
 /**

package/dist/src/index.js

@@ -36,9 +36,9 @@ export default { componentAnalysis, stackAnalysis, imageAnalysis, validateToken
  * TRUSTIFY_DA_SYFT_PATH?: string | undefined,
  * TRUSTIFY_DA_YARN_PATH?: string | undefined,
  * MATCH_MANIFEST_VERSIONS?: string | undefined,
- * RHDA_SOURCE?: string | undefined,
- * RHDA_TOKEN?: string | undefined,
- * RHDA_TELEMETRY_ID?: string | undefined,
+ * TRUSTIFY_DA_SOURCE?: string | undefined,
+ * TRUSTIFY_DA_TOKEN?: string | undefined,
+ * TRUSTIFY_DA_TELEMETRY_ID?: string | undefined,
  * [key: string]: string | undefined,
  * }} Options
  */

package/dist/src/provider.d.ts

@@ -10,7 +10,7 @@
  */
 export function match(manifest: string, providers: [Provider]): Provider;
 /** @typedef {{ecosystem: string, contentType: string, content: string}} Provided */
-/** @typedef {{isSupported: function(string): boolean, validateLockFile: function(string): void, provideComponent: function(string, {}): Provided, provideStack: function(string, {}): Provided}} Provider */
+/** @typedef {{isSupported: function(string): boolean, validateLockFile: function(string): void, provideComponent: function(string, {}): Provided | Promise<Provided>, provideStack: function(string, {}): Provided | Promise<Provided>}} Provider */
 /**
  * MUST include all providers here.
  * @type {[Provider]}
@@ -24,6 +24,6 @@ export type Provided = {
 export type Provider = {
     isSupported: (arg0: string) => boolean;
     validateLockFile: (arg0: string) => void;
-    provideComponent: (arg0: string, arg1: {}) => Provided;
-    provideStack: (arg0: string, arg1: {}) => Provided;
+    provideComponent: (arg0: string, arg1: {}) => Provided | Promise<Provided>;
+    provideStack: (arg0: string, arg1: {}) => Provided | Promise<Provided>;
 };

package/dist/src/provider.js

@@ -8,7 +8,7 @@ import Javascript_pnpm from './providers/javascript_pnpm.js';
 import Javascript_yarn from './providers/javascript_yarn.js';
 import pythonPipProvider from './providers/python_pip.js';
 /** @typedef {{ecosystem: string, contentType: string, content: string}} Provided */
-/** @typedef {{isSupported: function(string): boolean, validateLockFile: function(string): void, provideComponent: function(string, {}): Provided, provideStack: function(string, {}): Provided}} Provider */
+/** @typedef {{isSupported: function(string): boolean, validateLockFile: function(string): void, provideComponent: function(string, {}): Provided | Promise<Provided>, provideStack: function(string, {}): Provided | Promise<Provided>}} Provider */
 /**
  * MUST include all providers here.
  * @type {[Provider]}

package/dist/src/providers/python_controller.d.ts

@@ -15,13 +15,16 @@ export default class Python_controller {
     realEnvironment: boolean;
     pathToRequirements: string;
     options: {};
+    parser: Promise<import("web-tree-sitter").Parser>;
+    requirementsQuery: Promise<import("web-tree-sitter").Query>;
+    pinnedVersionQuery: Promise<import("web-tree-sitter").Query>;
     prepareEnvironment(): void;
     /**
      *
      * @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not
-     * @return {[DependencyEntry]}
+     * @return {Promise<[DependencyEntry]>}
      */
-    getDependencies(includeTransitive: boolean): [DependencyEntry];
+    getDependencies(includeTransitive: boolean): Promise<[DependencyEntry]>;
     #private;
 }
 export type DependencyEntry = {

package/dist/src/providers/python_controller.js

@@ -2,6 +2,7 @@ import fs from "node:fs";
 import path from 'node:path';
 import os, { EOL } from "os";
 import { environmentVariableIsPopulated, getCustom, invokeCommand } from "../tools.js";
+import { getParser, getRequirementQuery, getPinnedVersionQuery } from './requirements_parser.js';
 function getPipFreezeOutput() {
     try {
         return environmentVariableIsPopulated("TRUSTIFY_DA_PIP_FREEZE") ? new Buffer.from(process.env["TRUSTIFY_DA_PIP_FREEZE"], 'base64').toString('ascii') : invokeCommand(this.pathToPipBin, ['freeze', '--all']).toString();
@@ -26,6 +27,9 @@ export default class Python_controller {
     realEnvironment;
     pathToRequirements;
     options;
+    parser;
+    requirementsQuery;
+    pinnedVersionQuery;
     /**
      * Constructor to create new python controller instance to interact with pip package manager
      * @param {boolean} realEnvironment - whether to use real environment supplied by client or to create virtual environment
@@ -41,6 +45,9 @@ export default class Python_controller {
         this.prepareEnvironment();
         this.pathToRequirements = pathToRequirements;
         this.options = options;
+        this.parser = getParser();
+        this.requirementsQuery = getRequirementQuery();
+        this.pinnedVersionQuery = getPinnedVersionQuery();
     }
     prepareEnvironment() {
         if (!this.realEnvironment) {
@@ -86,6 +93,23 @@ export default class Python_controller {
             }
         }
     }
+    /**
+     * Parse the requirements.txt file using tree-sitter and return structured requirement data.
+     * @return {Promise<{name: string, version: string|null}[]>}
+     */
+    async #parseRequirements() {
+        const content = fs.readFileSync(this.pathToRequirements).toString();
+        const tree = (await this.parser).parse(content);
+        return Promise.all((await this.requirementsQuery).matches(tree.rootNode).map(async (match) => {
+            const reqNode = match.captures.find(c => c.name === 'req').node;
+            const name = match.captures.find(c => c.name === 'name').node.text;
+            const versionMatches = (await this.pinnedVersionQuery).matches(reqNode);
+            const version = versionMatches.length > 0
+                ? versionMatches[0].captures.find(c => c.name === 'version').node.text
+                : null;
+            return { name, version };
+        }));
+    }
     #decideIfWindowsOrLinuxPath(fileName) {
         if (os.platform() === "win32") {
             return fileName + ".exe";
@@ -97,9 +121,9 @@ export default class Python_controller {
     /**
      *
      * @param {boolean} includeTransitive - whether to return include in returned object transitive dependencies or not
-     * @return {[DependencyEntry]}
+     * @return {Promise<[DependencyEntry]>}
      */
-    getDependencies(includeTransitive) {
+    async getDependencies(includeTransitive) {
         let startingTime;
         let endingTime;
         if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
@@ -124,10 +148,10 @@ export default class Python_controller {
                 if (matchManifestVersions === "true") {
                     throw new Error("Conflicting settings, TRUSTIFY_DA_PYTHON_INSTALL_BEST_EFFORTS=true can only work with MATCH_MANIFEST_VERSIONS=false");
                 }
-                this.#installingRequirementsOneByOne();
+                await this.#installingRequirementsOneByOne();
             }
         }
-        let dependencies = this.#getDependenciesImpl(includeTransitive);
+        let dependencies = await this.#getDependenciesImpl(includeTransitive);
         this.#cleanEnvironment();
         if (process.env["TRUSTIFY_DA_DEBUG"] === "true") {
             endingTime = new Date();
@@ -137,16 +161,14 @@ export default class Python_controller {
         }
         return dependencies;
     }
-    #installingRequirementsOneByOne() {
-        let requirementsContent = fs.readFileSync(this.pathToRequirements);
-        let requirementsRows = requirementsContent.toString().split(EOL);
-        requirementsRows.filter((line) => !line.trim().startsWith("#")).filter((line) => line.trim() !== "").forEach((dependency) => {
-            let dependencyName = getDependencyName(dependency);
+    async #installingRequirementsOneByOne() {
+        const requirements = await this.#parseRequirements();
+        requirements.forEach(({ name }) => {
             try {
-                invokeCommand(this.pathToPipBin, ['install', dependencyName]);
+                invokeCommand(this.pathToPipBin, ['install', name]);
             }
             catch (error) {
-                throw new Error(`Failed in best-effort installing ${dependencyName} in virtual python environment`, { cause: error });
+                throw new Error(`Failed in best-effort installing ${name} in virtual python environment`, { cause: error });
             }
         });
     }
@@ -163,34 +185,26 @@ export default class Python_controller {
             }
         }
     }
-    #getDependenciesImpl(includeTransitive) {
-        let dependencies = new Array();
+    async #getDependenciesImpl(includeTransitive) {
+        let dependencies = [];
         let usePipDepTree = getCustom("TRUSTIFY_DA_PIP_USE_DEP_TREE", "false", this.options);
-        let freezeOutput;
-        let lines;
-        let depNames;
-        let pipShowOutput;
         let allPipShowDeps;
         let pipDepTreeJsonArrayOutput;
         if (usePipDepTree !== "true") {
-            freezeOutput = getPipFreezeOutput.call(this);
-            lines = freezeOutput.split(EOL);
-            depNames = lines.map(line => getDependencyName(line));
+            const freezeOutput = getPipFreezeOutput.call(this);
+            const lines = freezeOutput.split(EOL);
+            const depNames = lines.map(line => getDependencyName(line));
+            const pipShowOutput = getPipShowOutput.call(this, depNames);
+            allPipShowDeps = pipShowOutput.split(EOL + "---" + EOL);
         }
         else {
             pipDepTreeJsonArrayOutput = getDependencyTreeJsonFromPipDepTree(this.pathToPipBin, this.pathToPythonBin);
         }
-        if (usePipDepTree !== "true") {
-            pipShowOutput = getPipShowOutput.call(this, depNames);
-            allPipShowDeps = pipShowOutput.split(EOL + "---" + EOL);
-        }
-        //debug
-        // pipShowOutput = "alternative pip show output goes here for debugging"
         let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS", "true", this.options);
-        let linesOfRequirements = fs.readFileSync(this.pathToRequirements).toString().split(EOL).filter((line) => !line.trim().startsWith("#")).map(line => line.trim());
+        let parsedRequirements = await this.#parseRequirements();
         let CachedEnvironmentDeps = {};
         if (usePipDepTree !== "true") {
-            allPipShowDeps.forEach((record) => {
+            allPipShowDeps.forEach(record => {
                 let dependencyName = getDependencyNameShow(record).toLowerCase();
                 CachedEnvironmentDeps[dependencyName] = record;
                 CachedEnvironmentDeps[dependencyName.replace("-", "_")] = record;
@@ -210,40 +224,24 @@ export default class Python_controller {
                 CachedEnvironmentDeps[packageName.replace("_", "-")] = pipDepTreeEntryForCache;
             });
         }
-        linesOfRequirements.forEach((dep) => {
-            // if matchManifestVersions setting is turned on , then
-            if (matchManifestVersions === "true") {
-                let dependencyName;
-                let manifestVersion;
+        parsedRequirements.forEach(({ name: depName, version: manifestVersion }) => {
+            if (matchManifestVersions === "true" && manifestVersion != null) {
                 let installedVersion;
-                let doubleEqualSignPosition;
-                if (dep.includes("==")) {
-                    doubleEqualSignPosition = dep.indexOf("==");
-                    manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim();
-                    if (manifestVersion.includes("#")) {
-                        let hashCharIndex = manifestVersion.indexOf("#");
-                        manifestVersion = manifestVersion.substring(0, hashCharIndex);
+                if (CachedEnvironmentDeps[depName.toLowerCase()] !== undefined) {
+                    if (usePipDepTree !== "true") {
+                        installedVersion = getDependencyVersion(CachedEnvironmentDeps[depName.toLowerCase()]);
                     }
-                    dependencyName = getDependencyName(dep);
-                    // only compare between declared version in manifest to installed version , if the package is installed.
-                    if (CachedEnvironmentDeps[dependencyName.toLowerCase()] !== undefined) {
-                        if (usePipDepTree !== "true") {
-                            installedVersion = getDependencyVersion(CachedEnvironmentDeps[dependencyName.toLowerCase()]);
-                        }
-                        else {
-                            installedVersion = CachedEnvironmentDeps[dependencyName.toLowerCase()].version;
-                        }
+                    else {
+                        installedVersion = CachedEnvironmentDeps[depName.toLowerCase()].version;
                     }
-                    if (installedVersion) {
-                        if (manifestVersion.trim() !== installedVersion.trim()) {
-                            throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${dependencyName} (manifest version=${manifestVersion}, installed version=${installedVersion}).If you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting MATCH_MANIFEST_VERSIONS=false`);
-                        }
+                }
+                if (installedVersion) {
+                    if (manifestVersion.trim() !== installedVersion.trim()) {
+                        throw new Error(`Can't continue with analysis - versions mismatch for dependency name ${depName} (manifest version=${manifestVersion}, installed version=${installedVersion}).If you want to allow version mismatch for analysis between installed and requested packages, set environment variable/setting MATCH_MANIFEST_VERSIONS=false`);
                     }
                 }
             }
-            let path = new Array();
-            let depName = getDependencyName(dep);
-            //array to track a path for each branch in the dependency tree
+            let path = [];
             path.push(depName.toLowerCase());
             bringAllDependencies(dependencies, depName, CachedEnvironmentDeps, includeTransitive, path, usePipDepTree);
         });
@@ -347,11 +345,11 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
         version = record.version;
         directDeps = record.dependencies;
     }
-    let targetDeps = new Array();
+    let targetDeps = [];
     let entry = { "name": depName, "version": version, "dependencies": [] };
     dependencies.push(entry);
     directDeps.forEach((dep) => {
-        let depArray = new Array();
+        let depArray = [];
         // to avoid infinite loop, check if the dependency not already on current path, before going recursively resolving its dependencies.
         if (!path.includes(dep.toLowerCase())) {
             // send to recurrsion the path + the current dep

package/dist/src/providers/python_pip.d.ts

@@ -23,13 +23,13 @@ declare function validateLockFile(): boolean;
  * Provide content and content type for python-pip component analysis.
  * @param {string} manifest - path to requirements.txt for component report
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {Provided}
+ * @returns {Promise<Provided>}
  */
-declare function provideComponent(manifest: string, opts?: {}): Provided;
+declare function provideComponent(manifest: string, opts?: {}): Promise<Provided>;
 /**
  * Provide content and content type for python-pip stack analysis.
  * @param {string} manifest - the manifest path or name
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {Provided}
+ * @returns {Promise<Provided>}
  */
-declare function provideStack(manifest: string, opts?: {}): Provided;
+declare function provideStack(manifest: string, opts?: {}): Promise<Provided>;

package/dist/src/providers/python_pip.js

@@ -1,9 +1,9 @@
 import fs from 'node:fs';
-import { EOL } from 'os';
 import { PackageURL } from 'packageurl-js';
 import Sbom from '../sbom.js';
 import { environmentVariableIsPopulated, getCustom, getCustomPath, invokeCommand } from "../tools.js";
 import Python_controller from './python_controller.js';
+import { getParser, getIgnoreQuery, getPinnedVersionQuery } from './requirements_parser.js';
 export default { isSupported, validateLockFile, provideComponent, provideStack };
 /** @typedef {{name: string, version: string, dependencies: DependencyEntry[]}} DependencyEntry */
 /**
@@ -26,12 +26,12 @@ function validateLockFile() { return true; }
  * Provide content and content type for python-pip stack analysis.
  * @param {string} manifest - the manifest path or name
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {Provided}
+ * @returns {Promise<Provided>}
  */
-function provideStack(manifest, opts = {}) {
+async function provideStack(manifest, opts = {}) {
     return {
         ecosystem,
-        content: createSbomStackAnalysis(manifest, opts),
+        content: await createSbomStackAnalysis(manifest, opts),
         contentType: 'application/vnd.cyclonedx+json'
     };
 }
@@ -39,12 +39,12 @@ function provideStack(manifest, opts = {}) {
  * Provide content and content type for python-pip component analysis.
  * @param {string} manifest - path to requirements.txt for component report
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {Provided}
+ * @returns {Promise<Provided>}
  */
-function provideComponent(manifest, opts = {}) {
+async function provideComponent(manifest, opts = {}) {
     return {
         ecosystem,
-        content: getSbomForComponentAnalysis(manifest, opts),
+        content: await getSbomForComponentAnalysis(manifest, opts),
         contentType: 'application/vnd.cyclonedx+json'
     };
 }
@@ -66,49 +66,34 @@ function addAllDependencies(source, dep, sbom) {
 }
 /**
  *
- * @param nameVersion
- * @return {string}
- */
-function splitToNameVersion(nameVersion) {
-    let result = [];
-    if (nameVersion.includes("==")) {
-        return nameVersion.split("==");
-    }
-    const regex = /[^\w\s-_]/g;
-    let endIndex = nameVersion.search(regex);
-    if (endIndex === -1) {
-        return [nameVersion.trim()];
-    }
-    result.push(nameVersion.substring(0, endIndex).trim());
-    return result;
-}
-/**
- *
- * @param {string} requirementTxtContent
+ * @param {string} manifest - path to requirements.txt
  * @return {PackageURL []}
  */
-function getIgnoredDependencies(requirementTxtContent) {
-    let requirementsLines = requirementTxtContent.split(EOL);
-    return requirementsLines
-        .filter(line => line.includes("#exhortignore") || line.includes("# exhortignore"))
-        .map((line) => line.substring(0, line.indexOf("#")).trim())
-        .map((name) => {
-        let nameVersion = splitToNameVersion(name);
-        if (nameVersion.length === 2) {
-            return toPurl(nameVersion[0], nameVersion[1]);
-        }
-        return toPurl(nameVersion[0], undefined);
+async function getIgnoredDependencies(manifest) {
+    const [parser, ignoreQuery, pinnedVersionQuery] = await Promise.all([
+        getParser(), getIgnoreQuery(), getPinnedVersionQuery()
+    ]);
+    const content = fs.readFileSync(manifest).toString();
+    const tree = parser.parse(content);
+    return ignoreQuery.matches(tree.rootNode).map(match => {
+        const reqNode = match.captures.find(c => c.name === 'req').node;
+        const name = match.captures.find(c => c.name === 'name').node.text;
+        const versionMatches = pinnedVersionQuery.matches(reqNode);
+        const version = versionMatches.length > 0
+            ? versionMatches[0].captures.find(c => c.name === 'version').node.text
+            : undefined;
+        return toPurl(name, version);
     });
 }
 /**
  *
- * @param {string} requirementTxtContent content of requirments.txt in string
+ * @param {string} manifest - path to requirements.txt
  * @param {Sbom} sbom object to filter out from it exhortignore dependencies.
  * @param {{Object}} opts - various options and settings for the application
  * @private
  */
-function handleIgnoredDependencies(requirementTxtContent, sbom, opts = {}) {
-    let ignoredDeps = getIgnoredDependencies(requirementTxtContent);
+async function handleIgnoredDependencies(manifest, sbom, opts = {}) {
+    let ignoredDeps = await getIgnoredDependencies(manifest);
     let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS", "true", opts);
     if (matchManifestVersions === "true") {
         const ignoredDepsVersion = ignoredDeps.filter(dep => dep.version !== undefined);
@@ -172,22 +157,21 @@ const DEFAULT_PIP_ROOT_COMPONENT_VERSION = "0.0.0";
  * Create sbom json string out of a manifest path for stack analysis.
  * @param {string} manifest - path for requirements.txt
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {string} the sbom json string content
+ * @returns {Promise<string>} the sbom json string content
  * @private
  */
-function createSbomStackAnalysis(manifest, opts = {}) {
+async function createSbomStackAnalysis(manifest, opts = {}) {
     let binaries = {};
     let createVirtualPythonEnv = handlePythonEnvironment(binaries, opts);
     let pythonController = new Python_controller(createVirtualPythonEnv === "false", binaries.pip, binaries.python, manifest, opts);
-    let dependencies = pythonController.getDependencies(true);
+    let dependencies = await pythonController.getDependencies(true);
     let sbom = new Sbom();
     const rootPurl = toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION);
     sbom.addRoot(rootPurl);
     dependencies.forEach(dep => {
         addAllDependencies(rootPurl, dep, sbom);
     });
-    let requirementTxtContent = fs.readFileSync(manifest).toString();
-    handleIgnoredDependencies(requirementTxtContent, sbom, opts);
+    await handleIgnoredDependencies(manifest, sbom, opts);
     // In python there is no root component, then we must remove the dummy root we added, so the sbom json will be accepted by the DA backend
     // sbom.removeRootComponent()
     return sbom.getAsJsonString(opts);
@@ -196,22 +180,21 @@ function createSbomStackAnalysis(manifest, opts = {}) {
  * Create a sbom json string out of a manifest content for component analysis
  * @param {string} manifest - path to requirements.txt
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {string} the sbom json string content
+ * @returns {Promise<string>} the sbom json string content
  * @private
  */
-function getSbomForComponentAnalysis(manifest, opts = {}) {
+async function getSbomForComponentAnalysis(manifest, opts = {}) {
     let binaries = {};
     let createVirtualPythonEnv = handlePythonEnvironment(binaries, opts);
     let pythonController = new Python_controller(createVirtualPythonEnv === "false", binaries.pip, binaries.python, manifest, opts);
-    let dependencies = pythonController.getDependencies(false);
+    let dependencies = await pythonController.getDependencies(false);
     let sbom = new Sbom();
     const rootPurl = toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION);
     sbom.addRoot(rootPurl);
     dependencies.forEach(dep => {
         sbom.addDependency(rootPurl, toPurl(dep.name, dep.version));
     });
-    let requirementTxtContent = fs.readFileSync(manifest).toString();
-    handleIgnoredDependencies(requirementTxtContent, sbom, opts);
+    await handleIgnoredDependencies(manifest, sbom, opts);
     // In python there is no root component, then we must remove the dummy root we added, so the sbom json will be accepted by the DA backend
     // sbom.removeRootComponent()
     return sbom.getAsJsonString(opts);

package/dist/src/providers/requirements_parser.d.ts

@@ -0,0 +1,6 @@
+export function getParser(): Promise<Parser>;
+export function getRequirementQuery(): Promise<Query>;
+export function getIgnoreQuery(): Promise<Query>;
+export function getPinnedVersionQuery(): Promise<Query>;
+import { Parser } from 'web-tree-sitter';
+import { Query } from 'web-tree-sitter';

package/dist/src/providers/requirements_parser.js

@@ -0,0 +1,27 @@
+import { createRequire } from 'node:module';
+import { Language, Parser, Query } from 'web-tree-sitter';
+const require = createRequire(import.meta.url);
+async function init() {
+    await Parser.init({
+        locateFile() {
+            return require.resolve('web-tree-sitter/web-tree-sitter.wasm');
+        }
+    });
+    return await Language.load(require.resolve('tree-sitter-requirements/tree-sitter-requirements.wasm'));
+}
+export async function getParser() {
+    const language = await init();
+    return new Parser().setLanguage(language);
+}
+export async function getRequirementQuery() {
+    const language = await init();
+    return new Query(language, '(requirement (package) @name) @req');
+}
+export async function getIgnoreQuery() {
+    const language = await init();
+    return new Query(language, '((requirement (package) @name) @req . (comment) @comment (#match? @comment "^#[\\t ]*exhortignore"))');
+}
+export async function getPinnedVersionQuery() {
+    const language = await init();
+    return new Query(language, '(version_spec (version_cmp) @cmp (version) @version (#eq? @cmp "=="))');
+}

package/dist/src/tools.js

@@ -1,7 +1,7 @@
 import { execFileSync } from "child_process";
 import { EOL } from "os";
 import { PackageURL } from "packageurl-js";
-export const RegexNotToBeLogged = /TRUSTIFY_DA_.*_TOKEN|ex-.*-token/;
+export const RegexNotToBeLogged = /TRUSTIFY_DA_(.*_)?TOKEN|ex-.*-token|trust-.*-token/;
 /**
  *
  * @param {string} key to log its value from environment variables and from opts, if it exists

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@trustify-da/trustify-da-javascript-client",
-	"version": "0.3.0-ea.62f6bc7",
+	"version": "0.3.0-ea.63ae5c2",
 	"description": "Code-Ready Dependency Analytics JavaScript API.",
 	"license": "Apache-2.0",
 	"homepage": "https://github.com/guacsec/trustify-da-javascript-client#README.md",
@@ -46,13 +46,16 @@
 	"dependencies": {
 		"@babel/core": "^7.23.2",
 		"@cyclonedx/cyclonedx-library": "^6.13.0",
+		"eslint-import-resolver-typescript": "^4.4.4",
 		"fast-toml": "^0.5.4",
 		"fast-xml-parser": "^5.3.4",
 		"help": "^3.0.2",
 		"https-proxy-agent": "^7.0.6",
 		"node-fetch": "^3.3.2",
 		"packageurl-js": "~1.0.2",
-		"yargs": "^17.7.2"
+		"tree-sitter-requirements": "github:Strum355/tree-sitter-requirements#d0261ee76b84253997fe70d7d397e78c006c3801",
+		"web-tree-sitter": "^0.26.6",
+		"yargs": "^18.0.0"
 	},
 	"devDependencies": {
 		"@babel/core": "^7.23.2",
@@ -60,9 +63,10 @@
 		"@types/node": "^20.17.30",
 		"@types/which": "^3.0.4",
 		"babel-plugin-rewire": "^1.2.0",
-		"c8": "^10.1.3",
+		"c8": "^11.0.0",
 		"chai": "^4.3.7",
 		"eslint": "^8.42.0",
+		"eslint-import-resolver-typescript": "^4.4.4",
 		"eslint-plugin-editorconfig": "^4.0.3",
 		"eslint-plugin-import": "^2.29.1",
 		"esmock": "^2.6.2",