@trustify-da/trustify-da-javascript-client 0.3.0-ea.e12bc82 → 0.3.0-ea.ff266a3

package/dist/src/providers/python_pip.js

@@ -1,10 +1,10 @@
 import fs from 'node:fs';
-import { EOL } from 'os';
 import { PackageURL } from 'packageurl-js';
 import Sbom from '../sbom.js';
 import { environmentVariableIsPopulated, getCustom, getCustomPath, invokeCommand } from "../tools.js";
 import Python_controller from './python_controller.js';
-export default { isSupported, validateLockFile, provideComponent, provideStack };
+import { getParser, getIgnoreQuery, getPinnedVersionQuery } from './requirements_parser.js';
+export default { isSupported, validateLockFile, provideComponent, provideStack, readLicenseFromManifest };
 /** @typedef {{name: string, version: string, dependencies: DependencyEntry[]}} DependencyEntry */
 /**
  * @type {string} ecosystem for python-pip is 'pip'
@@ -18,6 +18,13 @@ const ecosystem = 'pip';
 function isSupported(manifestName) {
     return 'requirements.txt' === manifestName;
 }
+/**
+ * Python requirements.txt has no standard license field
+ * @param {string} manifestPath - path to requirements.txt
+ * @returns {string|null}
+ */
+// eslint-disable-next-line no-unused-vars
+function readLicenseFromManifest(manifestPath) { return null; }
 /**
  * @param {string} manifestDir - the directory where the manifest lies
  */
@@ -26,12 +33,12 @@ function validateLockFile() { return true; }
  * Provide content and content type for python-pip stack analysis.
  * @param {string} manifest - the manifest path or name
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {Provided}
+ * @returns {Promise<Provided>}
  */
-function provideStack(manifest, opts = {}) {
+async function provideStack(manifest, opts = {}) {
     return {
         ecosystem,
-        content: createSbomStackAnalysis(manifest, opts),
+        content: await createSbomStackAnalysis(manifest, opts),
         contentType: 'application/vnd.cyclonedx+json'
     };
 }
@@ -39,12 +46,12 @@ function provideStack(manifest, opts = {}) {
  * Provide content and content type for python-pip component analysis.
  * @param {string} manifest - path to requirements.txt for component report
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {Provided}
+ * @returns {Promise<Provided>}
  */
-function provideComponent(manifest, opts = {}) {
+async function provideComponent(manifest, opts = {}) {
     return {
         ecosystem,
-        content: getSbomForComponentAnalysis(manifest, opts),
+        content: await getSbomForComponentAnalysis(manifest, opts),
         contentType: 'application/vnd.cyclonedx+json'
     };
 }
@@ -66,49 +73,34 @@ function addAllDependencies(source, dep, sbom) {
 }
 /**
  *
- * @param nameVersion
- * @return {string}
- */
-function splitToNameVersion(nameVersion) {
-    let result = [];
-    if (nameVersion.includes("==")) {
-        return nameVersion.split("==");
-    }
-    const regex = /[^\w\s-_]/g;
-    let endIndex = nameVersion.search(regex);
-    if (endIndex === -1) {
-        return [nameVersion.trim()];
-    }
-    result.push(nameVersion.substring(0, endIndex).trim());
-    return result;
-}
-/**
- *
- * @param {string} requirementTxtContent
+ * @param {string} manifest - path to requirements.txt
  * @return {PackageURL []}
  */
-function getIgnoredDependencies(requirementTxtContent) {
-    let requirementsLines = requirementTxtContent.split(EOL);
-    return requirementsLines
-        .filter(line => line.includes("#exhortignore") || line.includes("# exhortignore"))
-        .map((line) => line.substring(0, line.indexOf("#")).trim())
-        .map((name) => {
-        let nameVersion = splitToNameVersion(name);
-        if (nameVersion.length === 2) {
-            return toPurl(nameVersion[0], nameVersion[1]);
-        }
-        return toPurl(nameVersion[0], undefined);
+async function getIgnoredDependencies(manifest) {
+    const [parser, ignoreQuery, pinnedVersionQuery] = await Promise.all([
+        getParser(), getIgnoreQuery(), getPinnedVersionQuery()
+    ]);
+    const content = fs.readFileSync(manifest).toString();
+    const tree = parser.parse(content);
+    return ignoreQuery.matches(tree.rootNode).map(match => {
+        const reqNode = match.captures.find(c => c.name === 'req').node;
+        const name = match.captures.find(c => c.name === 'name').node.text;
+        const versionMatches = pinnedVersionQuery.matches(reqNode);
+        const version = versionMatches.length > 0
+            ? versionMatches[0].captures.find(c => c.name === 'version').node.text
+            : undefined;
+        return toPurl(name, version);
     });
 }
 /**
  *
- * @param {string} requirementTxtContent content of requirments.txt in string
+ * @param {string} manifest - path to requirements.txt
  * @param {Sbom} sbom object to filter out from it exhortignore dependencies.
  * @param {{Object}} opts - various options and settings for the application
  * @private
  */
-function handleIgnoredDependencies(requirementTxtContent, sbom, opts = {}) {
-    let ignoredDeps = getIgnoredDependencies(requirementTxtContent);
+async function handleIgnoredDependencies(manifest, sbom, opts = {}) {
+    let ignoredDeps = await getIgnoredDependencies(manifest);
     let matchManifestVersions = getCustom("MATCH_MANIFEST_VERSIONS", "true", opts);
     if (matchManifestVersions === "true") {
         const ignoredDepsVersion = ignoredDeps.filter(dep => dep.version !== undefined);
@@ -172,22 +164,22 @@ const DEFAULT_PIP_ROOT_COMPONENT_VERSION = "0.0.0";
  * Create sbom json string out of a manifest path for stack analysis.
  * @param {string} manifest - path for requirements.txt
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {string} the sbom json string content
+ * @returns {Promise<string>} the sbom json string content
  * @private
  */
-function createSbomStackAnalysis(manifest, opts = {}) {
+async function createSbomStackAnalysis(manifest, opts = {}) {
     let binaries = {};
     let createVirtualPythonEnv = handlePythonEnvironment(binaries, opts);
     let pythonController = new Python_controller(createVirtualPythonEnv === "false", binaries.pip, binaries.python, manifest, opts);
-    let dependencies = pythonController.getDependencies(true);
+    let dependencies = await pythonController.getDependencies(true);
     let sbom = new Sbom();
     const rootPurl = toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION);
-    sbom.addRoot(rootPurl);
+    const license = readLicenseFromManifest(manifest);
+    sbom.addRoot(rootPurl, license);
     dependencies.forEach(dep => {
         addAllDependencies(rootPurl, dep, sbom);
     });
-    let requirementTxtContent = fs.readFileSync(manifest).toString();
-    handleIgnoredDependencies(requirementTxtContent, sbom, opts);
+    await handleIgnoredDependencies(manifest, sbom, opts);
     // In python there is no root component, then we must remove the dummy root we added, so the sbom json will be accepted by the DA backend
     // sbom.removeRootComponent()
     return sbom.getAsJsonString(opts);
@@ -196,22 +188,22 @@ function createSbomStackAnalysis(manifest, opts = {}) {
  * Create a sbom json string out of a manifest content for component analysis
  * @param {string} manifest - path to requirements.txt
  * @param {{}} [opts={}] - optional various options to pass along the application
- * @returns {string} the sbom json string content
+ * @returns {Promise<string>} the sbom json string content
  * @private
  */
-function getSbomForComponentAnalysis(manifest, opts = {}) {
+async function getSbomForComponentAnalysis(manifest, opts = {}) {
     let binaries = {};
     let createVirtualPythonEnv = handlePythonEnvironment(binaries, opts);
     let pythonController = new Python_controller(createVirtualPythonEnv === "false", binaries.pip, binaries.python, manifest, opts);
-    let dependencies = pythonController.getDependencies(false);
+    let dependencies = await pythonController.getDependencies(false);
     let sbom = new Sbom();
     const rootPurl = toPurl(DEFAULT_PIP_ROOT_COMPONENT_NAME, DEFAULT_PIP_ROOT_COMPONENT_VERSION);
-    sbom.addRoot(rootPurl);
+    const license = readLicenseFromManifest(manifest);
+    sbom.addRoot(rootPurl, license);
     dependencies.forEach(dep => {
         sbom.addDependency(rootPurl, toPurl(dep.name, dep.version));
     });
-    let requirementTxtContent = fs.readFileSync(manifest).toString();
-    handleIgnoredDependencies(requirementTxtContent, sbom, opts);
+    await handleIgnoredDependencies(manifest, sbom, opts);
     // In python there is no root component, then we must remove the dummy root we added, so the sbom json will be accepted by the DA backend
     // sbom.removeRootComponent()
     return sbom.getAsJsonString(opts);

package/dist/src/providers/requirements_parser.d.ts

@@ -0,0 +1,6 @@
+export function getParser(): Promise<Parser>;
+export function getRequirementQuery(): Promise<Query>;
+export function getIgnoreQuery(): Promise<Query>;
+export function getPinnedVersionQuery(): Promise<Query>;
+import { Parser } from 'web-tree-sitter';
+import { Query } from 'web-tree-sitter';

package/dist/src/providers/requirements_parser.js

@@ -0,0 +1,23 @@
+import { fileURLToPath } from 'url';
+import { Language, Parser, Query } from 'web-tree-sitter';
+const wasmPath = fileURLToPath(import.meta.resolve('tree-sitter-requirements/tree-sitter-requirements.wasm'));
+async function init() {
+    await Parser.init();
+    return await Language.load(wasmPath);
+}
+export async function getParser() {
+    const language = await init();
+    return new Parser().setLanguage(language);
+}
+export async function getRequirementQuery() {
+    const language = await init();
+    return new Query(language, '(requirement (package) @name) @req');
+}
+export async function getIgnoreQuery() {
+    const language = await init();
+    return new Query(language, '((requirement (package) @name) @req . (comment) @comment (#match? @comment "^#[\\t ]*exhortignore"))');
+}
+export async function getPinnedVersionQuery() {
+    const language = await init();
+    return new Query(language, '(version_spec (version_cmp) @cmp (version) @version (#eq? @cmp "=="))');
+}

package/dist/src/sbom.d.ts

@@ -2,9 +2,10 @@ export default class Sbom {
     sbomModel: CycloneDxSbom;
     /**
      * @param {PackageURL} root - add main/root component for sbom
+     * @param {string|Array} [licenses] - optional license(s) for the root component
      * @return Sbom
      */
-    addRoot(root: PackageURL): CycloneDxSbom;
+    addRoot(root: PackageURL, licenses?: string | any[]): CycloneDxSbom;
     /**
      * @return {{{"bom-ref": string, name, purl: string, type, version}}} root component of sbom.
      */
@@ -43,6 +44,7 @@ export default class Sbom {
         type: any;
         version: any;
         scope: any;
+        licenses?: any;
     };
     /** This method gets a component object, and a string name, and checks if the name is a substring of the component' purl.
      * @param {} component to search in its dependencies

package/dist/src/sbom.js

@@ -12,10 +12,11 @@ export default class Sbom {
     }
     /**
      * @param {PackageURL} root - add main/root component for sbom
+     * @param {string|Array} [licenses] - optional license(s) for the root component
      * @return Sbom
      */
-    addRoot(root) {
-        return this.sbomModel.addRoot(root);
+    addRoot(root, licenses) {
+        return this.sbomModel.addRoot(root, licenses);
     }
     /**
      * @return {{{"bom-ref": string, name, purl: string, type, version}}} root component of sbom.

package/dist/src/tools.d.ts

@@ -1,12 +1,10 @@
-/// <reference types="node" />
-/// <reference types="packageurl-js/src/package-url" />
 /**
  *
  * @param {string} key to log its value from environment variables and from opts, if it exists
  * @param {{}} [opts={}] different options of application, if key in it, log it.
  * @param {string }defValue default value of key in case there is no option and environment variable values for key
  */
-export function logValueFromObjects(key: string, opts?: {} | undefined, defValue: string): void;
+export function logValueFromObjects(key: string, opts?: {}, defValue: string): void;
 /**
  * Utility function will return the value for key from the environment variables,
  * if not present will return the value for key from the opts objects only if it's a string,
@@ -17,7 +15,7 @@ export function logValueFromObjects(key: string, opts?: {} | undefined, defValue
  * @returns {string|null} the value of the key found in the environment, options object, or the
  * 		default supplied
  */
-export function getCustom(key: string, def?: string | null | undefined, opts?: {} | undefined): string | null;
+export function getCustom(key: string, def?: string | null, opts?: {}): string | null;
 /**
  * Utility function for looking up custom variable for a binary path.
  * Will look in the environment variables (1) or in opts (2) for a key with TRUSTIFY_DA_x_PATH, x is an
@@ -28,7 +26,7 @@ export function getCustom(key: string, def?: string | null | undefined, opts?: {
  * @returns {string|null} the value of the key found in the environment, options object, or the
  * 		original name supplied
  */
-export function getCustomPath(name: any, opts?: {} | undefined): string | null;
+export function getCustomPath(name: any, opts?: {}): string | null;
 /**
  * Utility function for determining whether wrappers for build tools such as gradlew/mvnw should be
  * preferred over invoking the binary directly.
@@ -69,6 +67,24 @@ export function getGitRootDir(cwd: string): string | undefined;
  * @param {import('child_process').ExecFileOptionsWithStringEncoding} [opts={}]
  * @returns {string}
  */
-export function invokeCommand(bin: string, args: Array<string>, opts?: import("child_process").ExecFileOptionsWithStringEncoding | undefined): string;
+export function invokeCommand(bin: string, args: Array<string>, opts?: import("child_process").ExecFileOptionsWithStringEncoding): string;
+/**
+ * Adds proxy agent configuration to fetch options if a proxy URL is specified
+ * @param {RequestInit} options - The base fetch options
+ * @param {import("index.js").Options} opts - The trustify DA options that may contain proxy configuration
+ * @returns {RequestInit} The fetch options with proxy agent if applicable
+ */
+export function addProxyAgent(options: RequestInit, opts: import("index.js").Options): RequestInit;
+/**
+ * Utility function for fetching vendor tokens
+ * @param {import("index.js").Options} [opts={}] - optional various options to pass along the application
+ * @returns {{}}
+ */
+export function getTokenHeaders(opts?: import("index.js").Options): {};
 export const RegexNotToBeLogged: RegExp;
+export const TRUSTIFY_DA_TOKEN_HEADER: "trust-da-token";
+export const TRUSTIFY_DA_TELEMETRY_ID_HEADER: "telemetry-anonymous-id";
+export const TRUSTIFY_DA_SOURCE_HEADER: "trust-da-source";
+export const TRUSTIFY_DA_OPERATION_TYPE_HEADER: "trust-da-operation-type";
+export const TRUSTIFY_DA_PACKAGE_MANAGER_HEADER: "trust-da-pkg-manager";
 import { PackageURL } from "packageurl-js";

package/dist/src/tools.js

@@ -1,7 +1,8 @@
 import { execFileSync } from "child_process";
 import { EOL } from "os";
+import { HttpsProxyAgent } from "https-proxy-agent";
 import { PackageURL } from "packageurl-js";
-export const RegexNotToBeLogged = /TRUSTIFY_DA_.*_TOKEN|ex-.*-token/;
+export const RegexNotToBeLogged = /TRUSTIFY_DA_(.*_)?TOKEN|ex-.*-token|trust-.*-token/;
 /**
  *
  * @param {string} key to log its value from environment variables and from opts, if it exists
@@ -157,3 +158,57 @@ export function invokeCommand(bin, args, opts = {}) {
     };
     return execFileSync(bin, args, { ...{ stdio: 'pipe', encoding: 'utf-8' }, ...opts });
 }
+export const TRUSTIFY_DA_TOKEN_HEADER = "trust-da-token";
+export const TRUSTIFY_DA_TELEMETRY_ID_HEADER = "telemetry-anonymous-id";
+export const TRUSTIFY_DA_SOURCE_HEADER = "trust-da-source";
+export const TRUSTIFY_DA_OPERATION_TYPE_HEADER = "trust-da-operation-type";
+export const TRUSTIFY_DA_PACKAGE_MANAGER_HEADER = "trust-da-pkg-manager";
+/**
+ * Adds proxy agent configuration to fetch options if a proxy URL is specified
+ * @param {RequestInit} options - The base fetch options
+ * @param {import("index.js").Options} opts - The trustify DA options that may contain proxy configuration
+ * @returns {RequestInit} The fetch options with proxy agent if applicable
+ */
+export function addProxyAgent(options, opts) {
+    const proxyUrl = getCustom('TRUSTIFY_DA_PROXY_URL', null, opts);
+    if (proxyUrl) {
+        options.agent = new HttpsProxyAgent(proxyUrl);
+    }
+    return options;
+}
+/**
+ * Utility function for fetching vendor tokens
+ * @param {import("index.js").Options} [opts={}] - optional various options to pass along the application
+ * @returns {{}}
+ */
+export function getTokenHeaders(opts = {}) {
+    let headers = {};
+    setCustomHeader(TRUSTIFY_DA_TOKEN_HEADER, headers, 'TRUSTIFY_DA_TOKEN', opts);
+    setCustomHeader(TRUSTIFY_DA_SOURCE_HEADER, headers, 'TRUSTIFY_DA_SOURCE', opts);
+    setCustomHeader(TRUSTIFY_DA_OPERATION_TYPE_HEADER, headers, TRUSTIFY_DA_OPERATION_TYPE_HEADER.toUpperCase().replaceAll("-", "_"), opts);
+    setCustomHeader(TRUSTIFY_DA_PACKAGE_MANAGER_HEADER, headers, TRUSTIFY_DA_PACKAGE_MANAGER_HEADER.toUpperCase().replaceAll("-", "_"), opts);
+    setCustomHeader(TRUSTIFY_DA_TELEMETRY_ID_HEADER, headers, 'TRUSTIFY_DA_TELEMETRY_ID', opts);
+    if (getCustom("TRUSTIFY_DA_DEBUG", null, opts) === "true") {
+        console.log("Headers Values to be sent to Trustify DA backend:" + EOL);
+        for (const headerKey in headers) {
+            if (!headerKey.match(RegexNotToBeLogged)) {
+                console.log(`${headerKey}: ${headers[headerKey]}`);
+            }
+        }
+    }
+    return headers;
+}
+/**
+ *
+ * @param {string} headerName - the header name to populate in request
+ * @param headers
+ * @param {string} optsKey - key in the options object to use the value for
+ * @param {import("index.js").Options} [opts={}] - options input object to fetch header values from
+ * @private
+ */
+function setCustomHeader(headerName, headers, optsKey, opts) {
+    let customHeaderValue = getCustom(optsKey, null, opts);
+    if (customHeaderValue) {
+        headers[headerName] = customHeaderValue;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@trustify-da/trustify-da-javascript-client",
-	"version": "0.3.0-ea.e12bc82",
+	"version": "0.3.0-ea.ff266a3",
 	"description": "Code-Ready Dependency Analytics JavaScript API.",
 	"license": "Apache-2.0",
 	"homepage": "https://github.com/guacsec/trustify-da-javascript-client#README.md",
@@ -45,29 +45,33 @@
 	},
 	"dependencies": {
 		"@babel/core": "^7.23.2",
-		"@cyclonedx/cyclonedx-library": "~1.13.3",
+		"@cyclonedx/cyclonedx-library": "^6.13.0",
+		"eslint-import-resolver-typescript": "^4.4.4",
 		"fast-toml": "^0.5.4",
-		"fast-xml-parser": "^4.5.3",
+		"fast-xml-parser": "^5.3.4",
 		"help": "^3.0.2",
 		"https-proxy-agent": "^7.0.6",
-		"node-fetch": "^2.7.0",
-		"packageurl-js": "^1.0.2",
-		"yargs": "^17.7.2"
+		"node-fetch": "^3.3.2",
+		"packageurl-js": "~1.0.2",
+		"tree-sitter-requirements": "github:Strum355/tree-sitter-requirements#d0261ee76b84253997fe70d7d397e78c006c3801",
+		"web-tree-sitter": "^0.26.6",
+		"yargs": "^18.0.0"
 	},
 	"devDependencies": {
 		"@babel/core": "^7.23.2",
-		"@trustify-da/trustify-da-api-model": "^2.0.1",
+		"@trustify-da/trustify-da-api-model": "^2.0.7",
 		"@types/node": "^20.17.30",
 		"@types/which": "^3.0.4",
 		"babel-plugin-rewire": "^1.2.0",
-		"c8": "^8.0.0",
+		"c8": "^11.0.0",
 		"chai": "^4.3.7",
 		"eslint": "^8.42.0",
+		"eslint-import-resolver-typescript": "^4.4.4",
 		"eslint-plugin-editorconfig": "^4.0.3",
 		"eslint-plugin-import": "^2.29.1",
 		"esmock": "^2.6.2",
 		"mocha": "^10.2.0",
-		"msw": "^1.3.2",
+		"msw": "^2.12.7",
 		"sinon": "^15.1.2",
 		"sinon-chai": "^3.7.0",
 		"typescript": "^5.1.3",