@nodesecure/scanner 5.2.1 → 6.0.0

package/src/manifest.js

@@ -1,94 +0,0 @@
-// Import Node.js Dependencies
-import fs from "node:fs/promises";
-import path from "node:path";
-import crypto from "node:crypto";
-
-// Import Internal Dependencies
-import { parseAuthor } from "./utils/index.js";
-
-// CONSTANTS
-// PR welcome to contribute to this list!
-const kNativeNpmPackages = new Set([
-  "node-gyp", "node-pre-gyp", "node-gyp-build", "node-addon-api"
-]);
-const kNodemodulesBinPrefix = "node_modules/.bin/";
-
-/**
- * @see https://www.nerdycode.com/prevent-npm-executing-scripts-security/
- */
-const kUnsafeNpmScripts = new Set([
-  "install",
-  "preinstall",
-  "postinstall",
-  "preuninstall",
-  "postuninstall"
-]);
-
-/**
- * @param {!string} location
- * @returns {Promise<import("@npm/types").PackageJson>}
- */
-export async function read(location) {
-  const packageStr = await fs.readFile(
-    path.join(location, "package.json"),
-    "utf-8"
-  );
-
-  return JSON.parse(packageStr);
-}
-
-export async function readAnalyze(location) {
-  const {
-    name,
-    version,
-    description = "",
-    author = {},
-    scripts = {},
-    dependencies = {},
-    devDependencies = {},
-    gypfile = false,
-    engines = {},
-    repository = {},
-    imports = {},
-    license = ""
-  } = await read(location);
-
-  for (const [scriptName, scriptValue] of Object.entries(scripts)) {
-    if (scriptValue.startsWith(kNodemodulesBinPrefix)) {
-      scripts[scriptName] = scriptValue.replaceAll(kNodemodulesBinPrefix, "");
-    }
-  }
-
-  const integrityObj = {
-    name,
-    version,
-    dependencies,
-    license,
-    scripts
-  };
-
-  const integrity = crypto
-    .createHash("sha256")
-    .update(JSON.stringify(integrityObj))
-    .digest("hex");
-
-  const packageDeps = Object.keys(dependencies);
-  const packageDevDeps = Object.keys(devDependencies);
-  const hasNativePackage = [...packageDevDeps, ...packageDeps]
-    .some((pkg) => kNativeNpmPackages.has(pkg));
-
-  return {
-    author: parseAuthor(author),
-    description,
-    engines,
-    repository,
-    scripts,
-    hasScript: Object.keys(scripts)
-      .some((value) => kUnsafeNpmScripts.has(value.toLowerCase())),
-    packageDeps,
-    packageDevDeps,
-    nodejs: { imports },
-    hasNativeElements: hasNativePackage || gypfile,
-    integrity
-  };
-}

package/src/npmRegistry.js

@@ -1,136 +0,0 @@
-// Import Node.js Dependencies
-import crypto from "node:crypto";
-
-// Import Third-party Dependencies
-import semver from "semver";
-import { packument, packumentVersion } from "@nodesecure/npm-registry-sdk";
-
-// Import Internal Dependencies
-import { parseAuthor, getLinks } from "./utils/index.js";
-
-export async function manifestMetadata(
-  name,
-  version,
-  dependency
-) {
-  try {
-    const pkgVersion = await packumentVersion(name, version);
-
-    const integrity = getPackumentVersionIntegrity(pkgVersion);
-    Object.assign(
-      dependency.versions[version],
-      {
-        links: getLinks(pkgVersion)
-      }
-    );
-
-    dependency.metadata.integrity[version] = integrity;
-  }
-  catch {
-    // Ignore
-  }
-}
-
-export async function packageMetadata(name, version, options) {
-  const { ref, logger } = options;
-  const packageSpec = `${name}:${version}`;
-
-  try {
-    const pkg = await packument(name);
-
-    const oneYearFromToday = new Date();
-    oneYearFromToday.setFullYear(oneYearFromToday.getFullYear() - 1);
-
-    const lastVersion = pkg["dist-tags"].latest;
-    const lastUpdateAt = new Date(pkg.time[lastVersion]);
-    const metadata = {
-      author: parseAuthor(pkg.author),
-      homepage: pkg.homepage || null,
-      publishedCount: Object.values(pkg.versions).length,
-      lastVersion,
-      lastUpdateAt,
-      hasReceivedUpdateInOneYear: !(oneYearFromToday > lastUpdateAt),
-      maintainers: pkg.maintainers ?? [],
-      publishers: [],
-      integrity: {}
-    };
-
-    const isOutdated = semver.neq(version, lastVersion);
-    const flags = ref.versions[version].flags;
-    if (isOutdated) {
-      flags.push("isOutdated");
-    }
-
-    const publishers = new Set();
-    let searchForMaintainersInVersions = metadata.maintainers.length === 0;
-    for (const ver of Object.values(pkg.versions).reverse()) {
-      const versionSpec = `${ver.name}:${ver.version}`;
-      if (packageSpec === versionSpec) {
-        if (ver.deprecated && !flags.includes("isDeprecated")) {
-          flags.push("isDeprecated");
-        }
-
-        metadata.integrity[ver.version] = getPackumentVersionIntegrity(
-          ver
-        );
-      }
-
-      const { _npmUser: npmUser, version, maintainers = [] } = ver;
-
-      const isNullOrUndefined = typeof npmUser === "undefined" || npmUser === null;
-      if (isNullOrUndefined || !("name" in npmUser) || typeof npmUser.name !== "string") {
-        continue;
-      }
-
-      const authorName = metadata.author?.name ?? null;
-      if (authorName === null) {
-        metadata.author = npmUser;
-      }
-      else if (npmUser.name !== metadata.author.name) {
-        metadata.hasManyPublishers = true;
-      }
-
-      // TODO: add npmUser.email
-      if (!publishers.has(npmUser.name)) {
-        publishers.add(npmUser.name);
-        metadata.publishers.push({ ...npmUser, version, at: new Date(pkg.time[version]) });
-      }
-
-      if (searchForMaintainersInVersions) {
-        metadata.maintainers.push(...maintainers);
-        searchForMaintainersInVersions = false;
-      }
-    }
-
-    Object.assign(ref.versions[version], { links: getLinks(pkg.versions[version]) });
-    Object.assign(ref.metadata, metadata);
-  }
-  catch {
-    // ignore
-  }
-  finally {
-    logger.tick("registry");
-  }
-}
-
-function getPackumentVersionIntegrity(packumentVersion) {
-  const { name, version, dependencies = {}, license = "", scripts = {} } = packumentVersion;
-
-  // See https://github.com/npm/cli/issues/5234
-  if ("install" in dependencies && dependencies.install === "node-gyp rebuild") {
-    delete dependencies.install;
-  }
-
-  const integrityObj = {
-    name,
-    version,
-    dependencies,
-    license,
-    scripts
-  };
-
-  return crypto
-    .createHash("sha256")
-    .update(JSON.stringify(integrityObj))
-    .digest("hex");
-}

package/src/tarball.js

@@ -1,210 +0,0 @@
-// Import Node.js Dependencies
-import path from "path";
-import os from "os";
-import timers from "timers/promises";
-
-// Import Third-party Dependencies
-import { runASTAnalysisOnFile } from "@nodesecure/js-x-ray";
-import pacote from "pacote";
-import ntlp from "@nodesecure/ntlp";
-
-// Import Internal Dependencies
-import {
-  getTarballComposition,
-  isSensitiveFile,
-  filterDependencyKind,
-  analyzeDependencies,
-  booleanToFlags,
-  NPM_TOKEN,
-  getSemVerWarning
-} from "./utils/index.js";
-import * as manifest from "./manifest.js";
-
-// CONSTANTS
-const kNativeCodeExtensions = new Set([".gyp", ".c", ".cpp", ".node", ".so", ".h"]);
-const kJsExtname = new Set([".js", ".mjs", ".cjs"]);
-
-export async function scanJavascriptFile(dest, file, packageName) {
-  const result = await runASTAnalysisOnFile(path.join(dest, file), { packageName });
-
-  const warnings = result.warnings.map((curr) => Object.assign({}, curr, { file }));
-  if (!result.ok) {
-    return {
-      file,
-      warnings,
-      isMinified: false,
-      tryDependencies: [],
-      dependencies: [],
-      filesDependencies: []
-    };
-  }
-  const { packages, files } = filterDependencyKind(result.dependencies, path.dirname(file));
-
-  return {
-    file,
-    warnings,
-    isMinified: result.isMinified,
-    tryDependencies: [...result.dependencies.getDependenciesInTryStatement()],
-    dependencies: packages,
-    filesDependencies: files
-  };
-}
-
-export async function scanDirOrArchive(name, version, options) {
-  const { ref, location = process.cwd(), tmpLocation, locker, registry } = options;
-
-  const isNpmTarball = !(tmpLocation === null);
-  const dest = isNpmTarball ? path.join(tmpLocation, `${name}@${version}`) : location;
-  const free = await locker.acquireOne();
-
-  try {
-    // If this is an NPM tarball then we extract it on the disk with pacote.
-    if (isNpmTarball) {
-      await pacote.extract(ref.flags.includes("isGit") ? ref.gitUrl : `${name}@${version}`, dest, {
-        ...NPM_TOKEN,
-        registry,
-        cache: `${os.homedir()}/.npm`
-      });
-      await timers.setImmediate();
-    }
-    else {
-      // Set links to an empty object because theses are generated only for NPM tarballs
-      Object.assign(ref, { links: {} });
-    }
-
-    // Read the package.json at the root of the directory or archive.
-    const {
-      packageDeps,
-      packageDevDeps,
-      author,
-      description,
-      hasScript,
-      hasNativeElements,
-      nodejs,
-      engines,
-      repository,
-      scripts,
-      integrity
-    } = await manifest.readAnalyze(dest);
-    Object.assign(ref, {
-      author, description, engines, repository, scripts, integrity
-    });
-
-    // Get the composition of the (extracted) directory
-    const { ext, files, size } = await getTarballComposition(dest);
-    if (files.length === 1 && files.includes("package.json")) {
-      ref.warnings.push({
-        kind: "empty-package",
-        location: null,
-        i18n: "sast_warnings.emptyPackage",
-        severity: "Critical",
-        source: "Scanner",
-        experimental: false
-      });
-    }
-
-    ref.size = size;
-    ref.composition.extensions.push(...ext);
-    ref.composition.files.push(...files);
-    const hasBannedFile = files.some((path) => isSensitiveFile(path));
-    const hasNativeCode = hasNativeElements || files.some((file) => kNativeCodeExtensions.has(path.extname(file)));
-
-    // Search for minified and runtime dependencies
-    // Run a JS-X-Ray analysis on each JavaScript files of the project!
-    const fileAnalysisRaw = await Promise.allSettled(
-      files
-        .filter((name) => kJsExtname.has(path.extname(name)))
-        .map((file) => scanJavascriptFile(dest, file, name))
-    );
-
-    const fileAnalysisResults = fileAnalysisRaw
-      .filter((promiseSettledResult) => promiseSettledResult.status === "fulfilled")
-      .map((promiseSettledResult) => promiseSettledResult.value);
-
-    ref.warnings.push(...fileAnalysisResults.flatMap((row) => row.warnings));
-
-    if (/^0(\.\d+)*$/.test(version)) {
-      ref.warnings.push(getSemVerWarning(version));
-    }
-
-    const dependencies = [...new Set(fileAnalysisResults.flatMap((row) => row.dependencies))];
-    const filesDependencies = [...new Set(fileAnalysisResults.flatMap((row) => row.filesDependencies))];
-    const tryDependencies = new Set(fileAnalysisResults.flatMap((row) => row.tryDependencies));
-    const minifiedFiles = fileAnalysisResults.filter((row) => row.isMinified).flatMap((row) => row.file);
-
-    const {
-      nodeDependencies, thirdPartyDependencies, subpathImportsDependencies, missingDependencies, unusedDependencies, flags
-    } = analyzeDependencies(dependencies, { packageDeps, packageDevDeps, tryDependencies, nodeImports: nodejs.imports });
-
-    ref.composition.required_thirdparty = thirdPartyDependencies;
-    ref.composition.required_subpath = Object.fromEntries(subpathImportsDependencies);
-    ref.composition.unused.push(...unusedDependencies);
-    ref.composition.missing.push(...missingDependencies);
-    ref.composition.required_files = filesDependencies;
-    ref.composition.required_nodejs = nodeDependencies;
-    ref.composition.minified = minifiedFiles;
-
-    // License
-    await timers.setImmediate();
-    const licenses = await ntlp(dest);
-    const uniqueLicenseIds = Array.isArray(licenses.uniqueLicenseIds) ? licenses.uniqueLicenseIds : [];
-    ref.license = licenses;
-    ref.license.uniqueLicenseIds = uniqueLicenseIds;
-
-    ref.flags.push(...booleanToFlags({
-      ...flags,
-      hasNoLicense: uniqueLicenseIds.length === 0,
-      hasMultipleLicenses: licenses.hasMultipleLicenses,
-      hasMinifiedCode: minifiedFiles.length > 0,
-      hasWarnings: ref.warnings.length > 0 && !ref.flags.includes("hasWarnings"),
-      hasBannedFile,
-      hasNativeCode,
-      hasScript
-    }));
-  }
-  catch {
-    // Ignore
-  }
-  finally {
-    free();
-  }
-}
-
-export async function scanPackage(dest, packageName) {
-  const { type = "script", name } = await manifest.read(dest);
-
-  await timers.setImmediate();
-  const { ext, files, size } = await getTarballComposition(dest);
-  ext.delete("");
-
-  // Search for runtime dependencies
-  const dependencies = Object.create(null);
-  const [minified, warnings] = [[], []];
-
-  const JSFiles = files.filter((name) => kJsExtname.has(path.extname(name)));
-  for (const file of JSFiles) {
-    const result = await runASTAnalysisOnFile(path.join(dest, file), {
-      packageName: packageName ?? name,
-      module: type === "module"
-    });
-
-    warnings.push(...result.warnings.map((curr) => Object.assign({}, curr, { file })));
-    if (!result.ok) {
-      continue;
-    }
-
-    dependencies[file] = result.dependencies.dependencies;
-    result.isMinified && minified.push(file);
-  }
-
-  await timers.setImmediate();
-  const { uniqueLicenseIds, licenses } = await ntlp(dest);
-
-  return {
-    files: { list: files, extensions: [...ext], minified },
-    directorySize: size,
-    uniqueLicenseIds,
-    licenses,
-    ast: { dependencies, warnings }
-  };
-}

package/src/utils/addMissingVersionFlags.js

@@ -1,24 +0,0 @@
-/**
- * @param {Set<string>} flags
- * @param {import("../../types/scanner").Dependency} descriptor
- */
-export function* addMissingVersionFlags(flags, descriptor) {
-  const { metadata, vulnerabilities = [], versions } = descriptor;
-  const semverVersions = Object.keys(versions);
-
-  if (!metadata.hasReceivedUpdateInOneYear && flags.has("hasOutdatedDependency") && !flags.has("isDead")) {
-    yield "isDead";
-  }
-  if (metadata.hasManyPublishers && !flags.has("hasManyPublishers")) {
-    yield "hasManyPublishers";
-  }
-  if (metadata.hasChangedAuthor && !flags.has("hasChangedAuthor")) {
-    yield "hasChangedAuthor";
-  }
-  if (vulnerabilities.length > 0 && !flags.has("hasVulnerabilities")) {
-    yield "hasVulnerabilities";
-  }
-  if (semverVersions.length > 1 && !flags.has("hasDuplicate")) {
-    yield "hasDuplicate";
-  }
-}

package/src/utils/analyzeDependencies.js

@@ -1,71 +0,0 @@
-// Import Third-party Dependencies
-import difference from "lodash.difference";
-import builtins from "builtins";
-
-// Import Internal Dependencies
-import { getPackageName } from "./getPackageName.js";
-
-// CONSTANTS
-const kNodeModules = new Set(builtins({ experimental: true }));
-const kExternalModules = new Set(["http", "https", "net", "http2", "dgram", "child_process"]);
-
-export function analyzeDependencies(dependencies, deps = {}) {
-  const { packageDeps, packageDevDeps, tryDependencies, nodeImports = {} } = deps;
-
-  // See: https://nodejs.org/api/packages.html#subpath-imports
-  const subpathImportsDependencies = dependencies
-    .filter((name) => isAliasDependency(name) && name in nodeImports)
-    .map((name) => buildSubpathDependency(name, nodeImports));
-  const thirdPartyDependenciesAliased = new Set(
-    subpathImportsDependencies.flat().filter((name) => !isAliasDependency(name))
-  );
-
-  const thirdPartyDependencies = dependencies
-    .map((name) => (packageDeps.includes(name) ? name : getPackageName(name)))
-    .filter((name) => !name.startsWith("."))
-    .filter((name) => !isNodeCoreModule(name))
-    .filter((name) => !packageDevDeps.includes(name))
-    .filter((name) => !tryDependencies.has(name));
-
-  const unusedDependencies = difference(
-    packageDeps.filter((name) => !name.startsWith("@types")),
-    [...thirdPartyDependencies, ...thirdPartyDependenciesAliased]
-  );
-  const missingDependencies = [...new Set(difference(thirdPartyDependencies, packageDeps))]
-    .filter((name) => !(name in nodeImports));
-  const nodeDependencies = dependencies.filter((name) => isNodeCoreModule(name));
-
-  return {
-    nodeDependencies,
-    thirdPartyDependencies: [...new Set(thirdPartyDependencies)],
-    subpathImportsDependencies,
-    unusedDependencies,
-    missingDependencies,
-
-    flags: {
-      hasExternalCapacity: nodeDependencies.some((depName) => kExternalModules.has(depName)),
-      hasMissingOrUnusedDependency: unusedDependencies.length > 0 || missingDependencies.length > 0
-    }
-  };
-}
-
-/**
- * @param {!string} moduleName
- * @returns {boolean}
- */
-function isNodeCoreModule(moduleName) {
-  const cleanModuleName = moduleName.startsWith("node:") ? moduleName.slice(5) : moduleName;
-
-  // Note: We need to also check moduleName because builtins package only return true for 'node:test'.
-  return kNodeModules.has(cleanModuleName) || kNodeModules.has(moduleName);
-}
-
-function isAliasDependency(moduleName) {
-  return moduleName.charAt(0) === "#";
-}
-
-function buildSubpathDependency(alias, nodeImports) {
-  const importedDependency = nodeImports[alias].node ?? nodeImports[alias].default;
-
-  return [alias, importedDependency];
-}

package/src/utils/booleanToFlags.js

@@ -1,12 +0,0 @@
-/**
- * @param {Record<string, boolean>} flagsRecord
- * @example
- * console.log(...booleanToFlags({ hasScript: true })); // "hasScript"
- */
-export function* booleanToFlags(flagsRecord) {
-  for (const [flagName, boolValue] of Object.entries(flagsRecord)) {
-    if (boolValue) {
-      yield flagName;
-    }
-  }
-}

package/src/utils/dirname.js

@@ -1,9 +0,0 @@
-// Import Node.js Dependencies
-import { fileURLToPath } from "url";
-import { dirname } from "path";
-
-export function getDirNameFromUrl(url) {
-  const __filename = fileURLToPath(url);
-
-  return dirname(__filename);
-}

package/src/utils/filterDependencyKind.js

@@ -1,44 +0,0 @@
-// Import Node.js Dependencies
-import path from "path";
-
-// CONSTANTS
-const kRelativeImportPath = new Set([".", "..", "./", "../"]);
-
-/**
- * @see https://nodejs.org/docs/latest/api/modules.html#file-modules
- *
- * @param {IterableIterator<string>} dependencies
- * @param {!string} relativeFileLocation
- */
-export function filterDependencyKind(dependencies, relativeFileLocation) {
-  const packages = [];
-  const files = [];
-
-  for (const moduleNameOrPath of dependencies) {
-    const firstChar = moduleNameOrPath.charAt(0);
-
-    /**
-     * @example
-     * require("..");
-     * require("/home/marco/foo.js");
-     */
-    if (firstChar === "." || firstChar === "/") {
-      // Note: condition only possible for CJS
-      if (kRelativeImportPath.has(moduleNameOrPath)) {
-        files.push(path.join(moduleNameOrPath, "index.js"));
-      }
-      else {
-        // Note: we are speculating that the extension is .js (but it could be .json or .node)
-        const fixedFileName = path.extname(moduleNameOrPath) === "" ?
-          `${moduleNameOrPath}.js` : moduleNameOrPath;
-
-        files.push(path.join(relativeFileLocation, fixedFileName));
-      }
-    }
-    else {
-      packages.push(moduleNameOrPath);
-    }
-  }
-
-  return { packages, files };
-}

package/src/utils/getLinks.js

@@ -1,36 +0,0 @@
-// CONSTANTS
-const kVCSHosts = new Set(["github.com", "gitlab.com"]);
-
-function getVCSRepositoryURL(link) {
-  try {
-    const url = new URL(link);
-    const { hostname, pathname } = url;
-
-    if (kVCSHosts.has(hostname) === false) {
-      return null;
-    }
-
-    const [owner, repo] = pathname.split("/").filter(Boolean).map((curr) => curr.replace(".git", ""));
-
-    return `https://${hostname}/${owner}/${repo}`;
-  }
-  catch {
-    return null;
-  }
-}
-
-/**
- * @param {import("@nodesecure/npm-registry-sdk").PackumentVersion} packumentVersion
- */
-export function getLinks(
-  packumentVersion
-) {
-  const homepage = packumentVersion.homepage || null;
-  const repositoryUrl = packumentVersion.repository?.url || null;
-
-  return {
-    npm: `https://www.npmjs.com/package/${packumentVersion.name}/v/${packumentVersion.version}`,
-    homepage,
-    repository: getVCSRepositoryURL(homepage) ?? getVCSRepositoryURL(repositoryUrl)
-  };
-}

package/src/utils/getPackageName.js

@@ -1,21 +0,0 @@
-// CONSTANTS
-const kPackageSeparator = "/";
-const kPackageOrgSymbol = "@";
-
-/**
- * @see https://github.com/npm/validate-npm-package-name#naming-rules
- *
- * @param {!string} name full package name
- * @returns {string}
- *
- * @example
- * getPackageName("foo"); // foo
- * getPackageName("foo/bar"); // foo
- * getPackageName("@org/bar"); // @org/bar
- */
-export function getPackageName(name) {
-  const parts = name.split(kPackageSeparator);
-
-  // Note: only scoped package are allowed to start with @
-  return name.startsWith(kPackageOrgSymbol) ? `${parts[0]}/${parts[1]}` : parts[0];
-}