@nodesecure/scanner 5.2.1 → 6.0.0

package/index.js

@@ -1,74 +0,0 @@
-// Import Node.js Dependencies
-import path from "path";
-import fs from "fs/promises";
-import timers from "timers/promises";
-import os from "os";
-
-// Import Third-party Dependencies
-import pacote from "pacote";
-import { getLocalRegistryURL } from "@nodesecure/npm-registry-sdk";
-
-// Import Internal Dependencies
-import { depWalker } from "./src/depWalker.js";
-import { NPM_TOKEN } from "./src/utils/index.js";
-import { ScannerLoggerEvents } from "./src/constants.js";
-import Logger from "./src/class/logger.class.js";
-import * as tarball from "./src/tarball.js";
-
-// CONSTANTS
-const kDefaultCwdOptions = { forceRootAnalysis: true, usePackageLock: true, includeDevDeps: false };
-
-export async function cwd(location = process.cwd(), options = {}, logger = new Logger()) {
-  const registry = options.registry ? new URL(options.registry).toString() : getLocalRegistryURL();
-
-  const finalizedOptions = Object.assign(
-    { location },
-    kDefaultCwdOptions,
-    {
-      ...options,
-      registry
-    }
-  );
-
-  logger.start(ScannerLoggerEvents.manifest.read);
-  const packagePath = path.join(location, "package.json");
-  const str = await fs.readFile(packagePath, "utf-8");
-  logger.end(ScannerLoggerEvents.manifest.read);
-
-  return depWalker(JSON.parse(str), finalizedOptions, logger);
-}
-
-export async function from(packageName, options = {}, logger = new Logger()) {
-  const registry = options.registry ? new URL(options.registry).toString() : getLocalRegistryURL();
-
-  logger.start(ScannerLoggerEvents.manifest.fetch);
-  const manifest = await pacote.manifest(packageName, {
-    ...NPM_TOKEN, registry, cache: `${os.homedir()}/.npm`
-  });
-  logger.end(ScannerLoggerEvents.manifest.fetch);
-
-  return depWalker(manifest, Object.assign(options, { registry }), logger);
-}
-
-export async function verify(packageName = null) {
-  if (typeof packageName === "undefined" || packageName === null) {
-    return await tarball.scanPackage(process.cwd());
-  }
-
-  const tmpLocation = await fs.mkdtemp(path.join(os.tmpdir(), "/"));
-  const dest = path.join(tmpLocation, packageName);
-
-  try {
-    await pacote.extract(packageName, dest, {
-      ...NPM_TOKEN, registry: getLocalRegistryURL(), cache: `${os.homedir()}/.npm`
-    });
-
-    return await tarball.scanPackage(dest, packageName);
-  }
-  finally {
-    await timers.setImmediate();
-    await fs.rm(tmpLocation, { recursive: true, force: true });
-  }
-}
-
-export { depWalker, tarball, Logger, ScannerLoggerEvents };

package/src/class/dependency.class.js

@@ -1,113 +0,0 @@
-export default class Dependency {
-  #flags = new Set();
-  #parent = null;
-
-  constructor(name, version, parent = null) {
-    this.gitUrl = null;
-    this.dependencyCount = 0;
-    this.warnings = [];
-    this.name = name;
-    this.version = version;
-    this.dev = false;
-    this.existOnRemoteRegistry = true;
-    this.alias = {};
-
-    if (parent !== null) {
-      parent.addChildren();
-    }
-    this.#parent = parent;
-  }
-
-  addChildren() {
-    this.dependencyCount += 1;
-  }
-
-  get fullName() {
-    return `${this.name} ${this.version}`;
-  }
-
-  get flags() {
-    return [...this.#flags];
-  }
-
-  get parent() {
-    return this.#parent === null ? {} : { [this.#parent.name]: this.#parent.version };
-  }
-
-  addFlag(flagName, predicate = true) {
-    if (typeof flagName !== "string") {
-      throw new TypeError("flagName argument must be typeof string");
-    }
-
-    if (predicate) {
-      if (flagName === "hasDependencies" && this.#parent !== null) {
-        this.#parent.addFlag("hasIndirectDependencies");
-      }
-
-      this.#flags.add(flagName);
-    }
-  }
-
-  isGit(url) {
-    this.#flags.add("isGit");
-    if (typeof url === "string") {
-      this.gitUrl = url;
-    }
-
-    return this;
-  }
-
-  exportAsPlainObject(customId) {
-    if (this.warnings.length > 0) {
-      this.addFlag("hasWarnings");
-    }
-
-    return {
-      versions: {
-        [this.version]: {
-          id: typeof customId === "number" ? customId : Dependency.currentId++,
-          usedBy: this.parent,
-          isDevDependency: this.dev,
-          existOnRemoteRegistry: this.existOnRemoteRegistry,
-          flags: this.flags,
-          description: "",
-          size: 0,
-          author: {},
-          engines: {},
-          repository: {},
-          scripts: {},
-          warnings: this.warnings,
-          composition: {
-            extensions: [],
-            files: [],
-            minified: [],
-            unused: [],
-            missing: [],
-            alias: this.alias,
-            required_files: [],
-            required_nodejs: [],
-            required_thirdparty: [],
-            required_subpath: []
-          },
-          license: "unkown license",
-          gitUrl: this.gitUrl
-        }
-      },
-      vulnerabilities: [],
-      metadata: {
-        dependencyCount: this.dependencyCount,
-        publishedCount: 0,
-        lastUpdateAt: null,
-        lastVersion: null,
-        hasManyPublishers: false,
-        hasReceivedUpdateInOneYear: true,
-        homepage: null,
-        author: {},
-        publishers: [],
-        maintainers: []
-      }
-    };
-  }
-}
-
-Dependency.currentId = 1;

package/src/class/logger.class.js

@@ -1,54 +0,0 @@
-// Import Node.js Dependencies
-import { EventEmitter } from "events";
-import { performance } from "perf_hooks";
-
-export default class Logger extends EventEmitter {
-  constructor() {
-    super();
-
-    this.events = new Map();
-  }
-
-  start(eventName) {
-    if (this.events.has(eventName)) {
-      return this;
-    }
-
-    this.events.set(eventName, {
-      startedAt: performance.now(),
-      count: 0
-    });
-    this.emit("start", eventName);
-
-    return this;
-  }
-
-  tick(eventName) {
-    if (!this.events.has(eventName)) {
-      return this;
-    }
-
-    this.events.get(eventName).count++;
-    this.emit("tick", eventName);
-
-    return this;
-  }
-
-  count(eventName) {
-    return this.events.get(eventName)?.count ?? 0;
-  }
-
-  end(eventName) {
-    if (!this.events.has(eventName)) {
-      return this;
-    }
-
-    const data = this.events.get(eventName);
-    this.emit("end", eventName, {
-      ...data,
-      executionTime: performance.now() - data.startedAt
-    });
-
-    return this;
-  }
-}

package/src/constants.js

@@ -1,13 +0,0 @@
-
-export const ScannerLoggerEvents = {
-  done: "depWalkerFinished",
-  analysis: {
-    tree: "walkTree",
-    tarball: "tarball",
-    registry: "registry"
-  },
-  manifest: {
-    read: "readManifest",
-    fetch: "fetchManifest"
-  }
-};

package/src/depWalker.js

@@ -1,388 +0,0 @@
-// Import Node.js Dependencies
-import path from "path";
-import { readFileSync, promises as fs } from "fs";
-import timers from "timers/promises";
-import os from "os";
-
-// Import Third-party Dependencies
-import combineAsyncIterators from "combine-async-iterators";
-import * as iter from "itertools";
-import pacote from "pacote";
-import Arborist from "@npmcli/arborist";
-import Lock from "@slimio/lock";
-import * as vuln from "@nodesecure/vuln";
-import { ScannerLoggerEvents } from "./constants.js";
-
-// Import Internal Dependencies
-import {
-  mergeDependencies, getCleanDependencyName, getDependenciesWarnings, addMissingVersionFlags, isGitDependency,
-  NPM_TOKEN
-} from "./utils/index.js";
-import { scanDirOrArchive } from "./tarball.js";
-import { packageMetadata, manifestMetadata } from "./npmRegistry.js";
-import Dependency from "./class/dependency.class.js";
-import Logger from "./class/logger.class.js";
-
-const { version: packageVersion } = JSON.parse(
-  readFileSync(
-    new URL(path.join("..", "package.json"), import.meta.url)
-  )
-);
-
-export async function* searchDeepDependencies(packageName, gitURL, options) {
-  const { exclude, currDepth = 0, parent, maxDepth, registry } = options;
-
-  const { name, version, deprecated, ...pkg } = await pacote.manifest(gitURL ?? packageName, {
-    ...NPM_TOKEN,
-    registry,
-    cache: `${os.homedir()}/.npm`
-  });
-  const { dependencies, customResolvers, alias } = mergeDependencies(pkg);
-
-  const current = new Dependency(name, version, parent);
-  current.alias = Object.fromEntries(alias);
-
-  if (gitURL !== null) {
-    current.isGit(gitURL);
-    try {
-      await pacote.manifest(`${name}@${version}`, {
-        ...NPM_TOKEN,
-        registry,
-        cache: `${os.homedir()}/.npm`
-      });
-    }
-    catch {
-      current.existOnRemoteRegistry = false;
-    }
-  }
-  current.addFlag("isDeprecated", deprecated === true);
-  current.addFlag("hasCustomResolver", customResolvers.size > 0);
-  current.addFlag("hasDependencies", dependencies.size > 0);
-
-  if (currDepth !== maxDepth) {
-    const config = {
-      exclude, currDepth: currDepth + 1, parent: current, maxDepth, registry
-    };
-
-    const gitDependencies = iter.filter(customResolvers.entries(), ([, valueStr]) => isGitDependency(valueStr));
-    for (const [depName, valueStr] of gitDependencies) {
-      yield* searchDeepDependencies(depName, valueStr, config);
-    }
-
-    const depsNames = await Promise.all(iter.map(dependencies.entries(), getCleanDependencyName));
-    for (const [fullName, cleanName, isLatest] of depsNames) {
-      if (!isLatest) {
-        current.addFlag("hasOutdatedDependency");
-      }
-
-      if (exclude.has(cleanName)) {
-        current.addChildren();
-        exclude.get(cleanName).add(current.fullName);
-      }
-      else {
-        exclude.set(cleanName, new Set([current.fullName]));
-        yield* searchDeepDependencies(fullName, null, config);
-      }
-    }
-  }
-
-  yield current;
-}
-
-export async function* deepReadEdges(currentPackageName, options) {
-  const { to, parent, exclude, fullLockMode, includeDevDeps, registry } = options;
-  const { version, integrity = to.integrity } = to.package;
-
-  const updatedVersion = version === "*" || typeof version === "undefined" ? "latest" : version;
-  const current = new Dependency(currentPackageName, updatedVersion, parent);
-  current.dev = to.dev;
-
-  if (fullLockMode && !includeDevDeps) {
-    const { deprecated, _integrity, ...pkg } = await pacote.manifest(`${currentPackageName}@${updatedVersion}`, {
-      ...NPM_TOKEN,
-      registry,
-      cache: `${os.homedir()}/.npm`
-    });
-    const { customResolvers, alias } = mergeDependencies(pkg);
-
-    current.alias = Object.fromEntries(alias);
-    current.addFlag("hasValidIntegrity", _integrity === integrity);
-    current.addFlag("isDeprecated");
-    current.addFlag("hasCustomResolver", customResolvers.size > 0);
-
-    if (isGitDependency(to.resolved)) {
-      current.isGit(to.resolved);
-    }
-  }
-  current.addFlag("hasDependencies", to.edgesOut.size > 0);
-
-  for (const [packageName, { to: toNode }] of to.edgesOut) {
-    if (toNode === null || (!includeDevDeps && toNode.dev)) {
-      continue;
-    }
-    const cleanName = `${packageName}@${toNode.package.version}`;
-
-    if (exclude.has(cleanName)) {
-      current.addChildren();
-      exclude.get(cleanName).add(current.fullName);
-    }
-    else {
-      exclude.set(cleanName, new Set([current.fullName]));
-      yield* deepReadEdges(packageName, { parent: current, to: toNode, exclude, registry });
-    }
-  }
-  yield current;
-}
-
-export async function* getRootDependencies(manifest, options) {
-  const {
-    maxDepth = 4, exclude,
-    usePackageLock, fullLockMode, includeDevDeps,
-    location,
-    registry
-  } = options;
-
-  const { dependencies, customResolvers, alias } = mergeDependencies(manifest, void 0);
-  const parent = new Dependency(manifest.name, manifest.version);
-  parent.alias = Object.fromEntries(alias);
-
-  try {
-    await pacote.manifest(`${manifest.name}@${manifest.version}`, {
-      ...NPM_TOKEN,
-      registry,
-      cache: `${os.homedir()}/.npm`
-    });
-  }
-  catch {
-    parent.existOnRemoteRegistry = false;
-  }
-  parent.addFlag("hasCustomResolver", customResolvers.size > 0);
-  parent.addFlag("hasDependencies", dependencies.size > 0);
-
-  let iterators;
-  if (usePackageLock) {
-    const arb = new Arborist({
-      ...NPM_TOKEN,
-      path: location,
-      registry
-    });
-    let tree;
-    try {
-      await fs.access(path.join(location, "node_modules"));
-      tree = await arb.loadActual();
-    }
-    catch {
-      tree = await arb.loadVirtual();
-    }
-
-    iterators = [
-      ...iter
-        .filter(tree.edgesOut.entries(), ([, { to }]) => to !== null && (includeDevDeps ? true : (!to.dev || to.isWorkspace)))
-        .map(([packageName, { to }]) => [packageName, to.isWorkspace ? to.target : to])
-        .map(([packageName, to]) => deepReadEdges(packageName, {
-          to,
-          parent,
-          fullLockMode,
-          includeDevDeps,
-          exclude,
-          registry
-        }))
-    ];
-  }
-  else {
-    const configRef = { exclude, maxDepth, parent, registry };
-    iterators = [
-      ...iter.filter(customResolvers.entries(), ([, valueStr]) => isGitDependency(valueStr))
-        .map(([depName, valueStr]) => searchDeepDependencies(depName, valueStr, configRef)),
-      ...iter.map(dependencies.entries(), ([name, ver]) => searchDeepDependencies(`${name}@${ver}`, null, configRef))
-    ];
-  }
-  for await (const dep of combineAsyncIterators({}, ...iterators)) {
-    yield dep;
-  }
-
-  // Add root dependencies to the exclude Map (because the parent is not handled by searchDeepDependencies)
-  // if we skip this the code will fail to re-link properly dependencies in the following steps
-  const depsName = await Promise.all(iter.map(dependencies.entries(), getCleanDependencyName));
-  for (const [, fullRange, isLatest] of depsName) {
-    if (!isLatest) {
-      parent.addFlag("hasOutdatedDependency");
-    }
-    if (exclude.has(fullRange)) {
-      exclude.get(fullRange).add(parent.fullName);
-    }
-  }
-
-  yield parent;
-}
-
-/**
- * @param {*} manifest
- * @param {*} options
- * @param {Logger} logger
- */
-export async function depWalker(manifest, options = {}, logger = new Logger()) {
-  const {
-    forceRootAnalysis = false,
-    usePackageLock = false,
-    includeDevDeps = false,
-    fullLockMode = false,
-    maxDepth,
-    location,
-    vulnerabilityStrategy = vuln.strategies.NONE,
-    registry
-  } = options;
-
-  // Create TMP directory
-  const tmpLocation = await fs.mkdtemp(path.join(os.tmpdir(), "/"));
-
-  const payload = {
-    id: tmpLocation.slice(-6),
-    rootDependencyName: manifest.name,
-    scannerVersion: packageVersion,
-    vulnerabilityStrategy,
-    warnings: []
-  };
-
-  // We are dealing with an exclude Map to avoid checking a package more than one time in searchDeepDependencies
-  const exclude = new Map();
-  const dependencies = new Map();
-
-  {
-    logger
-      .start(ScannerLoggerEvents.analysis.tree)
-      .start(ScannerLoggerEvents.analysis.tarball)
-      .start(ScannerLoggerEvents.analysis.registry);
-    const fetchedMetadataPackages = new Set();
-    const promisesToWait = [];
-
-    const tarballLocker = new Lock({ maxConcurrent: 5 });
-    tarballLocker.on("freeOne", () => logger.tick(ScannerLoggerEvents.analysis.tarball));
-
-    const rootDepsOptions = { maxDepth, exclude, usePackageLock, fullLockMode, includeDevDeps, location, registry };
-    for await (const currentDep of getRootDependencies(manifest, rootDepsOptions)) {
-      const { name, version, dev } = currentDep;
-
-      const current = currentDep.exportAsPlainObject(name === manifest.name ? 0 : void 0);
-      let proceedDependencyAnalysis = true;
-
-      if (dependencies.has(name)) {
-        const dep = dependencies.get(name);
-        promisesToWait.push(manifestMetadata(name, version, dep));
-
-        const currVersion = Object.keys(current.versions)[0];
-        if (currVersion in dep.versions) {
-          // The dependency has already entered the analysis
-          // This happens if the package is used by multiple packages in the tree
-          proceedDependencyAnalysis = false;
-        }
-        else {
-          dep.versions[currVersion] = current.versions[currVersion];
-        }
-      }
-      else {
-        dependencies.set(name, current);
-      }
-
-      // If the dependency is a DevDependencies we ignore it.
-      if (dev) {
-        continue;
-      }
-
-      if (proceedDependencyAnalysis) {
-        logger.tick(ScannerLoggerEvents.analysis.tree);
-
-        // There is no need to fetch 'N' times the npm metadata for the same package.
-        if (fetchedMetadataPackages.has(name) || !current.versions[version].existOnRemoteRegistry) {
-          logger.tick(ScannerLoggerEvents.analysis.registry);
-        }
-        else {
-          fetchedMetadataPackages.add(name);
-          promisesToWait.push(packageMetadata(name, version, {
-            ref: current,
-            logger
-          }));
-        }
-
-        promisesToWait.push(scanDirOrArchive(name, version, {
-          ref: current.versions[version],
-          location,
-          tmpLocation: forceRootAnalysis && name === manifest.name ? null : tmpLocation,
-          locker: tarballLocker,
-          logger,
-          registry
-        }));
-      }
-    }
-
-    logger.end(ScannerLoggerEvents.analysis.tree);
-
-    // Wait for all extraction to be done!
-    await Promise.allSettled(promisesToWait);
-    await timers.setImmediate();
-
-    logger.end(ScannerLoggerEvents.analysis.tarball).end(ScannerLoggerEvents.analysis.registry);
-  }
-
-  const { hydratePayloadDependencies, strategy } = await vuln.setStrategy(vulnerabilityStrategy);
-  await hydratePayloadDependencies(dependencies, {
-    useStandardFormat: true,
-    path: location
-  });
-
-  payload.vulnerabilityStrategy = strategy;
-
-  // We do this because it "seem" impossible to link all dependencies in the first walk.
-  // Because we are dealing with package only one time it may happen sometimes.
-  const globalWarnings = [];
-  for (const [packageName, dependency] of dependencies) {
-    const metadataIntegrities = dependency.metadata?.integrity ?? {};
-
-    for (const [version, integrity] of Object.entries(metadataIntegrities)) {
-      const dependencyVer = dependency.versions[version];
-
-      const isEmptyPackage = dependencyVer.warnings
-        .some((warning) => warning.kind === "empty-package");
-      if (isEmptyPackage) {
-        globalWarnings.push(`${packageName}@${version} only contain a package.json file!`);
-      }
-
-      if (!("integrity" in dependencyVer) || dependencyVer.flags.includes("isGit")) {
-        continue;
-      }
-
-      if (dependencyVer.integrity !== integrity) {
-        globalWarnings.push(`${packageName}@${version} manifest & tarball integrity doesn't match!`);
-      }
-    }
-    for (const [verStr, verDescriptor] of Object.entries(dependency.versions)) {
-      verDescriptor.flags.push(...addMissingVersionFlags(new Set(verDescriptor.flags), dependency));
-
-      const usedDeps = exclude.get(`${packageName}@${verStr}`) || new Set();
-      if (usedDeps.size === 0) {
-        continue;
-      }
-
-      const usedBy = Object.create(null);
-      for (const [name, version] of [...usedDeps].map((name) => name.split(" "))) {
-        usedBy[name] = version;
-      }
-      Object.assign(verDescriptor.usedBy, usedBy);
-    }
-  }
-
-  try {
-    const { warnings, flaggedAuthors } = await getDependenciesWarnings(dependencies);
-    payload.warnings = globalWarnings.concat(warnings);
-    payload.flaggedAuthors = flaggedAuthors;
-    payload.dependencies = Object.fromEntries(dependencies);
-
-    return payload;
-  }
-  finally {
-    await timers.setImmediate();
-    await fs.rm(tmpLocation, { recursive: true, force: true });
-
-    logger.emit(ScannerLoggerEvents.done);
-  }
-}