@nodesecure/scanner 2.0.1 → 3.1.0

package/src/depWalker.js

@@ -8,18 +8,21 @@ import os from "os";
 import combineAsyncIterators from "combine-async-iterators";
 import iter from "itertools";
 import pacote from "pacote";
-import semver from "semver";
 import Arborist from "@npmcli/arborist";
 import Lock from "@slimio/lock";
-import { packument, getLocalRegistryURL } from "@nodesecure/npm-registry-sdk";
 import * as vuln from "@nodesecure/vuln";
-import { parseManifestAuthor } from "@nodesecure/utils";
+import { getLocalRegistryURL } from "@nodesecure/npm-registry-sdk";
+import { ScannerLoggerEvents } from "./constants.js";
 
 // Import Internal Dependencies
-import { mergeDependencies, constants, getCleanDependencyName, getDependenciesWarnings } from "./utils/index.js";
+import {
+  mergeDependencies, getCleanDependencyName, getDependenciesWarnings, addMissingVersionFlags, isGitDependency,
+  NPM_TOKEN
+} from "./utils/index.js";
 import { scanDirOrArchive } from "./tarball.js";
-import Dependency from "./dependency.class.js";
-import Logger from "./logger.class.js";
+import { packageMetadata } from "./npmRegistry.js";
+import Dependency from "./class/dependency.class.js";
+import Logger from "./class/logger.class.js";
 
 const { version: packageVersion } = JSON.parse(
   readFileSync(
@@ -27,20 +30,18 @@ const { version: packageVersion } = JSON.parse(
   )
 );
 
-
-async function* searchDeepDependencies(packageName, gitURL, options) {
-  const isGit = typeof gitURL === "string";
+export async function* searchDeepDependencies(packageName, gitURL, options) {
   const { exclude, currDepth = 0, parent, maxDepth } = options;
 
-  const { name, version, deprecated, ...pkg } = await pacote.manifest(isGit ? gitURL : packageName, {
-    ...constants.NPM_TOKEN,
+  const { name, version, deprecated, ...pkg } = await pacote.manifest(gitURL ?? packageName, {
+    ...NPM_TOKEN,
     registry: getLocalRegistryURL(),
     cache: `${os.homedir()}/.npm`
   });
   const { dependencies, customResolvers } = mergeDependencies(pkg);
 
   const current = new Dependency(name, version, parent);
-  isGit && current.isGit(gitURL);
+  gitURL !== null && current.isGit(gitURL);
   current.addFlag("isDeprecated", deprecated === true);
   current.addFlag("hasCustomResolver", customResolvers.size > 0);
   current.addFlag("hasDependencies", dependencies.size > 0);
@@ -50,9 +51,9 @@ async function* searchDeepDependencies(packageName, gitURL, options) {
       exclude, currDepth: currDepth + 1, parent: current, maxDepth
     };
 
-    const gitDependencies = iter.filter(customResolvers.entries(), ([, valueStr]) => valueStr.startsWith("git+"));
+    const gitDependencies = iter.filter(customResolvers.entries(), ([, valueStr]) => isGitDependency(valueStr));
     for (const [depName, valueStr] of gitDependencies) {
-      yield* searchDeepDependencies(depName, valueStr.slice(4), config);
+      yield* searchDeepDependencies(depName, valueStr, config);
     }
 
     const depsNames = await Promise.all(iter.map(dependencies.entries(), getCleanDependencyName));
@@ -66,7 +67,7 @@ async function* searchDeepDependencies(packageName, gitURL, options) {
       }
       else {
         exclude.set(cleanName, new Set([current.fullName]));
-        yield* searchDeepDependencies(fullName, void 0, config);
+        yield* searchDeepDependencies(fullName, null, config);
       }
     }
   }
@@ -74,7 +75,7 @@ async function* searchDeepDependencies(packageName, gitURL, options) {
   yield current;
 }
 
-async function* deepReadEdges(currentPackageName, { to, parent, exclude, fullLockMode }) {
+export async function* deepReadEdges(currentPackageName, { to, parent, exclude, fullLockMode }) {
   const { version, integrity = to.integrity } = to.package;
 
   const updatedVersion = version === "*" || typeof version === "undefined" ? "latest" : version;
@@ -82,7 +83,7 @@ async function* deepReadEdges(currentPackageName, { to, parent, exclude, fullLoc
 
   if (fullLockMode) {
     const { deprecated, _integrity, ...pkg } = await pacote.manifest(`${currentPackageName}@${updatedVersion}`, {
-      ...constants.NPM_TOKEN,
+      ...NPM_TOKEN,
       registry: getLocalRegistryURL(),
       cache: `${os.homedir()}/.npm`
     });
@@ -111,64 +112,7 @@ async function* deepReadEdges(currentPackageName, { to, parent, exclude, fullLoc
   yield current;
 }
 
-async function fetchPackageMetadata(name, version, options) {
-  const { ref, locker } = options;
-  const free = await locker.acquireOne();
-
-  try {
-    const pkg = await packument(name);
-
-    const publishers = new Set();
-    const oneYearFromToday = new Date();
-    oneYearFromToday.setFullYear(oneYearFromToday.getFullYear() - 1);
-
-    ref.metadata.lastVersion = pkg["dist-tags"].latest;
-    if (semver.neq(version, ref.metadata.lastVersion)) {
-      ref[version].flags.push("isOutdated");
-    }
-    ref.metadata.publishedCount = Object.values(pkg.versions).length;
-    ref.metadata.lastUpdateAt = new Date(pkg.time[ref.metadata.lastVersion]);
-    ref.metadata.hasReceivedUpdateInOneYear = !(oneYearFromToday > ref.metadata.lastUpdateAt);
-    ref.metadata.homepage = pkg.homepage || null;
-    ref.metadata.maintainers = pkg.maintainers;
-    if (typeof pkg.author === "string") {
-      ref.metadata.author = parseManifestAuthor(pkg.author);
-    }
-    else {
-      ref.metadata.author = pkg.author;
-    }
-    const authorName = ref.metadata.author?.name ?? null;
-
-    for (const ver of Object.values(pkg.versions)) {
-      const { _npmUser: npmUser, version } = ver;
-
-      const isNullOrUndefined = typeof npmUser === "undefined" || npmUser === null;
-      if (isNullOrUndefined || !("name" in npmUser) || typeof npmUser.name !== "string") {
-        continue;
-      }
-
-      if (authorName === null) {
-        ref.metadata.author.name = npmUser.name;
-      }
-      else if (npmUser.name !== ref.metadata.author.name) {
-        ref.metadata.hasManyPublishers = true;
-      }
-
-      if (!publishers.has(npmUser.name)) {
-        publishers.add(npmUser.name);
-        ref.metadata.publishers.push({ name: npmUser.name, version, at: new Date(pkg.time[version]) });
-      }
-    }
-  }
-  catch (err) {
-    // Ignore
-  }
-  finally {
-    free();
-  }
-}
-
-async function* getRootDependencies(manifest, options) {
+export async function* getRootDependencies(manifest, options) {
   const { maxDepth = 4, exclude, usePackageLock, fullLockMode } = options;
 
   const { dependencies, customResolvers } = mergeDependencies(manifest, void 0);
@@ -179,7 +123,7 @@ async function* getRootDependencies(manifest, options) {
   let iterators;
   if (usePackageLock) {
     const arb = new Arborist({
-      ...constants.NPM_TOKEN,
+      ...NPM_TOKEN,
       registry: getLocalRegistryURL()
     });
     let tree;
@@ -191,15 +135,15 @@ async function* getRootDependencies(manifest, options) {
       tree = await arb.loadVirtual();
     }
 
-    iterators = iter.filter(tree.edgesOut.entries(), ([, { to }]) => !to.dev)
+    iterators = iter.filter(tree.edgesOut.entries(), ([, { to }]) => to !== null && !to.dev)
       .map(([packageName, { to }]) => deepReadEdges(packageName, { to, parent, fullLockMode, exclude }));
   }
   else {
     const configRef = { exclude, maxDepth, parent };
     iterators = [
-      ...iter.filter(customResolvers.entries(), ([, valueStr]) => valueStr.startsWith("git+"))
-        .map(([depName, valueStr]) => searchDeepDependencies(depName, valueStr.slice(4), configRef)),
-      ...iter.map(dependencies.entries(), ([name, ver]) => searchDeepDependencies(`${name}@${ver}`, void 0, configRef))
+      ...iter.filter(customResolvers.entries(), ([, valueStr]) => isGitDependency(valueStr))
+        .map(([depName, valueStr]) => searchDeepDependencies(depName, valueStr, configRef)),
+      ...iter.map(dependencies.entries(), ([name, ver]) => searchDeepDependencies(`${name}@${ver}`, null, configRef))
     ];
   }
   for await (const dep of combineAsyncIterators({}, ...iterators)) {
@@ -241,58 +185,67 @@ export async function depWalker(manifest, options = {}, logger = new Logger()) {
   const payload = {
     id: tmpLocation.slice(-6),
     rootDepencyName: manifest.name,
-    warnings: [],
-    dependencies: new Map(),
-    version: packageVersion
+    version: packageVersion,
+    vulnerabilityStrategy,
+    warnings: []
   };
 
   // We are dealing with an exclude Map to avoid checking a package more than one time in searchDeepDependencies
   const exclude = new Map();
+  const dependencies = new Map();
 
   {
-    logger.start("walkTree").start("tarball").start("registry");
+    logger
+      .start(ScannerLoggerEvents.analysis.tree)
+      .start(ScannerLoggerEvents.analysis.tarball)
+      .start(ScannerLoggerEvents.analysis.registry);
+    const fetchedMetadataPackages = new Set();
     const promisesToWait = [];
-    const rootDepsOptions = { maxDepth, exclude, usePackageLock, fullLockMode };
 
     const tarballLocker = new Lock({ maxConcurrent: 5 });
-    const metadataLocker = new Lock({ maxConcurrent: 10 });
-    metadataLocker.on("freeOne", () => logger.tick("registry"));
-    tarballLocker.on("freeOne", () => logger.tick("tarball"));
+    tarballLocker.on("freeOne", () => logger.tick(ScannerLoggerEvents.analysis.tarball));
 
+    const rootDepsOptions = { maxDepth, exclude, usePackageLock, fullLockMode };
     for await (const currentDep of getRootDependencies(manifest, rootDepsOptions)) {
       const { name, version } = currentDep;
       const current = currentDep.exportAsPlainObject(name === manifest.name ? 0 : void 0);
       let proceedDependencyAnalysis = true;
 
-      if (payload.dependencies.has(name)) {
+      if (dependencies.has(name)) {
         // TODO: how to handle different metadata ?
-        const dep = payload.dependencies.get(name);
+        const dep = dependencies.get(name);
 
-        const currVersion = current.versions[0];
-        if (currVersion in dep) {
+        const currVersion = Object.keys(current.versions)[0];
+        if (currVersion in dep.versions) {
           // The dependency has already entered the analysis
           // This happens if the package is used by multiple packages in the tree
           proceedDependencyAnalysis = false;
         }
         else {
-          dep[currVersion] = current[currVersion];
-          dep.versions.push(currVersion);
+          dep.versions[currVersion] = current.versions[currVersion];
         }
       }
       else {
-        payload.dependencies.set(name, current);
+        dependencies.set(name, current);
       }
 
       if (proceedDependencyAnalysis) {
-        logger.tick("walkTree");
+        logger.tick(ScannerLoggerEvents.analysis.tree);
+
+        // There is no need to fetch 'N' times the npm metadata for the same package.
+        if (fetchedMetadataPackages.has(name)) {
+          logger.tick(ScannerLoggerEvents.analysis.registry);
+        }
+        else {
+          fetchedMetadataPackages.add(name);
+          promisesToWait.push(packageMetadata(name, version, {
+            ref: current,
+            logger
+          }));
+        }
 
-        promisesToWait.push(fetchPackageMetadata(name, version, {
-          ref: current,
-          locker: metadataLocker,
-          logger
-        }));
         promisesToWait.push(scanDirOrArchive(name, version, {
-          ref: current[version],
+          ref: current.versions[version],
           tmpLocation: forceRootAnalysis && name === manifest.name ? null : tmpLocation,
           locker: tarballLocker,
           logger
@@ -300,17 +253,17 @@ export async function depWalker(manifest, options = {}, logger = new Logger()) {
       }
     }
 
-    logger.end("walkTree");
+    logger.end(ScannerLoggerEvents.analysis.tree);
 
     // Wait for all extraction to be done!
     await Promise.allSettled(promisesToWait);
     await timers.setImmediate();
 
-    logger.end("tarball").end("registry");
+    logger.end(ScannerLoggerEvents.analysis.tarball).end(ScannerLoggerEvents.analysis.registry);
   }
 
   const { hydratePayloadDependencies, strategy } = await vuln.setStrategy(vulnerabilityStrategy);
-  await hydratePayloadDependencies(payload.dependencies, {
+  await hydratePayloadDependencies(dependencies, {
     useStandardFormat: true
   });
 
@@ -318,8 +271,10 @@ export async function depWalker(manifest, options = {}, logger = new Logger()) {
 
   // We do this because it "seem" impossible to link all dependencies in the first walk.
   // Because we are dealing with package only one time it may happen sometimes.
-  for (const [packageName, descriptor] of payload.dependencies) {
-    for (const verStr of descriptor.versions) {
+  for (const [packageName, dependency] of dependencies) {
+    for (const [verStr, verDescriptor] of Object.entries(dependency.versions)) {
+      verDescriptor.flags.push(...addMissingVersionFlags(new Set(verDescriptor.flags), dependency));
+
       const fullName = `${packageName}@${verStr}`;
       const usedDeps = exclude.get(fullName) || new Set();
       if (usedDeps.size === 0) {
@@ -330,18 +285,20 @@ export async function depWalker(manifest, options = {}, logger = new Logger()) {
       for (const [name, version] of [...usedDeps].map((name) => name.split(" "))) {
         usedBy[name] = version;
       }
-      Object.assign(descriptor[verStr].usedBy, usedBy);
+      Object.assign(verDescriptor.usedBy, usedBy);
     }
   }
 
   try {
-    payload.warnings = getDependenciesWarnings(payload.dependencies);
-    payload.dependencies = Object.fromEntries(payload.dependencies);
+    payload.warnings = getDependenciesWarnings(dependencies);
+    payload.dependencies = Object.fromEntries(dependencies);
 
     return payload;
   }
   finally {
     await timers.setImmediate();
     await fs.rm(tmpLocation, { recursive: true, force: true });
+
+    logger.emit(ScannerLoggerEvents.done);
   }
 }

package/src/manifest.js

@@ -0,0 +1,57 @@
+// Import Node.js Dependencies
+import fs from "fs/promises";
+import path from "path";
+
+// Import Third-party Dependencies
+import { parseManifestAuthor } from "@nodesecure/utils";
+
+// CONSTANTS
+// PR welcome to contribute to this list!
+const kNativeNpmPackages = new Set([
+  "node-gyp", "node-pre-gyp", "node-gyp-build", "node-addon-api"
+]);
+
+/**
+ * @see https://www.nerdycode.com/prevent-npm-executing-scripts-security/
+ */
+const kUnsafeNpmScripts = new Set([
+  "install",
+  "preinstall", "postinstall",
+  "preuninstall", "postuninstall"
+]);
+
+/**
+ * @param {!string} location
+ * @returns {import("@npm/types").PackageJson}
+ */
+export async function read(location) {
+  const packageStr = await fs.readFile(
+    path.join(location, "package.json"),
+    "utf-8"
+  );
+
+  return JSON.parse(packageStr);
+}
+
+// TODO: PR @npm/types to fix dependencies typo
+export async function readAnalyze(location) {
+  const {
+    description = "", author = {}, scripts = {},
+    dependencies = {}, devDependencies = {}, gypfile = false
+  } = await read(location);
+
+  const packageDeps = Object.keys(dependencies);
+  const packageDevDeps = Object.keys(devDependencies);
+  const hasNativePackage = [...packageDevDeps, ...packageDeps]
+    .some((pkg) => kNativeNpmPackages.has(pkg));
+
+  return {
+    author: typeof author === "string" ? parseManifestAuthor(author) : author,
+    description,
+    hasScript: Object.keys(scripts)
+      .some((value) => kUnsafeNpmScripts.has(value.toLowerCase())),
+    packageDeps,
+    packageDevDeps,
+    hasNativeElements: hasNativePackage || gypfile
+  };
+}

package/src/npmRegistry.js

@@ -0,0 +1,68 @@
+// Import Third-party Dependencies
+import semver from "semver";
+import { parseManifestAuthor } from "@nodesecure/utils";
+import { packument } from "@nodesecure/npm-registry-sdk";
+
+export function parseAuthor(author) {
+  return typeof author === "string" ? parseManifestAuthor(author) : author;
+}
+
+export async function packageMetadata(name, version, options) {
+  const { ref, logger } = options;
+
+  try {
+    const pkg = await packument(name);
+
+    const oneYearFromToday = new Date();
+    oneYearFromToday.setFullYear(oneYearFromToday.getFullYear() - 1);
+
+    const lastVersion = pkg["dist-tags"].latest;
+    const lastUpdateAt = new Date(pkg.time[lastVersion]);
+    const metadata = {
+      author: parseAuthor(pkg.author) ?? {},
+      homepage: pkg.homepage || null,
+      publishedCount: Object.values(pkg.versions).length,
+      lastVersion,
+      lastUpdateAt,
+      hasReceivedUpdateInOneYear: !(oneYearFromToday > lastUpdateAt),
+      maintainers: pkg.maintainers,
+      publishers: []
+    };
+
+    const isOutdated = semver.neq(version, lastVersion);
+    if (isOutdated) {
+      ref.versions[version].flags.push("isOutdated");
+    }
+
+    const publishers = new Set();
+    for (const ver of Object.values(pkg.versions).reverse()) {
+      const { _npmUser: npmUser, version } = ver;
+      const isNullOrUndefined = typeof npmUser === "undefined" || npmUser === null;
+      if (isNullOrUndefined || !("name" in npmUser) || typeof npmUser.name !== "string") {
+        continue;
+      }
+
+      const authorName = metadata.author?.name ?? null;
+      if (authorName === null) {
+        metadata.author = npmUser;
+      }
+      else if (npmUser.name !== metadata.author.name) {
+        metadata.hasManyPublishers = true;
+      }
+
+      // TODO: add npmUser.email
+      if (!publishers.has(npmUser.name)) {
+        publishers.add(npmUser.name);
+        metadata.publishers.push({ ...npmUser, version, at: new Date(pkg.time[version]) });
+      }
+    }
+
+    Object.assign(ref.metadata, metadata);
+  }
+  catch {
+    // ignore
+  }
+  finally {
+    logger.tick("registry");
+  }
+}