npq 3.8.0 → 3.9.0

package/lib/marshalls/age.marshall.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const BaseMarshall = require('./baseMarshall')
+const Warning = require('../helpers/warning')
 const { marshallCategories } = require('./constants')
 
 const MARSHALL_NAME = 'age'
@@ -32,13 +33,13 @@ class Marshall extends BaseMarshall {
           ageDateDiff = dateDiff
           if (dateDiff < PACKAGE_AGE_THRESHOLD) {
             throw new Error(
-              `detected a newly published package (created < ${PACKAGE_AGE_THRESHOLD} days) act carefully`
+              `Detected a newly published package (created < ${PACKAGE_AGE_THRESHOLD} days) act carefully`
             )
           }
 
           return pkg
         } else {
-          throw new Error('could not determine package age')
+          throw new Warning('Could not determine package age')
         }
       })
       .then((pkg) => {
@@ -59,7 +60,7 @@ class Marshall extends BaseMarshall {
         }
 
         if (versionDateDiffInDays >= PACKAGE_AGE_UNMAINTAINED_RISK) {
-          throw new Error(`detected an old package (created ${timeAgoNumber} ${timeAgoText} ago)`)
+          throw new Warning(`Detected an old package (created ${timeAgoNumber} ${timeAgoText} ago)`)
         }
       })
   }

package/lib/marshalls/author.marshall.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const BaseMarshall = require('./baseMarshall')
-const validator = require('validator')
+const Warning = require('../helpers/warning')
 const { marshallCategories } = require('./constants')
 
 const MARSHALL_NAME = 'author'
@@ -43,11 +43,14 @@ class Marshall extends BaseMarshall {
     // for dis-tags (latest)
     const npmUser = pakument.versions[packageVersion]._npmUser
     if (!npmUser || !npmUser.email) {
-      throw new Error('could not determine publishing user for this package version')
+      throw new Error('Could not determine publishing user for this package version')
     }
 
-    if (!validator.isEmail(npmUser.email)) {
-      throw new Error('the publishing user has no valid email address')
+    // Agree with Colin on keeping email regex simple: https://colinhacks.com/essays/reasonable-email-regex
+    const emailRegex =
+      /^(?!\.)(?!.*\.\.)([a-z0-9_'+\-\.]*)[a-z0-9_'+\-]@([a-z0-9][a-z0-9\-]*\.)+[a-z]{2,}$/i
+    if (!emailRegex.test(npmUser.email)) {
+      throw new Error('The publishing user has no valid email address')
     }
 
     let firstVersionForUser = null
@@ -60,9 +63,29 @@ class Marshall extends BaseMarshall {
     }
 
     if (!firstVersionForUser || firstVersionForUser.version === packageVersion) {
-      throw new Error(
-        `This is first version the user ${npmUser.name} <${npmUser.email}> published this package`
-      )
+      // Only throw the error if also the `packageVersion` was published less than 21 days ago:
+      if (versionPublishedDateString) {
+        const dateDiffInMs = new Date() - new Date(versionPublishedDateString)
+        let dateDiffInDays = 0
+
+        if (dateDiffInMs > 0) {
+          dateDiffInDays = Math.round(dateDiffInMs / (1000 * 60 * 60 * 24))
+        }
+
+        if (dateDiffInDays <= 21) {
+          throw new Error(
+            `The user ${npmUser.name} <${npmUser.email}> published this package for the first time only ${dateDiffInDays} days ago`
+          )
+        }
+      }
+
+      // otherwise, there's no point in throwing an error
+      // because this version already exists for a while. for e.g: package `ncp` latest version
+      // is from 10 years ago which was the first version published by the user, but that's
+      // hardly a risk at this point being 10 years old, so we don't throw the following error:
+      // throw new Error(
+      //   `This is the first version the user ${npmUser.name} <${npmUser.email}> published this package`
+      // )
     }
 
     const firstPublishedDateString = pakument.time[firstVersionForUser.version]
@@ -82,9 +105,15 @@ class Marshall extends BaseMarshall {
         dateDiffInDays = Math.round(dateDiffInMs / (1000 * 60 * 60 * 24))
       }
 
-      if (dateDiffInDays <= 30) {
+      if (dateDiffInDays <= 7) {
         throw new Error(
-          `The user ${npmUser.name} <${npmUser.email}> published this package for the first time only ${dateDiffInDays} days prior to this version.`
+          `The user ${npmUser.name} <${npmUser.email}> published this package for the first time only ${dateDiffInDays} days ago`
+        )
+      }
+
+      if (dateDiffInDays <= 30) {
+        throw new Warning(
+          `The user ${npmUser.name} <${npmUser.email}> published this package for the first time only ${dateDiffInDays} days ago`
         )
       }
     }

package/lib/marshalls/baseMarshall.js

@@ -1,38 +1,40 @@
 'use strict'
 
+const util = require('node:util')
 const Warning = require('../helpers/warning')
 const { marshallCategories } = require('./constants')
 
 class BaseMarshall {
   constructor(options) {
+    this.debug = util.debuglog('npq')
     this.packageRepoUtils = options.packageRepoUtils
     this.categoryId = marshallCategories.PackageHealth.id
   }
 
-  init(ctx, task) {
+  init(ctx) {
     this.ctx = ctx
-    this.task = task
 
     ctx.marshalls[this.name] = {
       status: null,
       errors: [],
       warnings: [],
-      data: {}
+      data: {},
+      marshall: this.name,
+      categoryId: this.categoryId
     }
   }
 
-  run(ctx, task) {
+  run(ctx) {
     const tasks = ctx.pkgs.reduce((prevPkg, currPkg) => {
-      return prevPkg.concat(this.checkPackage(currPkg, ctx, task))
+      return prevPkg.concat(this.checkPackage(currPkg, ctx))
     }, [])
 
     return Promise.all(tasks)
   }
 
-  checkPackage(pkg, ctx, task) {
+  checkPackage(pkg, ctx) {
     return this.validate(pkg)
       .then((data) => {
-        task.output = `querying ${pkg.packageString}...`
         ctx.marshalls[this.name].data[pkg.packageString] = data
 
         // not explicitly required, but a task can return its results
@@ -65,16 +67,6 @@ class BaseMarshall {
       message: msg.message
     })
   }
-
-  handleMessages() {
-    const errors = this.ctx.marshalls[this.name].errors
-    const warnings = this.ctx.marshalls[this.name].warnings
-    if ((errors && errors.length) || (warnings && warnings.length)) {
-      throw new Error()
-    }
-
-    return Promise.resolve()
-  }
 }
 
 module.exports = BaseMarshall

package/lib/marshalls/deprecation.marshall.js

@@ -0,0 +1,43 @@
+'use strict'
+
+const BaseMarshall = require('./baseMarshall')
+const { marshallCategories } = require('./constants')
+
+const MARSHALL_NAME = 'deprecation'
+
+class Marshall extends BaseMarshall {
+  constructor(options) {
+    super(options)
+    this.name = MARSHALL_NAME
+    this.categoryId = marshallCategories.PackageHealth.id
+  }
+
+  title() {
+    return 'Checking package for deprecation flag'
+  }
+
+  validate(pkg) {
+    return this.packageRepoUtils.getPackageInfo(pkg.packageName).then((data) => {
+      const packageVersion =
+        pkg.packageVersion === 'latest'
+          ? data['dist-tags'] && data['dist-tags'].latest
+          : this.packageRepoUtils.parsePackageVersion(pkg.packageVersion).version
+
+      if (!packageVersion) {
+        return true
+      }
+
+      const packageDeprecated =
+        data &&
+        data.versions &&
+        data.versions[packageVersion] &&
+        data.versions[packageVersion].deprecated
+
+      if (packageDeprecated) {
+        throw new Error(`Package deprecated: ${packageDeprecated}`)
+      }
+    })
+  }
+}
+
+module.exports = Marshall

package/lib/marshalls/downloads.marshall.js

@@ -12,7 +12,7 @@ class Marshall extends BaseMarshall {
   constructor(options) {
     super(options)
     this.name = MARSHALL_NAME
-    this.categoryId = marshallCategories.MalwareDetection.id
+    this.categoryId = marshallCategories.PackageHealth.id
   }
 
   title() {
@@ -23,13 +23,13 @@ class Marshall extends BaseMarshall {
     return this.packageRepoUtils.getDownloadInfo(pkg.packageName).then((downloadCount) => {
       if (downloadCount < DOWNLOAD_COUNT_THRESHOLD) {
         throw new Error(
-          `detected a low download-count package (downloads last month < ${DOWNLOAD_COUNT_THRESHOLD})`
+          `Detected a low download-count package (downloads last month < ${DOWNLOAD_COUNT_THRESHOLD})`
         )
       }
 
       if (downloadCount < DOWNLOAD_COUNT_UPPER_THRESHOLD) {
         throw new Warning(
-          `detected a low relatively low download-count package (${downloadCount} downloads last month)`
+          `Detected a low relatively low download-count package (${downloadCount} downloads last month)`
         )
       }
 

package/lib/marshalls/expiredDomains.marshall.js

@@ -2,15 +2,17 @@
 
 const BaseMarshall = require('./baseMarshall')
 const { marshallCategories } = require('./constants')
-const dns = require('dns').promises
+const { Resolver } = require('dns/promises')
 
+const dns = new Resolver()
+dns.setServers(['1.1.1.1', '8.8.8.8'])
 const MARSHALL_NAME = 'maintainers_expired_emails'
 
 class Marshall extends BaseMarshall {
   constructor(options) {
     super(options)
     this.name = MARSHALL_NAME
-    this.categoryId = marshallCategories.MalwareDetection.id
+    this.categoryId = marshallCategories.PackageHealth.id
   }
 
   title() {
@@ -30,16 +32,16 @@ class Marshall extends BaseMarshall {
         for (const maintainerInfo of maintainersAccounts) {
           const maintainerEmail = maintainerInfo.email
           const emailDomain = maintainerEmail.split('@')[1]
-          testEmails.push(dns.resolve(emailDomain))
+          testEmails.push(dns.resolve(emailDomain, 'NS'))
         }
 
         return Promise.all(testEmails)
       })
       .catch((error) => {
         const emailHostname = error.hostname ? error.hostname : '<unknown>'
+        this.debug('\nDetected error resolving domain for maintainer e-mail: %s', emailHostname)
         throw new Error(
-          'Unable to resolve domain for maintainer e-mail, could be an expired account: ' +
-            emailHostname
+          'Detected expired domain can be abused for account takeover: ' + emailHostname
         )
       })
   }

package/lib/marshalls/index.js

@@ -1,24 +1,28 @@
 'use strict'
 
-const { Listr } = require('listr2')
-const { glob } = require('glob')
-const { marshallCategories } = require('./constants')
+const fs = require('node:fs')
+const path = require('node:path')
 
 class Marshalls {
   static async collectMarshalls() {
-    const files = await glob(this.GLOB_MARSHALLS, {
-      cwd: __dirname,
-      absolute: true
-    })
+    const files = await fs.promises.readdir(__dirname)
+
+    const matchingFiles = files
+      .filter((file) => file.endsWith('.marshall.js'))
+      .map((file) => path.resolve(__dirname, file))
 
-    return files
+    return matchingFiles
   }
 
-  static async buildMarshallTasks(marshalls, config) {
+  static async buildMarshallTasks(marshalls, options) {
     if (!marshalls || !Array.isArray(marshalls) || marshalls.length === 0) {
       return Promise.reject(new Error('unable to collect marshalls, or no marshalls found'))
     }
 
+    const config = {
+      packageRepoUtils: options.packageRepoUtils
+    }
+
     const marshallTasks = marshalls.reduce((prev, curr) => {
       // eslint-disable-next-line security/detect-non-literal-require
       const Marshall = require(curr)
@@ -27,79 +31,114 @@ class Marshalls {
       return prev.concat({
         title: marshall.title(),
         categoryId: marshall.categoryId,
-        enabled: (ctx) => marshall.isEnabled(ctx),
-        task: async (ctx, task) => {
-          marshall.init(ctx, task)
-          await marshall.run(ctx, task)
-          const messages = marshall.handleMessages()
-          return messages
+        execute: async (options) => {
+          const ctx = {
+            pkgs: options.pkgs,
+            marshalls: {}
+          }
+          marshall.init(ctx)
+          await marshall.run(ctx)
+          return ctx
         }
       })
     }, [])
 
-    const allMarshallTasksByCategories = Object.keys(marshallCategories).map((categoryId) => {
-      const category = marshallCategories[categoryId]
-      const filteredMarshallTasks = marshallTasks.filter((marshall) => {
-        return marshall.categoryId === category.id
-      })
-
-      const marshallsInCategory = {
-        title: category.title,
-        enabled: true,
-        task: () => {
-          return new Listr(filteredMarshallTasks, {
-            exitOnError: false,
-            concurrent: true,
-            rendererOptions: { collapseSubtasks: false }
-          })
-        }
-      }
-      return marshallsInCategory
-    })
-
-    return new Listr(allMarshallTasksByCategories, {
-      exitOnError: false,
-      concurrent: true,
-      rendererOptions: { collapseSubtasks: false }
-    })
+    return marshallTasks
   }
 
-  static tasks(options) {
-    // console.log(options);
-    // process.exit(1);
-    return Marshalls.warmUpPackagesCache(options)
-      .then((packagesDataList) => {
-        // handle error in case we get just one package and it is not found
-        if (packagesDataList && packagesDataList.length === 1) {
-          if (packagesDataList[0].error && packagesDataList[0].error === 'Not found') {
-            throw new Error(`Package not found: ${options.pkgs[0].packageName}`)
+  /**
+   * Executes security and health audits on npm packages using registered marshalls
+   *
+   * @param {Object} options - Configuration object containing packages and utilities
+   * @param {Array<Object>} options.pkgs - Array of package objects with packageName, packageVersion, and packageString properties
+   * @param {PackageRepoUtils} options.packageRepoUtils - Instance for fetching package information from registry
+   * @param {Object} [progressManager] - Optional progress manager for displaying audit progress updates
+   * @returns {Promise<Array<Object>>} Array of marshall results containing audit findings for each package
+   */
+  static async tasks(options, progressManager) {
+    const marshallResults = []
+
+    // @TODO handle throw error because fetch didn't work due to network issues
+    // currently, errors here with fetch fail like this:
+    //   TypeError: fetch failed
+    //   at node:internal/deps/undici/undici:13502:13
+    //   at process.processTicksAndRejections (node:internal/process/task_queues:105:5)
+    //   at async Promise.all (index 0)
+    //   [cause]: Error: getaddrinfo ENOTFOUND registry.npmjs.org
+    //       at GetAddrInfoReqWrap.onlookupall [as oncomplete] (node:dns:120:26) {
+    //     errno: -3008,
+    //     code: 'ENOTFOUND',
+    //     syscall: 'getaddrinfo',
+    //     hostname: 'registry.npmjs.org'
+    //   }
+    // }
+    const packagesDataList = await Marshalls.warmUpPackagesCache(options)
+
+    // handle error in case we get just one package and it is not found
+    // if (packagesDataList && packagesDataList.length === 1) {
+    //   if (packagesDataList[0].error && packagesDataList[0].error === 'Not found') {
+    //   }
+    // }
+
+    // handle error in case we get more than one package and at least one is not found
+    // in which we case we simply remove the package from the `options` array
+    // which lists objects (each have a property of packageName)
+    if (packagesDataList && packagesDataList.length > 0) {
+      options.pkgs = options.pkgs.filter((pkg, index) => {
+        if (packagesDataList[index].error && packagesDataList[index].error === 'Not found') {
+          const errorResult = {
+            ['not_found']: {
+              marshall: 'not_found',
+              status: null,
+              errors: [{ pkg: options.pkgs[index].packageName, message: 'Package not found' }],
+              warnings: [],
+              data: {},
+              categoryId: 'PackageHealth'
+            }
           }
-        }
 
-        // handle error in case we get more than one package and at least one is not found
-        // in which we case we simply remove the package from the `options` array
-        // which lists objects (each have a property of packageName)
-        if (packagesDataList && packagesDataList.length > 1) {
-          options.pkgs = options.pkgs.filter((pkg, index) => {
-            return !packagesDataList[index].error || packagesDataList[index].error !== 'Not found'
-          })
+          marshallResults.push(errorResult)
+          return false
         }
+
+        return true
       })
-      .then(() => Marshalls.collectMarshalls())
-      .then((marshalls) => {
-        return Marshalls.buildMarshallTasks(marshalls, {
-          packageRepoUtils: options.packageRepoUtils
-        })
-      })
-      .then((tasks) => {
-        return Marshalls.runTasks(tasks, options)
-      })
-      .catch((err) => {
-        // console.error('Error:', err.message)
-        // avoid implementing a custom error message for a not found package
-        // because the package manager will yield its own error message
-        return { error: true, message: err.message }
+    }
+
+    const marshalls = await Marshalls.collectMarshalls()
+
+    if (progressManager && options.pkgs && options.pkgs.length > 0) {
+      const packageList = options.pkgs.map((pkg) => pkg.packageString).join(', ')
+      progressManager.update(`Auditing packages: ${packageList}`)
+    }
+
+    const packagesToProcess = options.pkgs
+    if (packagesToProcess.length > 0) {
+      const marshallTasks = await Marshalls.buildMarshallTasks(marshalls, {
+        packageRepoUtils: options.packageRepoUtils
       })
+
+      for (const marshall of marshallTasks) {
+        try {
+          const res = await marshall.execute({
+            pkgs: options.pkgs,
+            marshalls: {}
+          })
+          marshallResults.push(res.marshalls)
+        } catch (error) {
+          console.error(`Error running task ${marshall.title}:`, error)
+        }
+      }
+    }
+
+    return marshallResults
+
+    // .catch((err) => {
+    //   // console.error('Error:', err.message)
+    //   // avoid implementing a custom error message for a not found package
+    //   // because the package manager will yield its own error message
+    //   return { error: true, message: err.message }
+    // })
   }
 
   static warmUpPackagesCache(options) {
@@ -112,15 +151,6 @@ class Marshalls {
 
     return Promise.all(fetchPackagesInfoPromises)
   }
-
-  static runTasks(tasks, options) {
-    return tasks.run({
-      pkgs: options.pkgs,
-      marshalls: {}
-    })
-  }
 }
 
-Marshalls.GLOB_MARSHALLS = '*.marshall.js'
-
 module.exports = Marshalls

package/lib/marshalls/license.marshall.js

@@ -9,7 +9,7 @@ class Marshall extends BaseMarshall {
   constructor(options) {
     super(options)
     this.name = MARSHALL_NAME
-    this.categoryId = marshallCategories.PackageHealth.id
+    this.categoryId = marshallCategories.SupplyChainSecurity.id
   }
 
   title() {
@@ -19,11 +19,11 @@ class Marshall extends BaseMarshall {
   validate(pkg) {
     return this.packageRepoUtils.getLicenseInfo(pkg.packageName).then((licenseContents) => {
       if (!licenseContents || licenseContents === 'ERROR: No LICENSE data found!') {
-        throw new Error('package has no LICENSE file available')
+        throw new Error('Package has no LICENSE file available')
       }
 
       if (licenseContents && licenseContents.indexOf('# Security holding package') === 0) {
-        throw new Error('package flagged for security issues and served as place-holder')
+        throw new Error('Package flagged for security issues and served as place-holder')
       }
 
       return licenseContents

package/lib/marshalls/newbin.marshall.js

@@ -11,8 +11,7 @@ class NewBinMarshall extends BaseMarshall {
   constructor(options) {
     super(options)
     this.name = MARSHALL_NAME
-    // Decide on a category, e.g., PackageHealth or a new one if appropriate
-    this.categoryId = marshallCategories.PackageHealth.id
+    this.categoryId = marshallCategories.SupplyChainSecurity.id
   }
 
   title() {