@dependably/npm-check 1.7.0

package/src/usage-scanner.js

@@ -0,0 +1,215 @@
+// src/usage-scanner.js
+// Heuristic detection of dependencies declared in package.json but never
+// imported by the application's source code ("flag for removal").
+import fs from 'fs';
+import path from 'path';
+
+export class UsageScannerError extends Error {
+  constructor(message, code, context = {}) {
+    super(message);
+    this.name = 'UsageScannerError';
+    this.code = code;
+    this.context = context;
+  }
+}
+
+export const DEFAULT_EXTENSIONS = ['.js', '.mjs', '.cjs', '.jsx', '.ts', '.tsx', '.vue', '.svelte'];
+export const DEFAULT_IGNORE_DIRS = [
+  'node_modules', '.git', '.backups', 'dist', 'build', 'out', 'coverage', '.next', '.nuxt', 'vendor'
+];
+// Build/tooling output dirs that DEFAULT_IGNORE_DIRS skips during the app scan.
+// `findUnusedDependencies` scans these separately so a dependency imported only
+// by a hand-written build toolkit (e.g. a `build/` shipped as source) is not
+// mistaken for unused. Conservative by design: it can only rescue deps from a
+// removal suggestion, never add one.
+export const DEFAULT_BUILD_DIRS = ['build', 'dist', 'out'];
+
+// require('x') / require("x") / import('x') / import "x" / from 'x' / export ... from 'x'
+const IMPORT_PATTERNS = [
+  /\brequire\s*\(\s*['"]([^'"]+)['"]\s*\)/g,
+  /\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g,
+  /\bimport\s+['"]([^'"]+)['"]/g,
+  /\bfrom\s+['"]([^'"]+)['"]/g
+];
+
+/**
+ * Reduce an import specifier to its package name.
+ * 'lodash/fp' → 'lodash'; '@scope/pkg/sub' → '@scope/pkg';
+ * relative paths and node: builtins → null.
+ */
+export function specifierToPackageName(specifier) {
+  if (!specifier || typeof specifier !== 'string') return null;
+  if (specifier.startsWith('.') || specifier.startsWith('/') || specifier.startsWith('#')) return null;
+  if (specifier.startsWith('node:')) return null;
+  if (specifier.includes('://')) return null; // URLs (e.g. https: imports)
+
+  const parts = specifier.split('/');
+  if (specifier.startsWith('@')) {
+    return parts.length >= 2 ? `${parts[0]}/${parts[1]}` : null;
+  }
+  return parts[0];
+}
+
+function walkFiles(dir, extensions, ignoreDirs, files = []) {
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return files;
+  }
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      if (!ignoreDirs.includes(entry.name) && !entry.name.startsWith('.')) {
+        walkFiles(path.join(dir, entry.name), extensions, ignoreDirs, files);
+      }
+    } else if (entry.isFile() && extensions.includes(path.extname(entry.name))) {
+      files.push(path.join(dir, entry.name));
+    }
+  }
+  return files;
+}
+
+/**
+ * Scan a project's source files and collect the set of imported package names.
+ * @param {string} dir - Project root
+ * @param {object} options - { extensions, ignoreDirs }
+ * @returns {{used: Set<string>, scannedFiles: number}}
+ */
+export function scanUsedPackages(dir, options = {}) {
+  const { extensions = DEFAULT_EXTENSIONS, ignoreDirs = DEFAULT_IGNORE_DIRS } = options;
+
+  if (!fs.existsSync(dir)) {
+    throw new UsageScannerError(`Directory not found: ${dir}`, 'DIR_NOT_FOUND');
+  }
+
+  const files = walkFiles(dir, extensions, ignoreDirs);
+  const used = new Set();
+
+  for (const file of files) {
+    let content;
+    try {
+      content = fs.readFileSync(file, 'utf8');
+    } catch {
+      continue;
+    }
+    for (const pattern of IMPORT_PATTERNS) {
+      pattern.lastIndex = 0;
+      let match;
+      while ((match = pattern.exec(content)) !== null) {
+        const name = specifierToPackageName(match[1]);
+        if (name) used.add(name);
+      }
+    }
+  }
+
+  return { used, scannedFiles: files.length };
+}
+
+/**
+ * Decide whether a declared dependency counts as "used" — by an explicit
+ * ignore-list entry, a real import, an npm-script mention, or the
+ * @types/foo-follows-foo heuristic.
+ */
+function isDependencyUsed(name, { used, scriptsText, ignore }) {
+  return (
+    ignore.includes(name) ||
+    used.has(name) ||
+    // CLI tools invoked from npm scripts (eslint, jest, …) are used even
+    // though nothing imports them
+    scriptsText.includes(name) ||
+    // @types/foo is "used" when foo itself is
+    (name.startsWith('@types/') && used.has(name.slice('@types/'.length)))
+  );
+}
+
+/**
+ * Collect the unused entries from a single package.json dependency section.
+ */
+function collectUnusedInSection(deps, section, context) {
+  const unused = [];
+  if (!deps || typeof deps !== 'object') return unused;
+
+  for (const [name, version] of Object.entries(deps)) {
+    if (isDependencyUsed(name, context)) continue;
+    unused.push({ name, section, version });
+  }
+  return unused;
+}
+
+/**
+ * Scan each build/tooling dir that exists under `dir`, as its own pass. Returns
+ * the union of packages imported there plus a per-dir file/usage breakdown.
+ */
+function scanBuildDirs(dir, buildDirs, options) {
+  const used = new Set();
+  let scannedFiles = 0;
+  const dirsScanned = [];
+  for (const name of buildDirs) {
+    const full = path.join(dir, name);
+    if (!fs.existsSync(full)) continue;
+    const res = scanUsedPackages(full, options);
+    for (const pkg of res.used) used.add(pkg);
+    scannedFiles += res.scannedFiles;
+    dirsScanned.push(name);
+  }
+  return { used, scannedFiles, dirsScanned };
+}
+
+/**
+ * Find dependencies declared in package.json that the application never
+ * imports. Heuristic — results are flagged for removal, never auto-removed:
+ * packages used only via CLI, config files, or runtime magic can appear
+ * unused. Mentions in npm scripts count as used to reduce CLI-tool noise.
+ *
+ * Runs two passes: the application (with build/output dirs ignored), and the
+ * build/tooling dirs (`buildDirs`) separately. A dependency counts as used if
+ * either pass imports it, so a `build/` shipped as source no longer produces
+ * false "unused" flags; deps imported ONLY by the build pass are surfaced as
+ * `buildOnly` for visibility. Pass `buildDirs: []` to disable the second pass.
+ *
+ * @param {object} packageJson - Parsed package.json
+ * @param {string} dir - Project root to scan
+ * @param {object} options - { includeDev = false, ignore = [], buildDirs, extensions, ignoreDirs }
+ * @returns {{unused: Array<{name, section, version}>, used: Set<string>, usedByApp: Set<string>, usedByBuild: Set<string>, buildOnly: string[], scannedFiles: number, appFiles: number, buildFiles: number, buildDirsScanned: string[], sectionsChecked: string[]}}
+ */
+export function findUnusedDependencies(packageJson, dir, options = {}) {
+  const { includeDev = false, ignore = [], buildDirs = DEFAULT_BUILD_DIRS } = options;
+
+  if (!packageJson || typeof packageJson !== 'object') {
+    throw new UsageScannerError('package.json data is required', 'MISSING_PACKAGE_JSON');
+  }
+
+  // Pass 1: the application, with build/output dirs ignored (default behavior).
+  const app = scanUsedPackages(dir, options);
+  const usedByApp = app.used;
+
+  // Pass 2: the build/tooling dirs, scanned separately.
+  const build = scanBuildDirs(dir, buildDirs, options);
+  const usedByBuild = build.used;
+
+  const used = new Set([...usedByApp, ...usedByBuild]);
+  const scriptsText = Object.values(packageJson.scripts || {}).join('\n');
+  const context = { used, scriptsText, ignore };
+
+  const sectionsChecked = includeDev ? ['dependencies', 'devDependencies'] : ['dependencies'];
+  const unused = [];
+
+  for (const section of sectionsChecked) {
+    unused.push(...collectUnusedInSection(packageJson[section], section, context));
+  }
+
+  const buildOnly = [...usedByBuild].filter((name) => !usedByApp.has(name)).sort();
+
+  return {
+    unused,
+    used,
+    usedByApp,
+    usedByBuild,
+    buildOnly,
+    scannedFiles: app.scannedFiles + build.scannedFiles,
+    appFiles: app.scannedFiles,
+    buildFiles: build.scannedFiles,
+    buildDirsScanned: build.dirsScanned,
+    sectionsChecked
+  };
+}

package/src/validator.js

@@ -0,0 +1,282 @@
+// src/validator.js
+import { detectLockfileVersion, hasPackagesMap, hasDependenciesTree } from './format-library.js';
+
+export class ValidationError extends Error {
+  constructor(message, code) {
+    super(message);
+    this.code = code;
+  }
+}
+
+// Allow calling validatePackageLock(lockfile, options) when packageJson is omitted.
+function normalizeValidateArgs(argLength, packageJson, options) {
+  const looksLikeOptions = argLength === 2 && packageJson && typeof packageJson === 'object' &&
+    (packageJson.allowMissingIntegrity !== undefined || packageJson.validateAgainstPackageJson !== undefined);
+  if (looksLikeOptions) {
+    return { packageJson: null, options: packageJson };
+  }
+  return { packageJson, options };
+}
+
+// Basic top-level field checks (name, version, lockfileVersion).
+function validateRootFields(lockfile, errors) {
+  if (!lockfile.name || typeof lockfile.name !== 'string') {
+    errors.push(new ValidationError('Missing or invalid package name', 'INVALID_NAME'));
+  }
+  if (!lockfile.version || typeof lockfile.version !== 'string') {
+    errors.push(new ValidationError('Missing or invalid package version', 'INVALID_VERSION'));
+  }
+  // Basic semver-ish check (major.minor.patch). If format is wrong, mark invalid.
+  if (typeof lockfile.version === 'string') {
+    const semverLike = /^\d+\.\d+\.\d+(?:[-+].*)?$/;
+    if (!semverLike.test(lockfile.version)) {
+      errors.push(new ValidationError('Missing or invalid package version', 'INVALID_VERSION'));
+    }
+  }
+  if (typeof lockfile.lockfileVersion !== 'number') {
+    errors.push(new ValidationError('Missing or invalid lockfile version', 'INVALID_LOCKFILE_VERSION'));
+  }
+}
+
+// Validate the version-appropriate dependencies tree and/or packages map.
+function validateVersionStructure(lockfile, version, errors, warnings, options) {
+  if (hasDependenciesTree(version)) {
+    if (!lockfile.dependencies || typeof lockfile.dependencies !== 'object') {
+      errors.push(new ValidationError('Missing dependencies object', 'MISSING_DEPENDENCIES'));
+    } else {
+      validateDependenciesTree(lockfile.dependencies, errors, warnings);
+    }
+  }
+
+  if (hasPackagesMap(version)) {
+    if (!lockfile.packages || typeof lockfile.packages !== 'object') {
+      errors.push(new ValidationError('Missing packages map', 'MISSING_PACKAGES_MAP'));
+    } else {
+      validatePackagesMap(lockfile.packages, errors, warnings, options);
+    }
+  }
+}
+
+export function validatePackageLock(lockfile, packageJson = null, options = {}) {
+  ({ packageJson, options } = normalizeValidateArgs(arguments.length, packageJson, options));
+  const errors = [];
+  const warnings = [];
+  const info = {};
+
+  // Basic structure checks
+  validateRootFields(lockfile, errors);
+
+  let version;
+  try {
+    version = detectLockfileVersion(lockfile);
+    info.version = version;
+  } catch (e) {
+    // Normalize error code expected by tests
+    errors.push(new ValidationError(e.message, 'VERSION_MISMATCH'));
+    return { valid: false, errors, warnings, info };
+  }
+
+  validateVersionStructure(lockfile, version, errors, warnings, options);
+
+  // Validate against package.json if provided
+  if (packageJson && options.validateAgainstPackageJson) {
+    validateAgainstPackageJson(lockfile, packageJson, errors);
+  }
+
+  const valid = errors.length === 0 && !(options.strictMode && warnings.length > 0);
+  return { valid, errors, warnings, info };
+}
+
+// Recognise one SRI token: sha1/sha256/sha384/sha512.
+const SRI_TOKEN_REGEX = /^sha(?:1|256|384|512)-[A-Za-z0-9+/=]+$/;
+
+// Accept sha1/sha256/sha384/sha512 single hashes and space-separated multi-hash
+// SRI strings (e.g. 'sha512-... sha1-...'). Every whitespace-separated token
+// must match a recognised algorithm.
+function isValidIntegrityHash(value) {
+  if (typeof value !== 'string' || value.trim() === '') return false;
+  return value.trim().split(/\s+/).every(token => SRI_TOKEN_REGEX.test(token));
+}
+
+// Returns true when any token in a (valid) integrity value uses the legacy sha1
+// algorithm. Callers should emit a warning (not an error) suggesting an upgrade.
+function hasLegacySha1(value) {
+  if (typeof value !== 'string') return false;
+  return value.trim().split(/\s+/).some(token => token.startsWith('sha1-'));
+}
+
+// Classify a dependency entry by shape. Returns 'leaf' for string/boolean leaves,
+// 'object' for valid objects, or 'invalid' for anything that isn't an object.
+function classifyDependencyEntry(dep) {
+  if (typeof dep === 'string' || typeof dep === 'boolean') {
+    return 'leaf';
+  }
+  if (typeof dep !== 'object' || Array.isArray(dep)) {
+    return 'invalid';
+  }
+  return 'object';
+}
+
+// Validate a single dependency entry; recurses into nested dependencies.
+// warnings is threaded through so sha1 hashes can be flagged without an error.
+function validateDependencyEntry(name, dep, errors, warnings, depth) {
+  if (dep == null) {
+    errors.push(new ValidationError(`Dependency ${name} is not an object`, 'INVALID_DEPENDENCY'));
+    return;
+  }
+
+  const kind = classifyDependencyEntry(dep);
+  if (kind === 'invalid') {
+    errors.push(new ValidationError(`Dependency ${name} is not an object`, 'INVALID_DEPENDENCY'));
+    return;
+  }
+  if (kind === 'leaf') {
+    // Top-level string/boolean entries are missing required fields; nested
+    // string/boolean entries are valid range/flag leaves.
+    if (depth === 0) {
+      errors.push(new ValidationError(`Missing or invalid version for ${name}`, 'MISSING_DEP_VERSION'));
+    }
+    return;
+  }
+
+  // Validate version when dependency is an object
+  if (!dep.version || typeof dep.version !== 'string') {
+    errors.push(new ValidationError(`Missing or invalid version for ${name}`, 'MISSING_DEP_VERSION'));
+  }
+
+  // Validate integrity on dependency objects if present.
+  // sha1 is accepted as structurally valid but emits a LEGACY_INTEGRITY warning.
+  if (dep.integrity) {
+    if (!isValidIntegrityHash(dep.integrity)) {
+      errors.push(new ValidationError(`Invalid integrity hash for dependency ${name}`, 'INVALID_INTEGRITY'));
+    } else if (hasLegacySha1(dep.integrity)) {
+      warnings.push({ code: 'LEGACY_INTEGRITY', message: `Dependency ${name} uses sha1 integrity; run upgrade-hashes to convert to sha512` });
+    }
+  }
+
+  if (dep.dependencies) {
+    validateDependenciesTree(dep.dependencies, errors, warnings, depth + 1);
+  }
+}
+
+function validateDependenciesTree(dependencies, errors, warnings, depth = 0) {
+  for (const [name, dep] of Object.entries(dependencies)) {
+    validateDependencyEntry(name, dep, errors, warnings, depth);
+  }
+}
+
+// Validate the integrity field (and the allowMissingIntegrity policy) for a package entry.
+// sha1 integrity is accepted as structurally valid but emits a LEGACY_INTEGRITY warning.
+function validatePackageIntegrity(path, pkg, errors, warnings, options) {
+  if (pkg.integrity) {
+    if (!isValidIntegrityHash(pkg.integrity)) {
+      errors.push(new ValidationError(`Invalid integrity hash for package at ${path}`, 'INVALID_INTEGRITY'));
+    } else if (hasLegacySha1(pkg.integrity)) {
+      warnings.push({ code: 'LEGACY_INTEGRITY', message: `Package at ${path} uses sha1 integrity; run upgrade-hashes to convert to sha512` });
+    }
+  }
+  if (options.allowMissingIntegrity === false && !pkg.integrity) {
+    // Treat missing integrity as an error when not allowed, and also record a warning
+    errors.push(new ValidationError(`Missing integrity hash for package at ${path}`, 'MISSING_INTEGRITY'));
+    warnings.push({ code: 'MISSING_INTEGRITY', message: `Missing integrity hash for package at ${path}` });
+  }
+}
+
+// Validate the resolved-URL scheme for a package entry.
+// Guards typeof before calling startsWith to avoid a crash on corrupted
+// non-string resolved values (e.g. "resolved": 42 from a bad merge).
+function validatePackageResolved(path, pkg, warnings) {
+  if (!pkg.resolved) {
+    return;
+  }
+  if (typeof pkg.resolved !== 'string') {
+    warnings.push({ code: 'INVALID_RESOLVED', message: `Invalid resolved URL for package at ${path}: ${pkg.resolved}` });
+    return;
+  }
+  const validSchemes = ['https://', 'http://', 'git+', 'git://', 'file:'];
+  const hasValidScheme = validSchemes.some(scheme => pkg.resolved.startsWith(scheme));
+  if (!hasValidScheme) {
+    warnings.push({ code: 'INVALID_RESOLVED', message: `Invalid resolved URL for package at ${path}: ${pkg.resolved}` });
+  }
+}
+
+// Recurse into each dependency section of a package entry. packages-map
+// dependency values are version-range strings (depth 1 allows string/boolean
+// leaves), unlike the v1 top-level tree.
+function validatePackageDependencySections(pkg, errors, warnings) {
+  for (const section of ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']) {
+    if (pkg[section]) {
+      validateDependenciesTree(pkg[section], errors, warnings, 1);
+    }
+  }
+}
+
+// Validate a single packages-map entry.
+function validatePackageEntry(path, pkg, errors, warnings, options) {
+  if (!pkg || typeof pkg !== 'object') {
+    errors.push(new ValidationError(`Package at ${path} is not an object`, 'INVALID_PACKAGE'));
+    return;
+  }
+  // npm only writes `name` on the root entry (and aliased installs);
+  // requiring it elsewhere would reject every real lockfile
+  if (path === '' && (!pkg.name || typeof pkg.name !== 'string')) {
+    errors.push(new ValidationError(`Missing or invalid name for package at ${path}`, 'INVALID_PACKAGE_NAME'));
+  }
+  // link entries (`link: true`) carry no version by design
+  if (!pkg.link && (!pkg.version || typeof pkg.version !== 'string')) {
+    errors.push(new ValidationError(`Missing or invalid version for package at ${path}`, 'INVALID_PACKAGE_VERSION'));
+  }
+  validatePackageIntegrity(path, pkg, errors, warnings, options);
+  validatePackageResolved(path, pkg, warnings);
+  validatePackageDependencySections(pkg, errors, warnings);
+}
+
+function validatePackagesMap(packages, errors, warnings, options) {
+  for (const [path, pkg] of Object.entries(packages)) {
+    validatePackageEntry(path, pkg, errors, warnings, options);
+  }
+}
+
+// Check that every package.json dependency in a section is present in the
+// lockfile root entry's matching section (v2/v3 packages map path).
+function checkDependencySectionInLockfile(rootEntry, packageJson, section, label, code, errors) {
+  const lockDeps = rootEntry && rootEntry[section];
+  const pkgDeps = packageJson[section] || {};
+  for (const [name] of Object.entries(pkgDeps)) {
+    if (!lockDeps || !lockDeps[name]) {
+      errors.push(new ValidationError(`Missing ${label} ${name} in lockfile`, code));
+    }
+  }
+}
+
+// v1 lockfile cross-check: each package.json dep section is checked against
+// the top-level dependencies tree (v1 hoists all deps to the top level).
+function checkDependencySectionInV1Tree(v1Deps, packageJson, section, label, code, errors) {
+  const pkgDeps = packageJson[section] || {};
+  for (const [name] of Object.entries(pkgDeps)) {
+    if (!v1Deps[name]) {
+      errors.push(new ValidationError(`Missing ${label} ${name} in lockfile`, code));
+    }
+  }
+}
+
+function validateAgainstPackageJson(lockfile, packageJson, errors) {
+  const rootEntry = lockfile.packages && lockfile.packages[''];
+
+  if (!rootEntry) {
+    // v1 lockfile (or a degenerate v2/v3 missing the root packages[''] entry):
+    // fall back to the top-level dependencies tree for the cross-check.
+    if (lockfile.dependencies) {
+      checkDependencySectionInV1Tree(lockfile.dependencies, packageJson, 'dependencies', 'dependency', 'MISSING_IN_LOCKFILE', errors);
+      checkDependencySectionInV1Tree(lockfile.dependencies, packageJson, 'devDependencies', 'devDependency', 'MISSING_DEV_IN_LOCKFILE', errors);
+      checkDependencySectionInV1Tree(lockfile.dependencies, packageJson, 'optionalDependencies', 'optionalDependency', 'MISSING_OPT_IN_LOCKFILE', errors);
+      checkDependencySectionInV1Tree(lockfile.dependencies, packageJson, 'peerDependencies', 'peerDependency', 'MISSING_PEER_IN_LOCKFILE', errors);
+    }
+    return;
+  }
+
+  checkDependencySectionInLockfile(rootEntry, packageJson, 'dependencies', 'dependency', 'MISSING_IN_LOCKFILE', errors);
+  checkDependencySectionInLockfile(rootEntry, packageJson, 'devDependencies', 'devDependency', 'MISSING_DEV_IN_LOCKFILE', errors);
+  checkDependencySectionInLockfile(rootEntry, packageJson, 'optionalDependencies', 'optionalDependency', 'MISSING_OPT_IN_LOCKFILE', errors);
+  checkDependencySectionInLockfile(rootEntry, packageJson, 'peerDependencies', 'peerDependency', 'MISSING_PEER_IN_LOCKFILE', errors);
+}