@dependably/npm-check 1.7.0

package/src/schema.js

@@ -0,0 +1,76 @@
+// src/schema.js
+// The Dependably suite's shared finding JSON schema (v1). Every tool's
+// `--format json` emits the SAME top-level envelope so an AI or CI consumer can
+// parse any of the five tools identically; tool-specific data only ever rides
+// under `extra` (never as a new top-level key). See the suite schema spec.
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+export const TOOL_NAME = 'npm-check';
+export const SCHEMA_VERSION = '1.0';
+
+// The ONE severity ladder, most-severe first.
+export const SEVERITY_LADDER = ['critical', 'high', 'moderate', 'low', 'info'];
+
+let cachedVersion = null;
+
+// Resolve the tool's own semver from package.json (cached). Mirrors bin/cli.js's
+// getVersion so the envelope's toolVersion always matches `--version`.
+export function toolVersion() {
+  if (cachedVersion) return cachedVersion;
+  try {
+    const here = path.dirname(fileURLToPath(import.meta.url));
+    const pkg = JSON.parse(fs.readFileSync(path.join(here, '../package.json'), 'utf8'));
+    cachedVersion = pkg.version || 'unknown';
+  } catch {
+    cachedVersion = 'unknown';
+  }
+  return cachedVersion;
+}
+
+// A zeroed bySeverity histogram with every ladder key present.
+export function emptyBySeverity() {
+  return { critical: 0, high: 0, moderate: 0, low: 0, info: 0 };
+}
+
+// Count findings into the ladder histogram (ignores any off-ladder severity).
+export function tallyBySeverity(findings) {
+  const by = emptyBySeverity();
+  for (const f of findings) {
+    if (f && Object.prototype.hasOwnProperty.call(by, f.severity)) by[f.severity]++;
+  }
+  return by;
+}
+
+/**
+ * Assemble the shared envelope. `findings` MUST be the complete, schema-conformant
+ * list (never truncated); `summary.findings` is derived from it and `summary.exitCode`
+ * MUST equal the process exit code the run will return.
+ *
+ * @param {object} args
+ * @param {string} args.target   - path/manifest scanned, as given
+ * @param {number} args.scanned  - packages examined
+ * @param {Array}  args.findings - schema Finding objects
+ * @param {number} args.exitCode - the real process exit code (0/1/2)
+ * @param {object} [args.extra]  - the ONE sanctioned tool-specific escape hatch
+ * @returns {object} the envelope
+ */
+export function buildEnvelope({ target, scanned, findings, exitCode, extra }) {
+  const list = Array.isArray(findings) ? findings : [];
+  const envelope = {
+    tool: TOOL_NAME,
+    toolVersion: toolVersion(),
+    schemaVersion: SCHEMA_VERSION,
+    target,
+    summary: {
+      scanned,
+      findings: list.length,
+      bySeverity: tallyBySeverity(list),
+      exitCode
+    },
+    findings: list
+  };
+  if (extra !== undefined) envelope.extra = extra;
+  return envelope;
+}

package/src/streaming-parser.js

@@ -0,0 +1,251 @@
+/**
+ * Buffered parser for large package-lock.json files.
+ *
+ * NOTE: despite the "streaming" name, these helpers read the file in chunks (so
+ * progress can be reported) but parse the fully-buffered content with the shared
+ * `parseLockfile` from format-library — they do NOT parse incrementally. A true
+ * incremental JSON parser is future work. The buffered approach is honest about
+ * its result: it returns the actual parsed lockfile (or throws), never an empty
+ * skeleton that would make every downstream integrity/vuln/license check pass
+ * vacuously.
+ */
+
+import fs from 'fs';
+import { EventEmitter } from 'events';
+import { parseLockfile as parseLockfileFromFormat } from './format-library.js';
+
+// Record one incremental `package` event into the accumulating lockfile: the
+// root ('') merges into rootMetadata, others land in the packages map.
+function recordStreamPackage(path, pkg, lockfile, rootMetadata, options) {
+  if (path === '') {
+    Object.assign(rootMetadata, pkg); // root package → root metadata
+  } else {
+    lockfile.packages[path] = pkg;
+  }
+  if (options.onPackage) options.onPackage(path, pkg);
+}
+
+// Assemble the final result from the buffered `complete` event. The parsed result
+// is authoritative; anything the (currently no-op) incremental events collected is
+// layered on WITHOUT injecting empty skeleton keys — a v3 file must not gain a
+// spurious `dependencies: {}`, nor a v1 file a `packages: {}`, which a write-back
+// would then persist.
+function assembleStreamResult(result, lockfile, rootMetadata) {
+  const parsed = (result && typeof result === 'object') ? result : {};
+  const merged = { ...parsed };
+  for (const [k, v] of Object.entries(rootMetadata)) {
+    if (!(k in merged)) merged[k] = v;
+  }
+  if (Object.keys(lockfile.packages).length > 0) {
+    merged.packages = { ...(parsed.packages || {}), ...lockfile.packages };
+  }
+  return merged;
+}
+
+/**
+ * Streaming parser for package-lock.json files
+ * Handles large files by parsing incrementally
+ */
+export class StreamingParser extends EventEmitter {
+  /**
+   * Create a streaming parser
+   * @param {Object} options - Parser options
+   * @param {Function} options.onPackage - Callback when package is parsed (path, pkg)
+   * @param {Function} options.onProgress - Progress callback (bytesRead, totalBytes)
+   * @param {number} options.chunkSize - Read chunk size in bytes (default: 64KB)
+   */
+  constructor(options = {}) {
+    super();
+    this.onPackageCallback = options.onPackage || null;
+    this.onProgressCallback = options.onProgress || null;
+    this.chunkSize = options.chunkSize || 64 * 1024;
+    this.buffer = '';
+    this.state = 'initial';
+    this.depth = 0;
+    this.currentPath = null;
+    this.currentPackage = null;
+    this.inString = false;
+    this.escapeNext = false;
+    this.bytesRead = 0;
+    this.totalBytes = 0;
+  }
+
+  /**
+   * Parse a lockfile from file path using streaming
+   * @param {string} filePath - Path to lockfile
+   * @param {Object} options - Parser options
+   * @returns {Promise<Object>} Parsed lockfile object
+   */
+  static async parseLockfileStream(filePath, options = {}) {
+    const stats = fs.statSync(filePath);
+    const totalBytes = stats.size;
+
+    const parser = new StreamingParser({
+      ...options,
+      onProgress: (bytesRead) => {
+        if (options.onProgress) {
+          options.onProgress(bytesRead, totalBytes);
+        }
+      }
+    });
+
+    parser.totalBytes = totalBytes;
+
+    return new Promise((resolve, reject) => {
+      const lockfile = {
+        packages: {},
+        dependencies: {}
+      };
+
+      // Collect root metadata
+      let rootMetadata = {};
+
+      parser.on('package', (path, pkg) => recordStreamPackage(path, pkg, lockfile, rootMetadata, options));
+      parser.on('metadata', (key, value) => {
+        rootMetadata[key] = value;
+      });
+      parser.on('error', reject);
+      parser.on('complete', (result) => resolve(assembleStreamResult(result, lockfile, rootMetadata)));
+
+      const stream = fs.createReadStream(filePath, {
+        encoding: 'utf8',
+        highWaterMark: parser.chunkSize
+      });
+
+      stream.on('data', (chunk) => {
+        parser.processChunk(chunk);
+      });
+
+      stream.on('end', () => {
+        parser.finish();
+      });
+
+      stream.on('error', reject);
+    });
+  }
+
+  /**
+   * Process a chunk of data
+   * @param {string} chunk - Data chunk
+   */
+  processChunk(chunk) {
+    this.buffer += chunk;
+    this.bytesRead += Buffer.byteLength(chunk, 'utf8');
+
+    if (this.onProgressCallback) {
+      this.onProgressCallback(this.bytesRead);
+    }
+
+    // Simple approach: For very large files, fall back to standard parsing
+    // but do it in a way that doesn't block. For now, we'll use a hybrid approach.
+    // If buffer gets too large, parse what we have and continue.
+
+    // For package-lock.json, the structure is predictable enough that we can
+    // use a simpler incremental approach: read the file in chunks and parse
+    // the packages map incrementally.
+
+    // Since full streaming JSON parsing is complex, we'll use a pragmatic approach:
+    // For files under a certain size, use standard parsing.
+    // For larger files, we'll implement a simplified streaming parser that
+    // extracts packages one by one.
+  }
+
+  /**
+   * Finish parsing
+   */
+  finish() {
+    if (this.buffer.trim()) {
+      try {
+        // Parse the fully-buffered content through the shared format-library
+        // parser (consistent error messages; not raw JSON.parse).
+        const parsed = parseLockfileFromFormat(this.buffer);
+        this.emit('complete', parsed);
+      } catch (error) {
+        this.emit('error', error);
+      }
+    } else {
+      // An empty stream is not a valid lockfile. Fail loudly rather than
+      // resolving an empty (vacuously "clean") lockfile.
+      this.emit('error', new Error('Cannot parse lockfile: stream produced no data'));
+    }
+  }
+}
+
+/**
+ * Parse lockfile using streaming approach for large files
+ * Falls back to standard parsing for smaller files
+ * @param {string} filePath - Path to lockfile
+ * @param {Object} options - Options
+ * @param {Function} options.onPackage - Callback when package parsed
+ * @param {Function} options.onProgress - Progress callback
+ * @param {number} options.streamingThreshold - File size threshold in bytes for streaming (default: 10MB)
+ * @returns {Promise<Object>} Parsed lockfile
+ */
+export async function parseLockfileStream(filePath, options = {}) {
+  const stats = fs.statSync(filePath);
+  const fileSize = stats.size;
+  const threshold = options.streamingThreshold || 10 * 1024 * 1024; // 10MB default
+
+  // For files smaller than threshold, use standard parsing
+  if (fileSize < threshold) {
+    const content = fs.readFileSync(filePath, 'utf8');
+    return parseLockfileFromFormat(content);
+  }
+
+  // For larger files, read the file in chunks (so we can report progress) and then
+  // parse the fully-buffered content. This is buffered, not truly incremental, but
+  // it returns the ACTUAL parsed lockfile (or rejects) — never an empty skeleton.
+  return new Promise((resolve, reject) => {
+    const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
+    let buffer = '';
+    let bytesRead = 0;
+
+    stream.on('data', (chunk) => {
+      buffer += chunk;
+      bytesRead += Buffer.byteLength(chunk, 'utf8');
+
+      if (options.onProgress) {
+        options.onProgress(bytesRead, fileSize);
+      }
+    });
+
+    stream.on('end', () => {
+      try {
+        resolve(parseLockfileFromFormat(buffer));
+      } catch (error) {
+        reject(error);
+      }
+    });
+
+    stream.on('error', reject);
+  });
+}
+
+/**
+ * Synchronous version with callbacks (for compatibility)
+ * @param {string} filePath - Path to lockfile
+ * @param {Object} options - Options
+ * @returns {Object} Parsed lockfile
+ */
+export function parseLockfileStreamSync(filePath, options = {}) {
+  const stats = fs.statSync(filePath);
+  const fileSize = stats.size;
+  const threshold = options.streamingThreshold || 10 * 1024 * 1024;
+
+  // For smaller files, use standard parsing
+  if (fileSize < threshold) {
+    const content = fs.readFileSync(filePath, 'utf8');
+    return parseLockfileFromFormat(content);
+  }
+
+  // For larger files, read in chunks but still parse at once
+  // This is a compromise - true streaming would require async or more complex parsing
+  const content = fs.readFileSync(filePath, 'utf8');
+  return parseLockfileFromFormat(content);
+}
+
+export default {
+  StreamingParser,
+  parseLockfileStream,
+  parseLockfileStreamSync
+};

package/src/updater.js

@@ -0,0 +1,251 @@
+/**
+ * Updater module for package-lock.json files
+ * Optimized implementations for handling large lockfiles efficiently.
+ */
+
+import { detectLockfileVersion, hasPackagesMap, resolvePackageName } from './format-library.js';
+import {
+  shallowCopyLockfile,
+  filterPackagesLazy,
+  isLargeLockfile
+} from './performance.js';
+import { parallelUpgradeIntegrityHashes as parallelUpgrade, parallelDeduplicatePackages as parallelDedupe } from './parallel-processor.js';
+
+// Dependency sections within a package entry whose integrity hashes are
+// upgraded recursively alongside the entry's own integrity.
+const DEPENDENCY_SECTIONS = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies'];
+
+// Emit a progress update for the integrity-hash upgrade stage.
+function reportUpgradeProgress(onProgress, processed, total) {
+  if (!onProgress) return;
+  onProgress({ current: processed, total, percentage: Math.round((processed / total) * 100), stage: 'Upgrading integrity hashes' });
+}
+
+// Promote a single integrity hash to sha512. sha1 hashes are always promoted;
+// other non-sha512 hashes are promoted only when `all` is set.
+function upgradeHash(hash, all) {
+  if (!hash) return hash;
+  if (hash.startsWith('sha1-')) {
+    return 'sha512-' + hash.slice(5);
+  }
+  if (all && !hash.startsWith('sha512-')) {
+    return 'sha512-' + hash.slice(hash.indexOf('-') + 1);
+  }
+  return hash;
+}
+
+// Recursively upgrade integrity hashes in a dependency tree, preserving
+// non-object entries untouched.
+function upgradeDependencies(deps, all) {
+  if (!deps || typeof deps !== 'object') return deps;
+
+  const upgraded = {};
+  for (const [depName, dep] of Object.entries(deps)) {
+    if (!dep || typeof dep !== 'object') {
+      upgraded[depName] = dep;
+      continue;
+    }
+
+    // Upgrade this dependency and recursively upgrade nested ones
+    upgraded[depName] = {
+      ...dep,
+      ...(dep.integrity && { integrity: upgradeHash(dep.integrity, all) }),
+      ...(dep.dependencies && { dependencies: upgradeDependencies(dep.dependencies, all) })
+    };
+  }
+  return upgraded;
+}
+
+// Upgrade a single package entry's own integrity plus each of its nested
+// dependency sections, returning a shallow copy.
+function upgradePackageEntry(pkg, all) {
+  const upgradedPkg = { ...pkg };
+
+  if (upgradedPkg.integrity) {
+    upgradedPkg.integrity = upgradeHash(upgradedPkg.integrity, all);
+  }
+
+  for (const section of DEPENDENCY_SECTIONS) {
+    if (upgradedPkg[section] && typeof upgradedPkg[section] === 'object') {
+      upgradedPkg[section] = upgradeDependencies(upgradedPkg[section], all);
+    }
+  }
+
+  return upgradedPkg;
+}
+
+/**
+ * Upgrade integrity hashes in a lockfile
+ * @param {object} lockfileData - Lockfile data
+ * @param {object} options - Options for upgrade
+ * @param {boolean} options.all - Upgrade all hashes (default: false, only sha1)
+ * @param {Function} options.onProgress - Progress callback function(progressInfo)
+ * @param {boolean} options.parallel - Use parallel processing for large files (default: false)
+ * @returns {object|Promise<object>} Updated lockfile data (immutable - returns Promise if parallel)
+ */
+export function upgradeIntegrityHashes(lockfileData, options = {}) {
+  const { all = false, onProgress = null, parallel = false } = options;
+
+  // Use parallel processing if requested and file is large
+  if (parallel && isLargeLockfile(lockfileData, 10)) {
+    return parallelUpgrade(lockfileData, {
+      all,
+      onProgress,
+      ...options
+    });
+  }
+  const version = detectLockfileVersion(lockfileData);
+
+  if (!hasPackagesMap(version)) {
+    return lockfileData;
+  }
+
+  // Use shallow copy to avoid deep cloning
+  const result = shallowCopyLockfile(lockfileData);
+
+  // Process packages directly without deep recursion
+  const packageEntries = Object.entries(result.packages || {});
+  const total = packageEntries.length;
+  const upgradedPackages = {};
+  let processed = 0;
+
+  for (const [path, pkg] of packageEntries) {
+    upgradedPackages[path] = pkg ? upgradePackageEntry(pkg, all) : pkg;
+    processed++;
+
+    // Null entries report every step; real entries report every 100th.
+    if (!pkg || processed % 100 === 0) {
+      reportUpgradeProgress(onProgress, processed, total);
+    }
+  }
+
+  result.packages = upgradedPackages;
+  return result;
+}
+
+/**
+ * Deduplicate packages in a lockfile
+ * Uses Map-based deduplication for O(1) lookups
+ * @param {object} lockfileData - Lockfile data
+ * @param {object} options - Options for deduplication
+ * @param {boolean} options.keepLatest - Keep only latest version of duplicates (default: false)
+ * @param {Function} options.onProgress - Progress callback function(progressInfo)
+ * @param {boolean} options.parallel - Use parallel processing for large files (default: false)
+ * @returns {object|Promise<object>} Updated lockfile data (immutable - returns Promise if parallel)
+ */
+export function deduplicatePackages(lockfileData, options = {}) {
+  const { keepLatest = false, onProgress = null, parallel = false } = options;
+
+  // Use parallel processing if requested and file is large
+  if (parallel && isLargeLockfile(lockfileData, 10)) {
+    return parallelDedupe(lockfileData, {
+      keepLatest,
+      onProgress,
+      ...options
+    });
+  }
+  const version = detectLockfileVersion(lockfileData);
+
+  const result = shallowCopyLockfile(lockfileData);
+
+  // Packages map (v2/v3): keyed by *install path*. Every entry is a distinct,
+  // required node — the package name is encoded in the path, so most entries
+  // carry no `.name` field at all. There is no safe way to "remove duplicates"
+  // here: two entries that share a name@version live at different paths because
+  // npm could not hoist them to one location (conflicting dependents), and
+  // collapsing them produces an un-installable lockfile. Real npm deduplication
+  // is tree hoisting, which requires full re-resolution and is out of scope.
+  //
+  // So we PRESERVE every path entry. (The previous implementation keyed a map by
+  // `name#version` and rebuilt from it, which silently dropped every entry that
+  // lacked a `.name` field — i.e. effectively the entire packages map — gutting
+  // the lockfile down to the root. See deduplicatePackages preserve tests.)
+  // Use `prune` to remove genuinely orphaned (unreachable) entries.
+  if (hasPackagesMap(version) && result.packages) {
+    const total = Object.keys(result.packages).length;
+    if (onProgress) {
+      onProgress({ current: total, total, percentage: 100, stage: 'Deduplication complete' });
+    }
+  }
+
+  // v1 dependencies tree: top-level keys are unique by JS object construction,
+  // so a Set-based filter at this level is a no-op. Real v1 "deduplication" is
+  // tree hoisting from nested sub-dependency objects — that requires full
+  // re-resolution and is out of scope here (use `npm install --prefer-dedupe`).
+  // We preserve the tree as-is, consistent with the packages-map preserve-only
+  // policy above. No entries are removed.
+
+  return result;
+}
+
+/**
+ * Find packages matching a predicate without full copy (lazy evaluation)
+ * Useful for filtering operations on large lockfiles
+ * @param {object} lockfileData - Lockfile data
+ * @param {Function} predicate - Function(path, pkg) => boolean
+ * @returns {object} Filtered packages object
+ */
+export function findPackagesMatching(lockfileData, predicate) {
+  return filterPackagesLazy(lockfileData.packages || {}, predicate);
+}
+
+/**
+ * Count unique packages in lockfile (efficient for large files)
+ * @param {object} lockfileData - Lockfile data
+ * @returns {number} Number of unique packages
+ */
+export function countUniquePackages(lockfileData) {
+  if (!lockfileData.packages || typeof lockfileData.packages !== 'object') {
+    return 0;
+  }
+
+  const uniqueNames = new Set();
+  for (const [path, pkg] of Object.entries(lockfileData.packages)) {
+    if (!pkg || typeof pkg !== 'object') continue;
+    // Derive the name from the path — v2/v3 entries usually have no `.name` field,
+    // so reading `pkg.name` alone would miss nearly every real dependency.
+    const name = resolvePackageName(path, pkg);
+    if (name && name !== '(root)') uniqueNames.add(name);
+  }
+
+  return uniqueNames.size;
+}
+
+/**
+ * Find duplicate packages efficiently
+ * @param {object} lockfileData - Lockfile data
+ * @returns {Map} Map of package names to array of {path, version} objects
+ */
+export function findDuplicatePackages(lockfileData) {
+  const duplicates = new Map();
+
+  for (const [path, pkg] of Object.entries(lockfileData.packages || {})) {
+    if (!pkg || typeof pkg !== 'object') continue;
+    // Derive the name from the path — v2/v3 entries usually have no `.name` field.
+    const name = resolvePackageName(path, pkg);
+    if (!name || name === '(root)') continue;
+
+    if (!duplicates.has(name)) {
+      duplicates.set(name, []);
+    }
+
+    duplicates.get(name).push({ path, version: pkg.version });
+  }
+
+  // Keep only actual duplicates
+  for (const [name, entries] of duplicates.entries()) {
+    if (entries.length === 1) {
+      duplicates.delete(name);
+    }
+  }
+
+  return duplicates;
+}
+
+export default {
+  upgradeIntegrityHashes,
+  deduplicatePackages,
+  findPackagesMatching,
+  countUniquePackages,
+  findDuplicatePackages
+};