@dependably/npm-check 1.7.0

package/src/performance.js

@@ -0,0 +1,283 @@
+/**
+ * Performance utilities for handling large lockfiles efficiently.
+ * Includes streaming, memory optimization, and batch processing strategies.
+ */
+import { resolvePackageName } from './format-library.js';
+
+/**
+ * Create a shallow copy of a lockfile to avoid deep cloning
+ * Useful when only modifying specific nested properties
+ * @param {object} lockfile - The lockfile to copy
+ * @returns {object} Shallow copy of lockfile
+ */
+export function shallowCopyLockfile(lockfile) {
+  if (!lockfile || typeof lockfile !== 'object') return lockfile;
+
+  const copy = { ...lockfile };
+
+  // Shallow copy packages if present
+  if (lockfile.packages && typeof lockfile.packages === 'object') {
+    copy.packages = { ...lockfile.packages };
+  }
+
+  // Shallow copy dependencies if present
+  if (lockfile.dependencies && typeof lockfile.dependencies === 'object') {
+    copy.dependencies = { ...lockfile.dependencies };
+  }
+
+  return copy;
+}
+
+/**
+ * Invoke a progress callback for a completed batch, computing the percentage
+ * @param {Function|null} onProgress - Progress callback function(progressInfo)
+ * @param {number} processed - Number of packages processed so far
+ * @param {number} total - Total number of packages
+ * @param {string} stage - Stage name for progress reporting
+ * @returns {void}
+ */
+function reportBatchProgress(onProgress, processed, total, stage) {
+  if (!onProgress) return;
+
+  const percentage = total > 0 ? Math.round((processed / total) * 100) : 0;
+  onProgress({
+    current: processed,
+    total,
+    percentage,
+    stage
+  });
+}
+
+/**
+ * Process packages in batches to reduce memory pressure
+ * @param {object} packagesMap - The packages object to process
+ * @param {Function} processor - Function to apply to each package (called as processor(path, pkg))
+ * @param {number|Object} batchSizeOrOptions - Batch size (number) or options object
+ * @param {number} batchSizeOrOptions.batchSize - Number of packages per batch (default: 1000)
+ * @param {Function} batchSizeOrOptions.onProgress - Progress callback function(progressInfo)
+ * @param {string} batchSizeOrOptions.stage - Stage name for progress reporting
+ * @returns {Promise<void>}
+ */
+export async function processBatchedPackages(packagesMap, processor, batchSizeOrOptions = 1000) {
+  if (!packagesMap || typeof packagesMap !== 'object') {
+    return;
+  }
+
+  // Handle both old signature (batchSize as number) and new signature (options object)
+  const options = typeof batchSizeOrOptions === 'object' ? batchSizeOrOptions : { batchSize: batchSizeOrOptions };
+  const batchSize = options.batchSize || 1000;
+  const onProgress = options.onProgress || null;
+  const stage = options.stage || 'Processing packages';
+
+  const entries = Object.entries(packagesMap);
+  const total = entries.length;
+  let processed = 0;
+
+  for (let i = 0; i < entries.length; i += batchSize) {
+    const batch = entries.slice(i, i + batchSize);
+
+    for (const [path, pkg] of batch) {
+      processor(path, pkg);
+      processed++;
+    }
+
+    // Report progress if callback provided
+    reportBatchProgress(onProgress, processed, total, stage);
+
+    // Yield control to allow garbage collection
+    if (i + batchSize < entries.length) {
+      await new Promise(resolve => setImmediate(resolve));
+    }
+  }
+}
+
+/**
+ * Get memory usage estimate for current process
+ * @returns {object} Object with memory stats (heapUsed, heapTotal, external, rss in MB)
+ */
+export function getMemoryStats() {
+  if (typeof process !== 'undefined' && process.memoryUsage) {
+    const mem = process.memoryUsage();
+    return {
+      heapUsed: Math.round(mem.heapUsed / 1024 / 1024 * 100) / 100,
+      heapTotal: Math.round(mem.heapTotal / 1024 / 1024 * 100) / 100,
+      external: Math.round(mem.external / 1024 / 1024 * 100) / 100,
+      rss: Math.round(mem.rss / 1024 / 1024 * 100) / 100
+    };
+  }
+  return null;
+}
+
+/**
+ * Create a filtered view of packages without full copy
+ * Useful for operations that only need to inspect certain packages
+ * @param {object} packagesMap - The packages map to filter
+ * @param {Function} predicate - Function that returns true for packages to include
+ * @returns {object} New object with only matching packages
+ */
+export function filterPackagesLazy(packagesMap, predicate) {
+  const filtered = {};
+
+  for (const [path, pkg] of Object.entries(packagesMap || {})) {
+    if (predicate(path, pkg)) {
+      filtered[path] = pkg;
+    }
+  }
+
+  return filtered;
+}
+
+/**
+ * Optimize deduplication by using a Map instead of nested objects
+ * Faster lookups for large collections
+ * @param {object} packagesMap - The packages object to deduplicate
+ * @returns {Map} Map with deduped entries (key: packageName#version, value: {path, pkg})
+ */
+export function createDedupeMap(packagesMap) {
+  const dedupeMap = new Map();
+
+  for (const [path, pkg] of Object.entries(packagesMap || {})) {
+    if (!pkg || typeof pkg !== 'object') continue;
+    // Derive the name from the path — v2/v3 entries usually have no `.name` field,
+    // so keying off `pkg.name` alone silently drops nearly every real entry.
+    const name = resolvePackageName(path, pkg);
+    if (!name) continue;
+
+    const key = `${name}#${pkg.version || 'unknown'}`;
+
+    if (!dedupeMap.has(key)) {
+      dedupeMap.set(key, { path, pkg });
+    }
+  }
+
+  return dedupeMap;
+}
+
+/**
+ * Reconstruct packages object from a dedupe map
+ * @param {Map} dedupeMap - Map created by createDedupeMap
+ * @returns {object} Reconstructed packages object
+ */
+export function reconstructFromDedupeMap(dedupeMap) {
+  const packages = {};
+
+  for (const { path, pkg } of dedupeMap.values()) {
+    packages[path] = pkg;
+  }
+
+  return packages;
+}
+
+/**
+ * Split a large lockfile into manageable chunks for processing
+ * Useful for parallel processing or streaming
+ * @param {object} lockfile - The lockfile to chunk
+ * @param {number} chunkSize - Number of packages per chunk (default: 5000)
+ * @returns {Array<object>} Array of partial lockfile objects
+ */
+export function chunkLockfile(lockfile, chunkSize = 5000) {
+  if (!lockfile.packages) {
+    return [lockfile];
+  }
+
+  const chunks = [];
+  const entries = Object.entries(lockfile.packages);
+
+  for (let i = 0; i < entries.length; i += chunkSize) {
+    const packageChunk = Object.fromEntries(
+      entries.slice(i, i + chunkSize)
+    );
+
+    const chunk = {
+      ...lockfile,
+      packages: packageChunk
+    };
+
+    chunks.push(chunk);
+  }
+
+  return chunks.length > 0 ? chunks : [lockfile];
+}
+
+/**
+ * Merge multiple processed lockfile chunks back into a single lockfile
+ * Assumes chunks have the same metadata (lockfileVersion, etc)
+ * @param {Array<object>} chunks - Array of lockfile chunks
+ * @returns {object} Merged lockfile
+ */
+export function mergeLockfileChunks(chunks) {
+  if (chunks.length === 0) {
+    return {};
+  }
+
+  if (chunks.length === 1) {
+    return chunks[0];
+  }
+
+  // Use first chunk as base
+  const merged = { ...chunks[0] };
+  merged.packages = {};
+
+  // Merge all packages from all chunks
+  for (const chunk of chunks) {
+    if (chunk.packages && typeof chunk.packages === 'object') {
+      Object.assign(merged.packages, chunk.packages);
+    }
+  }
+
+  return merged;
+}
+
+/**
+ * Estimate the size of a lockfile in memory (approximate)
+ * @param {object} lockfile - The lockfile to measure
+ * @returns {number} Approximate size in bytes
+ */
+export function estimateLockfileSize(lockfile) {
+  // This is a rough estimate using JSON.stringify
+  try {
+    const json = JSON.stringify(lockfile);
+    return json.length;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Check if a lockfile is considered "large" (over threshold)
+ * @param {object} lockfile - The lockfile to check
+ * @param {number} thresholdMB - Size threshold in MB (default: 10)
+ * @returns {boolean} True if lockfile size exceeds threshold
+ */
+export function isLargeLockfile(lockfile, thresholdMB = 10) {
+  // Heuristics: use JSON size estimate, but also fallback to package count for very large maps
+  const estimatedBytes = estimateLockfileSize(lockfile);
+  const estimatedMB = estimatedBytes / 1024 / 1024;
+
+  if (estimatedMB > thresholdMB) return true;
+
+  // If packages map exists and is extremely large, consider it large regardless of JSON size estimate
+  try {
+    if (lockfile && lockfile.packages && typeof lockfile.packages === 'object') {
+      const pkgCount = Object.keys(lockfile.packages).length;
+      if (pkgCount > 10000) return true;
+    }
+  } catch {
+    // ignore and fall through
+  }
+
+  return false;
+}
+
+export default {
+  shallowCopyLockfile,
+  processBatchedPackages,
+  getMemoryStats,
+  filterPackagesLazy,
+  createDedupeMap,
+  reconstructFromDedupeMap,
+  chunkLockfile,
+  mergeLockfileChunks,
+  estimateLockfileSize,
+  isLargeLockfile
+};

package/src/pinner.js

@@ -0,0 +1,248 @@
+// src/pinner.js
+import { detectLockfileVersion, LOCKFILE_VERSIONS } from './format-library.js';
+import { walkOverrides } from './overrides.js';
+
+export class PinnerError extends Error {
+  constructor(message, code, context = {}) {
+    super(message);
+    this.name = 'PinnerError';
+    this.code = code;
+    this.context = context;
+  }
+}
+
+const EXACT_VERSION = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z-.+]+)?$/;
+const CARET_RANGE = /^\^\d+(?:\.\d+){0,2}(?:[-+][0-9A-Za-z-.+]+)?$/;
+const TILDE_RANGE = /^~\d+(?:\.\d+){0,2}(?:[-+][0-9A-Za-z-.+]+)?$/;
+
+/**
+ * Classify a package.json version range.
+ * @param {string} range - The range string
+ * @returns {string} 'exact' | 'caret' | 'tilde' | 'complex' | 'git' | 'file' | 'link' | 'workspace' | 'alias' | 'url'
+ */
+export function classifyRange(range) {
+  if (typeof range !== 'string' || range.trim() === '') return 'complex';
+  const r = range.trim();
+
+  if (r.startsWith('npm:')) return 'alias';
+  if (r.startsWith('workspace:')) return 'workspace';
+  if (r.startsWith('file:')) return 'file';
+  if (r.startsWith('link:')) return 'link';
+  if (r.startsWith('git+') || r.startsWith('git://') || /^(github|gitlab|bitbucket):/.test(r)) return 'git';
+  if (/^https?:\/\//.test(r)) return 'url';
+
+  if (EXACT_VERSION.test(r)) return 'exact';
+  if (CARET_RANGE.test(r)) return 'caret';
+  if (TILDE_RANGE.test(r)) return 'tilde';
+
+  // Everything else: >=, <=, ||, x-ranges, *, dist-tags, hyphen ranges,
+  // and GitHub shorthand like user/repo (indistinguishable from tags safely → complex)
+  return 'complex';
+}
+
+const DEFAULT_SECTIONS = ['dependencies', 'devDependencies', 'optionalDependencies'];
+
+/**
+ * Look up the version a package resolved to in the lockfile.
+ * v2/v3 use the packages map keyed by install path; v1 uses the dependencies tree.
+ * @param {object} lockfile - Parsed lockfile
+ * @param {string} name - Package name
+ * @param {boolean} hasPackages - Whether the lockfile has a packages map (v2/v3)
+ * @returns {string|undefined} The resolved version, or undefined if absent
+ */
+function resolvedVersionFor(lockfile, name, hasPackages) {
+  if (hasPackages) {
+    const entry = lockfile.packages && lockfile.packages[`node_modules/${name}`];
+    return entry && entry.version;
+  }
+  const entry = lockfile.dependencies && lockfile.dependencies[name];
+  return entry && entry.version;
+}
+
+/**
+ * Keep the lockfile root entry (packages['']) in sync after pinning (v2/v3 only).
+ * Only rewrites a range that's already present in the matching root section.
+ * @param {object} lockfile - The (cloned) lockfile being mutated
+ * @param {string} section - The dependency section
+ * @param {string} name - Package name
+ * @param {string} resolvedVersion - The exact version to pin to
+ */
+function syncLockfileRoot(lockfile, section, name, resolvedVersion) {
+  if (!lockfile.packages || !lockfile.packages['']) return;
+  const rootSection = lockfile.packages[''][section];
+  if (rootSection && Object.prototype.hasOwnProperty.call(rootSection, name)) {
+    rootSection[name] = resolvedVersion;
+  }
+}
+
+/**
+ * Pin a single dependency's range to the lockfile-resolved version, recording
+ * the result into changes/skipped and syncing the lockfile root on success.
+ * @param {object} ctx - { sourceLockfile, newLockfile, hasPackages, changes, skipped }
+ * @param {object} deps - The (cloned) package.json section being mutated
+ * @param {string} section - The dependency section
+ * @param {string} name - Package name
+ * @param {string} range - The original range
+ */
+function pinDependency(ctx, deps, section, name, range) {
+  const kind = classifyRange(range);
+
+  if (kind === 'exact') return;
+  if (kind !== 'caret' && kind !== 'tilde') {
+    ctx.skipped.push({ section, name, range, reason: `${kind}-range` });
+    return;
+  }
+
+  const resolvedVersion = resolvedVersionFor(ctx.sourceLockfile, name, ctx.hasPackages);
+  if (!resolvedVersion) {
+    ctx.skipped.push({ section, name, range, reason: 'not-in-lockfile' });
+    return;
+  }
+
+  deps[name] = resolvedVersion;
+  ctx.changes.push({ section, name, from: range, to: resolvedVersion });
+
+  // Keep the lockfile root entry in sync (v2/v3)
+  if (ctx.hasPackages) {
+    syncLockfileRoot(ctx.newLockfile, section, name, resolvedVersion);
+  }
+}
+
+/**
+ * Collect every DISTINCT version a package name resolves to anywhere in the
+ * lockfile. An override forces all instances and a nested selector targets a
+ * shadowed install path (e.g. node_modules/a/node_modules/b), so the top-level
+ * `node_modules/<name>` entry alone can be the wrong instance to pin to.
+ * @param {object} lockfile - The source lockfile
+ * @param {string} name - Package name (selector already stripped)
+ * @param {boolean} hasPackages - v2/v3 (packages map) vs v1 (dependencies tree)
+ * @returns {Set<string>} Distinct resolved versions
+ */
+function versionsFromPackagesMap(packages, name) {
+  const versions = new Set();
+  const suffix = `node_modules/${name}`;
+  for (const [key, entry] of Object.entries(packages || {})) {
+    if ((key === suffix || key.endsWith(`/${suffix}`)) && entry && entry.version) {
+      versions.add(entry.version);
+    }
+  }
+  return versions;
+}
+
+function versionsFromV1Tree(tree, name, versions = new Set()) {
+  if (!tree || typeof tree !== 'object') return versions;
+  for (const [depName, node] of Object.entries(tree)) {
+    if (!node || typeof node !== 'object') continue;
+    if (depName === name && node.version) versions.add(node.version);
+    if (node.dependencies) versionsFromV1Tree(node.dependencies, name, versions);
+  }
+  return versions;
+}
+
+function collectResolvedVersions(lockfile, name, hasPackages) {
+  return hasPackages
+    ? versionsFromPackagesMap(lockfile.packages, name)
+    : versionsFromV1Tree(lockfile.dependencies, name);
+}
+
+/**
+ * Pin caret/tilde ranges inside the npm `overrides` field to their lockfile-
+ * resolved versions. `overrides` is nested and never mirrored into packages[''],
+ * so there is no lockfile-root sync — the caller runs `npm install` to reconcile.
+ * `$`-references and non-caret/tilde forms are left alone (reported in skipped).
+ * Pins only when the name resolves to a SINGLE version tree-wide; a name present
+ * at multiple versions is skipped `ambiguous-resolution` rather than pinned to a
+ * possibly-wrong instance.
+ * @param {object} ctx - Shared context (see pinDependency)
+ * @param {object} overrides - The (cloned) package.json `overrides` object
+ */
+function pinOverrides(ctx, overrides) {
+  if (!overrides || typeof overrides !== 'object') return;
+  for (const { path, name, range, container, key } of walkOverrides(overrides)) {
+    const kind = classifyRange(range);
+    if (kind === 'exact') continue;
+    if (kind !== 'caret' && kind !== 'tilde') {
+      ctx.skipped.push({ section: 'overrides', name: path, range, reason: `${kind}-range` });
+      continue;
+    }
+    const versions = collectResolvedVersions(ctx.sourceLockfile, name, ctx.hasPackages);
+    if (versions.size === 0) {
+      ctx.skipped.push({ section: 'overrides', name: path, range, reason: 'not-in-lockfile' });
+      continue;
+    }
+    if (versions.size > 1) {
+      ctx.skipped.push({ section: 'overrides', name: path, range, reason: 'ambiguous-resolution' });
+      continue;
+    }
+    const resolvedVersion = [...versions][0];
+    container[key] = resolvedVersion;
+    ctx.changes.push({ section: 'overrides', name: path, from: range, to: resolvedVersion });
+  }
+}
+
+/**
+ * Pin caret/tilde ranges in package.json to the exact versions resolved in
+ * the lockfile, and keep the lockfile's root entry (packages['']) in sync.
+ * All other range forms are left alone and reported in `skipped`.
+ *
+ * @param {object} packageJson - Parsed package.json
+ * @param {object} lockfile - Parsed lockfile
+ * @param {object} options - { sections, includePeer }
+ * @returns {{packageJson, lockfile, changes, skipped, warnings}}
+ */
+export function pinVersions(packageJson, lockfile, options = {}) {
+  const { sections = DEFAULT_SECTIONS, includePeer = false } = options;
+
+  if (!packageJson || typeof packageJson !== 'object') {
+    throw new PinnerError('package.json data is required', 'MISSING_PACKAGE_JSON');
+  }
+  if (!lockfile || typeof lockfile !== 'object') {
+    throw new PinnerError('lockfile data is required', 'MISSING_LOCKFILE');
+  }
+
+  const hasPackages = detectLockfileVersion(lockfile) !== LOCKFILE_VERSIONS.V1;
+  const activeSections = includePeer ? [...sections, 'peerDependencies'] : sections;
+
+  const newPackageJson = JSON.parse(JSON.stringify(packageJson));
+  const newLockfile = JSON.parse(JSON.stringify(lockfile));
+
+  const changes = [];
+  const skipped = [];
+  const warnings = [];
+
+  if (!hasPackages) {
+    warnings.push('v1 lockfile has no packages map; pinned package.json only — consider `npm-check migrate 3`');
+  }
+
+  const ctx = { sourceLockfile: lockfile, newLockfile, hasPackages, changes, skipped };
+
+  for (const section of activeSections) {
+    const deps = newPackageJson[section];
+    if (!deps || typeof deps !== 'object') continue;
+
+    for (const [name, range] of Object.entries(deps)) {
+      pinDependency(ctx, deps, section, name, range);
+    }
+  }
+
+  // npm `overrides` (nested) — pnpm.overrides is deliberately left alone: `pin`
+  // refuses pnpm lockfiles, and its selector keys don't map to a resolvable
+  // package name here. The audit `pinned-versions` rule still flags both.
+  pinOverrides(ctx, newPackageJson.overrides);
+
+  if (skipped.some((s) => s.reason === 'not-in-lockfile')) {
+    warnings.push('some dependencies are missing from the lockfile; run `npm install` to sync it');
+  }
+
+  return { packageJson: newPackageJson, lockfile: newLockfile, changes, skipped, warnings };
+}
+
+/**
+ * Detect the indentation used in a JSON file's raw text (default two spaces).
+ * @param {string} rawText - Original file content
+ * @returns {string} Indentation string
+ */
+export function detectIndent(rawText) {
+  const match = typeof rawText === 'string' ? rawText.match(/^([ \t]+)["{[]/m) : null;
+  return match ? match[1] : '  ';
+}