flatlock 1.0.1 → 1.2.0

package/README.md

@@ -4,14 +4,14 @@ The Matlock of lockfile parsers - cuts through the complexity to get just the fa
 
 ## What makes `flatlock` different?
 
-![matlockish](https://github.com/indexzero/flatlock/raw/main/doc/matlockish.png)
+![matlockish](https://github.com/indexzero/flatlock/raw/main/doc/img/matlockish.png)
 
 Most lockfile parsers (like `@npmcli/arborist` or `snyk-nodejs-lockfile-parser`) build the full dependency graph with edges representing relationships between packages. This is necessary for dependency resolution but overkill for many use cases.
 
 **flatlock** takes a different approach: it extracts a flat stream of packages from any lockfile format. No trees, no graphs, no edges - just packages.
 
 ```javascript
-import * as `flatlock`from 'flatlock';
+import * as flatlock from 'flatlock';
 
 // Stream packages from any lockfile
 for await (const pkg of flatlock.fromPath('./package-lock.json')) {
@@ -89,6 +89,59 @@ Each yielded package has:
 }
 ```
 
+## FlatlockSet
+
+For more advanced use cases, `FlatlockSet` provides Set-like operations on lockfile dependencies:
+
+```javascript
+import { FlatlockSet } from 'flatlock';
+
+// Create from lockfile
+const set = await FlatlockSet.fromPath('./package-lock.json');
+console.log(set.size); // 1234
+console.log(set.has('lodash@4.17.21')); // true
+
+// Set operations (immutable - return new sets)
+const other = await FlatlockSet.fromPath('./other-lock.json');
+const common = set.intersection(other);  // packages in both
+const added = other.difference(set);     // packages only in other
+const all = set.union(other);            // packages in either
+
+// Predicates
+set.isSubsetOf(other);    // true if all packages in set are in other
+set.isSupersetOf(other);  // true if set contains all packages in other
+set.isDisjointFrom(other); // true if no packages in common
+
+// Iterate like a Set
+for (const dep of set) {
+  console.log(dep.name, dep.version);
+}
+```
+
+### Workspace-Specific SBOMs
+
+For monorepos, use `dependenciesOf()` to get only the dependencies of a specific workspace:
+
+```javascript
+import { readFile } from 'node:fs/promises';
+import { FlatlockSet } from 'flatlock';
+
+const lockfile = await FlatlockSet.fromPath('./package-lock.json');
+const pkg = JSON.parse(await readFile('./packages/api/package.json', 'utf8'));
+
+// Get only dependencies reachable from this workspace
+const subset = lockfile.dependenciesOf(pkg, {
+  workspacePath: 'packages/api',  // for correct resolution in monorepos
+  dev: false,                      // exclude devDependencies
+  optional: true,                  // include optionalDependencies
+  peer: false                      // exclude peerDependencies
+});
+
+console.log(`${pkg.name} has ${subset.size} production dependencies`);
+```
+
+**Note:** Sets created via `union()`, `intersection()`, or `difference()` cannot use `dependenciesOf()` because they lack the raw lockfile data needed for traversal. Check `set.canTraverse` before calling.
+
 ## License
 
 Apache-2.0

package/bin/flatlock-cmp.js

@@ -1,277 +1,10 @@
 #!/usr/bin/env node
 
-import { readFile, readdir, stat, mkdir, writeFile, rm } from 'node:fs/promises';
-import { join, dirname } from 'node:path';
+import { readdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
 import { parseArgs } from 'node:util';
-import { fileURLToPath } from 'node:url';
-import crypto from 'node:crypto';
 import * as flatlock from '../src/index.js';
 
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-// Unique run ID for this script execution (7 char hash of timestamp)
-const RUN_ID = Date.now().toString(36).slice(-7);
-const TMP_BASE = join(__dirname, 'tmp', RUN_ID);
-let tmpDirCreated = false;
-
-// Comparison parsers (lazy loaded)
-let Arborist, yarnLockfile, parseSyml, yaml;
-
-async function loadArborist() {
-  if (!Arborist) {
-    const mod = await import('@npmcli/arborist');
-    Arborist = mod.default;
-  }
-  return Arborist;
-}
-
-async function loadYarnClassic() {
-  if (!yarnLockfile) {
-    const mod = await import('@yarnpkg/lockfile');
-    yarnLockfile = mod.default || mod;
-  }
-  return yarnLockfile;
-}
-
-async function loadYarnBerry() {
-  if (!parseSyml) {
-    const mod = await import('@yarnpkg/parsers');
-    parseSyml = mod.parseSyml;
-  }
-  return parseSyml;
-}
-
-async function loadYaml() {
-  if (!yaml) {
-    const mod = await import('js-yaml');
-    yaml = mod.default;
-  }
-  return yaml;
-}
-
-/**
- * Ensure the temp directory for this run exists
- */
-async function ensureTmpDir() {
-  if (!tmpDirCreated) {
-    await mkdir(TMP_BASE, { recursive: true });
-    tmpDirCreated = true;
-  }
-}
-
-/**
- * Cleanup the temp directory for this run
- */
-async function cleanup() {
-  if (tmpDirCreated) {
-    try {
-      await rm(TMP_BASE, { recursive: true, force: true });
-    } catch {
-      // Ignore cleanup errors
-    }
-  }
-}
-
-/**
- * Get packages set using @npmcli/arborist
- *
- * Arborist requires a directory with package-lock.json (and optionally package.json).
- * We create a temp directory structure for each lockfile:
- *   bin/tmp/<run-id>/<file-hash>/package-lock.json
- */
-async function getPackagesFromNpm(content, filepath) {
-  const ArboristClass = await loadArborist();
-  await ensureTmpDir();
-
-  // Create unique subdir for this lockfile
-  const fileId = crypto.createHash('md5').update(filepath).digest('hex').slice(0, 7);
-  const tmpDir = join(TMP_BASE, fileId);
-
-  await mkdir(tmpDir, { recursive: true });
-  await writeFile(join(tmpDir, 'package-lock.json'), content);
-
-  // Create minimal package.json from lockfile root entry
-  const lockfile = JSON.parse(content);
-  const root = lockfile.packages?.[''] || {};
-  const pkg = {
-    name: root.name || 'arborist-temp',
-    version: root.version || '1.0.0'
-  };
-  await writeFile(join(tmpDir, 'package.json'), JSON.stringify(pkg));
-
-  try {
-    const arb = new ArboristClass({ path: tmpDir });
-    const tree = await arb.loadVirtual();
-
-    const result = new Set();
-    let workspaceCount = 0;
-    for (const node of tree.inventory.values()) {
-      if (node.isRoot) continue;
-      // Skip workspace symlinks (link:true, no version in raw lockfile)
-      if (node.isLink) {
-        workspaceCount++;
-        continue;
-      }
-      // Skip workspace package definitions (not in node_modules)
-      // Flatlock only yields packages from node_modules/ paths
-      if (node.location && !node.location.includes('node_modules')) {
-        workspaceCount++;
-        continue;
-      }
-      if (node.name && node.version) {
-        result.add(`${node.name}@${node.version}`);
-      }
-    }
-    return { packages: result, workspaceCount };
-  } finally {
-    // Cleanup this specific lockfile's temp dir
-    await rm(tmpDir, { recursive: true, force: true });
-  }
-}
-
-/**
- * Get packages set using @yarnpkg/lockfile (classic)
- */
-async function getPackagesFromYarnClassic(content) {
-  const yarnLock = await loadYarnClassic();
-  const parse = yarnLock.parse || yarnLock.default?.parse;
-  const { object: lockfile } = parse(content);
-
-  const result = new Set();
-  let workspaceCount = 0;
-  for (const [key, pkg] of Object.entries(lockfile)) {
-    if (key === '__metadata') continue;
-
-    // Skip workspace/link entries - flatlock only cares about external dependencies
-    const resolved = pkg.resolved || '';
-    if (resolved.startsWith('file:') || resolved.startsWith('link:')) {
-      workspaceCount++;
-      continue;
-    }
-
-    let name;
-    if (key.startsWith('@')) {
-      const idx = key.indexOf('@', 1);
-      name = key.slice(0, idx);
-    } else {
-      name = key.split('@')[0];
-    }
-    if (name && pkg.version) result.add(`${name}@${pkg.version}`);
-  }
-
-  return { packages: result, workspaceCount };
-}
-
-/**
- * Get packages set using @yarnpkg/parsers (berry)
- */
-async function getPackagesFromYarnBerry(content) {
-  const parse = await loadYarnBerry();
-  const lockfile = parse(content);
-
-  const result = new Set();
-  let workspaceCount = 0;
-  for (const [key, pkg] of Object.entries(lockfile)) {
-    if (key === '__metadata') continue;
-
-    // Skip workspace/link entries - flatlock only cares about external dependencies
-    const resolution = pkg.resolution || '';
-    if (resolution.startsWith('workspace:') ||
-        resolution.startsWith('portal:') ||
-        resolution.startsWith('link:')) {
-      workspaceCount++;
-      continue;
-    }
-
-    let name;
-    if (key.startsWith('@')) {
-      const idx = key.indexOf('@', 1);
-      name = key.slice(0, idx);
-    } else {
-      name = key.split('@')[0];
-    }
-    if (name && pkg.version) result.add(`${name}@${pkg.version}`);
-  }
-
-  return { packages: result, workspaceCount };
-}
-
-/**
- * Get packages set using js-yaml (pnpm)
- */
-async function getPackagesFromPnpm(content) {
-  const y = await loadYaml();
-  const lockfile = y.load(content);
-  const packages = lockfile.packages || {};
-
-  const result = new Set();
-  let workspaceCount = 0;
-  for (const [key, pkg] of Object.entries(packages)) {
-    // Skip link/file entries - flatlock only cares about external dependencies
-    // Keys can be: link:path, file:path, or @pkg@file:path
-    if (key.startsWith('link:') || key.startsWith('file:') ||
-        key.includes('@link:') || key.includes('@file:')) {
-      workspaceCount++;
-      continue;
-    }
-    // Also skip if resolution.type is 'directory' (workspace)
-    if (pkg.resolution?.type === 'directory') {
-      workspaceCount++;
-      continue;
-    }
-    // pnpm keys are like /lodash@4.17.21 or /@babel/core@7.0.0
-    const match = key.match(/^\/?(@?[^@]+)@(.+)$/);
-    if (match) result.add(`${match[1]}@${match[2]}`);
-  }
-
-  return { packages: result, workspaceCount };
-}
-
-/**
- * Get packages set with flatlock
- */
-async function getPackagesFromFlatlock(filepath) {
-  const result = new Set();
-  for await (const dep of flatlock.fromPath(filepath)) {
-    if (dep.name && dep.version) result.add(`${dep.name}@${dep.version}`);
-  }
-  return result;
-}
-
-/**
- * Get comparison parser name for type
- */
-function getComparisonName(type) {
-  switch (type) {
-    case 'npm': return '@npmcli/arborist';
-    case 'yarn-classic': return '@yarnpkg/lockfile';
-    case 'yarn-berry': return '@yarnpkg/parsers';
-    case 'pnpm': return 'js-yaml';
-    default: return 'unknown';
-  }
-}
-
-/**
- * Get packages with comparison parser based on type
- */
-async function getPackagesFromComparison(type, content, filepath) {
-  switch (type) {
-    case 'npm': return getPackagesFromNpm(content, filepath);
-    case 'yarn-classic': return getPackagesFromYarnClassic(content);
-    case 'yarn-berry': return getPackagesFromYarnBerry(content);
-    case 'pnpm': return getPackagesFromPnpm(content);
-    default: return null;
-  }
-}
-
-/**
- * Compare two sets and return differences
- */
-function compareSets(setA, setB) {
-  const onlyInA = new Set([...setA].filter(x => !setB.has(x)));
-  const onlyInB = new Set([...setB].filter(x => !setA.has(x)));
-  return { onlyInA, onlyInB };
-}
 
 /**
  * Convert glob pattern to regex
@@ -308,52 +41,38 @@ async function findFiles(dir, pattern) {
 }
 
 /**
- * Process a single lockfile - compare sets, not just counts
+ * Process a single lockfile using flatlock.compare()
  */
 async function processFile(filepath, baseDir) {
   try {
-    const content = await readFile(filepath, 'utf8');
-    const type = flatlock.detectType({ path: filepath, content });
+    const result = await flatlock.compare(filepath);
     const rel = baseDir ? filepath.replace(baseDir + '/', '') : filepath;
-    const comparisonName = getComparisonName(type);
 
-    const flatlockSet = await getPackagesFromFlatlock(filepath);
-    let comparisonResult;
-
-    try {
-      comparisonResult = await getPackagesFromComparison(type, content, filepath);
-    } catch (err) {
-      comparisonResult = null;
-    }
-
-    if (!comparisonResult) {
+    if (result.equinumerous === null) {
+      // Unsupported type or no comparison available
       return {
-        type,
+        type: result.type,
         path: rel,
-        comparisonName,
-        flatlockCount: flatlockSet.size,
+        source: result.source || 'unknown',
+        flatlockCount: result.flatlockCount,
         comparisonCount: null,
         workspaceCount: 0,
-        identical: null,
+        equinumerous: null,
         onlyInFlatlock: null,
         onlyInComparison: null
       };
     }
 
-    const { packages: comparisonSet, workspaceCount } = comparisonResult;
-    const { onlyInA: onlyInFlatlock, onlyInB: onlyInComparison } = compareSets(flatlockSet, comparisonSet);
-    const identical = onlyInFlatlock.size === 0 && onlyInComparison.size === 0;
-
     return {
-      type,
+      type: result.type,
       path: rel,
-      comparisonName,
-      flatlockCount: flatlockSet.size,
-      comparisonCount: comparisonSet.size,
-      workspaceCount,
-      identical,
-      onlyInFlatlock,
-      onlyInComparison
+      source: result.source,
+      flatlockCount: result.flatlockCount,
+      comparisonCount: result.comparisonCount,
+      workspaceCount: result.workspaceCount,
+      equinumerous: result.equinumerous,
+      onlyInFlatlock: result.onlyInFlatlock,
+      onlyInComparison: result.onlyInComparison
     };
   } catch (err) {
     const rel = baseDir ? filepath.replace(baseDir + '/', '') : filepath;
@@ -386,10 +105,21 @@ Options:
   -h, --help           Show this help
 
 Comparison parsers (workspace/link entries excluded from all):
-  npm:          @npmcli/arborist (loadVirtual)
+  npm:          @npmcli/arborist (preferred) or @cyclonedx/cyclonedx-npm
   yarn-classic: @yarnpkg/lockfile
   yarn-berry:   @yarnpkg/parsers
-  pnpm:         js-yaml
+  pnpm:         @pnpm/lockfile.fs (preferred) or js-yaml
+
+Result types:
+  ✓ equinumerous  Same packages in both (exact match)
+  ⊃ SUPERSET      flatlock found MORE packages (expected for pnpm)
+  ❌ MISMATCH      Unexpected difference (comparison found packages flatlock missed)
+
+Note on pnpm supersets:
+  flatlock performs full reachability analysis on lockfiles, finding all
+  transitive dependencies. pnpm's official tools don't enumerate all reachable
+  packages - they omit some transitive deps from their API output. When flatlock
+  finds MORE packages than pnpm, this is expected and correct behavior.
 
 Examples:
   flatlock-cmp package-lock.json
@@ -421,94 +151,117 @@ Examples:
   let fileCount = 0;
   let errorCount = 0;
   let matchCount = 0;
+  let supersetCount = 0;  // flatlock found more (expected for pnpm reachability)
   let mismatchCount = 0;
 
-  try {
-    for (const file of files) {
-      const result = await processFile(file, baseDir);
+  for (const file of files) {
+    const result = await processFile(file, baseDir);
 
-      if (result.error) {
-        errorCount++;
-        if (!values.quiet) {
-          console.log(`\n❌ ERROR: ${result.path}`);
-          console.log(`   ${result.error}`);
-        }
-        continue;
+    if (result.error) {
+      errorCount++;
+      if (!values.quiet) {
+        console.log(`\n❌ ERROR: ${result.path}`);
+        console.log(`   ${result.error}`);
       }
+      continue;
+    }
 
-      fileCount++;
-      totalFlatlock += result.flatlockCount;
-      totalWorkspaces += result.workspaceCount || 0;
+    fileCount++;
+    totalFlatlock += result.flatlockCount;
+    totalWorkspaces += result.workspaceCount || 0;
 
-      if (result.comparisonCount === null) {
-        if (!values.quiet) {
-          console.log(`\n⚠️  ${result.path}`);
-          console.log(`   flatlock: ${result.flatlockCount} packages`);
-          console.log(`   ${result.comparisonName}: unavailable`);
-        }
-        continue;
+    if (result.comparisonCount === null) {
+      if (!values.quiet) {
+        console.log(`\n⚠️  ${result.path}`);
+        console.log(`   flatlock: ${result.flatlockCount} packages`);
+        console.log(`   ${result.source}: unavailable`);
       }
+      continue;
+    }
 
-      totalComparison += result.comparisonCount;
+    totalComparison += result.comparisonCount;
 
-      if (result.identical) {
-        matchCount++;
+    if (result.equinumerous) {
+      matchCount++;
+      if (!values.quiet) {
+        const wsNote = result.workspaceCount > 0 ? ` (${result.workspaceCount} workspaces excluded)` : '';
+        console.log(`✓  ${result.path}${wsNote}`);
+        console.log(`   count: flatlock=${result.flatlockCount} ${result.source}=${result.comparisonCount}`);
+        console.log(`   sets:  equinumerous`);
+      }
+    } else {
+      // Determine if this is a "superset" (flatlock found more, expected for pnpm)
+      // or a true "mismatch" (comparison found packages flatlock missed)
+      const isPnpm = result.type === 'pnpm' || result.path.includes('pnpm-lock');
+      const isSuperset = result.onlyInFlatlock.length > 0 && result.onlyInComparison.length === 0;
+
+      if (isPnpm && isSuperset) {
+        // Expected behavior: flatlock's reachability analysis found more packages
+        supersetCount++;
         if (!values.quiet) {
           const wsNote = result.workspaceCount > 0 ? ` (${result.workspaceCount} workspaces excluded)` : '';
-          console.log(`✓  ${result.path}${wsNote}`);
-          console.log(`   count: flatlock=${result.flatlockCount} ${result.comparisonName}=${result.comparisonCount}`);
-          console.log(`   sets:  identical`);
+          console.log(`⊃  ${result.path}${wsNote}`);
+          console.log(`   count: flatlock=${result.flatlockCount} ${result.source}=${result.comparisonCount}`);
+          console.log(`   sets:  SUPERSET (+${result.onlyInFlatlock.length} reachable deps)`);
+          console.log(`   note:  flatlock's reachability analysis found transitive deps pnpm omits`);
         }
       } else {
         mismatchCount++;
         console.log(`\n❌ ${result.path}`);
-        console.log(`   count: flatlock=${result.flatlockCount} ${result.comparisonName}=${result.comparisonCount}`);
+        console.log(`   count: flatlock=${result.flatlockCount} ${result.source}=${result.comparisonCount}`);
         console.log(`   sets:  MISMATCH`);
 
-        if (result.onlyInFlatlock.size > 0) {
-          console.log(`   only in flatlock (${result.onlyInFlatlock.size}):`);
-          for (const pkg of [...result.onlyInFlatlock].slice(0, 10)) {
+        if (result.onlyInFlatlock.length > 0) {
+          console.log(`   only in flatlock (${result.onlyInFlatlock.length}):`);
+          for (const pkg of result.onlyInFlatlock.slice(0, 10)) {
             console.log(`     + ${pkg}`);
           }
-          if (result.onlyInFlatlock.size > 10) {
-            console.log(`     ... and ${result.onlyInFlatlock.size - 10} more`);
+          if (result.onlyInFlatlock.length > 10) {
+            console.log(`     ... and ${result.onlyInFlatlock.length - 10} more`);
           }
         }
 
-        if (result.onlyInComparison.size > 0) {
-          console.log(`   only in ${result.comparisonName} (${result.onlyInComparison.size}):`);
-          for (const pkg of [...result.onlyInComparison].slice(0, 10)) {
+        if (result.onlyInComparison.length > 0) {
+          console.log(`   only in ${result.source} (${result.onlyInComparison.length}):`);
+          for (const pkg of result.onlyInComparison.slice(0, 10)) {
             console.log(`     - ${pkg}`);
           }
-          if (result.onlyInComparison.size > 10) {
-            console.log(`     ... and ${result.onlyInComparison.size - 10} more`);
+          if (result.onlyInComparison.length > 10) {
+            console.log(`     ... and ${result.onlyInComparison.length - 10} more`);
           }
         }
       }
     }
+  }
 
-    // Summary
-    console.log('\n' + '='.repeat(70));
-    console.log(`SUMMARY: ${fileCount} files, ${matchCount} identical, ${mismatchCount} mismatches, ${errorCount} errors`);
-    console.log(`  flatlock total:    ${totalFlatlock.toString().padStart(8)} packages`);
-    if (totalComparison > 0) {
-      console.log(`  comparison total:  ${totalComparison.toString().padStart(8)} packages`);
-    }
-    if (totalWorkspaces > 0) {
-      console.log(`  workspaces:        ${totalWorkspaces.toString().padStart(8)} excluded (local/workspace refs)`);
-    }
+  // Summary
+  console.log('\n' + '='.repeat(70));
+  const summaryParts = [`${fileCount} files`, `${matchCount} equinumerous`];
+  if (supersetCount > 0) {
+    summaryParts.push(`${supersetCount} supersets`);
+  }
+  summaryParts.push(`${mismatchCount} mismatches`, `${errorCount} errors`);
+  console.log(`SUMMARY: ${summaryParts.join(', ')}`);
 
-    // Exit with error if any mismatches
-    if (mismatchCount > 0) {
-      process.exit(1);
-    }
-  } finally {
-    await cleanup();
+  console.log(`  flatlock total:    ${totalFlatlock.toString().padStart(8)} packages`);
+  if (totalComparison > 0) {
+    console.log(`  comparison total:  ${totalComparison.toString().padStart(8)} packages`);
+  }
+  if (totalWorkspaces > 0) {
+    console.log(`  workspaces:        ${totalWorkspaces.toString().padStart(8)} excluded (local/workspace refs)`);
+  }
+  if (supersetCount > 0) {
+    console.log(`  supersets:         ${supersetCount.toString().padStart(8)} (flatlock found more via reachability)`);
+  }
+
+  // Exit with error only for true mismatches (not supersets)
+  // Supersets are expected: flatlock's reachability analysis is more thorough
+  if (mismatchCount > 0) {
+    process.exit(1);
   }
 }
 
-main().catch(async err => {
-  await cleanup();
+main().catch(err => {
   console.error('Fatal error:', err.message);
   process.exit(1);
 });