@antongolub/lockfile 0.0.0-snapshot.2 → 0.0.0-snapshot.21

package/README.md

@@ -1,109 +1,184 @@
-# lockfile
-Read, write, and convert npm (v1, v2) and yarn (classic and berry) lockfiles in any directions with reasonable losses.
+# @antongolub/lockfile
+> Read and write lockfiles with reasonable losses
 
-### Motivation
-Every package manager brings its own philosophy of how to describe, store and control projects dependencies.
-This is awesome for developers, but literally becomes a ~~pain in *** ***~~ headache for isec, devops and release engineers.
+## Motivation
+Each package manager brings its own philosophy of how to describe, store and control project dependencies.
+It _seems_ acceptable for developers, but literally becomes a ~~pain in *** ***~~ headache for isec, devops and release engineers.
 This lib is a naive attempt to build a pm-independent, generic, extensible and reliable deps representation.
 
-The package manifest contains its own deps requirements, the lockfile defines the deps resolution snapshot<sup>*</sup>,
-so both of them are required to build a dependency graph. We can convert this data into a normalized representation for further analysis and processing (for example, to fix vulnerabilities).
-And then, if necessary, convert back to the original format.
+The `package.json` manifest contains its own deps requirements, the `lockfile` holds the deps resolution snapshot<sup>*</sup>,
+so both of them are required to build a dependency graph. We can try to convert this data into a normalized representation for further analysis and processing (for example, to fix vulnerabilities).
+And then, if necessary, try convert it back to the original/another format.
 
-### Status
-⚠️ Initial draft. Alpha-version
+## Status
+Proof of concept. The API may vary significantly ⚠️
 
-### Getting started
-#### Install
+## Getting started
+### Install
 ```shell
-yarn add @antongolub/lockfile
+yarn add @antongolub/lockfile@snapshot
 ```
 
-#### Usage
-```js
-import { parse, format } from '@antongolub/lockfile'
-
-const parsed = parse({
-  lockfile: './yarn.lock',
-  workspaces: {'': './package.json', 'foo': './packages/foo/package.json'},
-})
-
-// output
-{
-  entries: {
-    '@babel/code-frame@7.10.4': {
-      name: '@babel/code-frame',
-      version: '7.10.4',
-      scope: 'prod/dev/peer/opt',
-      integrities: {
-        sha512: 'hashsum',
-        sha256: '...',
-        sha1: '...',
-        md5: '...'
-      },
-      reference: {
-        sourceType: 'npm/git/file/workspace'
-        source: 'uri://remote/address',
-        linkType: 'hard/soft',
-        link: '<root>path/to/package'
-      },
-      dependencies: {
-        '@babel/highlight': '^7.10.4'
-      }
-    },
-    ...
-  },
-  meta: {
-    lockfile: {
-      type: 'yarn',
-      version: '5', // metadata format version
-    },
-    packageJson: {...},
-    workspaces: {
-      patterns: ['./packages/*'],
-      packages: {
-        '@qiwi/pijma-core': '<root>/packages/core/package.json'
-      }
-    }
-  },
-}
+## Usage
+_tl;dr_
+```ts
+import fs from 'fs/promises'
+import {parse, analyze} from '@antongolub/lockfile'
+
+const lf = await fs.readFile('yarn.lock', 'utf-8')
+const pkg = await fs.readFile('package.json', 'utf-8')
+
+const snapshot = parse(lf, pkg) // Holds JSON-friendly TEntries[]
+const idx = analyze(snapshot)   // An index to represent repo dep graphs
 
-const data = format({
-  ...parsed,
-  lockfileType: 'yarn-2'
-})
-// output
-`
-# This file is generated by running "yarn install" inside your project.
-# Manual changes might be lost - proceed with caution!
-
-__metadata:
-  version: 5
-  cacheKey: 8
-
-"@babel/code-frame@npm:7.10.4":
-  version: 7.10.4
-  resolution: "@babel/code-frame@npm:7.10.4"
-...
-`
+// idx.entries
+// idx.prod
+// idx.edges
 ```
 
-### Lockfile (meta) versions
-| Package manager  | Meta format | Supported | 
-|------------------|-------------|-----------|
-| npm <7           | 1           | x         |
-| npm >=7          | 2           |           |
-| yarn 1 (classic) | 1           | x         |
-| yarn 3           | 5, 6        | x         |
-| yarn 4           | 6, 7        |           |
+## API
+```ts
+import { parse, format, analyze } from '@antongolub/lockfile'
+
+const snapshot = parse('yarn.lock <raw contents>', 'package.json <raw contents>', './packages/foo/package.json <raw contents>')
+
+const lf = format(snapshot)
+const lf2 = format(snapshot, 'npm-1')         // Throws err: npm v1 meta does not support workspaces
+
+const meta = await readMeta()                 // reads local package.jsons data to gather required data like `engines`, `license`, `bins`, etc
+const meta2 = await fetchMeta(snapshot)       // does the same, but from the remote registry
+const lf3 = format(snapshot, 'npm-3', {meta}) // format with options
+
+const idx = analyze(snapshot)
+idx.edges
+// [
+//  [ '', '@antongolub/npm-test@4.0.1' ],
+//  [ '@antongolub/npm-test@4.0.1', '@antongolub/npm-test@3.0.1' ],
+//  [ '@antongolub/npm-test@3.0.1', '@antongolub/npm-test@2.0.1' ],
+//  [ '@antongolub/npm-test@2.0.1', '@antongolub/npm-test@1.0.0' ]
+// ]
+```
+
+### Terms
+`nmtree` — fs projection of deps, directories structure  
+`deptree` — bounds full dep paths with their resolved packages  
+`depgraph` — describes how resolved pkgs are related with each other
+
+### Lockfiles types
+| Package manager      | Meta format | Read | Write |
+|----------------------|-------------|------|-------|
+| npm <7               | 1           | ✓    | ✓     |
+| npm >=7              | 2           | ✓    |       |
+| npm >=9              | 3           | ✓    |       | 
+| yarn 1 (classic)     | 1           | ✓    | ✓     |
+| yarn 2, 3, 4 (berry) | 5, 6, 7     | ✓    | ✓     |
+
+### Dependency protocols
+| Type      | Supported | Example                                 | Description                                                    |
+|-----------|-----------|-----------------------------------------|----------------------------------------------------------------|
+| semver    | ✓         | `^1.2.3`                                | Resolves from the default registry                             |
+| tag       |           | `latest`                                | Resolves from the default registry                             |
+| npm       | ✓         | `npm:name@...`                          | Resolves from the npm registry                                 |
+| git       |           | `git@github.com:foo/bar.git`            | Downloads a public package from a Git repository               |
+| github    |           | `github:foo/bar`                        | Downloads a public package from GitHub                         |
+| github    | ✓         | `foo/bar`                               | Alias for the github: protocol                                 |
+| file      |           | `file:./my-package`                     | Copies the target location into the cache                      |
+| link      |           | `link:./my-folder`                      | Creates a link to the ./my-folder folder (ignore dependencies) |
+| patch     |           | `patch:left-pad@1.0.0#./my-patch.patch` | Creates a patched copy of the original package                 |
+| portal    |           | `portal:./my-folder`                    | Creates a link to the ./my-folder folder (follow dependencies) |
+| workspace | _limited_ | `workspace:*`                           | Creates a link to a package in another workspace               |
+
+https://v3.yarnpkg.com/features/protocols  
+https://yarnpkg.com/protocols  
+https://docs.npmjs.com/cli/v10/configuring-npm/package-json#dependencies
+
+### `TSnapshot`
+```ts
+export type TSnapshot = Record<string, TEntry>
+
+export type TEntry = {
+  name:       string
+  version:    string
+  ranges:     string[]
+  hashes:     {
+    sha512?:  string
+    sha256?:  string
+    sha1?:    string
+    checksum?: string
+    md5?:     string
+  }
+  source:     {
+    type:     TSourceType // npm, workspace, gh, patch, etc
+    id:       string
+    registry?: string
+  }
+  // optional pm-specific lockfile meta
+  manifest?:              TManifest
+  conditions?:            string
+  dependencies?:          TDependencies
+  dependenciesMeta?:      TDependenciesMeta
+  devDependencies?:       TDependencies
+  optionalDependencies?:  TDependencies
+  peerDependencies?:      TDependencies
+  peerDependenciesMeta?:  TDependenciesMeta
+  bin?:                   Record<string, string>
+  engines?:               Record<string, string>
+  funding?:               Record<string, string>
+}
+```
+
+### `TSnapshotIndex`
+```ts
+export interface TSnapshotIndex {
+  snapshot: TSnapshot
+  entries:  TEntry[]
+  roots:    TEntry[]
+  edges:    [string, string][]
+  tree:       Record<string, {
+    key:      string
+    chunks:   string[]
+    parents:  TEntry[]
+    id:       string
+    name:     string
+    version:  string
+    entry:    TEntry
+  }>
+  prod: Set<TEntry>
+  getEntryId ({name, version}: TEntry): string
+  getEntry (name: string, version?: string): TEntry | undefined,
+  getEntryByRange (name: string, range: string): TEntry | undefined
+  getEntryDeps(entry: TEntry): TEntry[]
+}
+```
 
 ### Caveats
-* Only `npm` links are supported for now
-* npm1: `optional: true` label is not supported by lockfile formatter
+* There is an infinite number of `nmtrees` that corresponds to the specified `deptree`, but among them there is a finite set of effective (sufficient) for the target criterion — for example, nesting, size, homogeneity of versions
+* npm1: `optional: true` label is not supported yet
+* yarn berry: no idea how to resolve and inject PnP patches https://github.com/yarnpkg/berry/tree/master/packages/plugin-compat
+* npm2 and npm3 requires `engines` and `funding` data, while yarn* or npm1 does not contain it
+* many `nmtree` projections may correspond to the specified `depgraph`
+* pkg.json `resolutions` and `overrides` directives are completely ignored for now
+* pkg aliases are not _fully_ supported yet [#2](https://github.com/antongolub/lockfile/issues/2#issuecomment-1786613893)
 
 ### Inspired by
 * [synp](https://github.com/imsnif/synp)
 * [snyk-nodejs-lockfile-parser](https://github.com/snyk/nodejs-lockfile-parser)
+* [yarn](https://github.com/yarnpkg/yarn/blob/master/src/lockfile/parse.js)
+* [yarn-lockfile](https://github.com/yarnpkg/yarn/tree/master/packages/lockfile)
+
+### Refs
+* [yarn.lock](https://classic.yarnpkg.com/en/docs/yarn-lock/)
+* [package-lock-json](https://docs.npmjs.com/cli/v10/configuring-npm/package-lock-json)
+* [what-is-package-lock-json](https://snyk.io/blog/what-is-package-lock-json/)
+* [the-ultimate-guide-to-yarn-lock-lockfiles](https://www.arahansen.com/the-ultimate-guide-to-yarn-lock-lockfiles/)
+* [package-lock-json-the-complete-guide](https://medium.com/helpshift-engineering/package-lock-json-the-complete-guide-2ae40175ebdd)
+<details>
+  <summary>more</summary>
+
+* [cvent/pnpm-lock-export](https://github.com/cvent/pnpm-lock-export)
+* [why-keep-package-lockjson](https://blog.npmjs.org/post/621733939456933888/npm-v7-series-why-keep-package-lockjson.html)
+* [pnpm/lockfile-utils](https://github.com/pnpm/pnpm/tree/main/lockfile/lockfile-utils)
+</details>
 
-### License
+## License
 [MIT](./LICENSE)

package/package.json

@@ -1,23 +1,26 @@
 {
   "name": "@antongolub/lockfile",
-  "version": "0.0.0-snapshot.2",
+  "version": "0.0.0-snapshot.21",
   "publishConfig": {
     "access": "public"
   },
-  "description": "Read, write, and convert npm (v1, v2) and yarn (classic and berry) lockfiles in any directions with reasonable losses.",
-  "main": "./target/es6/index.mjs",
+  "description": "Read and write lockfiles with reasonable losses",
+  "main": "./target/es6/index.js",
   "type": "module",
-  "exports": "./target/es6/index.mjs",
+  "exports": "./target/es6/index.js",
   "dependencies": {
-    "js-yaml": "^4.1.0"
+    "@semrel-extra/topo": "^1.14.0",
+    "js-yaml": "^4.1.0",
+    "semver": "^7.5.4"
   },
   "devDependencies": {
-    "@types/js-yaml": "^4.0.5",
-    "esbuild": "^0.15.12",
-    "esbuild-node-externals": "^1.5.0",
-    "tsc-esm-fix": "^2.20.5",
-    "tsm": "^2.2.2",
-    "typescript": "^4.8.4",
+    "@types/js-yaml": "^4.0.8",
+    "@types/node": "^20.8.10",
+    "@types/semver": "^7.5.4",
+    "esbuild": "^0.19.5",
+    "esbuild-node-externals": "^1.9.0",
+    "tsm": "^2.3.0",
+    "typescript": "^5.2.2",
     "uvu": "^0.5.6"
   },
   "scripts": {
@@ -25,8 +28,9 @@
     "build:es6": "node ./build.js",
     "build:libdef": "tsc --emitDeclarationOnly --outDir target/es6",
     "build:esmfix": "tsc-esm-fix --target=target/es6 --ext=.mjs",
-    "test": "uvu -r tsm -i helpers 'src/test/ts/'",
-    "publish:snapshot": "npm publish --no-git-tag-version --tag snapshot"
+    "test": "yarn test:unit",
+    "test:unit": "DEBUG=true uvu -r tsm -i helpers 'src/test/ts/'",
+    "publish:snapshot": "yarn build && npm publish --no-git-tag-version --tag snapshot"
   },
   "files": [
     "target",

package/target/es6/analyze.d.ts

@@ -0,0 +1,7 @@
+import { TEntry, TSnapshot, TSnapshotIndex } from './interface';
+export declare const getDeps: {
+    (entry: TEntry): Record<string, string>;
+    cache: WeakMap<TEntry, Record<string, string>>;
+};
+export declare const getId: (name?: string, version?: string) => string;
+export declare const analyze: (snapshot: TSnapshot) => TSnapshotIndex;

package/target/es6/common.d.ts

@@ -1,3 +1,28 @@
-import { TSnapshot } from './interface';
-export declare const normalizeOptions: () => void;
+import { IFormatReference, TDependencies, THashes, TSnapshot, TSource } from './interface';
 export declare const getSources: (snapshot: TSnapshot) => string[];
+export declare const parseIntegrity: (integrity?: string) => THashes;
+export declare const formatIntegrity: (hashes: THashes) => string;
+export interface IReference {
+    protocol: string;
+    id: string;
+    name?: string;
+    host?: string;
+    [extra: string]: any;
+}
+export declare const parseTarballUrl: (resolution: string) => TSource | null;
+export declare const formatTarballUrl: (name: string, version: string, registry?: string, hash?: string) => string;
+export declare const normalizeDeps: (deps?: TDependencies) => TDependencies | undefined;
+export declare const processDeps: (deps?: TDependencies, formatter?: IFormatReference, opts?: any) => TDependencies | undefined;
+export declare const normalizeReference: (ref: string) => string;
+export declare const parseReference: (raw?: any) => IReference;
+export declare const referenceKeysSorter: (a: string, b: string) => number;
+export declare const mapReference: (current: string, targetProtocol: string, strategy?: string) => string;
+/**
+ * Replaces local monorepo cross-refs with the target protocol: workspace or semver
+ */
+export declare const switchMonorefs: ({ cwd, strategy, protocol, dryrun }: {
+    cwd?: string | undefined;
+    strategy?: string | undefined;
+    protocol?: string | undefined;
+    dryrun?: boolean | undefined;
+}) => Promise<Record<string, import("@semrel-extra/topo").IPackageEntry>>;

package/target/es6/format.d.ts

@@ -0,0 +1,2 @@
+import { IFormatOpts, TSnapshot } from './interface';
+export declare const format: (snapshot: TSnapshot, version: string, opts?: IFormatOpts) => string;

package/target/es6/formats/npm-1.d.ts

@@ -0,0 +1,24 @@
+import { ICheck, IFormat, IParse, IPreformat, IParseResolution, IFormatResolution } from '../interface';
+export declare const version = "npm-1";
+export type TNpm1LockfileEntry = {
+    version: string;
+    resolved: string;
+    integrity: string;
+    dev?: boolean;
+    requires?: Record<string, string>;
+    dependencies?: TNpm1LockfileDeps;
+};
+export type TNpm1LockfileDeps = Record<string, TNpm1LockfileEntry>;
+export type TNpm1Lockfile = {
+    lockfileVersion: 1;
+    name: string;
+    version: string;
+    requires?: true;
+    dependencies: TNpm1LockfileDeps;
+};
+export declare const check: ICheck;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TNpm1Lockfile>;
+export declare const format: IFormat;
+export declare const parseResolution: IParseResolution;
+export declare const formatResolution: IFormatResolution;

package/target/es6/formats/npm-2.d.ts

@@ -0,0 +1,16 @@
+import { ICheck, IFormat, IPreformat } from '../interface';
+import { TNpm1LockfileDeps } from './npm-1';
+import { TNpm3LockfileDeps } from './npm-3';
+export type TNpm2Lockfile = {
+    lockfileVersion: 2;
+    name: string;
+    version: string;
+    requires?: true;
+    packages: TNpm3LockfileDeps;
+    dependencies: TNpm1LockfileDeps;
+};
+export declare const version = "npm-2";
+export { parse } from './npm-3';
+export declare const check: ICheck;
+export declare const preformat: IPreformat<TNpm2Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/npm-3.d.ts

@@ -0,0 +1,36 @@
+import { ICheck, IFormat, IFormatResolution, IParseResolution, IPreformat, TEntry, TSnapshot, TSnapshotIndex } from '../interface';
+export type TNpm3LockfileEntry = {
+    name?: string;
+    version?: string;
+    resolved?: string;
+    integrity?: string;
+    dev?: boolean;
+    link?: boolean;
+    dependencies?: Record<string, string>;
+    engines?: Record<string, string>;
+    funding?: Record<string, string>;
+    peerDependencies?: Record<string, string>;
+    devDependencies?: Record<string, string>;
+    optionalDependencies?: Record<string, string>;
+    bin?: any;
+    license?: string;
+};
+export type TNpm3LockfileDeps = Record<string, TNpm3LockfileEntry>;
+export type TNpm3Lockfile = {
+    lockfileVersion: 3;
+    name: string;
+    version: string;
+    requires?: true;
+    packages: TNpm3LockfileDeps;
+};
+export declare const version = "npm-3";
+export declare const check: ICheck;
+export declare const parse: (lockfile: string) => TSnapshot;
+export declare const buildNmtree: (idx: TSnapshotIndex) => Record<string, {
+    entry: TEntry;
+    parent: string;
+}>;
+export declare const preformat: IPreformat<TNpm3Lockfile>;
+export declare const format: IFormat;
+export declare const parseResolution: IParseResolution;
+export declare const formatResolution: IFormatResolution;

package/target/es6/formats/yarn-1.d.ts

@@ -0,0 +1,14 @@
+import { TDependencies, ICheck, IFormat, IParse, IPreformat } from '../interface';
+export type TYarn1Lockfile = Record<string, {
+    version: string;
+    resolved: string;
+    integrity: string;
+    dependencies?: TDependencies;
+    optionalDependencies?: TDependencies;
+}>;
+export declare const version = "yarn-1";
+export declare const check: ICheck;
+export declare const preparse: (value: string) => TYarn1Lockfile;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn1Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-5.d.ts

@@ -0,0 +1,20 @@
+import { ICheck, IFormat, IParse, IPreformat, TDependencies, TDependenciesMeta } from '../interface';
+export type TYarn5Lockfile = Record<string, {
+    version: string;
+    resolution: string;
+    conditions?: string;
+    checksum: string;
+    languageName: string;
+    linkType: string;
+    dependencies?: TDependencies;
+    dependenciesMeta?: TDependenciesMeta;
+    optionalDependencies?: TDependencies;
+    peerDependencies?: TDependencies;
+    peerDependenciesMeta?: TDependenciesMeta;
+    bin?: Record<string, string>;
+}>;
+export declare const version = "yarn-5";
+export declare const check: ICheck;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn5Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-berry.d.ts

@@ -0,0 +1,22 @@
+import { ICheck, IFormat, IParse, IParseResolution, IPreformat, TDependencies, TDependenciesMeta, TSource } from '../interface';
+export type TYarn5Lockfile = Record<string, {
+    version: string;
+    resolution: string;
+    conditions?: string;
+    checksum: string;
+    languageName: string;
+    linkType: string;
+    dependencies?: TDependencies;
+    dependenciesMeta?: TDependenciesMeta;
+    optionalDependencies?: TDependencies;
+    peerDependencies?: TDependencies;
+    peerDependenciesMeta?: TDependenciesMeta;
+    bin?: Record<string, string>;
+}>;
+export declare const version = "yarn-berry";
+export declare const check: ICheck;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn5Lockfile>;
+export declare const format: IFormat;
+export declare const parseResolution: IParseResolution;
+export declare const formatResolution: ({ name, id, type, alias, hash }: TSource) => string;

package/target/es6/formats/yarn-classic.d.ts

@@ -0,0 +1,16 @@
+import { TDependencies, ICheck, IFormat, IParse, IPreformat, IParseResolution, IFormatResolution } from '../interface';
+export type TYarn1Lockfile = Record<string, {
+    version: string;
+    resolved: string;
+    integrity: string;
+    dependencies?: TDependencies;
+    optionalDependencies?: TDependencies;
+}>;
+export declare const version = "yarn-classic";
+export declare const check: ICheck;
+export declare const preparse: (value: string) => TYarn1Lockfile;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn1Lockfile>;
+export declare const format: IFormat;
+export declare const parseResolution: IParseResolution;
+export declare const formatResolution: IFormatResolution;

package/target/es6/index.d.ts

@@ -1,8 +1,6 @@
-import { parse as parseNpm1, format as formatNpm1 } from './npm-1';
-import { parse as parseYarn1, format as formatYarn1 } from './yarn-1';
-import { parse as parseYarn5, format as formatYarn5 } from './yarn-5';
-import { TSnapshot } from './interface';
-export declare const foo = "bar";
+export { TSnapshot } from './interface';
 export { getSources } from './common';
-export { parseNpm1, formatNpm1, parseYarn1, formatYarn1, parseYarn5, formatYarn5 };
-export declare const parse: (lockfile: string, pkg: string) => Promise<TSnapshot>;
+export { analyze } from './analyze';
+export { parse } from './parse';
+export { format } from './format';
+export { buildNmtree } from './formats/npm-3';