@antongolub/lockfile 0.0.0-snapshot.1 → 0.0.0-snapshot.10

package/README.md

@@ -1,105 +1,157 @@
 # lockfile
-Read, write, and convert npm (v1, v2) and yarn (classic and berry) lockfiles in any directions with reasonable losses.
+> Read and write lockfiles with reasonable losses
 
-### Motivation
-Every package manager brings its own philosophy of how to describe, store and control projects dependencies.
-This is awesome for developers, but literally becomes a ~~pain in *** ***~~ headache for isec, devops and release engineers.
+## Motivation
+Each package manager brings its own philosophy of how to describe, store and control project dependencies.
+It _seems_ acceptable for developers, but literally becomes a ~~pain in *** ***~~ headache for isec, devops and release engineers.
 This lib is a naive attempt to build a pm-independent, generic, extensible and reliable deps representation.
 
-The package manifest contains its own deps requirements, the lockfile defines the deps resolution snapshot<sup>*</sup>,
-so both of them are required to build a dependency graph. We can convert this data into a normalized representation for further analysis and processing (for example, to fix vulnerabilities).
-And then, if necessary, convert back to the original format.
+The `package.json` manifest contains its own deps requirements, the `lockfile` holds the deps resolution snapshot<sup>*</sup>,
+so both of them are required to build a dependency graph. We can try to convert this data into a normalized representation for further analysis and processing (for example, to fix vulnerabilities).
+And then, if necessary, try convert it back to the original/other format.
 
-### Status
+## Status
 ⚠️ Initial draft. Alpha-version
 
-### Getting started
-#### Install
+## Getting started
+### Install
 ```shell
 yarn add @antongolub/lockfile
 ```
 
-#### Usage
-```js
-import { parse, format } from '@antongolub/lockfile'
-
-const parsed = parse({
-  lockfile: './yarn.lock',
-  workspaces: {'': './package.json', 'foo': './packages/foo/package.json'},
-})
-
-// output
-{
-  entries: {
-    '@babel/code-frame@7.10.4': {
-      name: '@babel/code-frame',
-      version: '7.10.4',
-      scope: 'prod/dev/peer/opt',
-      integrities: {
-        sha512: 'hashsum',
-        sha256: '...',
-        sha1: '...',
-        md5: '...'
-      },
-      reference: {
-        sourceType: 'npm/git/file/workspace'
-        source: 'uri://remote/address',
-        linkType: 'hard/soft',
-        link: '<root>path/to/package'
-      },
-      dependencies: {
-        '@babel/highlight': '^7.10.4'
-      }
-    },
-    ...
-  },
-  meta: {
-    lockfile: {
-      type: 'yarn',
-      version: '5', // metadata format version
-    },
-    packageJson: {...},
-    workspaces: {
-      patterns: ['./packages/*'],
-      packages: {
-        '@qiwi/pijma-core': '<root>/packages/core/package.json'
-      }
-    }
-  },
-}
+## Usage
+tl;dr
+```ts
+import fs from 'fs/promises'
+import {parse, analyze} from '@antongolub/lockfile'
+
+const lf = await fs.readFile('yarn.lock', 'utf-8')
+const pkg = await fs.readFile('package.json', 'utf-8')
+
+const snapshot = parse(lf, pkg)
+const idx = analyze(snapshot)
+
+// idx.entries
+// idx.prod
+// idx.edges
+```
+
+## API
+```ts
+import { parse, format, analyze } from '@antongolub/lockfile'
+
+const snapshot = parse('yarn.lock <raw contents>', 'package.json <raw contents>', './packages/foo/package.json <raw contents>')
+
+const lf = format(snapshot)
+const lf2 = format(snapshot, 'npm-1')         // Throws err: npm v1 meta does not support workspaces
+
+const meta = await readMeta()                 // reads local package.jsons data to gather required data like `engines`, `license`, `bins`, etc
+const meta2 = await fetchMeta(snapshot)       // does the same, but from the remote registry
+const lf3 = format(snapshot, 'npm-3', {meta}) // format with options
 
-const data = format({
-  ...parsed,
-  lockfileType: 'yarn-2'
-})
-// output
-`
-# This file is generated by running "yarn install" inside your project.
-# Manual changes might be lost - proceed with caution!
-
-__metadata:
-  version: 5
-  cacheKey: 8
-
-"@babel/code-frame@npm:7.10.4":
-  version: 7.10.4
-  resolution: "@babel/code-frame@npm:7.10.4"
-...
-`
+const idx = analyze(snapshot)
+idx.edges
+// [
+//  [ '', '@antongolub/npm-test@4.0.1' ],
+//  [ '@antongolub/npm-test@4.0.1', '@antongolub/npm-test@3.0.1' ],
+//  [ '@antongolub/npm-test@3.0.1', '@antongolub/npm-test@2.0.1' ],
+//  [ '@antongolub/npm-test@2.0.1', '@antongolub/npm-test@1.0.0' ]
+// ]
 ```
 
-### Lockfile (meta) versions
-| Package manager  | Meta format | Supported | 
-|------------------|-------------|-----------|
-| npm <7           | 1           | x         |
-| npm >=7          | 2           |           |
-| yarn 1 (classic) | 1           | x         |
-| yarn 3           | 5, 6        | x         |
-| yarn 4           | 6, 7        |           |
+### Terms
+* `nmtree` — fs projection of deps, directories structure
+* `deptree` — bounds full dep paths with their resolved packages
+* `depgraph` — describes how resolved pkgs are related with each other
+
+### Lockfiles types
+| Package manager      | Meta format | Parse | Format |
+|----------------------|-------------|-------|--------|
+| npm <7               | 1           | ✓     | ✓      |
+| npm >=7              | 2           | ✓     |        |
+| npm >=9              | 3           | ✓     |        | 
+| yarn 1 (classic)     | 1           | ✓     | ✓      |
+| yarn 2, 3, 4 (berry) | 5, 6, 7     | ✓     | ✓      |
+
+### Reference protocols
+| Type      | Supported |
+|-----------|-----------|
+| semver    | ✓         |
+| npm       | ✓         |
+| workspace | _limited_ |
+| patch     | _limited_ |
+| file      |           |
+| github    |           |
+| tag       |           |
+
+https://yarnpkg.com/features/protocols
+
+### `TSnapshot`
+```ts
+export type TSnapshot = Record<string, TEntry>
+
+export type TEntry = {
+  name:       string
+  version:    string
+  ranges:     string[]
+  hashes:     {
+    sha512?:  string
+    sha256?:  string
+    sha1?:    string
+    checksum?: string
+    md5?:     string
+  }
+  source:     {
+    type: TSourceType // npm, workspace, gh, patch, etc
+    id: string
+    registry?: string
+  }
+  // optional pm-specific lockfile meta
+  manifest?: TManifest
+  conditions?: string
+  dependencies?: TDependencies
+  dependenciesMeta?: TDependenciesMeta
+  devDependencies?: TDependencies
+  optionalDependencies?: TDependencies
+  peerDependencies?: TDependencies
+  peerDependenciesMeta?: TDependenciesMeta
+  bin?: Record<string, string>
+  engines?: Record<string, string>
+  funding?: Record<string, string>
+}>
+```
+
+### `TSnapshotIndex`
+```ts
+export interface TSnapshotIndex {
+  snapshot: TSnapshot
+  entries: TEntry[]
+  roots: TEntry[]
+  edges: [string, string][]
+  tree: Record<string, {
+    key: string
+    chunks: string[]
+    parents: TEntry[]
+    id: string
+    name: string
+    version: string
+    entry: TEntry
+  }>
+  prod: Set<TEntry>
+  getEntryId ({name, version}: TEntry): string
+  getEntry (name: string, version?: string): TEntry | undefined,
+  getEntryByRange (name: string, range: string): TEntry | undefined
+  getEntryDeps(entry: TEntry): TEntry[]
+}
+```
 
 ### Caveats
-* Only `npm` links are supported for now
-* npm1: `optional: true` label is not supported by lockfile formatter
+* There is an infinite number of `nmtrees` that corresponds to the specified `deptree`, but among them there is a finite set of effective (sufficient) for the target criterion — for example, nesting, size, homogeneity of versions
+* npm1: `optional: true` label is not supported yet
+* yarn berry: no idea how to resolve and inject PnP patches https://github.com/yarnpkg/berry/tree/master/packages/plugin-compat
+* npm2 and npm3 requires `engines` and `funding` data, while yarn* or npm1 does not contain it
+* many `nmtree` projections may correspond to the specified `depgraph`
+* no idea what todo with yarn `patch`
 
 ### Inspired by
 * [synp](https://github.com/imsnif/synp)

package/package.json

@@ -1,23 +1,26 @@
 {
   "name": "@antongolub/lockfile",
-  "version": "0.0.0-snapshot.1",
+  "version": "0.0.0-snapshot.10",
   "publishConfig": {
     "access": "public"
   },
-  "description": "Read, write, and convert npm (v1, v2) and yarn (classic and berry) lockfiles in any directions with reasonable losses.",
-  "main": "./target/es6/index.mjs",
+  "description": "Read and write lockfiles with reasonable losses",
+  "main": "./target/es6/index.js",
   "type": "module",
-  "exports": "./target/es6/index.mjs",
+  "exports": "./target/es6/index.js",
   "dependencies": {
-    "js-yaml": "^4.1.0"
+    "@semrel-extra/topo": "^1.13.1",
+    "js-yaml": "^4.1.0",
+    "semver": "^7.5.4"
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.5",
-    "esbuild": "^0.15.12",
-    "esbuild-node-externals": "^1.5.0",
-    "tsc-esm-fix": "^2.20.5",
-    "tsm": "^2.2.2",
-    "typescript": "^4.8.4",
+    "@types/node": "^20.5.0",
+    "@types/semver": "^7.5.0",
+    "esbuild": "^0.19.2",
+    "esbuild-node-externals": "^1.8.0",
+    "tsm": "^2.3.0",
+    "typescript": "^5.1.6",
     "uvu": "^0.5.6"
   },
   "scripts": {
@@ -25,7 +28,8 @@
     "build:es6": "node ./build.js",
     "build:libdef": "tsc --emitDeclarationOnly --outDir target/es6",
     "build:esmfix": "tsc-esm-fix --target=target/es6 --ext=.mjs",
-    "test": "uvu -r tsm -i helpers 'src/test/ts/'",
+    "test": "yarn test:unit",
+    "test:unit": "DEBUG=true uvu -r tsm -i helpers 'src/test/ts/'",
     "publish:snapshot": "npm publish --no-git-tag-version --tag snapshot"
   },
   "files": [

package/target/es6/analyze.d.ts

@@ -0,0 +1,7 @@
+import { TEntry, TSnapshot, TSnapshotIndex } from './interface';
+export declare const getDeps: {
+    (entry: TEntry): Record<string, string>;
+    cache: WeakMap<TEntry, Record<string, string>>;
+};
+export declare const getId: (name?: string, version?: string) => string;
+export declare const analyze: (snapshot: TSnapshot) => TSnapshotIndex;

package/target/es6/common.d.ts

@@ -1,3 +1,26 @@
-import { TSnapshot } from './interface';
-export declare const normalizeOptions: () => void;
+import { THashes, TManifest, TSnapshot } from './interface';
+export declare const formatTarballUrl: (name: string, version: string, registry?: string) => string;
 export declare const getSources: (snapshot: TSnapshot) => string[];
+export declare const isProd: (manifest: TManifest, name: string) => boolean;
+export declare const isDev: (manifest: TManifest, name: string) => boolean;
+export declare const isPeer: (manifest: TManifest, name: string) => boolean;
+export declare const isOptional: (manifest: TManifest, name: string) => boolean;
+export declare const parseIntegrity: (integrity?: string) => THashes;
+export declare const formatIntegrity: (hashes: THashes) => string;
+export type IReferenceDeclaration = {
+    protocol: string;
+    raw: string;
+    version: string | null;
+    [extra: string]: any;
+};
+export declare const parseReference: (raw?: any) => IReferenceDeclaration;
+export declare const mapReference: (current: string, targetProtocol: string, strategy?: string) => string;
+/**
+ * Replaces local monorepo cross-refs with the target protocol: workspace or semver
+ */
+export declare const switchMonorefs: ({ cwd, strategy, protocol, dryrun }: {
+    cwd?: string | undefined;
+    strategy?: string | undefined;
+    protocol?: string | undefined;
+    dryrun?: boolean | undefined;
+}) => Promise<Record<string, import("@semrel-extra/topo").IPackageEntry>>;

package/target/es6/format.d.ts

@@ -0,0 +1,2 @@
+import { IFormatOpts, TSnapshot } from './interface';
+export declare const format: (snapshot: TSnapshot, version: string, opts?: IFormatOpts) => string;

package/target/es6/formats/npm-1.d.ts

@@ -0,0 +1,22 @@
+import { ICheck, IFormat, IParse, IPreformat } from '../interface';
+export declare const version = "npm-1";
+export type TNpm1LockfileEntry = {
+    version: string;
+    resolved: string;
+    integrity: string;
+    dev?: boolean;
+    requires?: Record<string, string>;
+    dependencies?: TNpm1LockfileDeps;
+};
+export type TNpm1LockfileDeps = Record<string, TNpm1LockfileEntry>;
+export type TNpm1Lockfile = {
+    lockfileVersion: 1;
+    name: string;
+    version: string;
+    requires?: true;
+    dependencies: TNpm1LockfileDeps;
+};
+export declare const check: ICheck;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TNpm1Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/npm-2.d.ts

@@ -0,0 +1,16 @@
+import { ICheck, IFormat, IPreformat } from '../interface';
+import { TNpm1LockfileDeps } from './npm-1';
+import { TNpm3LockfileDeps } from './npm-3';
+export type TNpm2Lockfile = {
+    lockfileVersion: 2;
+    name: string;
+    version: string;
+    requires?: true;
+    packages: TNpm3LockfileDeps;
+    dependencies: TNpm1LockfileDeps;
+};
+export declare const version = "npm-2";
+export { parse } from './npm-3';
+export declare const check: ICheck;
+export declare const preformat: IPreformat<TNpm2Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/npm-3.d.ts

@@ -0,0 +1,30 @@
+import { ICheck, IFormat, IPreformat, TSnapshot } from '../interface';
+export type TNpm3LockfileEntry = {
+    name?: string;
+    version?: string;
+    resolved?: string;
+    integrity?: string;
+    dev?: boolean;
+    link?: boolean;
+    dependencies?: Record<string, string>;
+    engines?: Record<string, string>;
+    funding?: Record<string, string>;
+    peerDependencies?: Record<string, string>;
+    devDependencies?: Record<string, string>;
+    optionalDependencies?: Record<string, string>;
+    bin?: any;
+    license?: string;
+};
+export type TNpm3LockfileDeps = Record<string, TNpm3LockfileEntry>;
+export type TNpm3Lockfile = {
+    lockfileVersion: 3;
+    name: string;
+    version: string;
+    requires?: true;
+    packages: TNpm3LockfileDeps;
+};
+export declare const version = "npm-3";
+export declare const check: ICheck;
+export declare const parse: (lockfile: string) => TSnapshot;
+export declare const preformat: IPreformat<TNpm3Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-1.d.ts

@@ -0,0 +1,14 @@
+import { TDependencies, ICheck, IFormat, IParse, IPreformat } from '../interface';
+export type TYarn1Lockfile = Record<string, {
+    version: string;
+    resolved: string;
+    integrity: string;
+    dependencies?: TDependencies;
+    optionalDependencies?: TDependencies;
+}>;
+export declare const version = "yarn-1";
+export declare const check: ICheck;
+export declare const preparse: (value: string) => TYarn1Lockfile;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn1Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-5.d.ts

@@ -0,0 +1,20 @@
+import { ICheck, IFormat, IParse, IPreformat, TDependencies, TDependenciesMeta } from '../interface';
+export type TYarn5Lockfile = Record<string, {
+    version: string;
+    resolution: string;
+    conditions?: string;
+    checksum: string;
+    languageName: string;
+    linkType: string;
+    dependencies?: TDependencies;
+    dependenciesMeta?: TDependenciesMeta;
+    optionalDependencies?: TDependencies;
+    peerDependencies?: TDependencies;
+    peerDependenciesMeta?: TDependenciesMeta;
+    bin?: Record<string, string>;
+}>;
+export declare const version = "yarn-5";
+export declare const check: ICheck;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn5Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-berry.d.ts

@@ -0,0 +1,20 @@
+import { ICheck, IFormat, IParse, IPreformat, TDependencies, TDependenciesMeta } from '../interface';
+export type TYarn5Lockfile = Record<string, {
+    version: string;
+    resolution: string;
+    conditions?: string;
+    checksum: string;
+    languageName: string;
+    linkType: string;
+    dependencies?: TDependencies;
+    dependenciesMeta?: TDependenciesMeta;
+    optionalDependencies?: TDependencies;
+    peerDependencies?: TDependencies;
+    peerDependenciesMeta?: TDependenciesMeta;
+    bin?: Record<string, string>;
+}>;
+export declare const version = "yarn-berry";
+export declare const check: ICheck;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn5Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-classic.d.ts

@@ -0,0 +1,14 @@
+import { TDependencies, ICheck, IFormat, IParse, IPreformat } from '../interface';
+export type TYarn1Lockfile = Record<string, {
+    version: string;
+    resolved: string;
+    integrity: string;
+    dependencies?: TDependencies;
+    optionalDependencies?: TDependencies;
+}>;
+export declare const version = "yarn-1";
+export declare const check: ICheck;
+export declare const preparse: (value: string) => TYarn1Lockfile;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn1Lockfile>;
+export declare const format: IFormat;

package/target/es6/index.d.ts

@@ -1,8 +1,5 @@
-import { parse as parseNpm1, format as formatNpm1 } from './npm-1';
-import { parse as parseYarn1, format as formatYarn1 } from './yarn-1';
-import { parse as parseYarn5, format as formatYarn5 } from './yarn-5';
-import { TSnapshot } from './interface';
-export declare const foo = "bar";
+export { TSnapshot } from './interface';
 export { getSources } from './common';
-export { parseNpm1, formatNpm1, parseYarn1, formatYarn1, parseYarn5, formatYarn5 };
-export declare const parse: (lockfile: string, pkg: string) => Promise<TSnapshot>;
+export { analyze } from './analyze';
+export { parse } from './parse';
+export { format } from './format';