@antongolub/lockfile 0.0.0-snapshot.2 → 0.0.0-snapshot.5

package/README.md

@@ -1,105 +1,110 @@
 # lockfile
-Read, write, and convert npm (v1, v2) and yarn (classic and berry) lockfiles in any directions with reasonable losses.
+> Read and write lockfiles with reasonable losses
 
-### Motivation
-Every package manager brings its own philosophy of how to describe, store and control projects dependencies.
-This is awesome for developers, but literally becomes a ~~pain in *** ***~~ headache for isec, devops and release engineers.
+## Motivation
+Each package manager brings its own philosophy of how to describe, store and control project dependencies.
+It _seems_ acceptable for developers, but literally becomes a ~~pain in *** ***~~ headache for isec, devops and release engineers.
 This lib is a naive attempt to build a pm-independent, generic, extensible and reliable deps representation.
 
-The package manifest contains its own deps requirements, the lockfile defines the deps resolution snapshot<sup>*</sup>,
-so both of them are required to build a dependency graph. We can convert this data into a normalized representation for further analysis and processing (for example, to fix vulnerabilities).
-And then, if necessary, convert back to the original format.
+The `package.json` manifest contains its own deps requirements, the `lockfile` holds the deps resolution snapshot<sup>*</sup>,
+so both of them are required to build a dependency graph. We can try to convert this data into a normalized representation for further analysis and processing (for example, to fix vulnerabilities).
+And then, if necessary, try convert it back to the original/other format.
 
-### Status
+## Status
 ⚠️ Initial draft. Alpha-version
 
-### Getting started
-#### Install
+## Getting started
+### Install
 ```shell
 yarn add @antongolub/lockfile
 ```
 
-#### Usage
-```js
-import { parse, format } from '@antongolub/lockfile'
-
-const parsed = parse({
-  lockfile: './yarn.lock',
-  workspaces: {'': './package.json', 'foo': './packages/foo/package.json'},
-})
-
-// output
-{
-  entries: {
-    '@babel/code-frame@7.10.4': {
-      name: '@babel/code-frame',
-      version: '7.10.4',
-      scope: 'prod/dev/peer/opt',
-      integrities: {
-        sha512: 'hashsum',
-        sha256: '...',
-        sha1: '...',
-        md5: '...'
-      },
-      reference: {
-        sourceType: 'npm/git/file/workspace'
-        source: 'uri://remote/address',
-        linkType: 'hard/soft',
-        link: '<root>path/to/package'
-      },
-      dependencies: {
-        '@babel/highlight': '^7.10.4'
-      }
-    },
-    ...
-  },
-  meta: {
-    lockfile: {
-      type: 'yarn',
-      version: '5', // metadata format version
-    },
-    packageJson: {...},
-    workspaces: {
-      patterns: ['./packages/*'],
-      packages: {
-        '@qiwi/pijma-core': '<root>/packages/core/package.json'
-      }
-    }
-  },
-}
-
-const data = format({
-  ...parsed,
-  lockfileType: 'yarn-2'
-})
-// output
-`
-# This file is generated by running "yarn install" inside your project.
-# Manual changes might be lost - proceed with caution!
-
-__metadata:
-  version: 5
-  cacheKey: 8
-
-"@babel/code-frame@npm:7.10.4":
-  version: 7.10.4
-  resolution: "@babel/code-frame@npm:7.10.4"
-...
-`
+## Usage
+```ts
+import { parse, format, analyze } from '@antongolub/lockfile'
+
+const snapshot = parse('yarn.lock <raw contents>', 'package.json <raw contents>', './packages/foo/package.json <raw contents>')
+
+const lf = format(snapshot)
+const lf2 = format(snapshot, 'npm-1')         // Throws err: npm v1 meta does not support workspaces
+
+const meta = await readMeta()                 // reads local package.jsons data to gather required data like `engines`, `license`, `bins`, etc
+const meta2 = await fetchMeta(snapshot)       // does the same, but from the remote registry
+const lf3 = format(snapshot, 'npm-3', {meta}) // format with options
+
+const idx = analyze(snapshot)
+idx.edges
+// [
+//  [ '', '@antongolub/npm-test@4.0.1' ],
+//  [ '@antongolub/npm-test@4.0.1', '@antongolub/npm-test@3.0.1' ],
+//  [ '@antongolub/npm-test@3.0.1', '@antongolub/npm-test@2.0.1' ],
+//  [ '@antongolub/npm-test@2.0.1', '@antongolub/npm-test@1.0.0' ]
+// ]
 ```
 
-### Lockfile (meta) versions
-| Package manager  | Meta format | Supported | 
-|------------------|-------------|-----------|
-| npm <7           | 1           | x         |
-| npm >=7          | 2           |           |
-| yarn 1 (classic) | 1           | x         |
-| yarn 3           | 5, 6        | x         |
-| yarn 4           | 6, 7        |           |
+### Terms
+* `nmtree` — fs projection of deps, directories structure
+* `deptree` — bounds full dep paths with their resolved packages
+* `depgraph` — describes how resolved pkgs are related with each other
+
+### Lockfiles formats
+| Package manager  | Meta format | Read | Write |
+|------------------|-------------|------|-------|
+| npm <7           | 1           | ✓    | ✓     |
+| npm >=7          | 2           | ✓    |       |
+| npm >=9          | 3           | ✓    |       | 
+| yarn 1 (classic) | 1           | ✓    | ✓     |
+| yarn 2 (berry)   | 5, 6, 7     | ✓    | ✓     |
+
+### Reference protocols
+| Type      | Supported |
+|-----------|-----------|
+| semver    | ✓         |
+| npm       | ✓         |
+| workspace |           |
+| patch     |           |
+| file      |           |
+| github    |           |
+| tag       |           |
+
+https://yarnpkg.com/features/protocols
+
+### `TSnapshot`
+```ts
+export type TSnapshot = Record<string, {
+  name:       string
+  version:    string
+  ranges:     string[]
+  hashes:     {
+    sha512?:  string
+    sha256?:  string
+    sha1?:    string
+    checksum?: string
+    md5?:     string
+  }
+  source:     string
+  sourceType: TSourceType
+
+  // optional pm-specific lockfile meta
+  manifest?: TManifest
+  conditions?: string
+  dependencies?: TDependencies
+  dependenciesMeta?: TDependenciesMeta
+  optionalDependencies?: TDependencies
+  peerDependencies?: TDependencies
+  peerDependenciesMeta?: TDependenciesMeta
+  bin?: Record<string, string>
+  engines?: Record<string, string>
+  funding?: Record<string, string>
+}>
+```
 
 ### Caveats
-* Only `npm` links are supported for now
-* npm1: `optional: true` label is not supported by lockfile formatter
+* There is an infinite number of `nmtrees` that corresponds to the specified `deptree`, but among them there is a finite set of effective (sufficient) for the target criterion — for example, nesting, size, homogeneity of versions
+* npm1: `optional: true` label is not supported by `format`
+* yarn berry: no idea how to resolve and inject PnP patches https://github.com/yarnpkg/berry/tree/master/packages/plugin-compat
+* npm2 and npm3 requires `engines` and `funding` data, while yarn* or npm1 does not contain it
+* many `nmtree` projections may correspond the specified `depgraph`
 
 ### Inspired by
 * [synp](https://github.com/imsnif/synp)

package/package.json

@@ -1,23 +1,26 @@
 {
   "name": "@antongolub/lockfile",
-  "version": "0.0.0-snapshot.2",
+  "version": "0.0.0-snapshot.5",
   "publishConfig": {
     "access": "public"
   },
-  "description": "Read, write, and convert npm (v1, v2) and yarn (classic and berry) lockfiles in any directions with reasonable losses.",
-  "main": "./target/es6/index.mjs",
+  "description": "Read and write lockfiles with reasonable losses",
+  "main": "./target/es6/index.js",
   "type": "module",
-  "exports": "./target/es6/index.mjs",
+  "exports": "./target/es6/index.js",
   "dependencies": {
-    "js-yaml": "^4.1.0"
+    "@semrel-extra/topo": "^1.13.1",
+    "js-yaml": "^4.1.0",
+    "semver": "^7.5.4"
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.5",
-    "esbuild": "^0.15.12",
-    "esbuild-node-externals": "^1.5.0",
-    "tsc-esm-fix": "^2.20.5",
-    "tsm": "^2.2.2",
-    "typescript": "^4.8.4",
+    "@types/node": "^20.5.0",
+    "@types/semver": "^7.5.0",
+    "esbuild": "^0.19.2",
+    "esbuild-node-externals": "^1.8.0",
+    "tsm": "^2.3.0",
+    "typescript": "^5.1.6",
     "uvu": "^0.5.6"
   },
   "scripts": {
@@ -25,7 +28,8 @@
     "build:es6": "node ./build.js",
     "build:libdef": "tsc --emitDeclarationOnly --outDir target/es6",
     "build:esmfix": "tsc-esm-fix --target=target/es6 --ext=.mjs",
-    "test": "uvu -r tsm -i helpers 'src/test/ts/'",
+    "test": "yarn test:unit",
+    "test:unit": "DEBUG=true uvu -r tsm -i helpers 'src/test/ts/'",
     "publish:snapshot": "npm publish --no-git-tag-version --tag snapshot"
   },
   "files": [

package/target/es6/analyze.d.ts

@@ -0,0 +1,2 @@
+import { TSnapshot, TSnapshotIndex } from './interface';
+export declare const analyze: (snapshot: TSnapshot) => TSnapshotIndex;

package/target/es6/common.d.ts

@@ -1,3 +1,26 @@
-import { TSnapshot } from './interface';
-export declare const normalizeOptions: () => void;
+import { THashes, TManifest, TSnapshot } from './interface';
+export declare const formatTarballUrl: (name: string, version: string, registry?: string) => string;
 export declare const getSources: (snapshot: TSnapshot) => string[];
+export declare const isProd: (manifest: TManifest, name: string) => boolean;
+export declare const isDev: (manifest: TManifest, name: string) => boolean;
+export declare const isPeer: (manifest: TManifest, name: string) => boolean;
+export declare const isOptional: (manifest: TManifest, name: string) => boolean;
+export declare const parseIntegrity: (integrity: string) => THashes;
+export declare const formatIntegrity: (hashes: THashes) => string;
+export type IReferenceDeclaration = {
+    protocol: string;
+    raw: string;
+    version: string | null;
+    [extra: string]: any;
+};
+export declare const parseReference: (raw?: any) => IReferenceDeclaration;
+export declare const mapReference: (current: string, targetProtocol: string, strategy?: string) => string;
+/**
+ * Replaces local monorepo cross-refs with the target protocol: workspace or semver
+ */
+export declare const switchMonorefs: ({ cwd, strategy, protocol, dryrun }: {
+    cwd?: string | undefined;
+    strategy?: string | undefined;
+    protocol?: string | undefined;
+    dryrun?: boolean | undefined;
+}) => Promise<Record<string, import("@semrel-extra/topo").IPackageEntry>>;

package/target/es6/format.d.ts

@@ -0,0 +1,2 @@
+import { IFormatOpts, TSnapshot } from './interface';
+export declare const format: (snapshot: TSnapshot, version: string, opts?: IFormatOpts) => string;

package/target/es6/formats/npm-1.d.ts

@@ -0,0 +1,22 @@
+import { ICheck, IFormat, IParse, IPreformat } from '../interface';
+export declare const version = "npm-1";
+export type TNpm1LockfileEntry = {
+    version: string;
+    resolved: string;
+    integrity: string;
+    dev?: boolean;
+    requires?: Record<string, string>;
+    dependencies?: TNpm1LockfileDeps;
+};
+export type TNpm1LockfileDeps = Record<string, TNpm1LockfileEntry>;
+export type TNpm1Lockfile = {
+    lockfileVersion: 1;
+    name: string;
+    version: string;
+    requires?: true;
+    dependencies: TNpm1LockfileDeps;
+};
+export declare const check: ICheck;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TNpm1Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/npm-2.d.ts

@@ -0,0 +1,16 @@
+import { ICheck, IFormat, IPreformat } from '../interface';
+import { TNpm1LockfileDeps } from './npm-1';
+import { TNpm3LockfileDeps } from './npm-3';
+export type TNpm2Lockfile = {
+    lockfileVersion: 2;
+    name: string;
+    version: string;
+    requires?: true;
+    packages: TNpm3LockfileDeps;
+    dependencies: TNpm1LockfileDeps;
+};
+export declare const version = "npm-2";
+export { parse } from './npm-3';
+export declare const check: ICheck;
+export declare const preformat: IPreformat<TNpm2Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/npm-3.d.ts

@@ -0,0 +1,29 @@
+import { ICheck, IFormat, IPreformat, TSnapshot } from '../interface';
+export type TNpm3LockfileEntry = {
+    name?: string;
+    version: string;
+    resolved: string;
+    integrity: string;
+    dev?: boolean;
+    link?: boolean;
+    dependencies?: Record<string, string>;
+    engines?: Record<string, string>;
+    funding?: Record<string, string>;
+    peerDependencies?: Record<string, string>;
+    devDependencies?: Record<string, string>;
+    optionalDependencies?: Record<string, string>;
+    bin?: any;
+};
+export type TNpm3LockfileDeps = Record<string, TNpm3LockfileEntry>;
+export type TNpm3Lockfile = {
+    lockfileVersion: 3;
+    name: string;
+    version: string;
+    requires?: true;
+    packages: TNpm3LockfileDeps;
+};
+export declare const version = "npm-3";
+export declare const check: ICheck;
+export declare const parse: (lockfile: string) => TSnapshot;
+export declare const preformat: IPreformat<TNpm3Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-1.d.ts

@@ -0,0 +1,14 @@
+import { TDependencies, ICheck, IFormat, IParse, IPreformat } from '../interface';
+export type TYarn1Lockfile = Record<string, {
+    version: string;
+    resolved: string;
+    integrity: string;
+    dependencies?: TDependencies;
+    optionalDependencies?: TDependencies;
+}>;
+export declare const version = "yarn-1";
+export declare const check: ICheck;
+export declare const preparse: (value: string) => TYarn1Lockfile;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn1Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-5.d.ts

@@ -0,0 +1,20 @@
+import { ICheck, IFormat, IParse, IPreformat, TDependencies, TDependenciesMeta } from '../interface';
+export type TYarn5Lockfile = Record<string, {
+    version: string;
+    resolution: string;
+    conditions?: string;
+    checksum: string;
+    languageName: string;
+    linkType: string;
+    dependencies?: TDependencies;
+    dependenciesMeta?: TDependenciesMeta;
+    optionalDependencies?: TDependencies;
+    peerDependencies?: TDependencies;
+    peerDependenciesMeta?: TDependenciesMeta;
+    bin?: Record<string, string>;
+}>;
+export declare const version = "yarn-5";
+export declare const check: ICheck;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn5Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-berry.d.ts

@@ -0,0 +1,20 @@
+import { ICheck, IFormat, IParse, IPreformat, TDependencies, TDependenciesMeta } from '../interface';
+export type TYarn5Lockfile = Record<string, {
+    version: string;
+    resolution: string;
+    conditions?: string;
+    checksum: string;
+    languageName: string;
+    linkType: string;
+    dependencies?: TDependencies;
+    dependenciesMeta?: TDependenciesMeta;
+    optionalDependencies?: TDependencies;
+    peerDependencies?: TDependencies;
+    peerDependenciesMeta?: TDependenciesMeta;
+    bin?: Record<string, string>;
+}>;
+export declare const version = "yarn-berry";
+export declare const check: ICheck;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn5Lockfile>;
+export declare const format: IFormat;

package/target/es6/formats/yarn-classic.d.ts

@@ -0,0 +1,14 @@
+import { TDependencies, ICheck, IFormat, IParse, IPreformat } from '../interface';
+export type TYarn1Lockfile = Record<string, {
+    version: string;
+    resolved: string;
+    integrity: string;
+    dependencies?: TDependencies;
+    optionalDependencies?: TDependencies;
+}>;
+export declare const version = "yarn-1";
+export declare const check: ICheck;
+export declare const preparse: (value: string) => TYarn1Lockfile;
+export declare const parse: IParse;
+export declare const preformat: IPreformat<TYarn1Lockfile>;
+export declare const format: IFormat;

package/target/es6/index.d.ts

@@ -1,8 +1,5 @@
-import { parse as parseNpm1, format as formatNpm1 } from './npm-1';
-import { parse as parseYarn1, format as formatYarn1 } from './yarn-1';
-import { parse as parseYarn5, format as formatYarn5 } from './yarn-5';
-import { TSnapshot } from './interface';
-export declare const foo = "bar";
+export { TSnapshot } from './interface';
 export { getSources } from './common';
-export { parseNpm1, formatNpm1, parseYarn1, formatYarn1, parseYarn5, formatYarn5 };
-export declare const parse: (lockfile: string, pkg: string) => Promise<TSnapshot>;
+export { analyze } from './analyze';
+export { parse } from './parse';
+export { format } from './format';