@vltpkg/cli-sdk 1.1.1 → 1.2.0

package/dist/commands/cache.d.ts

@@ -4,12 +4,20 @@ import type { CommandFn, CommandUsage } from '../index.ts';
 import type { ViewOptions, Views } from '../view.ts';
 import { ViewClass } from '../view.ts';
 export type CacheMap = Record<string, ReturnType<CacheEntry['toJSON']>>;
+/** `verify`, `prune-store`: entries checked, and why each was removed */
+export type StoreResult = {
+    checked: number;
+    removed: Record<string, string>;
+    /** `verify <spec>`: specs not in the global store */
+    missing?: string[];
+};
+export type CacheResult = void | CacheMap | StoreResult;
 export type CacheSubcommands = keyof (typeof usageDef)['subcommands'];
 export declare class CacheView extends ViewClass {
     constructor(options: ViewOptions, conf: LoadedConfig);
     stdout(...args: unknown[]): void;
 }
-export declare const views: Views<void | CacheMap>;
+export declare const views: Views<CacheResult>;
 declare const usageDef: {
     readonly command: "cache";
     readonly usage: "<command> [flags]";
@@ -37,12 +45,20 @@ declare const usageDef: {
         };
         readonly 'delete-before': {
             readonly usage: "<date>";
-            readonly description: "Purge all cache items from before a given date. Date can be\n                    provided in any format that JavaScript can parse.";
+            readonly description: "Purge all cache items from before a given date, and\n                    global store entries written before it. Date can be\n                    provided in any format that JavaScript can parse.";
         };
         readonly 'delete-all': {
             readonly usage: "";
             readonly description: "Delete the entire cache folder to make vlt slower.";
         };
+        readonly verify: {
+            readonly usage: ["<package-spec> [<package-spec>...]", "--all"];
+            readonly description: "Check global store entries against their cached\n                    tarballs (file list, contents, exec bits) and remove\n                    any that differ or have no cached tarball, e.g. after\n                    a file in `node_modules` was edited in place.";
+        };
+        readonly 'prune-store': {
+            readonly usage: "";
+            readonly description: "Remove global store entries that no `node_modules`\n                    folder links to. Entries ever copied from (install\n                    scripts, `store-linker=copy`, a cache on another\n                    drive) stay: copies do not show use.";
+        };
     };
     readonly examples: {
         readonly 'vlt cache ls https://registry.npmjs.org/typescript': {
@@ -57,9 +73,17 @@ declare const usageDef: {
         readonly 'vlt cache delete-before 2025-01-01': {
             readonly description: "Delete all entries created before Jan 1, 2025";
         };
+        readonly 'verify --all': {
+            readonly description: "Check every global store entry";
+        };
+    };
+    readonly options: {
+        readonly all: {
+            readonly description: "With `verify`, check every global store entry.";
+        };
     };
 };
 export declare const needsRegistry = true;
 export declare const usage: CommandUsage;
-export declare const command: CommandFn<void | CacheMap>;
+export declare const command: CommandFn<CacheResult>;
 export {};

package/dist/commands/cache.js

@@ -1,7 +1,12 @@
 import { error } from '@vltpkg/error-cause';
 import { CacheEntry, assertOk } from '@vltpkg/registry-client';
 import { Spec } from '@vltpkg/spec';
+import { removeStoreEntry, storeEntryCopied, storeEntryLinked, storeEntryNames, storeEntryTime, verifyStoreEntry, } from '@vltpkg/tar/store-entry';
+import { readStoreIndex } from '@vltpkg/tar/store-index';
+import { integrityHex } from '@vltpkg/types';
+import { readFileSync } from 'node:fs';
 import { mkdir, rm } from 'node:fs/promises';
+import { resolve } from 'node:path';
 import prettyBytes from 'pretty-bytes';
 import { commandUsage } from "../config/usage.js";
 import { stderr, stdout } from "../output.js";
@@ -55,13 +60,28 @@ const usageDef = {
         },
         'delete-before': {
             usage: '<date>',
-            description: `Purge all cache items from before a given date. Date can be
+            description: `Purge all cache items from before a given date, and
+                    global store entries written before it. Date can be
                     provided in any format that JavaScript can parse.`,
         },
         'delete-all': {
             usage: '',
             description: `Delete the entire cache folder to make vlt slower.`,
         },
+        verify: {
+            usage: ['<package-spec> [<package-spec>...]', '--all'],
+            description: `Check global store entries against their cached
+                    tarballs (file list, contents, exec bits) and remove
+                    any that differ or have no cached tarball, e.g. after
+                    a file in \`node_modules\` was edited in place.`,
+        },
+        'prune-store': {
+            usage: '',
+            description: `Remove global store entries that no \`node_modules\`
+                    folder links to. Entries ever copied from (install
+                    scripts, \`store-linker=copy\`, a cache on another
+                    drive) stay: copies do not show use.`,
+        },
     },
     examples: {
         'vlt cache ls https://registry.npmjs.org/typescript': {
@@ -78,6 +98,14 @@ const usageDef = {
         'vlt cache delete-before 2025-01-01': {
             description: 'Delete all entries created before Jan 1, 2025',
         },
+        'verify --all': {
+            description: 'Check every global store entry',
+        },
+    },
+    options: {
+        all: {
+            description: 'With `verify`, check every global store entry.',
+        },
     },
 };
 export const needsRegistry = true;
@@ -99,6 +127,10 @@ export const command = async (conf) => {
             return deleteBefore(conf, args, view);
         case 'delete-all':
             return deleteAll(conf, args, view);
+        case 'verify':
+            return verify(conf, args, view);
+        case 'prune-store':
+            return pruneStore(conf, args, view);
         default: {
             throw error('Unrecognized cache command', {
                 code: 'EUSAGE',
@@ -212,7 +244,18 @@ const deleteBefore = async (conf, args, view) => {
             found: before,
         });
     }
-    return deleteEntries(conf, [], entry => !!entry.date && entry.date < before, view);
+    const map = await deleteEntries(conf, [], entry => !!entry.date && entry.date < before, view);
+    const { storeRoot } = conf.options;
+    let count = 0;
+    for (const hex of storeEntryNames(storeRoot)) {
+        const entry = resolve(storeRoot, hex);
+        if (storeEntryTime(entry) < before.getTime()) {
+            removeStoreEntry(entry);
+            count++;
+        }
+    }
+    view?.stdout(`Removed ${entries(count)}`);
+    return map;
 };
 const deleteKeys = async (conf, keys, view) => {
     if (!keys.length) {
@@ -225,9 +268,91 @@ const deleteKeys = async (conf, keys, view) => {
 const deleteAll = async (conf, _, view) => {
     const { cache } = await conf.options.packageInfo.getRegistryClient();
     await rm(cache.path(), { recursive: true, force: true });
+    // every global store layout version, and a store root set elsewhere
+    const rf = { recursive: true, force: true };
+    await rm(resolve(conf.options.cache, 'store'), rf);
+    if (cache.store)
+        await rm(cache.store, rf);
     await mkdir(cache.path(), { recursive: true });
     view?.stdout('Deleted all cache entries.');
 };
+const entries = (n) => `${n} global store entr${n === 1 ? 'y' : 'ies'}`;
+/** `name@version` from the entry's sidecar, else `hex` */
+const entryLabel = (entry, hex) => {
+    const i = readStoreIndex(entry);
+    return i?.name && i.version ? `${i.name}@${i.version}` : hex;
+};
+const verify = async (conf, specs, view) => {
+    const all = !!conf.values.all;
+    if (!specs.length && !all) {
+        throw error('Must provide specs to verify, or --all', {
+            code: 'EUSAGE',
+        });
+    }
+    const { packageInfo, storeRoot } = conf.options;
+    const cachePath = (await packageInfo.getRegistryClient()).cache.path();
+    const present = new Set(storeEntryNames(storeRoot));
+    const checks = [];
+    const missing = [];
+    if (all) {
+        for (const hex of present)
+            checks.push([hex, hex]);
+    }
+    else {
+        for (const spec of specs) {
+            const { integrity } = await packageInfo.resolve(Spec.parseArgs(spec, conf.options));
+            const hex = integrityHex(integrity);
+            if (hex && present.has(hex))
+                checks.push([spec, hex]);
+            else {
+                missing.push(spec);
+                view?.stdout('Not in the global store:', spec);
+            }
+        }
+    }
+    const removed = {};
+    for (const [key, hex] of checks) {
+        const entry = resolve(storeRoot, hex);
+        let tarball;
+        try {
+            tarball = CacheEntry.decode(readFileSync(resolve(cachePath, hex))).buffer();
+        }
+        catch { }
+        const reason = tarball ? verifyStoreEntry(entry, tarball) : 'no cached tarball';
+        if (reason) {
+            // read before the sidecar goes
+            const label = all ? entryLabel(entry, hex) : key;
+            removeStoreEntry(entry);
+            removed[key] = reason;
+            view?.stdout('-', label, reason);
+        }
+    }
+    const n = Object.keys(removed).length;
+    view?.stdout(`Checked ${entries(checks.length)}, removed ${n}`);
+    return {
+        checked: checks.length,
+        removed,
+        ...(!all && { missing }),
+    };
+};
+const pruneStore = async (conf, _, view) => {
+    const { storeRoot } = conf.options;
+    const names = storeEntryNames(storeRoot);
+    const removed = {};
+    for (const hex of names) {
+        const entry = resolve(storeRoot, hex);
+        const index = readStoreIndex(entry);
+        if (storeEntryCopied(entry, index) ||
+            storeEntryLinked(entry, index)) {
+            continue;
+        }
+        removeStoreEntry(entry);
+        removed[hex] = 'unused';
+    }
+    const n = Object.keys(removed).length;
+    view?.stdout(`Removed ${n} of ${entries(names.length)}`);
+    return { checked: names.length, removed };
+};
 const add = async (conf, specs, view) => {
     if (!specs.length) {
         throw error('Must provide specs to add to the cache', {

package/dist/commands/install/reporter.js

@@ -38,7 +38,9 @@ const App = ({ trailer }) => {
             if (state === 'start') {
                 setRequests(p => p + 1);
             }
-            else if (state === 'cache' || state === 'stale') {
+            else if (state === 'cache' ||
+                state === 'stale' ||
+                state === 'store') {
                 setCacheHit(p => p + 1);
             }
         };

package/dist/commands/view.js

@@ -301,7 +301,7 @@ export const command = async (conf) => {
     const pic = new PackageInfoClient(conf.options);
     // Fetch the full packument (needs time, maintainers) and resolved manifest
     const [packument, resolvedManifest] = await Promise.all([
-        pic.packument(spec),
+        pic.packument(spec, { full: true }),
         pic.manifest(spec),
     ]);
     const manifest = resolvedManifest;

package/dist/config/definition.d.ts

@@ -73,6 +73,8 @@ export declare const getCommand: (s?: string) => Commands[keyof Commands] | unde
 export declare const recordFields: readonly ["git-hosts", "registries", "git-host-archives", "scoped-registries", "jsr-registries"];
 export type RecordField = (typeof recordFields)[number];
 export declare const isRecordField: (s: string) => s is RecordField;
+/** Valid `store-linker` values. */
+export declare const storeLinkers: readonly ["auto", "hardlink", "copy", "unpack"];
 export declare const definition: import("jackspeak").Jack<{
     color: {
         short: string;
@@ -133,6 +135,12 @@ export declare const definition: import("jackspeak").Jack<{
         description: string;
         default: string;
     };
+    'store-linker': {
+        hint: string;
+        description: string;
+        validOptions: readonly ["auto", "hardlink", "copy", "unpack"];
+        default: string;
+    };
     tag: {
         description: string;
         default: string;
@@ -159,6 +167,7 @@ export declare const definition: import("jackspeak").Jack<{
     };
 } & {
     cache: import("jackspeak").ConfigOption<"string", false, readonly string[] | undefined>;
+    'store-linker': import("jackspeak").ConfigOption<"string", false, readonly string[] | undefined>;
     tag: import("jackspeak").ConfigOption<"string", false, readonly string[] | undefined>;
     before: import("jackspeak").ConfigOption<"string", false, readonly string[] | undefined>;
     os: import("jackspeak").ConfigOption<"string", false, readonly string[] | undefined>;

package/dist/config/definition.js

@@ -109,6 +109,13 @@ export const recordFields = [
     'jsr-registries',
 ];
 export const isRecordField = (s) => recordFields.includes(s);
+/** Valid `store-linker` values. */
+export const storeLinkers = [
+    'auto',
+    'hardlink',
+    'copy',
+    'unpack',
+];
 const j = jack({
     envPrefix: 'VLT',
     allowPositionals: true,
@@ -308,6 +315,32 @@ export const definition = j
       `,
         default: cacheDir,
     },
+    'store-linker': {
+        hint: 'auto | hardlink | copy | unpack',
+        description: `How packages are placed in \`node_modules\`. Every
+                    linker but \`unpack\` fills a global store under
+                    \`cache\` in the background, then places packages
+                    from it.
+
+                    - auto: Default. On Linux, hardlink from the global
+                      store, copying any file that cannot be linked.
+                      Elsewhere, same as \`unpack\`: a hardlink is the
+                      slowest way to place a file on APFS, so macOS and
+                      Windows unpack by default.
+                    - hardlink: Hardlink from the global store on every
+                      platform.
+                    - copy: Copy from the global store. Use this when
+                      editing files in \`node_modules\` in place, since
+                      a hardlink shares its content with every project.
+                    - unpack: Unpack each package tarball, skipping the
+                      global store.
+
+                    Packages with install scripts are always copied.
+                    An invalid \`VLT_STORE_LINKER\` warns and uses
+                    \`unpack\`.`,
+        validOptions: storeLinkers,
+        default: 'auto',
+    },
     tag: {
         description: `Default \`dist-tag\` to install or publish`,
         default: 'latest',

package/dist/config/index.d.ts

@@ -81,6 +81,8 @@ export type ConfigOptions = ConfigOptionsNoExtras & Pick<SpecOptions, 'catalog'
     packageJson: PackageJson;
     scurry: PathScurry;
     projectRoot: string;
+    /** global store root, under `cache` */
+    storeRoot: string;
     monorepo?: Monorepo;
     packageInfo: PackageInfoClient;
 };

package/dist/config/index.js

@@ -23,6 +23,7 @@
 import { error } from '@vltpkg/error-cause';
 import { PackageInfoClient } from '@vltpkg/package-info';
 import { PackageJson } from '@vltpkg/package-json';
+import { storeRoot } from '@vltpkg/registry-client/store-root';
 import { resetCaches } from '@vltpkg/dep-id';
 import { getOptions } from '@vltpkg/spec';
 import { assertRecordStringString, assertRecordStringT, isObject, isRecordStringString, } from '@vltpkg/types';
@@ -31,7 +32,7 @@ import { Monorepo } from '@vltpkg/workspaces';
 import { readFile, rm, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
 import { PathScurry } from 'path-scurry';
-import { commands, definition, defaultValues, getCommand, isRecordField, recordFields, } from "./definition.js";
+import { commands, definition, defaultValues, getCommand, isRecordField, recordFields, storeLinkers, } from "./definition.js";
 import { merge } from "./merge.js";
 import { cloneLayer, mergeLayers } from "./merge-layers.js";
 export { commands, definition, isRecordField, recordFields, };
@@ -92,6 +93,7 @@ export const recordsToPairs = (obj) => {
         k === 'packageJson' ||
         k === 'monorepo' ||
         k === 'projectRoot' ||
+        k === 'storeRoot' ||
         k === 'packageInfo'))
         .map(([k, v]) => [
         k,
@@ -170,6 +172,7 @@ export class Config {
         const asRecords = pairsToRecords(this.parse().values);
         const extras = {
             projectRoot: this.projectRoot,
+            storeRoot: storeRoot(asRecords.cache),
             scurry,
             packageJson,
             monorepo: Monorepo.maybeLoad(this.projectRoot, {
@@ -250,6 +253,12 @@ export class Config {
             return this;
         // Store the original args for potential reload
         this.#originalArgs = [...args];
+        // an invalid env linker is treated as unpack
+        const badLinker = process.env.VLT_STORE_LINKER;
+        const fixLinker = badLinker !== undefined &&
+            !storeLinkers.includes(badLinker);
+        if (fixLinker)
+            process.env.VLT_STORE_LINKER = 'unpack';
         const envKeys = Object.keys(defaultValues).filter(k => process.env[envKey(k)] !== undefined);
         const parent = parentEnv();
         // file layer values, before env replaces them
@@ -267,6 +276,15 @@ export class Config {
             this.jack.setConfigValues(cmdPairs);
         // ok, applied cmd-specific defaults, do rest of the parse
         this.jack.applyDefaults(p);
+        // `auto` only hardlinks from the global store on linux: on APFS a
+        // hardlink is the slowest way to place a file, so darwin and win32
+        // unpack by default. the env layer is rewritten too, so `explicit`
+        // records the resolved value.
+        if (p.values['store-linker'] === 'auto' &&
+            process.platform !== 'linux') {
+            p.values['store-linker'] = 'unpack';
+            envValues['store-linker'] = 'unpack';
+        }
         // what was set on the cli or env for this run. record fields hold
         // only those pairs, env then cli, so later ones win per key.
         const values = p.values;
@@ -318,6 +336,11 @@ export class Config {
         if (p.values.verbose && p.values.loglevel === 'info') {
             p.values.loglevel = 'verbose';
         }
+        const { loglevel } = p.values;
+        if (fixLinker && loglevel !== 'silent' && loglevel !== 'error') {
+            // eslint-disable-next-line no-console
+            console.error(`Warning: invalid VLT_STORE_LINKER ${JSON.stringify(badLinker)}, using ${p.values['store-linker']}`);
+        }
         /* c8 ignore start - unpossible */
         if (!isParsed(this))
             throw error('failed to parse config');

package/dist/custom-help.js

@@ -63,7 +63,7 @@ const commandHelp = {
         defaultOrder: 4,
     },
     cache: {
-        args: '[add|ls|info|clean|delete|delete-before|delete-all]',
+        args: '[add|ls|info|clean|delete|delete-before|delete-all|verify|prune-store]',
         desc: 'Manage the package cache',
     },
     ci: {

package/dist/print-err.js

@@ -16,7 +16,9 @@ const formatURL = (v, format) => v instanceof URL ? v.toString() : /* c8 ignore
 // True when `v` overrides the default `Object.prototype.toString`, i.e.
 // stringifying it yields something meaningful (like a `Spec` → `next@*`)
 // rather than `[object Object]`.
-const hasCustomToString = (v) => isObject(v) && v.toString !== Object.prototype.toString;
+const hasCustomToString = (v) => typeof v === 'object' &&
+    v !== null &&
+    v.toString !== Object.prototype.toString;
 // Render a value that has a meaningful custom `toString()` (such as a
 // `Spec`, e.g. `next@*`) as a concise string rather than dumping its
 // entire internal structure via util.inspect. Falls back to `format`
@@ -161,8 +163,10 @@ const printCode = (err, usage, stderr, format) => {
             if (from) {
                 stderr(indent(`From: ${format(from)}`));
             }
-            if (response) {
-                stderr(indent(`Response: ${format(response)}`));
+            // the full response is in the error log; here only the status
+            const status = response?.statusCode;
+            if (status) {
+                stderr(indent(`Status: ${format(status)}`));
             }
             return { file: true };
         }

package/dist/verbose-log.js

@@ -15,6 +15,7 @@ const tagStyle = (state, statusCode) => {
     switch (state) {
         case 'cache':
         case 'stale':
+        case 'store':
         case '304':
             return 'green';
         case 'start':

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@vltpkg/cli-sdk",
   "description": "The source for the vlt CLI",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/vltpkg/vltpkg.git",
@@ -14,30 +14,31 @@
   },
   "dependencies": {
     "@resvg/resvg-wasm": "^2.6.2",
-    "@vltpkg/config": "1.1.1",
-    "@vltpkg/dep-id": "1.1.1",
-    "@vltpkg/dot-prop": "1.1.1",
-    "@vltpkg/error-cause": "1.1.1",
-    "@vltpkg/git": "1.1.1",
-    "@vltpkg/graph": "1.1.1",
-    "@vltpkg/graph-run": "1.1.1",
-    "@vltpkg/init": "1.1.1",
-    "@vltpkg/output": "1.1.1",
-    "@vltpkg/package-info": "1.1.1",
-    "@vltpkg/package-json": "1.1.1",
-    "@vltpkg/promise-spawn": "1.1.1",
-    "@vltpkg/query": "1.1.1",
-    "@vltpkg/registry-client": "1.1.1",
-    "@vltpkg/rollback-remove": "1.1.1",
-    "@vltpkg/run": "1.1.1",
-    "@vltpkg/security-archive": "1.1.1",
-    "@vltpkg/spec": "1.1.1",
-    "@vltpkg/types": "1.1.1",
-    "@vltpkg/url-open": "1.1.1",
-    "@vltpkg/vlt-json": "1.1.1",
-    "@vltpkg/vlx": "1.1.1",
-    "@vltpkg/workspaces": "1.1.1",
-    "@vltpkg/xdg": "1.1.1",
+    "@vltpkg/config": "1.2.0",
+    "@vltpkg/dep-id": "1.2.0",
+    "@vltpkg/dot-prop": "1.2.0",
+    "@vltpkg/error-cause": "1.2.0",
+    "@vltpkg/git": "1.2.0",
+    "@vltpkg/graph": "1.2.0",
+    "@vltpkg/graph-run": "1.2.0",
+    "@vltpkg/init": "1.2.0",
+    "@vltpkg/output": "1.2.0",
+    "@vltpkg/package-info": "1.2.0",
+    "@vltpkg/package-json": "1.2.0",
+    "@vltpkg/promise-spawn": "1.2.0",
+    "@vltpkg/query": "1.2.0",
+    "@vltpkg/registry-client": "1.2.0",
+    "@vltpkg/rollback-remove": "1.2.0",
+    "@vltpkg/run": "1.2.0",
+    "@vltpkg/security-archive": "1.2.0",
+    "@vltpkg/spec": "1.2.0",
+    "@vltpkg/tar": "1.2.0",
+    "@vltpkg/types": "1.2.0",
+    "@vltpkg/url-open": "1.2.0",
+    "@vltpkg/vlt-json": "1.2.0",
+    "@vltpkg/vlx": "1.2.0",
+    "@vltpkg/workspaces": "1.2.0",
+    "@vltpkg/xdg": "1.2.0",
     "ansi-to-pre": "^1.0.6",
     "beautiful-mermaid": "^1.1.3",
     "chalk": "^5.6.2",
@@ -66,7 +67,7 @@
     "@types/ssri": "^7.1.5",
     "eslint": "^9.39.1",
     "prettier": "^3.7.4",
-    "tap": "^21.5.0",
+    "tap": "^21.8.0",
     "typedoc": "~0.27.9",
     "typescript": "5.7.3",
     "typescript-eslint": "^8.49.0"