@pnpm/worker 1100.0.1 → 1100.1.0

package/lib/index.d.ts

@@ -2,7 +2,7 @@ import { PnpmError } from '@pnpm/error';
 import type { FilesMap, PackageFilesResponse } from '@pnpm/store.cafs-types';
 import type { StoreIndex } from '@pnpm/store.index';
 import type { BundledManifest } from '@pnpm/types';
-import type { AddDirToStoreMessage, LinkPkgMessage, SymlinkAllModulesMessage, TarballExtractMessage } from './types.js';
+import type { AddDirToStoreMessage, FetchAndWriteCafsMessage, LinkPkgMessage, SymlinkAllModulesMessage, TarballExtractMessage } from './types.js';
 export declare function restartWorkerPool(): Promise<void>;
 export declare function finishWorkers(): Promise<void>;
 export declare function calcMaxWorkers(): number;
@@ -31,11 +31,16 @@ export declare class TarballIntegrityError extends PnpmError {
         url: string;
     });
 }
-type AddFilesFromTarballOptions = Pick<TarballExtractMessage, 'buffer' | 'storeDir' | 'filesIndexFile' | 'integrity' | 'readManifest' | 'pkg' | 'appendManifest'> & {
+type AddFilesFromTarballOptions = Pick<TarballExtractMessage, 'buffer' | 'storeDir' | 'filesIndexFile' | 'integrity' | 'readManifest' | 'pkg' | 'appendManifest' | 'ignoreFilePattern'> & {
     storeIndex: StoreIndex;
     url: string;
 };
 export declare function addFilesFromTarball(opts: AddFilesFromTarballOptions): Promise<AddFilesResult>;
+export declare function fetchAndWriteCafsFiles(opts: {
+    registryUrl: string;
+    storeDir: string;
+    digests: FetchAndWriteCafsMessage['digests'];
+}): Promise<number>;
 export interface ReadPkgFromCafsContext {
     storeDir: string;
     verifyStoreIntegrity: boolean;
@@ -54,6 +59,19 @@ export interface ReadPkgFromCafsResult {
     bundledManifest?: BundledManifest;
 }
 export declare function readPkgFromCafs(ctx: ReadPkgFromCafsContext, filesIndexFile: string, opts?: ReadPkgFromCafsOptions): Promise<ReadPkgFromCafsResult>;
+/**
+ * Temporarily change import concurrency. Called by the pnpm agent code path
+ * where there's no concurrent fetching competing for workers. Returns a
+ * disposer that restores the previous limiter — callers must invoke it (in a
+ * finally block) to avoid leaking the mutation to other installs in the same
+ * process (e.g. test suites).
+ *
+ * If two installs overlap, the disposer for the outer install would otherwise
+ * clobber the inner one's still-active limiter. Each disposer captures the
+ * limiter it installed and only restores when it's still the active one,
+ * leaving any newer override in place.
+ */
+export declare function setImportConcurrency(concurrency: number): () => void;
 export declare function importPackage(opts: Omit<LinkPkgMessage, 'type'>): Promise<{
     isBuilt: boolean;
     importMethod: string | undefined;

package/lib/index.js

@@ -144,6 +144,29 @@ export async function addFilesFromTarball(opts) {
             readManifest: opts.readManifest,
             pkg: opts.pkg,
             appendManifest: opts.appendManifest,
+            ignoreFilePattern: opts.ignoreFilePattern,
+        });
+    });
+}
+export async function fetchAndWriteCafsFiles(opts) {
+    if (!workerPool) {
+        workerPool = createTarballWorkerPool();
+    }
+    const localWorker = await workerPool.checkoutWorkerAsync(true);
+    return new Promise((resolve, reject) => {
+        localWorker.once('message', ({ status, error, filesWritten }) => {
+            workerPool.checkinWorker(localWorker);
+            if (status === 'error') {
+                reject(new PnpmError('CAFS_FETCH_WRITE', error.message));
+                return;
+            }
+            resolve(filesWritten);
+        });
+        localWorker.postMessage({
+            type: 'fetch-and-write-cafs',
+            registryUrl: opts.registryUrl,
+            storeDir: opts.storeDir,
+            digests: opts.digests,
         });
     });
 }
@@ -178,7 +201,32 @@ export async function readPkgFromCafs(ctx, filesIndexFile, opts) {
 // so, running them in parallel helps only to a point.
 // With local experimenting it was discovered that running 4 workers gives the best results.
 // Adding more workers actually makes installation slower.
-const limitImportingPackage = pLimit(4);
+let limitImportingPackage = pLimit(4);
+/**
+ * Temporarily change import concurrency. Called by the pnpm agent code path
+ * where there's no concurrent fetching competing for workers. Returns a
+ * disposer that restores the previous limiter — callers must invoke it (in a
+ * finally block) to avoid leaking the mutation to other installs in the same
+ * process (e.g. test suites).
+ *
+ * If two installs overlap, the disposer for the outer install would otherwise
+ * clobber the inner one's still-active limiter. Each disposer captures the
+ * limiter it installed and only restores when it's still the active one,
+ * leaving any newer override in place.
+ */
+export function setImportConcurrency(concurrency) {
+    if (!Number.isInteger(concurrency) || concurrency < 1) {
+        throw new Error(`setImportConcurrency: expected a positive integer, got ${concurrency}`);
+    }
+    const previous = limitImportingPackage;
+    const installed = pLimit(concurrency);
+    limitImportingPackage = installed;
+    return () => {
+        if (limitImportingPackage === installed) {
+            limitImportingPackage = previous;
+        }
+    };
+}
 export async function importPackage(opts) {
     return limitImportingPackage(async () => {
         if (!workerPool) {

package/lib/start.js

@@ -1,6 +1,7 @@
 import crypto from 'node:crypto';
 import fs from 'node:fs';
 import path from 'node:path';
+import util from 'node:util';
 import { parentPort } from 'node:worker_threads';
 import { pkgRequiresBuild } from '@pnpm/building.pkg-requires-build';
 import { formatIntegrity, parseIntegrity } from '@pnpm/crypto.integrity';
@@ -124,6 +125,10 @@ async function handleMessage(message) {
                 parentPort.postMessage({ status: 'success' });
                 break;
             }
+            case 'fetch-and-write-cafs': {
+                parentPort.postMessage(await fetchAndWriteCafs(message));
+                break;
+            }
         }
     }
     catch (e) { // eslint-disable-line
@@ -137,7 +142,7 @@ async function handleMessage(message) {
         });
     }
 }
-function addTarballToStore({ buffer, storeDir, integrity, filesIndexFile, appendManifest }) {
+function addTarballToStore({ buffer, storeDir, integrity, filesIndexFile, appendManifest, ignoreFilePattern }) {
     if (integrity) {
         const { algorithm, hexDigest } = parseIntegrity(integrity);
         const calculatedHash = crypto.hash(algorithm, buffer, 'hex');
@@ -157,7 +162,8 @@ function addTarballToStore({ buffer, storeDir, integrity, filesIndexFile, append
         cafsCache.set(storeDir, createCafs(storeDir));
     }
     const cafs = cafsCache.get(storeDir);
-    let { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true);
+    const ignore = ignoreFilePattern ? makeIgnoreFromPattern(ignoreFilePattern) : undefined;
+    let { filesIndex, manifest } = cafs.addFilesFromTarball(buffer, true, ignore);
     if (appendManifest && manifest == null) {
         manifest = appendManifest;
         addManifestToCafs(cafs, filesIndex, appendManifest);
@@ -189,6 +195,21 @@ function calcIntegrity(buffer) {
     const calculatedHash = crypto.hash('sha512', buffer, 'hex');
     return formatIntegrity('sha512', calculatedHash);
 }
+function makeIgnoreFromPattern(pattern) {
+    // `ignoreFilePattern` is a public field on FetchOptions, so callers that don't go
+    // through the binary-fetcher's validated `archiveFilters` path could still supply a
+    // bad regex. Convert the SyntaxError into a PnpmError with a stable code so it's
+    // actionable for users.
+    let regex;
+    try {
+        regex = new RegExp(pattern);
+    }
+    catch (err) {
+        const detail = util.types.isNativeError(err) ? `: ${err.message}` : '';
+        throw new PnpmError('INVALID_IGNORE_FILE_PATTERN', `Invalid ignoreFilePattern regex${detail}: ${pattern}`);
+    }
+    return (filename) => regex.test(filename);
+}
 function packToShared(data) {
     const packed = packForStorage(data);
     const shared = new SharedArrayBuffer(packed.byteLength);
@@ -374,4 +395,182 @@ function symlinkAllModules(opts) {
     }
     return { status: 'success' };
 }
+async function fetchAndWriteCafs(message) {
+    const http = await import('node:http');
+    const https = await import('node:https');
+    const { URL } = await import('node:url');
+    const { createGunzip } = await import('node:zlib');
+    const { contentPathFromHex } = await import('@pnpm/store.cafs');
+    // Preserve any path prefix on the agent URL (e.g. https://host/pnpm-agent/)
+    // by normalizing the base and using a relative URL.
+    const base = message.registryUrl.endsWith('/') ? message.registryUrl : `${message.registryUrl}/`;
+    const url = new URL('v1/files', base);
+    const requestFn = url.protocol === 'https:' ? https.request : http.request;
+    const body = JSON.stringify({ digests: message.digests });
+    const createdDirs = new Set();
+    // Build a set of digests we actually requested, so we can reject a
+    // misbehaving agent that streams unrelated entries and tries to write
+    // unbounded files into our CAFS. The set is keyed by `${digest}:${exec}`
+    // because the same digest may appear with different modes.
+    const requestedDigests = new Set();
+    for (const d of message.digests) {
+        requestedDigests.add(`${d.digest}:${d.executable ? 'x' : ''}`);
+    }
+    // Stream: HTTP response → gunzip → parse entries → write to CAFS.
+    // No buffering — files are written as data arrives.
+    return new Promise((resolve, reject) => {
+        let filesWritten = 0;
+        let buf = Buffer.alloc(0);
+        let headerSkipped = false;
+        let endMarkerSeen = false;
+        const END_MARKER = Buffer.alloc(64, 0);
+        const processBuffer = () => {
+            // Skip JSON header on first chunk
+            if (!headerSkipped && buf.length >= 4) {
+                const jsonLen = buf.readUInt32BE(0);
+                if (buf.length >= 4 + jsonLen) {
+                    buf = buf.subarray(4 + jsonLen);
+                    headerSkipped = true;
+                }
+                else {
+                    return;
+                }
+            }
+            // Parse complete file entries from the buffer
+            while (headerSkipped && !endMarkerSeen) {
+                if (buf.length < 64)
+                    break;
+                if (buf.subarray(0, 64).equals(END_MARKER)) {
+                    buf = buf.subarray(64);
+                    endMarkerSeen = true;
+                    break;
+                }
+                if (buf.length < 69)
+                    break; // 64 digest + 4 size + 1 mode
+                const size = buf.readUInt32BE(64);
+                const entryLen = 69 + size;
+                if (buf.length < entryLen)
+                    break; // incomplete entry
+                const digest = buf.subarray(0, 64).toString('hex');
+                const executable = (buf[68] & 0x01) !== 0;
+                const requestKey = `${digest}:${executable ? 'x' : ''}`;
+                if (!requestedDigests.has(requestKey)) {
+                    throw new Error(`pnpm agent /v1/files returned an entry that was not requested: digest=${digest} executable=${String(executable)}`);
+                }
+                // Consume the request so duplicates past the requested count also fail.
+                requestedDigests.delete(requestKey);
+                const content = buf.subarray(69, entryLen);
+                const relPath = contentPathFromHex(executable ? 'exec' : 'nonexec', digest);
+                const fullPath = path.join(message.storeDir, relPath);
+                const dir = path.dirname(fullPath);
+                if (!createdDirs.has(dir)) {
+                    fs.mkdirSync(dir, { recursive: true });
+                    createdDirs.add(dir);
+                }
+                try {
+                    fs.writeFileSync(fullPath, content, { flag: 'wx', mode: executable ? 0o755 : 0o644 });
+                }
+                catch (err) {
+                    if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST')) {
+                        throw err;
+                    }
+                    // EEXIST means the same digest is already at this CAFS path. CAFS
+                    // is content-addressed, so a complete file is by definition correct.
+                    // But a previous process could have crashed mid-write and left a
+                    // truncated file — the agent path skips integrity verification, so
+                    // we'd silently install garbage. Detect truncation by size and
+                    // overwrite atomically if the on-disk file is the wrong length.
+                    const onDiskSize = fs.statSync(fullPath).size;
+                    if (onDiskSize !== content.length) {
+                        const tmpPath = `${fullPath}.tmp-${process.pid}-${Date.now()}`;
+                        fs.writeFileSync(tmpPath, content, { mode: executable ? 0o755 : 0o644 });
+                        fs.renameSync(tmpPath, fullPath);
+                    }
+                }
+                filesWritten++;
+                buf = buf.subarray(entryLen);
+            }
+        };
+        // processBuffer is called from stream event handlers where a thrown
+        // exception would become `uncaughtException` and crash the worker.
+        // Surface errors via the Promise rejection instead.
+        const safeProcessBuffer = () => {
+            try {
+                processBuffer();
+                return true;
+            }
+            catch (err) {
+                reject(err);
+                req.destroy();
+                return false;
+            }
+        };
+        // Match the NDJSON client timeout — a stalled connection would otherwise
+        // hang the install indefinitely waiting on `fileDownloads`.
+        const FILES_REQUEST_TIMEOUT_MS = 600_000;
+        const req = requestFn(url, {
+            method: 'POST',
+            timeout: FILES_REQUEST_TIMEOUT_MS,
+            headers: {
+                'Content-Type': 'application/json',
+                'Content-Length': Buffer.byteLength(body),
+                'Accept-Encoding': 'gzip',
+            },
+        }, (res) => {
+            // Non-2xx responses are JSON error bodies from the agent server; read
+            // and reject so we never try to gunzip an error payload as a file stream.
+            if (typeof res.statusCode === 'number' && (res.statusCode < 200 || res.statusCode >= 300)) {
+                const chunks = [];
+                res.on('data', (chunk) => chunks.push(chunk));
+                res.on('end', () => {
+                    reject(new Error(`pnpm agent /v1/files responded with ${res.statusCode}: ${Buffer.concat(chunks).toString('utf-8')}`));
+                });
+                res.on('error', reject);
+                return;
+            }
+            let stream = res;
+            if (res.headers['content-encoding'] === 'gzip') {
+                const gunzip = createGunzip();
+                res.pipe(gunzip);
+                stream = gunzip;
+            }
+            stream.on('data', (chunk) => {
+                buf = Buffer.concat([buf, chunk]);
+                safeProcessBuffer();
+            });
+            stream.on('end', () => {
+                if (!safeProcessBuffer())
+                    return;
+                // Guard against a truncated response: the server must terminate the
+                // stream with the 64-byte end marker. If it didn't, or if there's a
+                // partial entry still in `buf`, fail — otherwise we'd silently leave
+                // the CAFS missing files.
+                if (!endMarkerSeen) {
+                    reject(new Error('pnpm agent /v1/files stream ended without the end marker'));
+                    return;
+                }
+                if (buf.length > 0) {
+                    reject(new Error(`pnpm agent /v1/files stream left ${buf.length} unparsed bytes after end marker`));
+                    return;
+                }
+                // Every received entry was drained from `requestedDigests` as it was
+                // parsed; anything still in the set means the server ended cleanly
+                // but omitted files, which would silently leave the CAFS incomplete.
+                if (requestedDigests.size > 0) {
+                    const sample = [...requestedDigests].slice(0, 3).join(', ');
+                    reject(new Error(`pnpm agent /v1/files omitted ${requestedDigests.size} requested entries (e.g. ${sample})`));
+                    return;
+                }
+                resolve({ status: 'success', filesWritten });
+            });
+            stream.on('error', reject);
+        });
+        req.on('timeout', () => {
+            req.destroy(new Error(`pnpm agent /v1/files request timed out after ${FILES_REQUEST_TIMEOUT_MS / 1000}s`));
+        });
+        req.on('error', reject);
+        req.write(body);
+        req.end();
+    });
+}
 //# sourceMappingURL=start.js.map

package/lib/types.d.ts

@@ -17,6 +17,12 @@ export interface TarballExtractMessage {
     readManifest?: boolean;
     pkg?: PkgNameVersion;
     appendManifest?: DependencyManifest;
+    /**
+     * Regex source matching the normalized relative path of files inside the tarball that
+     * should be skipped. Matching happens after the tar parser strips the top-level directory
+     * segment — i.e. against the same path form that is written to `filesIndex`.
+     */
+    ignoreFilePattern?: string;
 }
 export interface LinkPkgMessage {
     type: 'link';
@@ -65,3 +71,13 @@ export interface HardLinkDirMessage {
     src: string;
     destDirs: string[];
 }
+export interface FetchAndWriteCafsMessage {
+    type: 'fetch-and-write-cafs';
+    registryUrl: string;
+    storeDir: string;
+    digests: Array<{
+        digest: string;
+        size: number;
+        executable: boolean;
+    }>;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pnpm/worker",
-  "version": "1100.0.1",
+  "version": "1100.1.0",
   "description": "A worker for extracting package taralls to the store",
   "keywords": [
     "pnpm",
@@ -31,15 +31,15 @@
     "p-limit": "^7.1.0",
     "semver": "^7.7.2",
     "@pnpm/building.pkg-requires-build": "1100.0.1",
-    "@pnpm/crypto.integrity": "1100.0.0",
     "@pnpm/error": "1100.0.0",
-    "@pnpm/fs.hard-link-dir": "1100.0.0",
+    "@pnpm/crypto.integrity": "1100.0.0",
     "@pnpm/fs.graceful-fs": "1100.0.0",
+    "@pnpm/fs.hard-link-dir": "1100.0.0",
+    "@pnpm/store.cafs": "1100.1.0",
     "@pnpm/fs.symlink-dependency": "1100.0.1",
-    "@pnpm/store.cafs": "1100.0.1",
     "@pnpm/store.cafs-types": "1100.0.0",
-    "@pnpm/store.index": "1100.0.0",
-    "@pnpm/store.create-cafs-store": "1100.0.1"
+    "@pnpm/store.create-cafs-store": "1100.0.3",
+    "@pnpm/store.index": "1100.0.0"
   },
   "peerDependencies": {
     "@pnpm/logger": ">=1001.0.0 <1002.0.0"
@@ -47,9 +47,9 @@
   "devDependencies": {
     "@types/is-windows": "^1.0.2",
     "@types/semver": "7.7.1",
-    "@pnpm/types": "1101.0.0",
     "@pnpm/logger": "1100.0.0",
-    "@pnpm/worker": "1100.0.1"
+    "@pnpm/types": "1101.0.0",
+    "@pnpm/worker": "1100.1.0"
   },
   "engines": {
     "node": ">=22.13"