@pnpm/fetching.tarball-fetcher 1102.1.7 → 1102.1.8

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @pnpm/tarball-fetcher
 
+## 1102.1.8
+
+### Patch Changes
+
+- Large package downloads and large files inside gzip and bzip2 package archives now use bounded memory during installation. Package manifests and archive metadata larger than 64 MiB are rejected.
+
+- Updated dependencies:
+  - @pnpm/error@1100.2.2
+  - @pnpm/exec.prepare-package@1100.0.42
+  - @pnpm/fs.graceful-fs@1100.2.5
+  - @pnpm/fs.packlist@1100.0.8
+  - @pnpm/network.fetch@1100.1.22
+  - @pnpm/store.index@1101.0.1
+
 ## 1102.1.7
 
 ### Patch Changes

package/lib/localTarballFetcher.js

@@ -1,15 +1,18 @@
 import path from 'node:path';
 import gfs from '@pnpm/fs.graceful-fs';
 import { addFilesFromTarball } from '@pnpm/worker';
+import { MAX_BUFFERED_DOWNLOAD_SIZE } from './remoteTarballFetcher.js';
 const isAbsolutePath = /^\/|^[A-Z]:/i;
 export function createLocalTarballFetcher(storeIndex) {
     const fetch = async (cafs, resolution, opts) => {
         const tarball = resolvePath(opts.lockfileDir, resolution.tarball.slice(5));
-        const buffer = gfs.readFileSync(tarball);
+        const source = gfs.statSync(tarball).size > MAX_BUFFERED_DOWNLOAD_SIZE
+            ? { tarballFile: tarball }
+            : { buffer: gfs.readFileSync(tarball) };
         const result = await addFilesFromTarball({
             storeDir: cafs.storeDir,
             storeIndex,
-            buffer,
+            ...source,
             filesIndexFile: opts.filesIndexFile,
             integrity: resolution.integrity,
             readManifest: opts.readManifest,

package/lib/remoteTarballFetcher.d.ts

@@ -3,6 +3,7 @@ import type { FetchOptions, FetchResult } from '@pnpm/fetching.fetcher-base';
 import type { FetchFromRegistry, GetAuthHeader, RetryTimeoutOptions } from '@pnpm/fetching.types';
 import type { Cafs } from '@pnpm/store.cafs-types';
 import type { StoreIndex } from '@pnpm/store.index';
+export declare const MAX_BUFFERED_DOWNLOAD_SIZE: number;
 export interface HttpResponse {
     body: string;
 }

package/lib/remoteTarballFetcher.js

@@ -1,4 +1,6 @@
+import { promises as fs } from 'node:fs';
 import { isIP } from 'node:net';
+import path from 'node:path';
 import { requestRetryLogger } from '@pnpm/core-loggers';
 import { FetchError, FetchTimeoutError, isError, isFetchTimeoutError, redactUrlForDisplay } from '@pnpm/error';
 import { globalWarn } from '@pnpm/logger';
@@ -7,6 +9,7 @@ import { addFilesFromTarball } from '@pnpm/worker';
 import * as retry from '@zkochan/retry';
 import throttle from 'lodash.throttle';
 import { BadTarballError } from './errorTypes/index.js';
+export const MAX_BUFFERED_DOWNLOAD_SIZE = 64 * 1024 * 1024;
 const BIG_TARBALL_SIZE = 1024 * 1024 * 5; // 5 MB
 export function createDownloader(fetchFromRegistry, gotOpts) {
     const defaultRetryOpts = {
@@ -88,25 +91,35 @@ async function fetchTarball(ctx) {
             fetchMinSpeedKiBps: ctx.fetchMinSpeedKiBps,
             onProgress,
             size,
+            storeDir: ctx.opts.cafs.storeDir,
             url: ctx.url,
         });
     }
     catch (err) {
         throw wrapFetchError(err, ctx.url, ctx.attempt, ctx.gotOpts.timeout);
     }
-    return addFilesFromTarball({
-        buffer: data,
-        storeDir: ctx.opts.cafs.storeDir,
-        storeIndex: ctx.opts.storeIndex,
-        readManifest: ctx.opts.readManifest,
-        integrity: ctx.opts.integrity,
-        filesIndexFile: ctx.opts.filesIndexFile,
-        pkgId: ctx.opts.pkgId,
-        url: ctx.url,
-        pkg: ctx.opts.pkg,
-        appendManifest: ctx.opts.appendManifest,
-        ignoreFilePattern: ctx.opts.ignoreFilePattern,
-    });
+    return importDownloadedTarball(ctx, data);
+}
+async function importDownloadedTarball(ctx, data) {
+    try {
+        return await addFilesFromTarball({
+            buffer: data.buffer,
+            tarballFile: data.tarballFile,
+            storeDir: ctx.opts.cafs.storeDir,
+            storeIndex: ctx.opts.storeIndex,
+            readManifest: ctx.opts.readManifest,
+            integrity: ctx.opts.integrity,
+            filesIndexFile: ctx.opts.filesIndexFile,
+            pkgId: ctx.opts.pkgId,
+            url: ctx.url,
+            pkg: ctx.opts.pkg,
+            appendManifest: ctx.opts.appendManifest,
+            ignoreFilePattern: ctx.opts.ignoreFilePattern,
+        });
+    }
+    finally {
+        await data.cleanup?.();
+    }
 }
 function parseTarballContentLength(res) {
     const isEncoded = isContentEncoded(res.headers.get('content-encoding'));
@@ -116,54 +129,85 @@ function parseTarballContentLength(res) {
 }
 async function readResponseBody(opts) {
     const startTime = Date.now();
-    let downloaded = 0;
-    let data;
-    if (opts.size !== null) {
-        data = await readKnownSizeBody(opts.body, opts.size, opts.url, opts.onProgress);
-        downloaded = opts.size;
+    if (opts.size != null && opts.size <= MAX_BUFFERED_DOWNLOAD_SIZE) {
+        const buffer = await readKnownSizeBody(opts);
+        checkDownloadSpeed(opts.size, startTime, opts.fetchMinSpeedKiBps, opts.url);
+        return { buffer };
     }
-    else {
-        const result = await readChunkedBody(opts.body, opts.onProgress);
-        data = result.data;
-        downloaded = result.downloaded;
+    const state = { chunks: [], downloaded: 0 };
+    try {
+        for await (const chunk of opts.body) {
+            const bytes = chunk;
+            state.downloaded += bytes.byteLength;
+            checkDownloadedSize(opts, state.downloaded, false);
+            await bufferDownloadChunk(state, bytes, opts.storeDir);
+            opts.onProgress?.(state.downloaded);
+        }
+        checkDownloadedSize(opts, state.downloaded, true);
+        checkDownloadSpeed(state.downloaded, startTime, opts.fetchMinSpeedKiBps, opts.url);
+        await state.file?.close();
+        return finishDownloadBuffer(state);
+    }
+    catch (err) {
+        await state.file?.close().catch(() => { });
+        if (state.directory)
+            await fs.rm(state.directory, { recursive: true, force: true });
+        throw err;
     }
-    checkDownloadSpeed(downloaded, startTime, opts.fetchMinSpeedKiBps, opts.url);
-    return data;
 }
-async function readKnownSizeBody(body, size, url, onProgress) {
-    const data = Buffer.from(new SharedArrayBuffer(size));
+async function readKnownSizeBody(opts) {
+    const buffer = Buffer.from(new SharedArrayBuffer(opts.size));
     let downloaded = 0;
-    for await (const chunk of body) {
+    for await (const chunk of opts.body) {
         const bytes = chunk;
-        const nextDownloaded = downloaded + bytes.byteLength;
-        if (nextDownloaded > size) {
-            throw new BadTarballError({ expectedSize: size, receivedSize: nextDownloaded, tarballUrl: url });
+        checkDownloadedSize(opts, downloaded + bytes.byteLength, false);
+        buffer.set(bytes, downloaded);
+        downloaded += bytes.byteLength;
+        opts.onProgress?.(downloaded);
+    }
+    checkDownloadedSize(opts, downloaded, true);
+    return buffer;
+}
+function checkDownloadedSize(opts, downloaded, finished) {
+    if (opts.size == null)
+        return;
+    if (downloaded > opts.size || (finished && downloaded !== opts.size)) {
+        throw new BadTarballError({ expectedSize: opts.size, receivedSize: downloaded, tarballUrl: opts.url });
+    }
+}
+async function bufferDownloadChunk(state, bytes, storeDir) {
+    if (!state.file && state.downloaded > MAX_BUFFERED_DOWNLOAD_SIZE) {
+        await fs.mkdir(storeDir, { recursive: true });
+        state.directory = await fs.mkdtemp(path.join(storeDir, 'download-'));
+        state.file = await fs.open(path.join(state.directory, 'archive'), 'wx', 0o600);
+        for (const buffered of state.chunks) {
+            // eslint-disable-next-line no-await-in-loop -- writes preserve tarball byte order
+            await state.file.writeFile(buffered);
         }
-        data.set(bytes, downloaded);
-        downloaded = nextDownloaded;
-        onProgress?.(downloaded);
+        state.chunks.length = 0;
+    }
+    if (state.file) {
+        await state.file.writeFile(bytes);
     }
-    if (size !== downloaded) {
-        throw new BadTarballError({ expectedSize: size, receivedSize: downloaded, tarballUrl: url });
+    else {
+        state.chunks.push(bytes);
     }
-    return data;
 }
-async function readChunkedBody(body, onProgress) {
-    const chunks = [];
-    let downloaded = 0;
-    for await (const chunk of body) {
-        const bytes = chunk;
-        chunks.push(bytes);
-        downloaded += bytes.byteLength;
-        onProgress?.(downloaded);
+function finishDownloadBuffer(state) {
+    if (state.directory) {
+        const tempDirectory = state.directory;
+        return {
+            tarballFile: path.join(tempDirectory, 'archive'),
+            cleanup: async () => fs.rm(tempDirectory, { recursive: true, force: true }),
+        };
     }
-    const data = Buffer.from(new SharedArrayBuffer(downloaded));
+    const buffer = Buffer.from(new SharedArrayBuffer(state.downloaded));
     let offset = 0;
-    for (const chunk of chunks) {
-        data.set(chunk, offset);
+    for (const chunk of state.chunks) {
+        buffer.set(chunk, offset);
         offset += chunk.byteLength;
     }
-    return { data, downloaded };
+    return { buffer };
 }
 function checkDownloadSpeed(downloaded, startTime, fetchMinSpeedKiBps, url) {
     const elapsedSec = (Date.now() - startTime) / 1000;
@@ -191,6 +235,7 @@ function handleDownloadRetry(opts) {
         status === 403 ||
         status === 404 ||
         opts.error.code === 'ERR_PNPM_PREPARE_PKG_FAILURE' ||
+        opts.error.code === 'ERR_PNPM_TARBALL_ENTRY_TOO_LARGE' ||
         isNonRetryableError(opts.error);
     if (isFatal) {
         opts.reject(opts.error);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pnpm/fetching.tarball-fetcher",
-  "version": "1102.1.7",
+  "version": "1102.1.8",
   "description": "Fetcher for packages hosted as tarballs",
   "keywords": [
     "pnpm",
@@ -30,14 +30,14 @@
   ],
   "dependencies": {
     "@pnpm/core-loggers": "1101.0.2",
-    "@pnpm/error": "1100.2.1",
-    "@pnpm/exec.prepare-package": "1100.0.41",
+    "@pnpm/error": "1100.2.2",
+    "@pnpm/exec.prepare-package": "1100.0.42",
     "@pnpm/fetching.fetcher-base": "1100.2.12",
     "@pnpm/fetching.types": "1100.0.3",
-    "@pnpm/fs.graceful-fs": "1100.2.4",
-    "@pnpm/fs.packlist": "1100.0.7",
-    "@pnpm/network.fetch": "1100.1.21",
-    "@pnpm/store.index": "1101.0.0",
+    "@pnpm/fs.graceful-fs": "1100.2.5",
+    "@pnpm/fs.packlist": "1100.0.8",
+    "@pnpm/network.fetch": "1100.1.22",
+    "@pnpm/store.index": "1101.0.1",
     "@pnpm/types": "1102.1.1",
     "@zkochan/retry": "^0.2.0",
     "lodash.throttle": "4.1.1",
@@ -45,14 +45,14 @@
   },
   "peerDependencies": {
     "@pnpm/logger": "^1100.0.0",
-    "@pnpm/worker": "^1100.4.7"
+    "@pnpm/worker": "^1100.4.8"
   },
   "devDependencies": {
     "@jest/globals": "30.4.1",
-    "@pnpm/fetching.tarball-fetcher": "1102.1.7",
+    "@pnpm/fetching.tarball-fetcher": "1102.1.8",
     "@pnpm/logger": "1100.0.0",
     "@pnpm/store.cafs-types": "1100.1.1",
-    "@pnpm/store.create-cafs-store": "1100.0.33",
+    "@pnpm/store.create-cafs-store": "1100.0.34",
     "@pnpm/test-fixtures": "1100.0.1",
     "@pnpm/text.ordinal-comparator": "1100.0.0",
     "@pnpm/types": "1102.1.1",