@pnpm/fetching.tarball-fetcher 1102.1.5 → 1102.1.7

package/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @pnpm/tarball-fetcher
 
+## 1102.1.7
+
+### Patch Changes
+
+- Updated dependencies:
+  - @pnpm/core-loggers@1101.0.2
+  - @pnpm/exec.prepare-package@1100.0.41
+  - @pnpm/fs.packlist@1100.0.7
+  - @pnpm/network.fetch@1100.1.21
+
+## 1102.1.6
+
+### Patch Changes
+
+- Updated dependencies:
+  - @pnpm/exec.prepare-package@1100.0.40
+  - @pnpm/network.fetch@1100.1.20
+  - @pnpm/store.index@1101.0.0
+
 ## 1102.1.5
 
 ### Patch Changes

package/lib/gitHostedTarballFetcher.js

@@ -45,6 +45,58 @@ export function createGitHostedTarballFetcher(fetchRemoteTarball, fetcherOpts) {
     });
 }
 async function prepareGitHostedPkg(filesMap, cafs, rawFilesIndexFile, filesIndexFile, opts, fetcherOpts, resolution) {
+    const { shouldBeBuilt, pkgDir, ignoredBuild } = await unpackAndPreparePackage(cafs, filesMap, opts, fetcherOpts, resolution);
+    filesIndexFile = resolveGitHostedIndexFile({
+        filesIndexFile,
+        ignoredBuild,
+        ignoreScripts: opts.ignoreScripts,
+        pkgResolutionId: fetcherOpts.pkgResolutionId ?? createGitHostedTarballPkgResolutionId(resolution),
+        shouldBeBuilt,
+    });
+    const files = await packlist(pkgDir);
+    const isUnchanged = !resolution.path && files.length === filesMap.size && (!shouldBeBuilt || ignoredBuild);
+    if (isUnchanged) {
+        return handleUnchangedGitPkg({
+            filesIndexFile,
+            filesMap,
+            ignoredBuild,
+            ignoreScripts: opts.ignoreScripts,
+            rawFilesIndexFile,
+            shouldBeBuilt,
+            storeIndex: opts.storeIndex,
+        });
+    }
+    return reindexAndAddGitPkgFiles({
+        cafs,
+        fetcherOpts,
+        files,
+        filesIndexFile,
+        ignoredBuild,
+        opts,
+        pkgDir,
+        rawFilesIndexFile,
+        shouldBeBuilt,
+    });
+}
+async function reindexAndAddGitPkgFiles(opts) {
+    opts.opts.storeIndex.delete(opts.rawFilesIndexFile);
+    return {
+        filesIndexFile: opts.filesIndexFile,
+        ...await addFilesFromDir({
+            storeDir: opts.cafs.storeDir,
+            storeIndex: opts.opts.storeIndex,
+            dir: opts.pkgDir,
+            files: opts.files,
+            filesIndexFile: opts.filesIndexFile,
+            pkg: opts.fetcherOpts.pkg,
+            readManifest: opts.fetcherOpts.readManifest,
+            requiresPrepare: opts.shouldBeBuilt,
+        }),
+        ignoredBuild: opts.ignoredBuild,
+        requiresPrepare: opts.shouldBeBuilt,
+    };
+}
+async function unpackAndPreparePackage(cafs, filesMap, opts, fetcherOpts, resolution) {
     const tempLocation = await cafs.tempDir();
     cafs.importPackage(tempLocation, {
         filesResponse: {
@@ -59,47 +111,30 @@ async function prepareGitHostedPkg(filesMap, cafs, rawFilesIndexFile, filesIndex
         allowBuild: fetcherOpts.allowBuild,
         pkgResolutionId: fetcherOpts.pkgResolutionId ?? createGitHostedTarballPkgResolutionId(resolution),
     }, tempLocation, resolution.path ?? '');
-    if (shouldBeBuilt && ((ignoredBuild && !opts.ignoreScripts) || (!ignoredBuild && filesIndexFile.endsWith('\tnot-built')))) {
-        filesIndexFile = gitHostedStoreIndexKey(fetcherOpts.pkgResolutionId ?? createGitHostedTarballPkgResolutionId(resolution), { built: !ignoredBuild });
+    return { shouldBeBuilt, pkgDir, ignoredBuild };
+}
+function resolveGitHostedIndexFile(opts) {
+    const shouldReindex = opts.shouldBeBuilt &&
+        ((opts.ignoredBuild && !opts.ignoreScripts) || (!opts.ignoredBuild && opts.filesIndexFile.endsWith('\tnot-built')));
+    if (shouldReindex) {
+        return gitHostedStoreIndexKey(opts.pkgResolutionId, { built: !opts.ignoredBuild });
     }
-    const files = await packlist(pkgDir);
-    const { storeIndex } = opts;
-    if (!resolution.path && files.length === filesMap.size) {
-        if (!shouldBeBuilt || ignoredBuild) {
-            if (!ignoredBuild || !opts.ignoreScripts) {
-                const data = storeIndex.get(rawFilesIndexFile);
-                if (data) {
-                    data.requiresPrepare = shouldBeBuilt;
-                    storeIndex.set(filesIndexFile, data);
-                }
-            }
-            storeIndex.delete(rawFilesIndexFile);
-            return {
-                filesIndexFile,
-                filesMap,
-                ignoredBuild,
-                requiresPrepare: shouldBeBuilt,
-            };
+    return opts.filesIndexFile;
+}
+function handleUnchangedGitPkg(opts) {
+    if (!opts.ignoredBuild || !opts.ignoreScripts) {
+        const data = opts.storeIndex.get(opts.rawFilesIndexFile);
+        if (data) {
+            data.requiresPrepare = opts.shouldBeBuilt;
+            opts.storeIndex.set(opts.filesIndexFile, data);
         }
     }
-    storeIndex.delete(rawFilesIndexFile);
-    // Important! We cannot remove the temp location at this stage.
-    // Even though we have the index of the package,
-    // the linking of files to the store is in progress.
+    opts.storeIndex.delete(opts.rawFilesIndexFile);
     return {
-        filesIndexFile,
-        ...await addFilesFromDir({
-            storeDir: cafs.storeDir,
-            storeIndex: opts.storeIndex,
-            dir: pkgDir,
-            files,
-            filesIndexFile,
-            pkg: fetcherOpts.pkg,
-            readManifest: fetcherOpts.readManifest,
-            requiresPrepare: shouldBeBuilt,
-        }),
-        ignoredBuild,
-        requiresPrepare: shouldBeBuilt,
+        filesIndexFile: opts.filesIndexFile,
+        filesMap: opts.filesMap,
+        ignoredBuild: opts.ignoredBuild,
+        requiresPrepare: opts.shouldBeBuilt,
     };
 }
 function createGitHostedTarballPkgResolutionId(resolution) {

package/lib/remoteTarballFetcher.js

@@ -9,7 +9,7 @@ import throttle from 'lodash.throttle';
 import { BadTarballError } from './errorTypes/index.js';
 const BIG_TARBALL_SIZE = 1024 * 1024 * 5; // 5 MB
 export function createDownloader(fetchFromRegistry, gotOpts) {
-    const retryOpts = {
+    const defaultRetryOpts = {
         factor: 10,
         maxTimeout: 6e4, // 1 minute
         minTimeout: 1e4, // 10 seconds
@@ -19,164 +19,212 @@ export function createDownloader(fetchFromRegistry, gotOpts) {
     const fetchMinSpeedKiBps = gotOpts.fetchMinSpeedKiBps ?? 50; // 50 KiB/s
     return async function download(url, opts) {
         const authHeaderValue = getSecureNodeMirrorAuthHeader(opts.getAuthHeaderByURI(url, { pkgName: opts.pkg?.name }), url, opts.appendManifest?.name);
-        const downloadRetryOpts = { ...retryOpts, ...opts.retry };
+        const downloadRetryOpts = { ...defaultRetryOpts, ...opts.retry };
         const op = retry.operation(downloadRetryOpts);
         return new Promise((resolve, reject) => {
-            op.attempt(async (attempt) => {
-                try {
-                    resolve(await fetch(attempt));
-                }
-                catch (error) { // eslint-disable-line
-                    if ((opts.redirect === 'manual' && error.response?.status >= 300 && error.response.status < 400) ||
-                        error.response?.status === 401 ||
-                        error.response?.status === 403 ||
-                        error.response?.status === 404 ||
-                        error.code === 'ERR_PNPM_PREPARE_PKG_FAILURE' ||
-                        isNonRetryableError(error)) {
-                        reject(error);
-                        return;
-                    }
-                    const timeout = op.retry(error);
-                    if (timeout === false) {
-                        reject(op.mainError());
-                        return;
-                    }
-                    // Extract error properties into a plain object because Error properties
-                    // are non-enumerable and don't serialize well through the logging system
-                    const errorInfo = {
-                        name: error.name,
-                        message: error.message,
-                        code: error.code,
-                        errno: error.errno,
-                        // For HTTP errors from our ResponseError class
-                        status: error.status,
-                        statusCode: error.statusCode,
-                        // undici wraps the actual network error in a cause property
-                        cause: error.cause ? {
-                            code: error.cause.code,
-                            errno: error.cause.errno,
-                        } : undefined,
-                    };
-                    requestRetryLogger.debug({
-                        attempt,
-                        error: errorInfo,
-                        maxRetries: downloadRetryOpts.retries,
-                        method: 'GET',
-                        timeout,
-                        url: redactUrlForDisplay(url),
-                    });
-                }
-            });
-        });
-        async function fetch(currentAttempt) {
-            let data;
-            try {
-                const res = await fetchFromRegistry(url, {
+            op.attempt((attempt) => {
+                void executeDownloadAttempt({
+                    attempt,
                     authHeaderValue,
-                    // Tarballs are already compressed; ask the server not to apply an additional
-                    // Content-Encoding so Content-Length matches the body we receive and we don't
-                    // waste CPU on round-trip re-compression. See https://github.com/pnpm/pnpm/issues/11506
-                    headers: { 'accept-encoding': 'identity' },
-                    // The fetch library can retry requests on bad HTTP responses.
-                    // However, it is not enough to retry on bad HTTP responses only.
-                    // Requests should also be retried when the tarball's integrity check fails.
-                    // Hence, we tell fetch to not retry,
-                    // and we perform the retries from this function instead.
-                    retry: { retries: 0 },
-                    redirect: opts.redirect,
-                    timeout: gotOpts.timeout,
-                });
-                if (res.status !== 200) {
-                    throw new FetchError({ url, authHeaderValue }, res);
-                }
-                // When Content-Encoding is present, Content-Length refers to the encoded form
-                // of the data, not the decoded bytes that the fetch implementation yields.
-                // See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Content-Encoding
-                const isEncoded = isContentEncoded(res.headers.get('content-encoding'));
-                const contentLength = !isEncoded && res.headers.has('content-length') && res.headers.get('content-length');
-                const parsedLength = typeof contentLength === 'string' ? parseInt(contentLength, 10) : NaN;
-                const size = Number.isFinite(parsedLength) && parsedLength >= 0 ? parsedLength : null;
-                if (opts.onStart != null) {
-                    opts.onStart(size, currentAttempt);
-                }
-                // In order to reduce the amount of logs, we only report the download progress of big tarballs
-                const onProgress = (size != null && size >= BIG_TARBALL_SIZE && opts.onProgress)
-                    ? throttle(opts.onProgress, 500)
-                    : undefined;
-                const startTime = Date.now();
-                let downloaded = 0;
-                if (size !== null) {
-                    // Known size: pre-allocate and copy directly (avoids intermediate array + second copy pass)
-                    data = Buffer.from(new SharedArrayBuffer(size));
-                    for await (const chunk of res.body) {
-                        const c = chunk;
-                        const nextDownloaded = downloaded + c.byteLength;
-                        if (nextDownloaded > size) {
-                            throw new BadTarballError({
-                                expectedSize: size,
-                                receivedSize: nextDownloaded,
-                                tarballUrl: url,
-                            });
-                        }
-                        data.set(c, downloaded);
-                        downloaded = nextDownloaded;
-                        onProgress?.(downloaded);
-                    }
-                    if (size !== downloaded) {
-                        throw new BadTarballError({
-                            expectedSize: size,
-                            receivedSize: downloaded,
-                            tarballUrl: url,
-                        });
-                    }
-                }
-                else {
-                    const chunks = [];
-                    for await (const chunk of res.body) {
-                        const c = chunk;
-                        chunks.push(c);
-                        downloaded += c.byteLength;
-                        onProgress?.(downloaded);
-                    }
-                    data = Buffer.from(new SharedArrayBuffer(downloaded));
-                    let offset = 0;
-                    for (const chunk of chunks) {
-                        data.set(chunk, offset);
-                        offset += chunk.byteLength;
-                    }
-                }
-                const elapsedSec = (Date.now() - startTime) / 1000;
-                const avgKiBps = Math.floor((downloaded / elapsedSec) / 1024);
-                if (downloaded > 0 && elapsedSec > 1 && avgKiBps < fetchMinSpeedKiBps) {
-                    const sizeKb = Math.floor(downloaded / 1024);
-                    globalWarn(`Tarball download average speed ${avgKiBps} KiB/s (size ${sizeKb} KiB) is below ${fetchMinSpeedKiBps} KiB/s: ${redactUrlForDisplay(url)} (GET)`);
-                }
-            }
-            catch (err) {
-                const error = isFetchTimeoutError(err)
-                    ? new FetchTimeoutError('FETCH_TIMEOUT', url, gotOpts.timeout, { cause: err })
-                    : isError(err) ? err : new Error(String(err), { cause: err });
-                Object.assign(error, {
-                    attempts: currentAttempt,
-                    resource: url,
+                    downloadRetryOpts,
+                    fetchFromRegistry,
+                    fetchMinSpeedKiBps,
+                    gotOpts,
+                    op,
+                    opts,
+                    reject,
+                    resolve,
+                    url,
                 });
-                throw error;
-            }
-            return addFilesFromTarball({
-                buffer: data,
-                storeDir: opts.cafs.storeDir,
-                storeIndex: opts.storeIndex,
-                readManifest: opts.readManifest,
-                integrity: opts.integrity,
-                filesIndexFile: opts.filesIndexFile,
-                pkgId: opts.pkgId,
-                url,
-                pkg: opts.pkg,
-                appendManifest: opts.appendManifest,
-                ignoreFilePattern: opts.ignoreFilePattern,
             });
+        });
+    };
+}
+async function executeDownloadAttempt(ctx) {
+    try {
+        const result = await fetchTarball({
+            attempt: ctx.attempt,
+            authHeaderValue: ctx.authHeaderValue,
+            fetchFromRegistry: ctx.fetchFromRegistry,
+            fetchMinSpeedKiBps: ctx.fetchMinSpeedKiBps,
+            gotOpts: ctx.gotOpts,
+            opts: ctx.opts,
+            url: ctx.url,
+        });
+        ctx.resolve(result);
+    }
+    catch (error) { // eslint-disable-line
+        handleDownloadRetry({
+            attempt: ctx.attempt,
+            downloadRetryOpts: ctx.downloadRetryOpts,
+            error,
+            op: ctx.op,
+            opts: ctx.opts,
+            reject: ctx.reject,
+            url: ctx.url,
+        });
+    }
+}
+async function fetchTarball(ctx) {
+    let data;
+    try {
+        const res = await ctx.fetchFromRegistry(ctx.url, {
+            authHeaderValue: ctx.authHeaderValue,
+            headers: { 'accept-encoding': 'identity' },
+            retry: { retries: 0 },
+            redirect: ctx.opts.redirect,
+            timeout: ctx.gotOpts.timeout,
+        });
+        if (res.status !== 200) {
+            throw new FetchError({ url: ctx.url, authHeaderValue: ctx.authHeaderValue }, res);
         }
+        const size = parseTarballContentLength(res);
+        ctx.opts.onStart?.(size, ctx.attempt);
+        const onProgress = (size != null && size >= BIG_TARBALL_SIZE && ctx.opts.onProgress)
+            ? throttle(ctx.opts.onProgress, 500)
+            : undefined;
+        data = await readResponseBody({
+            body: res.body,
+            fetchMinSpeedKiBps: ctx.fetchMinSpeedKiBps,
+            onProgress,
+            size,
+            url: ctx.url,
+        });
+    }
+    catch (err) {
+        throw wrapFetchError(err, ctx.url, ctx.attempt, ctx.gotOpts.timeout);
+    }
+    return addFilesFromTarball({
+        buffer: data,
+        storeDir: ctx.opts.cafs.storeDir,
+        storeIndex: ctx.opts.storeIndex,
+        readManifest: ctx.opts.readManifest,
+        integrity: ctx.opts.integrity,
+        filesIndexFile: ctx.opts.filesIndexFile,
+        pkgId: ctx.opts.pkgId,
+        url: ctx.url,
+        pkg: ctx.opts.pkg,
+        appendManifest: ctx.opts.appendManifest,
+        ignoreFilePattern: ctx.opts.ignoreFilePattern,
+    });
+}
+function parseTarballContentLength(res) {
+    const isEncoded = isContentEncoded(res.headers.get('content-encoding'));
+    const contentLength = !isEncoded && res.headers.has('content-length') && res.headers.get('content-length');
+    const parsedLength = typeof contentLength === 'string' ? parseInt(contentLength, 10) : NaN;
+    return Number.isFinite(parsedLength) && parsedLength >= 0 ? parsedLength : null;
+}
+async function readResponseBody(opts) {
+    const startTime = Date.now();
+    let downloaded = 0;
+    let data;
+    if (opts.size !== null) {
+        data = await readKnownSizeBody(opts.body, opts.size, opts.url, opts.onProgress);
+        downloaded = opts.size;
+    }
+    else {
+        const result = await readChunkedBody(opts.body, opts.onProgress);
+        data = result.data;
+        downloaded = result.downloaded;
+    }
+    checkDownloadSpeed(downloaded, startTime, opts.fetchMinSpeedKiBps, opts.url);
+    return data;
+}
+async function readKnownSizeBody(body, size, url, onProgress) {
+    const data = Buffer.from(new SharedArrayBuffer(size));
+    let downloaded = 0;
+    for await (const chunk of body) {
+        const bytes = chunk;
+        const nextDownloaded = downloaded + bytes.byteLength;
+        if (nextDownloaded > size) {
+            throw new BadTarballError({ expectedSize: size, receivedSize: nextDownloaded, tarballUrl: url });
+        }
+        data.set(bytes, downloaded);
+        downloaded = nextDownloaded;
+        onProgress?.(downloaded);
+    }
+    if (size !== downloaded) {
+        throw new BadTarballError({ expectedSize: size, receivedSize: downloaded, tarballUrl: url });
+    }
+    return data;
+}
+async function readChunkedBody(body, onProgress) {
+    const chunks = [];
+    let downloaded = 0;
+    for await (const chunk of body) {
+        const bytes = chunk;
+        chunks.push(bytes);
+        downloaded += bytes.byteLength;
+        onProgress?.(downloaded);
+    }
+    const data = Buffer.from(new SharedArrayBuffer(downloaded));
+    let offset = 0;
+    for (const chunk of chunks) {
+        data.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return { data, downloaded };
+}
+function checkDownloadSpeed(downloaded, startTime, fetchMinSpeedKiBps, url) {
+    const elapsedSec = (Date.now() - startTime) / 1000;
+    const avgKiBps = Math.floor((downloaded / elapsedSec) / 1024);
+    if (downloaded > 0 && elapsedSec > 1 && avgKiBps < fetchMinSpeedKiBps) {
+        const sizeKb = Math.floor(downloaded / 1024);
+        globalWarn(`Tarball download average speed ${avgKiBps} KiB/s (size ${sizeKb} KiB) is below ${fetchMinSpeedKiBps} KiB/s: ${redactUrlForDisplay(url)} (GET)`);
+    }
+}
+function wrapFetchError(err, url, attempt, timeout) {
+    const error = isFetchTimeoutError(err)
+        ? new FetchTimeoutError('FETCH_TIMEOUT', url, timeout, { cause: err })
+        : isError(err) ? err : new Error(String(err), { cause: err });
+    Object.assign(error, {
+        attempts: attempt,
+        resource: url,
+    });
+    return error;
+}
+function handleDownloadRetry(opts) {
+    const status = opts.error.response?.status;
+    const isManualRedirect = opts.opts.redirect === 'manual' && status >= 300 && status < 400;
+    const isFatal = isManualRedirect ||
+        status === 401 ||
+        status === 403 ||
+        status === 404 ||
+        opts.error.code === 'ERR_PNPM_PREPARE_PKG_FAILURE' ||
+        isNonRetryableError(opts.error);
+    if (isFatal) {
+        opts.reject(opts.error);
+        return;
+    }
+    const timeout = opts.op.retry(opts.error);
+    if (timeout === false) {
+        opts.reject(opts.op.mainError());
+        return;
+    }
+    logRetryAttempt(opts.attempt, opts.error, opts.downloadRetryOpts.retries ?? 0, timeout, opts.url);
+}
+function logRetryAttempt(attempt, error, // eslint-disable-line
+maxRetries, timeout, url) {
+    const errorInfo = {
+        name: error.name,
+        message: error.message,
+        code: error.code,
+        errno: error.errno,
+        status: error.status,
+        statusCode: error.statusCode,
+        cause: error.cause ? {
+            code: error.cause.code,
+            errno: error.cause.errno,
+        } : undefined,
     };
+    requestRetryLogger.debug({
+        attempt,
+        error: errorInfo,
+        maxRetries,
+        method: 'GET',
+        timeout,
+        url: redactUrlForDisplay(url),
+    });
 }
 function getSecureNodeMirrorAuthHeader(authHeaderValue, url, appendedPackageName) {
     if (authHeaderValue == null || appendedPackageName !== 'node')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pnpm/fetching.tarball-fetcher",
-  "version": "1102.1.5",
+  "version": "1102.1.7",
   "description": "Fetcher for packages hosted as tarballs",
   "keywords": [
     "pnpm",
@@ -29,15 +29,15 @@
     "!*.map"
   ],
   "dependencies": {
-    "@pnpm/core-loggers": "1101.0.1",
+    "@pnpm/core-loggers": "1101.0.2",
     "@pnpm/error": "1100.2.1",
-    "@pnpm/exec.prepare-package": "1100.0.39",
+    "@pnpm/exec.prepare-package": "1100.0.41",
     "@pnpm/fetching.fetcher-base": "1100.2.12",
     "@pnpm/fetching.types": "1100.0.3",
     "@pnpm/fs.graceful-fs": "1100.2.4",
-    "@pnpm/fs.packlist": "1100.0.6",
-    "@pnpm/network.fetch": "1100.1.19",
-    "@pnpm/store.index": "1100.3.3",
+    "@pnpm/fs.packlist": "1100.0.7",
+    "@pnpm/network.fetch": "1100.1.21",
+    "@pnpm/store.index": "1101.0.0",
     "@pnpm/types": "1102.1.1",
     "@zkochan/retry": "^0.2.0",
     "lodash.throttle": "4.1.1",
@@ -45,14 +45,14 @@
   },
   "peerDependencies": {
     "@pnpm/logger": "^1100.0.0",
-    "@pnpm/worker": "^1100.4.5"
+    "@pnpm/worker": "^1100.4.7"
   },
   "devDependencies": {
     "@jest/globals": "30.4.1",
-    "@pnpm/fetching.tarball-fetcher": "1102.1.5",
+    "@pnpm/fetching.tarball-fetcher": "1102.1.7",
     "@pnpm/logger": "1100.0.0",
     "@pnpm/store.cafs-types": "1100.1.1",
-    "@pnpm/store.create-cafs-store": "1100.0.32",
+    "@pnpm/store.create-cafs-store": "1100.0.33",
     "@pnpm/test-fixtures": "1100.0.1",
     "@pnpm/text.ordinal-comparator": "1100.0.0",
     "@pnpm/types": "1102.1.1",
@@ -62,7 +62,7 @@
     "@types/ssri": "^7.1.5",
     "ssri": "13.0.1",
     "tempy": "3.0.0",
-    "undici": "^7.29.0"
+    "undici": "^7.29.1"
   },
   "engines": {
     "node": ">=22.13"