@pnpm/fetching.tarball-fetcher 1102.0.4 → 1102.0.6

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @pnpm/tarball-fetcher
 
+## 1102.0.6
+
+### Patch Changes
+
+- Updated dependencies:
+  - @pnpm/exec.prepare-package@1100.0.24
+
+## 1102.0.5
+
+### Patch Changes
+
+- Updated dependencies:
+  - @pnpm/core-loggers@1100.2.3
+  - @pnpm/exec.prepare-package@1100.0.23
+  - @pnpm/fs.packlist@1100.0.3
+
 ## 1102.0.4
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pnpm/fetching.tarball-fetcher",
-  "version": "1102.0.4",
+  "version": "1102.0.6",
   "description": "Fetcher for packages hosted as tarballs",
   "keywords": [
     "pnpm",
@@ -29,13 +29,13 @@
     "!*.map"
   ],
   "dependencies": {
-    "@pnpm/core-loggers": "1100.2.2",
+    "@pnpm/core-loggers": "1100.2.3",
     "@pnpm/error": "1100.0.1",
-    "@pnpm/exec.prepare-package": "1100.0.22",
+    "@pnpm/exec.prepare-package": "1100.0.24",
     "@pnpm/fetching.fetcher-base": "1100.2.2",
     "@pnpm/fetching.types": "1100.0.2",
     "@pnpm/fs.graceful-fs": "1100.1.0",
-    "@pnpm/fs.packlist": "1100.0.2",
+    "@pnpm/fs.packlist": "1100.0.3",
     "@pnpm/store.index": "1100.2.1",
     "@pnpm/types": "1101.4.0",
     "@zkochan/retry": "^0.2.0",
@@ -45,15 +45,15 @@
   },
   "peerDependencies": {
     "@pnpm/logger": "^1100.0.0",
-    "@pnpm/worker": "^1100.2.4"
+    "@pnpm/worker": "^1100.2.5"
   },
   "devDependencies": {
     "@jest/globals": "30.4.1",
-    "@pnpm/fetching.tarball-fetcher": "1102.0.4",
+    "@pnpm/fetching.tarball-fetcher": "1102.0.6",
     "@pnpm/logger": "1100.0.0",
-    "@pnpm/network.fetch": "1100.1.5",
+    "@pnpm/network.fetch": "1100.1.6",
     "@pnpm/store.cafs-types": "1100.0.1",
-    "@pnpm/store.create-cafs-store": "1100.0.18",
+    "@pnpm/store.create-cafs-store": "1100.0.19",
     "@pnpm/test-fixtures": "1100.0.0",
     "@pnpm/types": "1101.4.0",
     "@pnpm/util.lex-comparator": "^4.0.1",

package/lib/errorTypes/BadTarballError.d.ts

@@ -1,11 +0,0 @@
-import { PnpmError } from '@pnpm/error';
-export declare class BadTarballError extends PnpmError {
-    expectedSize: number;
-    receivedSize: number;
-    constructor(opts: {
-        attempts?: number;
-        expectedSize: number;
-        receivedSize: number;
-        tarballUrl: string;
-    });
-}

package/lib/errorTypes/BadTarballError.js

@@ -1,14 +0,0 @@
-import { PnpmError } from '@pnpm/error';
-export class BadTarballError extends PnpmError {
-    expectedSize;
-    receivedSize;
-    constructor(opts) {
-        const message = `Actual size (${opts.receivedSize}) of tarball (${opts.tarballUrl}) did not match the one specified in 'Content-Length' header (${opts.expectedSize})`;
-        super('BAD_TARBALL_SIZE', message, {
-            attempts: opts?.attempts,
-        });
-        this.expectedSize = opts.expectedSize;
-        this.receivedSize = opts.receivedSize;
-    }
-}
-//# sourceMappingURL=BadTarballError.js.map

package/lib/errorTypes/index.d.ts

@@ -1 +0,0 @@
-export { BadTarballError } from './BadTarballError.js';

package/lib/errorTypes/index.js

@@ -1,2 +0,0 @@
-export { BadTarballError } from './BadTarballError.js';
-//# sourceMappingURL=index.js.map

package/lib/gitHostedTarballFetcher.d.ts

@@ -1,8 +0,0 @@
-import type { FetchFunction } from '@pnpm/fetching.fetcher-base';
-import type { StoreIndex } from '@pnpm/store.index';
-export interface CreateGitHostedTarballFetcher {
-    ignoreScripts?: boolean;
-    storeIndex: StoreIndex;
-    unsafePerm?: boolean;
-}
-export declare function createGitHostedTarballFetcher(fetchRemoteTarball: FetchFunction, fetcherOpts: CreateGitHostedTarballFetcher): FetchFunction;

package/lib/gitHostedTarballFetcher.js

@@ -1,100 +0,0 @@
-import assert from 'node:assert';
-import util from 'node:util';
-import { preparePackage } from '@pnpm/exec.prepare-package';
-import { packlist } from '@pnpm/fs.packlist';
-import { globalWarn } from '@pnpm/logger';
-import { addFilesFromDir } from '@pnpm/worker';
-export function createGitHostedTarballFetcher(fetchRemoteTarball, fetcherOpts) {
-    const fetch = async (cafs, resolution, opts) => {
-        const rawFilesIndexFile = `${opts.filesIndexFile}\traw`;
-        const { filesMap, manifest, requiresBuild, integrity } = await fetchRemoteTarball(cafs, resolution, {
-            ...opts,
-            filesIndexFile: rawFilesIndexFile,
-        });
-        // Flush any queued store index writes so that the raw files index entry
-        // written during tarball extraction is visible to subsequent reads.
-        fetcherOpts.storeIndex.flush();
-        try {
-            const prepareResult = await prepareGitHostedPkg(filesMap, cafs, rawFilesIndexFile, opts.filesIndexFile, fetcherOpts, opts, resolution);
-            if (prepareResult.ignoredBuild) {
-                globalWarn(`The git-hosted package fetched from "${resolution.tarball}" has to be built but the build scripts were ignored.`);
-            }
-            return {
-                filesMap: prepareResult.filesMap,
-                manifest: prepareResult.manifest ?? manifest,
-                requiresBuild,
-                // Propagate the raw tarball integrity so the lockfile pins it and
-                // future installs detect a tampered tarball from the git host.
-                integrity,
-            };
-        }
-        catch (err) {
-            assert(util.types.isNativeError(err));
-            err.message = `Failed to prepare git-hosted package fetched from "${resolution.tarball}": ${err.message}`;
-            throw err;
-        }
-    };
-    return fetch;
-}
-async function prepareGitHostedPkg(filesMap, cafs, rawFilesIndexFile, filesIndexFile, opts, fetcherOpts, resolution) {
-    const tempLocation = await cafs.tempDir();
-    cafs.importPackage(tempLocation, {
-        filesResponse: {
-            filesMap,
-            resolvedFrom: 'remote',
-            requiresBuild: false,
-        },
-        force: true,
-    });
-    const { shouldBeBuilt, pkgDir } = await preparePackage({
-        ...opts,
-        allowBuild: fetcherOpts.allowBuild,
-        pkgResolutionId: createGitHostedTarballPkgResolutionId(resolution),
-    }, tempLocation, resolution.path ?? '');
-    const files = await packlist(pkgDir);
-    const { storeIndex } = opts;
-    if (!resolution.path && files.length === filesMap.size) {
-        if (!shouldBeBuilt) {
-            const data = storeIndex.get(rawFilesIndexFile);
-            if (data) {
-                storeIndex.set(filesIndexFile, data);
-                storeIndex.delete(rawFilesIndexFile);
-            }
-            return {
-                filesMap,
-                ignoredBuild: false,
-            };
-        }
-        if (opts.ignoreScripts) {
-            storeIndex.delete(rawFilesIndexFile);
-            return {
-                filesMap,
-                ignoredBuild: true,
-            };
-        }
-    }
-    storeIndex.delete(rawFilesIndexFile);
-    // Important! We cannot remove the temp location at this stage.
-    // Even though we have the index of the package,
-    // the linking of files to the store is in progress.
-    return {
-        ...await addFilesFromDir({
-            storeDir: cafs.storeDir,
-            storeIndex: opts.storeIndex,
-            dir: pkgDir,
-            files,
-            filesIndexFile,
-            pkg: fetcherOpts.pkg,
-            readManifest: fetcherOpts.readManifest,
-        }),
-        ignoredBuild: Boolean(opts.ignoreScripts),
-    };
-}
-function createGitHostedTarballPkgResolutionId(resolution) {
-    let pkgResolutionId = resolution.tarball;
-    if (resolution.path) {
-        pkgResolutionId += `#path:${resolution.path}`;
-    }
-    return pkgResolutionId;
-}
-//# sourceMappingURL=gitHostedTarballFetcher.js.map

package/lib/index.d.ts

@@ -1,23 +0,0 @@
-import type { FetchFunction } from '@pnpm/fetching.fetcher-base';
-import type { FetchFromRegistry, GetAuthHeader, RetryTimeoutOptions } from '@pnpm/fetching.types';
-import type { StoreIndex } from '@pnpm/store.index';
-import { TarballIntegrityError } from '@pnpm/worker';
-import { type CreateDownloaderOptions } from './remoteTarballFetcher.js';
-export { BadTarballError } from './errorTypes/index.js';
-export { TarballIntegrityError };
-export { createGitHostedTarballFetcher } from './gitHostedTarballFetcher.js';
-export { createLocalTarballFetcher } from './localTarballFetcher.js';
-export { createDownloader, type CreateDownloaderOptions, type DownloadFunction } from './remoteTarballFetcher.js';
-export interface TarballFetchers {
-    localTarball: FetchFunction;
-    remoteTarball: FetchFunction;
-    gitHostedTarball: FetchFunction;
-}
-export declare function createTarballFetcher(fetchFromRegistry: FetchFromRegistry, getAuthHeader: GetAuthHeader, opts: {
-    unsafePerm?: boolean;
-    ignoreScripts?: boolean;
-    storeIndex: StoreIndex;
-    timeout?: number;
-    retry?: RetryTimeoutOptions;
-    offline?: boolean;
-} & Pick<CreateDownloaderOptions, 'fetchMinSpeedKiBps'>): TarballFetchers;

package/lib/localTarballFetcher.d.ts

@@ -1,3 +0,0 @@
-import type { FetchFunction } from '@pnpm/fetching.fetcher-base';
-import type { StoreIndex } from '@pnpm/store.index';
-export declare function createLocalTarballFetcher(storeIndex: StoreIndex): FetchFunction;

package/lib/localTarballFetcher.js

@@ -1,29 +0,0 @@
-import path from 'node:path';
-import gfs from '@pnpm/fs.graceful-fs';
-import { addFilesFromTarball } from '@pnpm/worker';
-const isAbsolutePath = /^\/|^[A-Z]:/i;
-export function createLocalTarballFetcher(storeIndex) {
-    const fetch = (cafs, resolution, opts) => {
-        const tarball = resolvePath(opts.lockfileDir, resolution.tarball.slice(5));
-        const buffer = gfs.readFileSync(tarball);
-        return addFilesFromTarball({
-            storeDir: cafs.storeDir,
-            storeIndex,
-            buffer,
-            filesIndexFile: opts.filesIndexFile,
-            integrity: resolution.integrity,
-            readManifest: opts.readManifest,
-            url: tarball,
-            pkg: opts.pkg,
-            appendManifest: opts.appendManifest,
-            ignoreFilePattern: opts.ignoreFilePattern,
-        });
-    };
-    return fetch;
-}
-function resolvePath(where, spec) {
-    if (isAbsolutePath.test(spec))
-        return spec;
-    return path.resolve(where, spec);
-}
-//# sourceMappingURL=localTarballFetcher.js.map

package/lib/remoteTarballFetcher.d.ts

@@ -1,37 +0,0 @@
-import type { IncomingMessage } from 'node:http';
-import type { FetchOptions, FetchResult } from '@pnpm/fetching.fetcher-base';
-import type { FetchFromRegistry, GetAuthHeader } from '@pnpm/fetching.types';
-import type { Cafs } from '@pnpm/store.cafs-types';
-import type { StoreIndex } from '@pnpm/store.index';
-export interface HttpResponse {
-    body: string;
-}
-export type DownloadOptions = {
-    getAuthHeaderByURI: GetAuthHeader;
-    cafs: Cafs;
-    registry?: string;
-    onStart?: (totalSize: number | null, attempt: number) => void;
-    onProgress?: (downloaded: number) => void;
-    integrity?: string;
-    storeIndex: StoreIndex;
-    pkg?: FetchOptions['pkg'];
-} & Pick<FetchOptions, 'appendManifest' | 'readManifest' | 'filesIndexFile' | 'ignoreFilePattern'>;
-export type DownloadFunction = (url: string, opts: DownloadOptions) => Promise<FetchResult>;
-export interface NpmRegistryClient {
-    get: (url: string, getOpts: object, cb: (err: Error, data: object, raw: object, res: HttpResponse) => void) => void;
-    fetch: (url: string, opts: {
-        auth?: object;
-    }, cb: (err: Error, res: IncomingMessage) => void) => void;
-}
-export interface CreateDownloaderOptions {
-    retry?: {
-        retries?: number;
-        factor?: number;
-        minTimeout?: number;
-        maxTimeout?: number;
-        randomize?: boolean;
-    };
-    timeout?: number;
-    fetchMinSpeedKiBps?: number;
-}
-export declare function createDownloader(fetchFromRegistry: FetchFromRegistry, gotOpts: CreateDownloaderOptions): DownloadFunction;

package/lib/remoteTarballFetcher.js

@@ -1,184 +0,0 @@
-import util from 'node:util';
-import { requestRetryLogger } from '@pnpm/core-loggers';
-import { FetchError } from '@pnpm/error';
-import { globalWarn } from '@pnpm/logger';
-import { addFilesFromTarball } from '@pnpm/worker';
-import * as retry from '@zkochan/retry';
-import throttle from 'lodash.throttle';
-import { BadTarballError } from './errorTypes/index.js';
-const BIG_TARBALL_SIZE = 1024 * 1024 * 5; // 5 MB
-export function createDownloader(fetchFromRegistry, gotOpts) {
-    const retryOpts = {
-        factor: 10,
-        maxTimeout: 6e4, // 1 minute
-        minTimeout: 1e4, // 10 seconds
-        retries: 2,
-        ...gotOpts.retry,
-    };
-    const fetchMinSpeedKiBps = gotOpts.fetchMinSpeedKiBps ?? 50; // 50 KiB/s
-    return async function download(url, opts) {
-        const authHeaderValue = opts.getAuthHeaderByURI(url, { pkgName: opts.pkg?.name });
-        const op = retry.operation(retryOpts);
-        return new Promise((resolve, reject) => {
-            op.attempt(async (attempt) => {
-                try {
-                    resolve(await fetch(attempt));
-                }
-                catch (error) { // eslint-disable-line
-                    if (error.response?.status === 401 ||
-                        error.response?.status === 403 ||
-                        error.response?.status === 404 ||
-                        error.code === 'ERR_PNPM_PREPARE_PKG_FAILURE') {
-                        reject(error);
-                        return;
-                    }
-                    const timeout = op.retry(error);
-                    if (timeout === false) {
-                        reject(op.mainError());
-                        return;
-                    }
-                    // Extract error properties into a plain object because Error properties
-                    // are non-enumerable and don't serialize well through the logging system
-                    const errorInfo = {
-                        name: error.name,
-                        message: error.message,
-                        code: error.code,
-                        errno: error.errno,
-                        // For HTTP errors from our ResponseError class
-                        status: error.status,
-                        statusCode: error.statusCode,
-                        // undici wraps the actual network error in a cause property
-                        cause: error.cause ? {
-                            code: error.cause.code,
-                            errno: error.cause.errno,
-                        } : undefined,
-                    };
-                    requestRetryLogger.debug({
-                        attempt,
-                        error: errorInfo,
-                        maxRetries: retryOpts.retries,
-                        method: 'GET',
-                        timeout,
-                        url,
-                    });
-                }
-            });
-        });
-        async function fetch(currentAttempt) {
-            let data;
-            try {
-                const res = await fetchFromRegistry(url, {
-                    authHeaderValue,
-                    // Tarballs are already compressed; ask the server not to apply an additional
-                    // Content-Encoding so Content-Length matches the body we receive and we don't
-                    // waste CPU on round-trip re-compression. See https://github.com/pnpm/pnpm/issues/11506
-                    headers: { 'accept-encoding': 'identity' },
-                    // The fetch library can retry requests on bad HTTP responses.
-                    // However, it is not enough to retry on bad HTTP responses only.
-                    // Requests should also be retried when the tarball's integrity check fails.
-                    // Hence, we tell fetch to not retry,
-                    // and we perform the retries from this function instead.
-                    retry: { retries: 0 },
-                    timeout: gotOpts.timeout,
-                });
-                if (res.status !== 200) {
-                    throw new FetchError({ url, authHeaderValue }, res);
-                }
-                // When Content-Encoding is present, Content-Length refers to the encoded form
-                // of the data, not the decoded bytes that the fetch implementation yields.
-                // See: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Content-Encoding
-                const isEncoded = isContentEncoded(res.headers.get('content-encoding'));
-                const contentLength = !isEncoded && res.headers.has('content-length') && res.headers.get('content-length');
-                const parsedLength = typeof contentLength === 'string' ? parseInt(contentLength, 10) : NaN;
-                const size = Number.isFinite(parsedLength) && parsedLength >= 0 ? parsedLength : null;
-                if (opts.onStart != null) {
-                    opts.onStart(size, currentAttempt);
-                }
-                // In order to reduce the amount of logs, we only report the download progress of big tarballs
-                const onProgress = (size != null && size >= BIG_TARBALL_SIZE && opts.onProgress)
-                    ? throttle(opts.onProgress, 500)
-                    : undefined;
-                const startTime = Date.now();
-                let downloaded = 0;
-                if (size !== null) {
-                    // Known size: pre-allocate and copy directly (avoids intermediate array + second copy pass)
-                    data = Buffer.from(new SharedArrayBuffer(size));
-                    for await (const chunk of res.body) {
-                        const c = chunk;
-                        const nextDownloaded = downloaded + c.byteLength;
-                        if (nextDownloaded > size) {
-                            throw new BadTarballError({
-                                expectedSize: size,
-                                receivedSize: nextDownloaded,
-                                tarballUrl: url,
-                            });
-                        }
-                        data.set(c, downloaded);
-                        downloaded = nextDownloaded;
-                        onProgress?.(downloaded);
-                    }
-                    if (size !== downloaded) {
-                        throw new BadTarballError({
-                            expectedSize: size,
-                            receivedSize: downloaded,
-                            tarballUrl: url,
-                        });
-                    }
-                }
-                else {
-                    const chunks = [];
-                    for await (const chunk of res.body) {
-                        const c = chunk;
-                        chunks.push(c);
-                        downloaded += c.byteLength;
-                        onProgress?.(downloaded);
-                    }
-                    data = Buffer.from(new SharedArrayBuffer(downloaded));
-                    let offset = 0;
-                    for (const chunk of chunks) {
-                        data.set(chunk, offset);
-                        offset += chunk.byteLength;
-                    }
-                }
-                const elapsedSec = (Date.now() - startTime) / 1000;
-                const avgKiBps = Math.floor((downloaded / elapsedSec) / 1024);
-                if (downloaded > 0 && elapsedSec > 1 && avgKiBps < fetchMinSpeedKiBps) {
-                    const sizeKb = Math.floor(downloaded / 1024);
-                    globalWarn(`Tarball download average speed ${avgKiBps} KiB/s (size ${sizeKb} KiB) is below ${fetchMinSpeedKiBps} KiB/s: ${url} (GET)`);
-                }
-            }
-            catch (err) {
-                const error = util.types.isNativeError(err) ? err : new Error(String(err), { cause: err });
-                Object.assign(error, {
-                    attempts: currentAttempt,
-                    resource: url,
-                });
-                throw error;
-            }
-            return addFilesFromTarball({
-                buffer: data,
-                storeDir: opts.cafs.storeDir,
-                storeIndex: opts.storeIndex,
-                readManifest: opts.readManifest,
-                integrity: opts.integrity,
-                filesIndexFile: opts.filesIndexFile,
-                url,
-                pkg: opts.pkg,
-                appendManifest: opts.appendManifest,
-                ignoreFilePattern: opts.ignoreFilePattern,
-            });
-        }
-    };
-}
-// Per RFC 9110 §8.4, Content-Encoding is a comma-separated list of codings.
-// The response is encoded unless every coding is `identity` (case-insensitive,
-// surrounding whitespace ignored).
-function isContentEncoded(header) {
-    if (header == null)
-        return false;
-    return header
-        .split(',')
-        .map(coding => coding.trim().toLowerCase())
-        .some(coding => coding !== '' && coding !== 'identity');
-}
-//# sourceMappingURL=remoteTarballFetcher.js.map