@pnpm/fetching.tarball-fetcher 1003.0.0

package/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015-2016 Rico Sta. Cruz and other contributors
+Copyright (c) 2016-2026 Zoltan Kochan and other contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,15 @@
+# @pnpm/tarball-fetcher
+
+> Fetcher for packages hosted as tarballs
+
+[![npm version](https://img.shields.io/npm/v/@pnpm/tarball-fetcher.svg)](https://www.npmjs.com/package/@pnpm/tarball-fetcher)
+
+## Installation
+
+```
+pnpm add @pnpm/tarball-fetcher
+```
+
+## License
+
+MIT

package/lib/errorTypes/BadTarballError.d.ts

@@ -0,0 +1,11 @@
+import { PnpmError } from '@pnpm/error';
+export declare class BadTarballError extends PnpmError {
+    expectedSize: number;
+    receivedSize: number;
+    constructor(opts: {
+        attempts?: number;
+        expectedSize: number;
+        receivedSize: number;
+        tarballUrl: string;
+    });
+}

package/lib/errorTypes/BadTarballError.js

@@ -0,0 +1,14 @@
+import { PnpmError } from '@pnpm/error';
+export class BadTarballError extends PnpmError {
+    expectedSize;
+    receivedSize;
+    constructor(opts) {
+        const message = `Actual size (${opts.receivedSize}) of tarball (${opts.tarballUrl}) did not match the one specified in 'Content-Length' header (${opts.expectedSize})`;
+        super('BAD_TARBALL_SIZE', message, {
+            attempts: opts?.attempts,
+        });
+        this.expectedSize = opts.expectedSize;
+        this.receivedSize = opts.receivedSize;
+    }
+}
+//# sourceMappingURL=BadTarballError.js.map

package/lib/errorTypes/index.d.ts

@@ -0,0 +1 @@
+export { BadTarballError } from './BadTarballError.js';

package/lib/errorTypes/index.js

@@ -0,0 +1,2 @@
+export { BadTarballError } from './BadTarballError.js';
+//# sourceMappingURL=index.js.map

package/lib/gitHostedTarballFetcher.d.ts

@@ -0,0 +1,9 @@
+import type { FetchFunction } from '@pnpm/fetching.fetcher-base';
+import type { StoreIndex } from '@pnpm/store.index';
+export interface CreateGitHostedTarballFetcher {
+    ignoreScripts?: boolean;
+    rawConfig: Record<string, unknown>;
+    storeIndex: StoreIndex;
+    unsafePerm?: boolean;
+}
+export declare function createGitHostedTarballFetcher(fetchRemoteTarball: FetchFunction, fetcherOpts: CreateGitHostedTarballFetcher): FetchFunction;

package/lib/gitHostedTarballFetcher.js

@@ -0,0 +1,89 @@
+import assert from 'node:assert';
+import util from 'node:util';
+import { preparePackage } from '@pnpm/exec.prepare-package';
+import { packlist } from '@pnpm/fs.packlist';
+import { globalWarn } from '@pnpm/logger';
+import { addFilesFromDir } from '@pnpm/worker';
+export function createGitHostedTarballFetcher(fetchRemoteTarball, fetcherOpts) {
+    const fetch = async (cafs, resolution, opts) => {
+        const rawFilesIndexFile = `${opts.filesIndexFile}\traw`;
+        const { filesMap, manifest, requiresBuild } = await fetchRemoteTarball(cafs, resolution, {
+            ...opts,
+            filesIndexFile: rawFilesIndexFile,
+        });
+        // Flush any queued store index writes so that the raw files index entry
+        // written during tarball extraction is visible to subsequent reads.
+        fetcherOpts.storeIndex.flush();
+        try {
+            const prepareResult = await prepareGitHostedPkg(filesMap, cafs, rawFilesIndexFile, opts.filesIndexFile, fetcherOpts, opts, resolution);
+            if (prepareResult.ignoredBuild) {
+                globalWarn(`The git-hosted package fetched from "${resolution.tarball}" has to be built but the build scripts were ignored.`);
+            }
+            return {
+                filesMap: prepareResult.filesMap,
+                manifest: prepareResult.manifest ?? manifest,
+                requiresBuild,
+            };
+        }
+        catch (err) {
+            assert(util.types.isNativeError(err));
+            err.message = `Failed to prepare git-hosted package fetched from "${resolution.tarball}": ${err.message}`;
+            throw err;
+        }
+    };
+    return fetch;
+}
+async function prepareGitHostedPkg(filesMap, cafs, rawFilesIndexFile, filesIndexFile, opts, fetcherOpts, resolution) {
+    const tempLocation = await cafs.tempDir();
+    cafs.importPackage(tempLocation, {
+        filesResponse: {
+            filesMap,
+            resolvedFrom: 'remote',
+            requiresBuild: false,
+        },
+        force: true,
+    });
+    const { shouldBeBuilt, pkgDir } = await preparePackage({
+        ...opts,
+        allowBuild: fetcherOpts.allowBuild,
+    }, tempLocation, resolution.path ?? '');
+    const files = await packlist(pkgDir);
+    const { storeIndex } = opts;
+    if (!resolution.path && files.length === filesMap.size) {
+        if (!shouldBeBuilt) {
+            const data = storeIndex.get(rawFilesIndexFile);
+            if (data) {
+                storeIndex.set(filesIndexFile, data);
+                storeIndex.delete(rawFilesIndexFile);
+            }
+            return {
+                filesMap,
+                ignoredBuild: false,
+            };
+        }
+        if (opts.ignoreScripts) {
+            storeIndex.delete(rawFilesIndexFile);
+            return {
+                filesMap,
+                ignoredBuild: true,
+            };
+        }
+    }
+    storeIndex.delete(rawFilesIndexFile);
+    // Important! We cannot remove the temp location at this stage.
+    // Even though we have the index of the package,
+    // the linking of files to the store is in progress.
+    return {
+        ...await addFilesFromDir({
+            storeDir: cafs.storeDir,
+            storeIndex: opts.storeIndex,
+            dir: pkgDir,
+            files,
+            filesIndexFile,
+            pkg: fetcherOpts.pkg,
+            readManifest: fetcherOpts.readManifest,
+        }),
+        ignoredBuild: Boolean(opts.ignoreScripts),
+    };
+}
+//# sourceMappingURL=gitHostedTarballFetcher.js.map

package/lib/index.d.ts

@@ -0,0 +1,24 @@
+import type { FetchFunction } from '@pnpm/fetching.fetcher-base';
+import type { FetchFromRegistry, GetAuthHeader, RetryTimeoutOptions } from '@pnpm/fetching.types';
+import type { StoreIndex } from '@pnpm/store.index';
+import { TarballIntegrityError } from '@pnpm/worker';
+import { type CreateDownloaderOptions } from './remoteTarballFetcher.js';
+export { BadTarballError } from './errorTypes/index.js';
+export { TarballIntegrityError };
+export { createGitHostedTarballFetcher } from './gitHostedTarballFetcher.js';
+export { createLocalTarballFetcher } from './localTarballFetcher.js';
+export { createDownloader, type CreateDownloaderOptions, type DownloadFunction } from './remoteTarballFetcher.js';
+export interface TarballFetchers {
+    localTarball: FetchFunction;
+    remoteTarball: FetchFunction;
+    gitHostedTarball: FetchFunction;
+}
+export declare function createTarballFetcher(fetchFromRegistry: FetchFromRegistry, getAuthHeader: GetAuthHeader, opts: {
+    rawConfig: Record<string, unknown>;
+    unsafePerm?: boolean;
+    ignoreScripts?: boolean;
+    storeIndex: StoreIndex;
+    timeout?: number;
+    retry?: RetryTimeoutOptions;
+    offline?: boolean;
+} & Pick<CreateDownloaderOptions, 'fetchMinSpeedKiBps'>): TarballFetchers;

package/lib/index.js

@@ -0,0 +1,48 @@
+import { PnpmError } from '@pnpm/error';
+import { TarballIntegrityError } from '@pnpm/worker';
+import { createGitHostedTarballFetcher } from './gitHostedTarballFetcher.js';
+import { createLocalTarballFetcher } from './localTarballFetcher.js';
+import { createDownloader, } from './remoteTarballFetcher.js';
+export { BadTarballError } from './errorTypes/index.js';
+export { TarballIntegrityError };
+// Export individual fetcher factories for custom fetcher authors
+export { createGitHostedTarballFetcher } from './gitHostedTarballFetcher.js';
+export { createLocalTarballFetcher } from './localTarballFetcher.js';
+export { createDownloader } from './remoteTarballFetcher.js';
+export function createTarballFetcher(fetchFromRegistry, getAuthHeader, opts) {
+    const download = createDownloader(fetchFromRegistry, {
+        retry: opts.retry,
+        timeout: opts.timeout,
+        fetchMinSpeedKiBps: opts.fetchMinSpeedKiBps,
+    });
+    const remoteTarballFetcher = fetchFromTarball.bind(null, {
+        download,
+        getAuthHeaderByURI: getAuthHeader,
+        offline: opts.offline,
+        storeIndex: opts.storeIndex,
+    });
+    return {
+        localTarball: createLocalTarballFetcher(opts.storeIndex),
+        remoteTarball: remoteTarballFetcher,
+        gitHostedTarball: createGitHostedTarballFetcher(remoteTarballFetcher, opts),
+    };
+}
+async function fetchFromTarball(ctx, cafs, resolution, opts) {
+    if (ctx.offline) {
+        throw new PnpmError('NO_OFFLINE_TARBALL', `A package is missing from the store but cannot download it in offline mode. The missing package may be downloaded from ${resolution.tarball}.`);
+    }
+    return ctx.download(resolution.tarball, {
+        getAuthHeaderByURI: ctx.getAuthHeaderByURI,
+        cafs,
+        storeIndex: ctx.storeIndex,
+        integrity: resolution.integrity,
+        readManifest: opts.readManifest,
+        onProgress: opts.onProgress,
+        onStart: opts.onStart,
+        registry: resolution.registry,
+        filesIndexFile: opts.filesIndexFile,
+        pkg: opts.pkg,
+        appendManifest: opts.appendManifest,
+    });
+}
+//# sourceMappingURL=index.js.map

package/lib/localTarballFetcher.d.ts

@@ -0,0 +1,3 @@
+import type { FetchFunction } from '@pnpm/fetching.fetcher-base';
+import type { StoreIndex } from '@pnpm/store.index';
+export declare function createLocalTarballFetcher(storeIndex: StoreIndex): FetchFunction;

package/lib/localTarballFetcher.js

@@ -0,0 +1,28 @@
+import path from 'node:path';
+import gfs from '@pnpm/fs.graceful-fs';
+import { addFilesFromTarball } from '@pnpm/worker';
+const isAbsolutePath = /^\/|^[A-Z]:/i;
+export function createLocalTarballFetcher(storeIndex) {
+    const fetch = (cafs, resolution, opts) => {
+        const tarball = resolvePath(opts.lockfileDir, resolution.tarball.slice(5));
+        const buffer = gfs.readFileSync(tarball);
+        return addFilesFromTarball({
+            storeDir: cafs.storeDir,
+            storeIndex,
+            buffer,
+            filesIndexFile: opts.filesIndexFile,
+            integrity: resolution.integrity,
+            readManifest: opts.readManifest,
+            url: tarball,
+            pkg: opts.pkg,
+            appendManifest: opts.appendManifest,
+        });
+    };
+    return fetch;
+}
+function resolvePath(where, spec) {
+    if (isAbsolutePath.test(spec))
+        return spec;
+    return path.resolve(where, spec);
+}
+//# sourceMappingURL=localTarballFetcher.js.map

package/lib/remoteTarballFetcher.d.ts

@@ -0,0 +1,36 @@
+import type { IncomingMessage } from 'node:http';
+import type { FetchOptions, FetchResult } from '@pnpm/fetching.fetcher-base';
+import type { FetchFromRegistry } from '@pnpm/fetching.types';
+import type { Cafs } from '@pnpm/store.cafs-types';
+import type { StoreIndex } from '@pnpm/store.index';
+export interface HttpResponse {
+    body: string;
+}
+export type DownloadOptions = {
+    getAuthHeaderByURI: (registry: string) => string | undefined;
+    cafs: Cafs;
+    registry?: string;
+    onStart?: (totalSize: number | null, attempt: number) => void;
+    onProgress?: (downloaded: number) => void;
+    integrity?: string;
+    storeIndex: StoreIndex;
+} & Pick<FetchOptions, 'pkg' | 'appendManifest' | 'readManifest' | 'filesIndexFile'>;
+export type DownloadFunction = (url: string, opts: DownloadOptions) => Promise<FetchResult>;
+export interface NpmRegistryClient {
+    get: (url: string, getOpts: object, cb: (err: Error, data: object, raw: object, res: HttpResponse) => void) => void;
+    fetch: (url: string, opts: {
+        auth?: object;
+    }, cb: (err: Error, res: IncomingMessage) => void) => void;
+}
+export interface CreateDownloaderOptions {
+    retry?: {
+        retries?: number;
+        factor?: number;
+        minTimeout?: number;
+        maxTimeout?: number;
+        randomize?: boolean;
+    };
+    timeout?: number;
+    fetchMinSpeedKiBps?: number;
+}
+export declare function createDownloader(fetchFromRegistry: FetchFromRegistry, gotOpts: CreateDownloaderOptions): DownloadFunction;

package/lib/remoteTarballFetcher.js

@@ -0,0 +1,130 @@
+import assert from 'node:assert';
+import util from 'node:util';
+import { requestRetryLogger } from '@pnpm/core-loggers';
+import { FetchError } from '@pnpm/error';
+import { globalWarn } from '@pnpm/logger';
+import { addFilesFromTarball } from '@pnpm/worker';
+import * as retry from '@zkochan/retry';
+import throttle from 'lodash.throttle';
+import { BadTarballError } from './errorTypes/index.js';
+const BIG_TARBALL_SIZE = 1024 * 1024 * 5; // 5 MB
+export function createDownloader(fetchFromRegistry, gotOpts) {
+    const retryOpts = {
+        factor: 10,
+        maxTimeout: 6e4, // 1 minute
+        minTimeout: 1e4, // 10 seconds
+        retries: 2,
+        ...gotOpts.retry,
+    };
+    const fetchMinSpeedKiBps = gotOpts.fetchMinSpeedKiBps ?? 50; // 50 KiB/s
+    return async function download(url, opts) {
+        const authHeaderValue = opts.getAuthHeaderByURI(url);
+        const op = retry.operation(retryOpts);
+        return new Promise((resolve, reject) => {
+            op.attempt(async (attempt) => {
+                try {
+                    resolve(await fetch(attempt));
+                }
+                catch (error) { // eslint-disable-line
+                    if (error.response?.status === 401 ||
+                        error.response?.status === 403 ||
+                        error.response?.status === 404 ||
+                        error.code === 'ERR_PNPM_PREPARE_PKG_FAILURE') {
+                        reject(error);
+                        return;
+                    }
+                    const timeout = op.retry(error);
+                    if (timeout === false) {
+                        reject(op.mainError());
+                        return;
+                    }
+                    requestRetryLogger.debug({
+                        attempt,
+                        error,
+                        maxRetries: retryOpts.retries,
+                        method: 'GET',
+                        timeout,
+                        url,
+                    });
+                }
+            });
+        });
+        async function fetch(currentAttempt) {
+            let data;
+            try {
+                const res = await fetchFromRegistry(url, {
+                    authHeaderValue,
+                    // The fetch library can retry requests on bad HTTP responses.
+                    // However, it is not enough to retry on bad HTTP responses only.
+                    // Requests should also be retried when the tarball's integrity check fails.
+                    // Hence, we tell fetch to not retry,
+                    // and we perform the retries from this function instead.
+                    retry: { retries: 0 },
+                    timeout: gotOpts.timeout,
+                });
+                if (res.status !== 200) {
+                    throw new FetchError({ url, authHeaderValue }, res);
+                }
+                const contentLength = res.headers.has('content-length') && res.headers.get('content-length');
+                const size = typeof contentLength === 'string'
+                    ? parseInt(contentLength, 10)
+                    : null;
+                if (opts.onStart != null) {
+                    opts.onStart(size, currentAttempt);
+                }
+                // In order to reduce the amount of logs, we only report the download progress of big tarballs
+                const onProgress = (size != null && size >= BIG_TARBALL_SIZE && opts.onProgress)
+                    ? throttle(opts.onProgress, 500)
+                    : undefined;
+                const startTime = Date.now();
+                let downloaded = 0;
+                const chunks = [];
+                // This will handle the 'data', 'error', and 'end' events.
+                for await (const chunk of res.body) {
+                    chunks.push(chunk);
+                    downloaded += chunk.length;
+                    onProgress?.(downloaded);
+                }
+                if (size !== null && size !== downloaded) {
+                    throw new BadTarballError({
+                        expectedSize: size,
+                        receivedSize: downloaded,
+                        tarballUrl: url,
+                    });
+                }
+                const elapsedSec = (Date.now() - startTime) / 1000;
+                const avgKiBps = Math.floor((downloaded / elapsedSec) / 1024);
+                if (downloaded > 0 && elapsedSec > 1 && avgKiBps < fetchMinSpeedKiBps) {
+                    const sizeKb = Math.floor(downloaded / 1024);
+                    globalWarn(`Tarball download average speed ${avgKiBps} KiB/s (size ${sizeKb} KiB) is below ${fetchMinSpeedKiBps} KiB/s: ${url} (GET)`);
+                }
+                data = Buffer.from(new SharedArrayBuffer(downloaded));
+                let offset = 0;
+                for (const chunk of chunks) {
+                    chunk.copy(data, offset);
+                    offset += chunk.length;
+                }
+            }
+            catch (err) {
+                assert(util.types.isNativeError(err));
+                Object.assign(err, {
+                    attempts: currentAttempt,
+                    resource: url,
+                });
+                throw err;
+            }
+            return addFilesFromTarball({
+                buffer: data,
+                storeDir: opts.cafs.storeDir,
+                storeIndex: opts.storeIndex,
+                readManifest: opts.readManifest,
+                integrity: opts.integrity,
+                filesIndexFile: opts.filesIndexFile,
+                url,
+                pkg: opts.pkg,
+                appendManifest: opts.appendManifest,
+            });
+        }
+    };
+}
+//# sourceMappingURL=remoteTarballFetcher.js.map

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "@pnpm/fetching.tarball-fetcher",
+  "version": "1003.0.0",
+  "description": "Fetcher for packages hosted as tarballs",
+  "keywords": [
+    "pnpm",
+    "pnpm11",
+    "fetcher",
+    "tarball"
+  ],
+  "license": "MIT",
+  "funding": "https://opencollective.com/pnpm",
+  "repository": "https://github.com/pnpm/pnpm/tree/main/fetching/tarball-fetcher",
+  "homepage": "https://github.com/pnpm/pnpm/tree/main/fetching/tarball-fetcher#readme",
+  "bugs": {
+    "url": "https://github.com/pnpm/pnpm/issues"
+  },
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": "./lib/index.js"
+  },
+  "files": [
+    "lib",
+    "!*.map"
+  ],
+  "dependencies": {
+    "@zkochan/retry": "^0.2.0",
+    "lodash.throttle": "4.1.1",
+    "p-map-values": "^0.1.0",
+    "ramda": "npm:@pnpm/ramda@0.28.1",
+    "@pnpm/core-loggers": "1001.0.4",
+    "@pnpm/error": "1000.0.5",
+    "@pnpm/fetching.fetcher-base": "1001.0.2",
+    "@pnpm/fs.graceful-fs": "1000.0.1",
+    "@pnpm/fs.packlist": "1000.0.0",
+    "@pnpm/fetching.types": "1000.2.0",
+    "@pnpm/exec.prepare-package": "1000.0.26",
+    "@pnpm/types": "1000.9.0",
+    "@pnpm/store.index": "1000.0.0-0"
+  },
+  "peerDependencies": {
+    "@pnpm/logger": ">=1001.0.0 <1002.0.0",
+    "@pnpm/worker": "^1000.3.0"
+  },
+  "devDependencies": {
+    "@jest/globals": "30.0.5",
+    "@pnpm/util.lex-comparator": "^3.0.2",
+    "@types/lodash.throttle": "4.1.7",
+    "@types/ramda": "0.29.12",
+    "@types/retry": "^0.12.5",
+    "@types/ssri": "^7.1.5",
+    "nock": "13.3.4",
+    "ssri": "13.0.0",
+    "tempy": "3.0.0",
+    "@pnpm/fetching.tarball-fetcher": "1003.0.0",
+    "@pnpm/network.fetch": "1000.2.6",
+    "@pnpm/store.cafs-types": "1000.0.0",
+    "@pnpm/store.create-cafs-store": "1000.0.20",
+    "@pnpm/logger": "1001.0.1",
+    "@pnpm/test-fixtures": "1000.0.0",
+    "@pnpm/types": "1000.9.0"
+  },
+  "engines": {
+    "node": ">=22.13"
+  },
+  "jest": {
+    "preset": "@pnpm/jest-config"
+  },
+  "scripts": {
+    "lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
+    "_test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest",
+    "test": "pnpm run compile && pnpm run _test",
+    "compile": "tsgo --build && pnpm run lint --fix"
+  }
+}