@pnpm/installing.package-requester 1008.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +17 -0
- package/lib/index.d.ts +2 -0
- package/lib/index.js +2 -0
- package/lib/packageRequester.d.ts +25 -0
- package/lib/packageRequester.js +468 -0
- package/package.json +88 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2015-2016 Rico Sta. Cruz and other contributors
|
|
4
|
+
Copyright (c) 2016-2026 Zoltan Kochan and other contributors
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# @pnpm/package-requester
|
|
2
|
+
|
|
3
|
+
> Concurrent downloader of npm-compatible packages
|
|
4
|
+
|
|
5
|
+
<!--@shields('npm')-->
|
|
6
|
+
[](https://www.npmjs.com/package/@pnpm/package-requester)
|
|
7
|
+
<!--/@-->
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
pnpm add @pnpm/logger @pnpm/package-requester
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## License
|
|
16
|
+
|
|
17
|
+
MIT
|
package/lib/index.d.ts
ADDED
package/lib/index.js
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { Fetchers } from '@pnpm/fetching.fetcher-base';
|
|
2
|
+
import type { CustomFetcher } from '@pnpm/hooks.types';
|
|
3
|
+
import type { ResolveFunction } from '@pnpm/resolving.resolver-base';
|
|
4
|
+
import type { Cafs } from '@pnpm/store.cafs-types';
|
|
5
|
+
import type { FetchPackageToStoreFunction, GetFilesIndexFilePath, RequestPackageFunction } from '@pnpm/store.controller-types';
|
|
6
|
+
export declare function createPackageRequester(opts: {
|
|
7
|
+
engineStrict?: boolean;
|
|
8
|
+
force?: boolean;
|
|
9
|
+
nodeVersion?: string;
|
|
10
|
+
pnpmVersion?: string;
|
|
11
|
+
resolve: ResolveFunction;
|
|
12
|
+
fetchers: Fetchers;
|
|
13
|
+
cafs: Cafs;
|
|
14
|
+
ignoreFile?: (filename: string) => boolean;
|
|
15
|
+
networkConcurrency?: number;
|
|
16
|
+
storeDir: string;
|
|
17
|
+
verifyStoreIntegrity: boolean;
|
|
18
|
+
virtualStoreDirMaxLength: number;
|
|
19
|
+
strictStorePkgContentCheck?: boolean;
|
|
20
|
+
customFetchers?: CustomFetcher[];
|
|
21
|
+
}): RequestPackageFunction & {
|
|
22
|
+
fetchPackageToStore: FetchPackageToStoreFunction;
|
|
23
|
+
getFilesIndexFilePath: GetFilesIndexFilePath;
|
|
24
|
+
requestPackage: RequestPackageFunction;
|
|
25
|
+
};
|
|
@@ -0,0 +1,468 @@
|
|
|
1
|
+
import { createReadStream, promises as fs } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { packageIsInstallable } from '@pnpm/config.package-is-installable';
|
|
4
|
+
import { fetchingProgressLogger, progressLogger } from '@pnpm/core-loggers';
|
|
5
|
+
import { depPathToFilename } from '@pnpm/deps.path';
|
|
6
|
+
import { PnpmError } from '@pnpm/error';
|
|
7
|
+
import { pickFetcher } from '@pnpm/fetching.pick-fetcher';
|
|
8
|
+
import gfs from '@pnpm/fs.graceful-fs';
|
|
9
|
+
import { logger } from '@pnpm/logger';
|
|
10
|
+
import { normalizeBundledManifest, } from '@pnpm/store.cafs';
|
|
11
|
+
import { gitHostedStoreIndexKey, storeIndexKey } from '@pnpm/store.index';
|
|
12
|
+
import { calcMaxWorkers, readPkgFromCafs as _readPkgFromCafs, } from '@pnpm/worker';
|
|
13
|
+
import { familySync } from 'detect-libc';
|
|
14
|
+
import { loadJsonFile } from 'load-json-file';
|
|
15
|
+
import pDefer, {} from 'p-defer';
|
|
16
|
+
import PQueue from 'p-queue';
|
|
17
|
+
import { pShare } from 'promise-share';
|
|
18
|
+
import { pick } from 'ramda';
|
|
19
|
+
import ssri from 'ssri';
|
|
20
|
+
let currentLibc;
|
|
21
|
+
function getLibcFamilySync() {
|
|
22
|
+
if (currentLibc === undefined) {
|
|
23
|
+
currentLibc = familySync();
|
|
24
|
+
}
|
|
25
|
+
return currentLibc;
|
|
26
|
+
}
|
|
27
|
+
const TARBALL_INTEGRITY_FILENAME = 'tarball-integrity';
|
|
28
|
+
const packageRequestLogger = logger('package-requester');
|
|
29
|
+
export function createPackageRequester(opts) {
|
|
30
|
+
opts = opts || {};
|
|
31
|
+
const networkConcurrency = opts.networkConcurrency ?? Math.min(64, Math.max(calcMaxWorkers() * 3, 16));
|
|
32
|
+
const requestsQueue = new PQueue({
|
|
33
|
+
concurrency: networkConcurrency,
|
|
34
|
+
});
|
|
35
|
+
const fetch = fetcher.bind(null, opts.fetchers, opts.cafs, opts.customFetchers);
|
|
36
|
+
const readPkgFromCafs = _readPkgFromCafs.bind(null, {
|
|
37
|
+
storeDir: opts.storeDir,
|
|
38
|
+
verifyStoreIntegrity: opts.verifyStoreIntegrity,
|
|
39
|
+
strictStorePkgContentCheck: opts.strictStorePkgContentCheck,
|
|
40
|
+
});
|
|
41
|
+
const fetchPackageToStore = fetchToStore.bind(null, {
|
|
42
|
+
readPkgFromCafs,
|
|
43
|
+
fetch,
|
|
44
|
+
fetchingLocker: new Map(),
|
|
45
|
+
requestsQueue: Object.assign(requestsQueue, {
|
|
46
|
+
counter: 0,
|
|
47
|
+
concurrency: networkConcurrency,
|
|
48
|
+
}),
|
|
49
|
+
storeDir: opts.storeDir,
|
|
50
|
+
virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
|
|
51
|
+
strictStorePkgContentCheck: opts.strictStorePkgContentCheck,
|
|
52
|
+
});
|
|
53
|
+
const requestPackage = resolveAndFetch.bind(null, {
|
|
54
|
+
engineStrict: opts.engineStrict,
|
|
55
|
+
nodeVersion: opts.nodeVersion,
|
|
56
|
+
pnpmVersion: opts.pnpmVersion,
|
|
57
|
+
force: opts.force,
|
|
58
|
+
fetchPackageToStore,
|
|
59
|
+
requestsQueue,
|
|
60
|
+
resolve: opts.resolve,
|
|
61
|
+
storeDir: opts.storeDir,
|
|
62
|
+
});
|
|
63
|
+
return Object.assign(requestPackage, {
|
|
64
|
+
fetchPackageToStore,
|
|
65
|
+
getFilesIndexFilePath: getFilesIndexFilePath.bind(null, {
|
|
66
|
+
storeDir: opts.storeDir,
|
|
67
|
+
virtualStoreDirMaxLength: opts.virtualStoreDirMaxLength,
|
|
68
|
+
}),
|
|
69
|
+
requestPackage,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
async function resolveAndFetch(ctx, wantedDependency, options) {
|
|
73
|
+
let resolution = options.currentPkg?.resolution;
|
|
74
|
+
let pkgId = options.currentPkg?.id;
|
|
75
|
+
// When we have a currentPkg but a resolution is still performed due to
|
|
76
|
+
// options.skipFetch, it's necessary to make sure the resolution doesn't
|
|
77
|
+
// accidentally return a newer version of the package. When skipFetch is
|
|
78
|
+
// set, the resolved package shouldn't be different. This is done by
|
|
79
|
+
// overriding the preferredVersions object to only contain the current
|
|
80
|
+
// package's version.
|
|
81
|
+
//
|
|
82
|
+
// A naive approach would be to change the bare specifier to be the exact
|
|
83
|
+
// version of the current pkg if the bare specifier is a range, but this
|
|
84
|
+
// would cause the version returned for calcSpecifier to be different.
|
|
85
|
+
const preferredVersions = (resolution && !options.update && options.currentPkg?.name != null && options.currentPkg?.version != null)
|
|
86
|
+
? {
|
|
87
|
+
...options.preferredVersions,
|
|
88
|
+
[options.currentPkg.name]: { [options.currentPkg.version]: 'version' },
|
|
89
|
+
}
|
|
90
|
+
: options.preferredVersions;
|
|
91
|
+
const resolveResult = await ctx.requestsQueue.add(async () => ctx.resolve(wantedDependency, {
|
|
92
|
+
...options,
|
|
93
|
+
preferredVersions,
|
|
94
|
+
currentPkg: (options.currentPkg?.id && options.currentPkg?.resolution)
|
|
95
|
+
? {
|
|
96
|
+
id: options.currentPkg.id,
|
|
97
|
+
name: options.currentPkg.name,
|
|
98
|
+
version: options.currentPkg.version,
|
|
99
|
+
resolution: options.currentPkg.resolution,
|
|
100
|
+
}
|
|
101
|
+
: undefined,
|
|
102
|
+
}), { priority: options.downloadPriority });
|
|
103
|
+
let { manifest } = resolveResult;
|
|
104
|
+
const { latest, resolvedVia, publishedAt, normalizedBareSpecifier, alias, } = resolveResult;
|
|
105
|
+
// Check if the integrity has changed between the current and newly resolved package
|
|
106
|
+
// Use 'in' check to safely access integrity from any resolution type that has it
|
|
107
|
+
const previousResolution = options.currentPkg?.resolution;
|
|
108
|
+
const previousIntegrity = previousResolution && 'integrity' in previousResolution ? previousResolution.integrity : undefined;
|
|
109
|
+
const newIntegrity = 'integrity' in resolveResult.resolution ? resolveResult.resolution.integrity : undefined;
|
|
110
|
+
const integrityChanged = previousIntegrity != null && newIntegrity != null && previousIntegrity !== newIntegrity;
|
|
111
|
+
const updated = pkgId !== resolveResult.id || !resolution || integrityChanged;
|
|
112
|
+
resolution = resolveResult.resolution;
|
|
113
|
+
pkgId = resolveResult.id;
|
|
114
|
+
const id = pkgId;
|
|
115
|
+
if ('type' in resolution && resolution.type === 'directory' && !id.startsWith('file:')) {
|
|
116
|
+
if (manifest == null) {
|
|
117
|
+
throw new Error(`Couldn't read package.json of local dependency ${wantedDependency.alias ? wantedDependency.alias + '@' : ''}${wantedDependency.bareSpecifier ?? ''}`);
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
body: {
|
|
121
|
+
id,
|
|
122
|
+
isLocal: true,
|
|
123
|
+
manifest,
|
|
124
|
+
resolution: resolution,
|
|
125
|
+
resolvedVia,
|
|
126
|
+
updated,
|
|
127
|
+
normalizedBareSpecifier,
|
|
128
|
+
alias,
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
let isInstallable = (ctx.force === true ||
|
|
133
|
+
(manifest == null
|
|
134
|
+
? undefined
|
|
135
|
+
: packageIsInstallable(id, manifest, {
|
|
136
|
+
engineStrict: ctx.engineStrict,
|
|
137
|
+
lockfileDir: options.lockfileDir,
|
|
138
|
+
nodeVersion: ctx.nodeVersion,
|
|
139
|
+
optional: wantedDependency.optional === true,
|
|
140
|
+
supportedArchitectures: options.supportedArchitectures,
|
|
141
|
+
})));
|
|
142
|
+
// We can skip fetching the package only if the manifest
|
|
143
|
+
// is present after resolution AND the content of the package has not changed
|
|
144
|
+
if ((options.skipFetch === true || isInstallable === false) && !integrityChanged && (manifest != null)) {
|
|
145
|
+
return {
|
|
146
|
+
body: {
|
|
147
|
+
id,
|
|
148
|
+
isLocal: false,
|
|
149
|
+
isInstallable: isInstallable ?? undefined,
|
|
150
|
+
latest,
|
|
151
|
+
manifest,
|
|
152
|
+
normalizedBareSpecifier,
|
|
153
|
+
resolution,
|
|
154
|
+
resolvedVia,
|
|
155
|
+
updated,
|
|
156
|
+
publishedAt,
|
|
157
|
+
alias,
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
const pkg = manifest != null ? pick(['name', 'version'], manifest) : {};
|
|
162
|
+
const fetchResult = ctx.fetchPackageToStore({
|
|
163
|
+
allowBuild: options.allowBuild,
|
|
164
|
+
fetchRawManifest: true,
|
|
165
|
+
force: integrityChanged,
|
|
166
|
+
ignoreScripts: options.ignoreScripts,
|
|
167
|
+
lockfileDir: options.lockfileDir,
|
|
168
|
+
pkg: {
|
|
169
|
+
...(options.expectedPkg?.name != null
|
|
170
|
+
? (updated ? { name: options.expectedPkg.name, version: pkg.version } : options.expectedPkg)
|
|
171
|
+
: pkg),
|
|
172
|
+
id,
|
|
173
|
+
resolution,
|
|
174
|
+
},
|
|
175
|
+
onFetchError: options.onFetchError,
|
|
176
|
+
supportedArchitectures: options.supportedArchitectures,
|
|
177
|
+
});
|
|
178
|
+
if (!manifest) {
|
|
179
|
+
const fetchedResult = await fetchResult.fetching();
|
|
180
|
+
if (fetchedResult.bundledManifest) {
|
|
181
|
+
manifest = fetchedResult.bundledManifest;
|
|
182
|
+
}
|
|
183
|
+
else if (fetchedResult.files.filesMap.has('package.json')) {
|
|
184
|
+
manifest = await loadJsonFile(fetchedResult.files.filesMap.get('package.json'));
|
|
185
|
+
}
|
|
186
|
+
// Add integrity to resolution if it was computed during fetching (only for TarballResolution)
|
|
187
|
+
if (fetchedResult.integrity && !resolution.type && !resolution.integrity) {
|
|
188
|
+
resolution.integrity = fetchedResult.integrity;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
// Check installability now that we have the manifest (for git/tarball packages without registry metadata)
|
|
192
|
+
if (isInstallable === undefined && manifest != null) {
|
|
193
|
+
isInstallable = ctx.force === true || packageIsInstallable(id, manifest, {
|
|
194
|
+
engineStrict: ctx.engineStrict,
|
|
195
|
+
lockfileDir: options.lockfileDir,
|
|
196
|
+
nodeVersion: ctx.nodeVersion,
|
|
197
|
+
optional: wantedDependency.optional === true,
|
|
198
|
+
supportedArchitectures: options.supportedArchitectures,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
body: {
|
|
203
|
+
id,
|
|
204
|
+
isLocal: false,
|
|
205
|
+
isInstallable: isInstallable ?? undefined,
|
|
206
|
+
latest,
|
|
207
|
+
manifest,
|
|
208
|
+
normalizedBareSpecifier,
|
|
209
|
+
resolution,
|
|
210
|
+
resolvedVia,
|
|
211
|
+
updated,
|
|
212
|
+
publishedAt,
|
|
213
|
+
alias,
|
|
214
|
+
},
|
|
215
|
+
fetching: fetchResult.fetching,
|
|
216
|
+
filesIndexFile: fetchResult.filesIndexFile,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
function getFilesIndexFilePath(ctx, opts) {
|
|
220
|
+
const targetRelative = depPathToFilename(opts.pkg.id, ctx.virtualStoreDirMaxLength);
|
|
221
|
+
const target = path.join(ctx.storeDir, targetRelative);
|
|
222
|
+
if (opts.pkg.resolution.integrity) {
|
|
223
|
+
return {
|
|
224
|
+
target,
|
|
225
|
+
filesIndexFile: storeIndexKey(opts.pkg.resolution.integrity, opts.pkg.id),
|
|
226
|
+
resolution: opts.pkg.resolution,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
let resolution;
|
|
230
|
+
if (opts.pkg.resolution.type === 'variations') {
|
|
231
|
+
resolution = findResolution(opts.pkg.resolution.variants, opts.supportedArchitectures);
|
|
232
|
+
if (resolution.integrity) {
|
|
233
|
+
return {
|
|
234
|
+
target,
|
|
235
|
+
filesIndexFile: storeIndexKey(resolution.integrity, opts.pkg.id),
|
|
236
|
+
resolution,
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
resolution = opts.pkg.resolution;
|
|
242
|
+
}
|
|
243
|
+
const filesIndexFile = gitHostedStoreIndexKey(opts.pkg.id, { built: !opts.ignoreScripts });
|
|
244
|
+
return { filesIndexFile, target, resolution };
|
|
245
|
+
}
|
|
246
|
+
function findResolution(resolutionVariants, supportedArchitectures) {
|
|
247
|
+
const platform = getOneIfNonCurrent(supportedArchitectures?.os) ?? process.platform;
|
|
248
|
+
const cpu = getOneIfNonCurrent(supportedArchitectures?.cpu) ?? process.arch;
|
|
249
|
+
const libc = getOneIfNonCurrent(supportedArchitectures?.libc) ?? getLibcFamilySync();
|
|
250
|
+
const resolutionVariant = resolutionVariants
|
|
251
|
+
.find((resolutionVariant) => resolutionVariant.targets.some((target) => target.os === platform &&
|
|
252
|
+
target.cpu === cpu &&
|
|
253
|
+
(target.libc == null || target.libc === libc)));
|
|
254
|
+
if (!resolutionVariant) {
|
|
255
|
+
const resolutionTargets = resolutionVariants.map((variant) => variant.targets);
|
|
256
|
+
throw new PnpmError('NO_RESOLUTION_MATCHED', `Cannot find a resolution variant for the current platform in these resolutions: ${JSON.stringify(resolutionTargets)}`);
|
|
257
|
+
}
|
|
258
|
+
return resolutionVariant.resolution;
|
|
259
|
+
}
|
|
260
|
+
function getOneIfNonCurrent(requirements) {
|
|
261
|
+
if (requirements?.length && requirements[0] !== 'current') {
|
|
262
|
+
return requirements[0];
|
|
263
|
+
}
|
|
264
|
+
return undefined;
|
|
265
|
+
}
|
|
266
|
+
function fetchToStore(ctx, opts) {
|
|
267
|
+
if (!opts.pkg.name) {
|
|
268
|
+
opts.fetchRawManifest = true;
|
|
269
|
+
}
|
|
270
|
+
if (!ctx.fetchingLocker.has(opts.pkg.id)) {
|
|
271
|
+
const fetching = pDefer();
|
|
272
|
+
const { filesIndexFile, target, resolution } = getFilesIndexFilePath(ctx, opts);
|
|
273
|
+
doFetchToStore(filesIndexFile, fetching, target, resolution);
|
|
274
|
+
ctx.fetchingLocker.set(opts.pkg.id, {
|
|
275
|
+
fetching: removeKeyOnFail(fetching.promise),
|
|
276
|
+
filesIndexFile,
|
|
277
|
+
fetchRawManifest: opts.fetchRawManifest,
|
|
278
|
+
});
|
|
279
|
+
// When files resolves, the cached result has to set fromStore to true, without
|
|
280
|
+
// affecting previous invocations: so we need to replace the cache.
|
|
281
|
+
//
|
|
282
|
+
// Changing the value of fromStore is needed for correct reporting of `pnpm server`.
|
|
283
|
+
// Otherwise, if a package was not in store when the server started, it will always be
|
|
284
|
+
// reported as "downloaded" instead of "reused".
|
|
285
|
+
fetching.promise.then((cache) => {
|
|
286
|
+
progressLogger.debug({
|
|
287
|
+
packageId: opts.pkg.id,
|
|
288
|
+
requester: opts.lockfileDir,
|
|
289
|
+
status: cache.files.resolvedFrom === 'remote'
|
|
290
|
+
? 'fetched'
|
|
291
|
+
: 'found_in_store',
|
|
292
|
+
});
|
|
293
|
+
// If it's already in the store, we don't need to update the cache
|
|
294
|
+
if (cache.files.resolvedFrom !== 'remote') {
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
const tmp = ctx.fetchingLocker.get(opts.pkg.id);
|
|
298
|
+
// If fetching failed then it was removed from the cache.
|
|
299
|
+
// It is OK. In that case there is no need to update it.
|
|
300
|
+
if (tmp == null)
|
|
301
|
+
return;
|
|
302
|
+
ctx.fetchingLocker.set(opts.pkg.id, {
|
|
303
|
+
...tmp,
|
|
304
|
+
fetching: Promise.resolve({
|
|
305
|
+
...cache,
|
|
306
|
+
files: {
|
|
307
|
+
...cache.files,
|
|
308
|
+
resolvedFrom: 'store',
|
|
309
|
+
},
|
|
310
|
+
}),
|
|
311
|
+
});
|
|
312
|
+
})
|
|
313
|
+
.catch(() => {
|
|
314
|
+
ctx.fetchingLocker.delete(opts.pkg.id);
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
const result = ctx.fetchingLocker.get(opts.pkg.id);
|
|
318
|
+
if (opts.fetchRawManifest && !result.fetchRawManifest) {
|
|
319
|
+
result.fetching = removeKeyOnFail(result.fetching.then(async ({ files }) => {
|
|
320
|
+
if (!files.filesMap.has('package.json'))
|
|
321
|
+
return {
|
|
322
|
+
files,
|
|
323
|
+
bundledManifest: undefined,
|
|
324
|
+
};
|
|
325
|
+
return {
|
|
326
|
+
files,
|
|
327
|
+
bundledManifest: await readBundledManifest(files.filesMap.get('package.json')),
|
|
328
|
+
};
|
|
329
|
+
}));
|
|
330
|
+
result.fetchRawManifest = true;
|
|
331
|
+
}
|
|
332
|
+
return {
|
|
333
|
+
fetching: pShare(result.fetching),
|
|
334
|
+
filesIndexFile: result.filesIndexFile,
|
|
335
|
+
};
|
|
336
|
+
async function removeKeyOnFail(p) {
|
|
337
|
+
try {
|
|
338
|
+
return await p;
|
|
339
|
+
}
|
|
340
|
+
catch (err) { // eslint-disable-line
|
|
341
|
+
ctx.fetchingLocker.delete(opts.pkg.id);
|
|
342
|
+
if (opts.onFetchError) {
|
|
343
|
+
throw opts.onFetchError(err);
|
|
344
|
+
}
|
|
345
|
+
throw err;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
async function doFetchToStore(filesIndexFile, fetching, target, resolution) {
|
|
349
|
+
try {
|
|
350
|
+
const isLocalTarballDep = opts.pkg.id.startsWith('file:');
|
|
351
|
+
const isLocalPkg = resolution.type === 'directory';
|
|
352
|
+
if (!opts.force &&
|
|
353
|
+
(!isLocalTarballDep ||
|
|
354
|
+
await tarballIsUpToDate(opts.pkg.resolution, target, opts.lockfileDir) // eslint-disable-line
|
|
355
|
+
) &&
|
|
356
|
+
!isLocalPkg) {
|
|
357
|
+
const { verified, files, bundledManifest } = await ctx.readPkgFromCafs(filesIndexFile, {
|
|
358
|
+
readManifest: opts.fetchRawManifest,
|
|
359
|
+
expectedPkg: opts.pkg,
|
|
360
|
+
});
|
|
361
|
+
if (verified) {
|
|
362
|
+
fetching.resolve({
|
|
363
|
+
files,
|
|
364
|
+
bundledManifest,
|
|
365
|
+
});
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
if ((files?.filesMap) != null) {
|
|
369
|
+
packageRequestLogger.warn({
|
|
370
|
+
message: `Refetching ${target} to store. It was either modified or had no integrity checksums`,
|
|
371
|
+
prefix: opts.lockfileDir,
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
// We fetch into targetStage directory first and then fs.rename() it to the
|
|
376
|
+
// target directory.
|
|
377
|
+
// Tarballs are requested first because they are bigger than metadata files.
|
|
378
|
+
// However, when one line is left available, allow it to be picked up by a metadata request.
|
|
379
|
+
// This is done in order to avoid situations when tarballs are downloaded in chunks
|
|
380
|
+
// As many tarballs should be downloaded simultaneously as possible.
|
|
381
|
+
const priority = (++ctx.requestsQueue.counter % ctx.requestsQueue.concurrency === 0 ? -1 : 1) * 1000;
|
|
382
|
+
const fetchedPackage = await ctx.requestsQueue.add(async () => ctx.fetch(opts.pkg.id, resolution, {
|
|
383
|
+
allowBuild: opts.allowBuild,
|
|
384
|
+
filesIndexFile,
|
|
385
|
+
lockfileDir: opts.lockfileDir,
|
|
386
|
+
readManifest: opts.fetchRawManifest,
|
|
387
|
+
onProgress: (downloaded) => {
|
|
388
|
+
fetchingProgressLogger.debug({
|
|
389
|
+
downloaded,
|
|
390
|
+
packageId: opts.pkg.id,
|
|
391
|
+
status: 'in_progress',
|
|
392
|
+
});
|
|
393
|
+
},
|
|
394
|
+
onStart: (size, attempt) => {
|
|
395
|
+
fetchingProgressLogger.debug({
|
|
396
|
+
attempt,
|
|
397
|
+
packageId: opts.pkg.id,
|
|
398
|
+
size,
|
|
399
|
+
status: 'started',
|
|
400
|
+
});
|
|
401
|
+
},
|
|
402
|
+
pkg: {
|
|
403
|
+
name: opts.pkg.name,
|
|
404
|
+
version: opts.pkg.version,
|
|
405
|
+
},
|
|
406
|
+
}), { priority });
|
|
407
|
+
const integrity = opts.pkg.resolution.integrity ?? fetchedPackage.integrity;
|
|
408
|
+
if (isLocalTarballDep && integrity) {
|
|
409
|
+
await fs.mkdir(target, { recursive: true });
|
|
410
|
+
await gfs.writeFile(path.join(target, TARBALL_INTEGRITY_FILENAME), integrity, 'utf8');
|
|
411
|
+
}
|
|
412
|
+
fetching.resolve({
|
|
413
|
+
files: {
|
|
414
|
+
resolvedFrom: fetchedPackage.local ? 'local-dir' : 'remote',
|
|
415
|
+
filesMap: fetchedPackage.filesMap,
|
|
416
|
+
packageImportMethod: fetchedPackage.packageImportMethod,
|
|
417
|
+
requiresBuild: fetchedPackage.requiresBuild,
|
|
418
|
+
},
|
|
419
|
+
bundledManifest: fetchedPackage.manifest,
|
|
420
|
+
integrity,
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
catch (err) { // eslint-disable-line
|
|
424
|
+
fetching.reject(err);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
async function readBundledManifest(pkgJsonPath) {
|
|
429
|
+
return normalizeBundledManifest(await loadJsonFile(pkgJsonPath));
|
|
430
|
+
}
|
|
431
|
+
async function tarballIsUpToDate(resolution, pkgInStoreLocation, lockfileDir) {
|
|
432
|
+
let currentIntegrity;
|
|
433
|
+
try {
|
|
434
|
+
currentIntegrity = (await gfs.readFile(path.join(pkgInStoreLocation, TARBALL_INTEGRITY_FILENAME), 'utf8'));
|
|
435
|
+
}
|
|
436
|
+
catch (err) { // eslint-disable-line
|
|
437
|
+
return false;
|
|
438
|
+
}
|
|
439
|
+
if (resolution.integrity && currentIntegrity !== resolution.integrity)
|
|
440
|
+
return false;
|
|
441
|
+
const tarball = path.join(lockfileDir, resolution.tarball.slice(5));
|
|
442
|
+
const tarballStream = createReadStream(tarball);
|
|
443
|
+
try {
|
|
444
|
+
return Boolean(await ssri.checkStream(tarballStream, currentIntegrity));
|
|
445
|
+
}
|
|
446
|
+
catch (err) { // eslint-disable-line
|
|
447
|
+
return false;
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
async function fetcher(fetcherByHostingType, cafs, customFetchers, packageId, resolution, opts) {
|
|
451
|
+
try {
|
|
452
|
+
// pickFetcher now handles custom fetcher hooks internally
|
|
453
|
+
const fetch = await pickFetcher(fetcherByHostingType, resolution, {
|
|
454
|
+
customFetchers,
|
|
455
|
+
packageId,
|
|
456
|
+
});
|
|
457
|
+
const result = await fetch(cafs, resolution, opts); // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
458
|
+
return result;
|
|
459
|
+
}
|
|
460
|
+
catch (err) { // eslint-disable-line
|
|
461
|
+
packageRequestLogger.warn({
|
|
462
|
+
message: `Fetching ${packageId} failed!`,
|
|
463
|
+
prefix: opts.lockfileDir,
|
|
464
|
+
});
|
|
465
|
+
throw err;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
//# sourceMappingURL=packageRequester.js.map
|
package/package.json
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@pnpm/installing.package-requester",
|
|
3
|
+
"version": "1008.0.0",
|
|
4
|
+
"description": "Concurrent downloader of npm-compatible packages",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"pnpm",
|
|
7
|
+
"pnpm11",
|
|
8
|
+
"npm",
|
|
9
|
+
"resolver"
|
|
10
|
+
],
|
|
11
|
+
"license": "MIT",
|
|
12
|
+
"funding": "https://opencollective.com/pnpm",
|
|
13
|
+
"repository": "https://github.com/pnpm/pnpm/tree/main/installing/package-requester",
|
|
14
|
+
"homepage": "https://github.com/pnpm/pnpm/tree/main/installing/package-requester#readme",
|
|
15
|
+
"bugs": {
|
|
16
|
+
"url": "https://github.com/pnpm/pnpm/issues"
|
|
17
|
+
},
|
|
18
|
+
"type": "module",
|
|
19
|
+
"main": "lib/index.js",
|
|
20
|
+
"types": "lib/index.d.ts",
|
|
21
|
+
"exports": {
|
|
22
|
+
".": "./lib/index.js"
|
|
23
|
+
},
|
|
24
|
+
"files": [
|
|
25
|
+
"lib",
|
|
26
|
+
"!*.map"
|
|
27
|
+
],
|
|
28
|
+
"dependencies": {
|
|
29
|
+
"detect-libc": "^2.0.3",
|
|
30
|
+
"load-json-file": "^7.0.1",
|
|
31
|
+
"p-defer": "^4.0.1",
|
|
32
|
+
"p-limit": "^7.1.0",
|
|
33
|
+
"p-queue": "^8.1.0",
|
|
34
|
+
"promise-share": "^2.0.0",
|
|
35
|
+
"ramda": "npm:@pnpm/ramda@0.28.1",
|
|
36
|
+
"semver": "^7.7.2",
|
|
37
|
+
"ssri": "13.0.0",
|
|
38
|
+
"@pnpm/config.package-is-installable": "1000.0.15",
|
|
39
|
+
"@pnpm/deps.path": "1001.1.3",
|
|
40
|
+
"@pnpm/error": "1000.0.5",
|
|
41
|
+
"@pnpm/fetching.fetcher-base": "1001.0.2",
|
|
42
|
+
"@pnpm/fetching.pick-fetcher": "1001.0.0",
|
|
43
|
+
"@pnpm/core-loggers": "1001.0.4",
|
|
44
|
+
"@pnpm/fs.graceful-fs": "1000.0.1",
|
|
45
|
+
"@pnpm/hooks.types": "1001.0.12",
|
|
46
|
+
"@pnpm/store.cafs": "1000.0.19",
|
|
47
|
+
"@pnpm/resolving.resolver-base": "1005.1.0",
|
|
48
|
+
"@pnpm/store.index": "1000.0.0-0",
|
|
49
|
+
"@pnpm/store.controller-types": "1004.1.0",
|
|
50
|
+
"@pnpm/types": "1000.9.0"
|
|
51
|
+
},
|
|
52
|
+
"peerDependencies": {
|
|
53
|
+
"@pnpm/logger": ">=1001.0.0 <1002.0.0",
|
|
54
|
+
"@pnpm/worker": "^1000.3.0"
|
|
55
|
+
},
|
|
56
|
+
"devDependencies": {
|
|
57
|
+
"@jest/globals": "30.0.5",
|
|
58
|
+
"@pnpm/registry-mock": "5.2.4",
|
|
59
|
+
"@types/normalize-path": "^3.0.2",
|
|
60
|
+
"@types/ramda": "0.29.12",
|
|
61
|
+
"@types/semver": "7.7.1",
|
|
62
|
+
"@types/ssri": "^7.1.5",
|
|
63
|
+
"delay": "^6.0.0",
|
|
64
|
+
"nock": "13.3.4",
|
|
65
|
+
"normalize-path": "^3.0.0",
|
|
66
|
+
"tempy": "3.0.0",
|
|
67
|
+
"@pnpm/fs.msgpack-file": "1100.0.0-0",
|
|
68
|
+
"@pnpm/installing.client": "1001.1.4",
|
|
69
|
+
"@pnpm/installing.package-requester": "1008.0.0",
|
|
70
|
+
"@pnpm/logger": "1001.0.1",
|
|
71
|
+
"@pnpm/store.cafs-types": "1000.0.0",
|
|
72
|
+
"@pnpm/store.create-cafs-store": "1000.0.20",
|
|
73
|
+
"@pnpm/test-fixtures": "1000.0.0"
|
|
74
|
+
},
|
|
75
|
+
"engines": {
|
|
76
|
+
"node": ">=22.13"
|
|
77
|
+
},
|
|
78
|
+
"jest": {
|
|
79
|
+
"preset": "@pnpm/jest-config/with-registry"
|
|
80
|
+
},
|
|
81
|
+
"scripts": {
|
|
82
|
+
"start": "tsgo --watch",
|
|
83
|
+
"lint": "eslint \"src/**/*.ts\" \"test/**/*.ts\"",
|
|
84
|
+
"_test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --disable-warning=ExperimentalWarning --disable-warning=DEP0169\" jest",
|
|
85
|
+
"test": "pnpm run compile && pnpm run _test",
|
|
86
|
+
"compile": "tsgo --build && pnpm run lint --fix"
|
|
87
|
+
}
|
|
88
|
+
}
|