@pnpm/network.fetch 1100.1.7 → 1100.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/lib/dispatcher.d.ts +33 -0
- package/lib/dispatcher.js +380 -0
- package/lib/fetch.d.ts +22 -0
- package/lib/fetch.js +109 -0
- package/lib/fetchFromRegistry.d.ts +29 -0
- package/lib/fetchFromRegistry.js +133 -0
- package/lib/index.d.ts +4 -0
- package/package.json +6 -6
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @pnpm/fetch
|
|
2
2
|
|
|
3
|
+
## 1100.1.8
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Republished every package: the tarballs published by the v11.13.1 through v11.16.0 releases were missing most of their compiled files due to a packing bug [#13164](https://github.com/pnpm/pnpm/issues/13164).
|
|
8
|
+
|
|
9
|
+
- Updated dependencies:
|
|
10
|
+
- @pnpm/core-loggers@1100.2.5
|
|
11
|
+
- @pnpm/error@1100.1.0
|
|
12
|
+
- @pnpm/fetching.types@1100.0.3
|
|
13
|
+
- @pnpm/types@1101.6.0
|
|
14
|
+
|
|
3
15
|
## 1100.1.7
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { TlsConfig } from '@pnpm/types';
|
|
2
|
+
import { type Dispatcher } from 'undici';
|
|
3
|
+
export type ClientCertificates = Record<string, TlsConfig>;
|
|
4
|
+
export interface DispatcherOptions {
|
|
5
|
+
ca?: string | string[] | Buffer;
|
|
6
|
+
cert?: string | string[] | Buffer;
|
|
7
|
+
key?: string | Buffer;
|
|
8
|
+
localAddress?: string;
|
|
9
|
+
maxSockets?: number;
|
|
10
|
+
strictSsl?: boolean;
|
|
11
|
+
timeout?: number;
|
|
12
|
+
httpProxy?: string;
|
|
13
|
+
httpsProxy?: string;
|
|
14
|
+
noProxy?: boolean | string;
|
|
15
|
+
clientCertificates?: ClientCertificates;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Clear the dispatcher cache. Useful for testing.
|
|
19
|
+
*/
|
|
20
|
+
export declare function clearDispatcherCache(): void;
|
|
21
|
+
/**
|
|
22
|
+
* Destroy the global dispatcher and every cached dispatcher, closing their open
|
|
23
|
+
* sockets. Intended for process shutdown only: once called, the module can no
|
|
24
|
+
* longer perform network requests. This is used to work around a Windows crash
|
|
25
|
+
* that happens when the process exits while sockets are still open
|
|
26
|
+
* (https://github.com/nodejs/node/issues/56645).
|
|
27
|
+
*/
|
|
28
|
+
export declare function destroyDispatchers(): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Get a dispatcher for the given URI and options.
|
|
31
|
+
* Returns undefined if no special configuration is needed (to use global dispatcher).
|
|
32
|
+
*/
|
|
33
|
+
export declare function getDispatcher(uri: string, opts: DispatcherOptions): Dispatcher | undefined;
|
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
import net from 'node:net';
|
|
2
|
+
import tls from 'node:tls';
|
|
3
|
+
import { URL } from 'node:url';
|
|
4
|
+
import { nerfDart } from '@pnpm/config.nerf-dart';
|
|
5
|
+
import { PnpmError } from '@pnpm/error';
|
|
6
|
+
import { LRUCache } from 'lru-cache';
|
|
7
|
+
import { SocksClient } from 'socks';
|
|
8
|
+
import { Agent, getGlobalDispatcher, ProxyAgent, setGlobalDispatcher } from 'undici';
|
|
9
|
+
const DEFAULT_MAX_SOCKETS = 50;
|
|
10
|
+
const KEEP_ALIVE_TIMEOUT = 30_000; // 30 seconds
|
|
11
|
+
const KEEP_ALIVE_MAX_TIMEOUT = 600_000; // 10 minutes
|
|
12
|
+
// Set an optimized global dispatcher so that requests without custom options
|
|
13
|
+
// (no proxy, no custom certs) still benefit from better keep-alive and Happy Eyeballs.
|
|
14
|
+
//
|
|
15
|
+
// Note: we intentionally do NOT enable HTTP/2 (allowH2) or HTTP/1.1 pipelining here.
|
|
16
|
+
// With HTTP/2, undici multiplexes many streams over 1-2 TCP connections sharing a single
|
|
17
|
+
// congestion window. In benchmarks this was slower than opening ~50 independent HTTP/1.1
|
|
18
|
+
// connections that each get their own congestion window and can saturate bandwidth in parallel.
|
|
19
|
+
const GLOBAL_DISPATCHER = new Agent({
|
|
20
|
+
connections: DEFAULT_MAX_SOCKETS,
|
|
21
|
+
keepAliveTimeout: KEEP_ALIVE_TIMEOUT,
|
|
22
|
+
keepAliveMaxTimeout: KEEP_ALIVE_MAX_TIMEOUT,
|
|
23
|
+
connect: {
|
|
24
|
+
autoSelectFamily: true,
|
|
25
|
+
},
|
|
26
|
+
}).compose(stripSecFetchHeaders);
|
|
27
|
+
setGlobalDispatcher(GLOBAL_DISPATCHER);
|
|
28
|
+
// undici's fetch() automatically adds sec-fetch-* headers (e.g. sec-fetch-mode: cors)
|
|
29
|
+
// per the Fetch spec. Some registries like Azure DevOps Artifacts interpret these as
|
|
30
|
+
// browser requests and reject them with HTTP 400. Since pnpm is a CLI tool, these
|
|
31
|
+
// headers serve no purpose and must be stripped.
|
|
32
|
+
// See https://github.com/pnpm/pnpm/issues/11572
|
|
33
|
+
function stripSecFetchHeaders(dispatch) {
|
|
34
|
+
return (opts, handler) => {
|
|
35
|
+
if (opts.headers) {
|
|
36
|
+
if (Array.isArray(opts.headers)) {
|
|
37
|
+
// Flat array format: [key1, val1, key2, val2, ...]
|
|
38
|
+
const filtered = [];
|
|
39
|
+
for (let i = 0; i < opts.headers.length; i += 2) {
|
|
40
|
+
if (!opts.headers[i].toLowerCase().startsWith('sec-fetch-')) {
|
|
41
|
+
filtered.push(opts.headers[i], opts.headers[i + 1]);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
opts = { ...opts, headers: filtered };
|
|
45
|
+
}
|
|
46
|
+
else if (typeof opts.headers === 'object') {
|
|
47
|
+
// undici also accepts an iterable of [key, value] pairs (e.g. a Map or
|
|
48
|
+
// web Headers). Use that iterator when present; otherwise fall back to
|
|
49
|
+
// Object.entries for plain IncomingHttpHeaders objects.
|
|
50
|
+
const entries = Symbol.iterator in opts.headers
|
|
51
|
+
? opts.headers
|
|
52
|
+
: Object.entries(opts.headers);
|
|
53
|
+
const headers = {};
|
|
54
|
+
for (const [key, value] of entries) {
|
|
55
|
+
if (!key.toLowerCase().startsWith('sec-fetch-')) {
|
|
56
|
+
headers[key] = value;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
opts = { ...opts, headers };
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return dispatch(opts, handler);
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
const DISPATCHER_CACHE = new LRUCache({
|
|
66
|
+
max: 50,
|
|
67
|
+
dispose: (dispatcher) => {
|
|
68
|
+
if (typeof dispatcher.close === 'function') {
|
|
69
|
+
void dispatcher.close();
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
/**
|
|
74
|
+
* Clear the dispatcher cache. Useful for testing.
|
|
75
|
+
*/
|
|
76
|
+
export function clearDispatcherCache() {
|
|
77
|
+
DISPATCHER_CACHE.clear();
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Destroy the global dispatcher and every cached dispatcher, closing their open
|
|
81
|
+
* sockets. Intended for process shutdown only: once called, the module can no
|
|
82
|
+
* longer perform network requests. This is used to work around a Windows crash
|
|
83
|
+
* that happens when the process exits while sockets are still open
|
|
84
|
+
* (https://github.com/nodejs/node/issues/56645).
|
|
85
|
+
*/
|
|
86
|
+
export async function destroyDispatchers() {
|
|
87
|
+
// getGlobalDispatcher() is included in case something replaced GLOBAL_DISPATCHER
|
|
88
|
+
// via setGlobalDispatcher(); the Set removes the duplicate when it is still our own instance.
|
|
89
|
+
const dispatchers = new Set([
|
|
90
|
+
GLOBAL_DISPATCHER,
|
|
91
|
+
getGlobalDispatcher(),
|
|
92
|
+
...DISPATCHER_CACHE.values(),
|
|
93
|
+
]);
|
|
94
|
+
await Promise.allSettled(Array.from(dispatchers, dispatcher => dispatcher.destroy()));
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Get a dispatcher for the given URI and options.
|
|
98
|
+
* Returns undefined if no special configuration is needed (to use global dispatcher).
|
|
99
|
+
*/
|
|
100
|
+
export function getDispatcher(uri, opts) {
|
|
101
|
+
// If no special options are set, use the global dispatcher
|
|
102
|
+
if (!needsCustomDispatcher(opts)) {
|
|
103
|
+
return undefined;
|
|
104
|
+
}
|
|
105
|
+
const parsedUri = new URL(uri);
|
|
106
|
+
if ((opts.httpProxy || opts.httpsProxy) && !checkNoProxy(parsedUri, opts)) {
|
|
107
|
+
const proxyDispatcher = getProxyDispatcher(parsedUri, opts);
|
|
108
|
+
if (proxyDispatcher)
|
|
109
|
+
return proxyDispatcher;
|
|
110
|
+
}
|
|
111
|
+
return getNonProxyDispatcher(parsedUri, opts);
|
|
112
|
+
}
|
|
113
|
+
function hasClientCertificates(certs) {
|
|
114
|
+
if (!certs)
|
|
115
|
+
return false;
|
|
116
|
+
for (const uri in certs) {
|
|
117
|
+
const entry = certs[uri];
|
|
118
|
+
if (entry.cert || entry.key || entry.ca)
|
|
119
|
+
return true;
|
|
120
|
+
}
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
function needsCustomDispatcher(opts) {
|
|
124
|
+
return Boolean(opts.httpProxy ||
|
|
125
|
+
opts.httpsProxy ||
|
|
126
|
+
opts.ca ||
|
|
127
|
+
opts.cert ||
|
|
128
|
+
opts.key ||
|
|
129
|
+
opts.localAddress ||
|
|
130
|
+
opts.strictSsl === false ||
|
|
131
|
+
hasClientCertificates(opts.clientCertificates) ||
|
|
132
|
+
opts.maxSockets);
|
|
133
|
+
}
|
|
134
|
+
function parseProxyUrl(proxy, protocol) {
|
|
135
|
+
let proxyUrl = proxy;
|
|
136
|
+
if (!proxyUrl.includes('://')) {
|
|
137
|
+
proxyUrl = `${protocol}//${proxyUrl}`;
|
|
138
|
+
}
|
|
139
|
+
try {
|
|
140
|
+
return new URL(proxyUrl);
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
throw new PnpmError('INVALID_PROXY', "Couldn't parse proxy URL", {
|
|
144
|
+
hint: 'If your proxy URL contains a username and password, make sure to URL-encode them ' +
|
|
145
|
+
'(you may use the encodeURIComponent function). For instance, ' +
|
|
146
|
+
'https-proxy=https://use%21r:pas%2As@my.proxy:1234/foo. ' +
|
|
147
|
+
'Do not encode the colon (:) between the username and password.',
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
function getSocksProxyType(protocol) {
|
|
152
|
+
switch (protocol.replace(':', '')) {
|
|
153
|
+
case 'socks4':
|
|
154
|
+
case 'socks4a':
|
|
155
|
+
return 4;
|
|
156
|
+
default:
|
|
157
|
+
return 5;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
function getProxyDispatcher(parsedUri, opts) {
|
|
161
|
+
const isHttps = parsedUri.protocol === 'https:';
|
|
162
|
+
const proxy = isHttps ? opts.httpsProxy : opts.httpProxy;
|
|
163
|
+
if (!proxy)
|
|
164
|
+
return null;
|
|
165
|
+
const proxyUrl = parseProxyUrl(proxy, parsedUri.protocol);
|
|
166
|
+
const sslConfig = pickSettingByUrl(opts.clientCertificates, parsedUri.href);
|
|
167
|
+
const { ca, cert, key: certKey } = { ...opts, ...sslConfig };
|
|
168
|
+
const key = [
|
|
169
|
+
`proxy:${proxyUrl.protocol}//${proxyUrl.username}:${proxyUrl.password}@${proxyUrl.host}:${proxyUrl.port}`,
|
|
170
|
+
`https:${isHttps.toString()}`,
|
|
171
|
+
`local-address:${opts.localAddress ?? '>no-local-address<'}`,
|
|
172
|
+
`max-sockets:${(opts.maxSockets ?? DEFAULT_MAX_SOCKETS).toString()}`,
|
|
173
|
+
`strict-ssl:${isHttps ? Boolean(opts.strictSsl).toString() : '>no-strict-ssl<'}`,
|
|
174
|
+
`ca:${(isHttps && ca?.toString()) || '-'}`,
|
|
175
|
+
`cert:${(isHttps && cert?.toString()) || '-'}`,
|
|
176
|
+
`key:${(isHttps && certKey?.toString()) || '-'}`,
|
|
177
|
+
].join(':');
|
|
178
|
+
if (DISPATCHER_CACHE.has(key)) {
|
|
179
|
+
return DISPATCHER_CACHE.get(key);
|
|
180
|
+
}
|
|
181
|
+
let dispatcher;
|
|
182
|
+
if (proxyUrl.protocol.startsWith('socks')) {
|
|
183
|
+
dispatcher = createSocksDispatcher(proxyUrl, parsedUri, opts, { ca, cert, key: certKey });
|
|
184
|
+
}
|
|
185
|
+
else {
|
|
186
|
+
dispatcher = createHttpProxyDispatcher(proxyUrl, isHttps, opts, { ca, cert, key: certKey });
|
|
187
|
+
}
|
|
188
|
+
dispatcher = dispatcher.compose(stripSecFetchHeaders);
|
|
189
|
+
DISPATCHER_CACHE.set(key, dispatcher);
|
|
190
|
+
return dispatcher;
|
|
191
|
+
}
|
|
192
|
+
function createHttpProxyDispatcher(proxyUrl, isHttps, opts, tlsConfig) {
|
|
193
|
+
return new ProxyAgent({
|
|
194
|
+
uri: proxyUrl.href,
|
|
195
|
+
token: proxyUrl.username
|
|
196
|
+
? `Basic ${Buffer.from(`${decodeURIComponent(proxyUrl.username)}:${decodeURIComponent(proxyUrl.password)}`).toString('base64')}`
|
|
197
|
+
: undefined,
|
|
198
|
+
connections: opts.maxSockets ?? DEFAULT_MAX_SOCKETS,
|
|
199
|
+
keepAliveTimeout: KEEP_ALIVE_TIMEOUT,
|
|
200
|
+
keepAliveMaxTimeout: KEEP_ALIVE_MAX_TIMEOUT,
|
|
201
|
+
requestTls: isHttps
|
|
202
|
+
? {
|
|
203
|
+
ca: tlsConfig.ca,
|
|
204
|
+
cert: tlsConfig.cert,
|
|
205
|
+
key: tlsConfig.key,
|
|
206
|
+
rejectUnauthorized: opts.strictSsl ?? true,
|
|
207
|
+
localAddress: opts.localAddress,
|
|
208
|
+
}
|
|
209
|
+
: undefined,
|
|
210
|
+
proxyTls: {
|
|
211
|
+
ca: opts.ca,
|
|
212
|
+
rejectUnauthorized: opts.strictSsl ?? true,
|
|
213
|
+
},
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
function createSocksDispatcher(proxyUrl, targetUri, opts, tlsConfig) {
|
|
217
|
+
const isHttps = targetUri.protocol === 'https:';
|
|
218
|
+
const socksType = getSocksProxyType(proxyUrl.protocol);
|
|
219
|
+
const proxyHost = proxyUrl.hostname;
|
|
220
|
+
const proxyPort = parseInt(proxyUrl.port, 10) || (socksType === 4 ? 1080 : 1080);
|
|
221
|
+
return new Agent({
|
|
222
|
+
connections: opts.maxSockets ?? DEFAULT_MAX_SOCKETS,
|
|
223
|
+
keepAliveTimeout: KEEP_ALIVE_TIMEOUT,
|
|
224
|
+
keepAliveMaxTimeout: KEEP_ALIVE_MAX_TIMEOUT,
|
|
225
|
+
connect: async (connectOpts, callback) => {
|
|
226
|
+
try {
|
|
227
|
+
const { socket } = await SocksClient.createConnection({
|
|
228
|
+
proxy: {
|
|
229
|
+
host: proxyHost,
|
|
230
|
+
port: proxyPort,
|
|
231
|
+
type: socksType,
|
|
232
|
+
userId: proxyUrl.username ? decodeURIComponent(proxyUrl.username) : undefined,
|
|
233
|
+
password: proxyUrl.password ? decodeURIComponent(proxyUrl.password) : undefined,
|
|
234
|
+
},
|
|
235
|
+
command: 'connect',
|
|
236
|
+
destination: {
|
|
237
|
+
host: connectOpts.hostname,
|
|
238
|
+
port: parseInt(String(connectOpts.port), 10),
|
|
239
|
+
},
|
|
240
|
+
});
|
|
241
|
+
if (isHttps) {
|
|
242
|
+
const tlsOpts = {
|
|
243
|
+
socket: socket,
|
|
244
|
+
servername: connectOpts.hostname,
|
|
245
|
+
ca: tlsConfig.ca,
|
|
246
|
+
cert: tlsConfig.cert,
|
|
247
|
+
key: tlsConfig.key,
|
|
248
|
+
rejectUnauthorized: opts.strictSsl ?? true,
|
|
249
|
+
};
|
|
250
|
+
const tlsSocket = tls.connect(tlsOpts);
|
|
251
|
+
tlsSocket.on('secureConnect', () => {
|
|
252
|
+
callback(null, tlsSocket);
|
|
253
|
+
});
|
|
254
|
+
tlsSocket.on('error', (err) => {
|
|
255
|
+
callback(err, null);
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
else {
|
|
259
|
+
callback(null, socket);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
catch (err) {
|
|
263
|
+
callback(err, null);
|
|
264
|
+
}
|
|
265
|
+
},
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
function getNonProxyDispatcher(parsedUri, opts) {
|
|
269
|
+
const isHttps = parsedUri.protocol === 'https:';
|
|
270
|
+
const sslConfig = pickSettingByUrl(opts.clientCertificates, parsedUri.href);
|
|
271
|
+
const { ca, cert, key: certKey } = { ...opts, ...sslConfig };
|
|
272
|
+
const key = [
|
|
273
|
+
`https:${isHttps.toString()}`,
|
|
274
|
+
`local-address:${opts.localAddress ?? '>no-local-address<'}`,
|
|
275
|
+
`max-sockets:${(opts.maxSockets ?? DEFAULT_MAX_SOCKETS).toString()}`,
|
|
276
|
+
`strict-ssl:${isHttps ? Boolean(opts.strictSsl).toString() : '>no-strict-ssl<'}`,
|
|
277
|
+
`ca:${(isHttps && ca?.toString()) || '-'}`,
|
|
278
|
+
`cert:${(isHttps && cert?.toString()) || '-'}`,
|
|
279
|
+
`key:${(isHttps && certKey?.toString()) || '-'}`,
|
|
280
|
+
].join(':');
|
|
281
|
+
if (DISPATCHER_CACHE.has(key)) {
|
|
282
|
+
return DISPATCHER_CACHE.get(key);
|
|
283
|
+
}
|
|
284
|
+
const connectTimeout = typeof opts.timeout !== 'number' || opts.timeout === 0
|
|
285
|
+
? 0
|
|
286
|
+
: opts.timeout + 1;
|
|
287
|
+
const agent = new Agent({
|
|
288
|
+
connections: opts.maxSockets ?? DEFAULT_MAX_SOCKETS,
|
|
289
|
+
connectTimeout,
|
|
290
|
+
keepAliveTimeout: KEEP_ALIVE_TIMEOUT,
|
|
291
|
+
keepAliveMaxTimeout: KEEP_ALIVE_MAX_TIMEOUT,
|
|
292
|
+
connect: isHttps
|
|
293
|
+
? {
|
|
294
|
+
autoSelectFamily: true,
|
|
295
|
+
ca,
|
|
296
|
+
cert,
|
|
297
|
+
key: certKey,
|
|
298
|
+
rejectUnauthorized: opts.strictSsl ?? true,
|
|
299
|
+
localAddress: opts.localAddress,
|
|
300
|
+
}
|
|
301
|
+
: {
|
|
302
|
+
autoSelectFamily: true,
|
|
303
|
+
localAddress: opts.localAddress,
|
|
304
|
+
},
|
|
305
|
+
});
|
|
306
|
+
const dispatcher = agent.compose(stripSecFetchHeaders);
|
|
307
|
+
DISPATCHER_CACHE.set(key, dispatcher);
|
|
308
|
+
return dispatcher;
|
|
309
|
+
}
|
|
310
|
+
function checkNoProxy(parsedUri, opts) {
|
|
311
|
+
const host = parsedUri.hostname
|
|
312
|
+
.split('.')
|
|
313
|
+
.filter(x => x)
|
|
314
|
+
.reverse();
|
|
315
|
+
if (typeof opts.noProxy === 'string') {
|
|
316
|
+
const noproxyArr = opts.noProxy.split(',').map(s => s.trim());
|
|
317
|
+
return noproxyArr.some(no => {
|
|
318
|
+
const noParts = no
|
|
319
|
+
.split('.')
|
|
320
|
+
.filter(x => x)
|
|
321
|
+
.reverse();
|
|
322
|
+
if (noParts.length === 0) {
|
|
323
|
+
return false;
|
|
324
|
+
}
|
|
325
|
+
for (let i = 0; i < noParts.length; i++) {
|
|
326
|
+
if (host[i] !== noParts[i]) {
|
|
327
|
+
return false;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return true;
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
return opts.noProxy === true;
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Pick SSL/TLS configuration by URL using nerf-dart matching.
|
|
337
|
+
* This matches the behavior of @pnpm/network.config's pickSettingByUrl.
|
|
338
|
+
*/
|
|
339
|
+
function pickSettingByUrl(settings, uri) {
|
|
340
|
+
if (!settings)
|
|
341
|
+
return undefined;
|
|
342
|
+
// Try exact match first
|
|
343
|
+
if (settings[uri])
|
|
344
|
+
return settings[uri];
|
|
345
|
+
// Use nerf-dart format for matching (e.g., //registry.npmjs.org/)
|
|
346
|
+
const nerf = nerfDart(uri);
|
|
347
|
+
if (settings[nerf])
|
|
348
|
+
return settings[nerf];
|
|
349
|
+
// Try without port
|
|
350
|
+
const parsedUrl = new URL(uri);
|
|
351
|
+
const withoutPort = removePort(parsedUrl);
|
|
352
|
+
if (settings[withoutPort])
|
|
353
|
+
return settings[withoutPort];
|
|
354
|
+
// Try progressively shorter nerf-dart paths
|
|
355
|
+
const maxParts = Object.keys(settings).reduce((max, key) => {
|
|
356
|
+
const parts = key.split('/').length;
|
|
357
|
+
return parts > max ? parts : max;
|
|
358
|
+
}, 0);
|
|
359
|
+
const parts = nerf.split('/');
|
|
360
|
+
for (let i = Math.min(parts.length, maxParts) - 1; i >= 3; i--) {
|
|
361
|
+
const key = `${parts.slice(0, i).join('/')}/`;
|
|
362
|
+
if (settings[key]) {
|
|
363
|
+
return settings[key];
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
// If the URL had a port, try again without it
|
|
367
|
+
if (withoutPort !== uri) {
|
|
368
|
+
return pickSettingByUrl(settings, withoutPort);
|
|
369
|
+
}
|
|
370
|
+
return undefined;
|
|
371
|
+
}
|
|
372
|
+
function removePort(parsedUrl) {
|
|
373
|
+
if (parsedUrl.port === '')
|
|
374
|
+
return parsedUrl.href;
|
|
375
|
+
const copy = new URL(parsedUrl.href);
|
|
376
|
+
copy.port = '';
|
|
377
|
+
const res = copy.toString();
|
|
378
|
+
return res.endsWith('/') ? res : `${res}/`;
|
|
379
|
+
}
|
|
380
|
+
//# sourceMappingURL=dispatcher.js.map
|
package/lib/fetch.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type RetryTimeoutOptions } from '@zkochan/retry';
|
|
2
|
+
import { type Dispatcher } from 'undici';
|
|
3
|
+
export { type RetryTimeoutOptions };
|
|
4
|
+
interface URLLike {
|
|
5
|
+
href: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function isRedirect(statusCode: number): boolean;
|
|
8
|
+
export type RequestInfo = string | URLLike | URL;
|
|
9
|
+
export interface RequestInit extends globalThis.RequestInit {
|
|
10
|
+
retry?: RetryTimeoutOptions;
|
|
11
|
+
timeout?: number;
|
|
12
|
+
dispatcher?: Dispatcher;
|
|
13
|
+
}
|
|
14
|
+
export declare function fetch(url: RequestInfo, opts?: RequestInit): Promise<Response>;
|
|
15
|
+
export declare class ResponseError extends Error {
|
|
16
|
+
res: Response;
|
|
17
|
+
code: number;
|
|
18
|
+
status: number;
|
|
19
|
+
statusCode: number;
|
|
20
|
+
url: string;
|
|
21
|
+
constructor(res: Response);
|
|
22
|
+
}
|
package/lib/fetch.js
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { requestRetryLogger } from '@pnpm/core-loggers';
|
|
2
|
+
import { operation } from '@zkochan/retry';
|
|
3
|
+
import { fetch as undiciFetch } from 'undici';
|
|
4
|
+
export {};
|
|
5
|
+
const NO_RETRY_ERROR_CODES = new Set([
|
|
6
|
+
'SELF_SIGNED_CERT_IN_CHAIN',
|
|
7
|
+
'ERR_OSSL_PEM_NO_START_LINE',
|
|
8
|
+
]);
|
|
9
|
+
const REDIRECT_CODES = new Set([301, 302, 303, 307, 308]);
|
|
10
|
+
export function isRedirect(statusCode) {
|
|
11
|
+
return REDIRECT_CODES.has(statusCode);
|
|
12
|
+
}
|
|
13
|
+
export async function fetch(url, opts = {}) {
|
|
14
|
+
const retryOpts = opts.retry ?? {};
|
|
15
|
+
const maxRetries = retryOpts.retries ?? 2;
|
|
16
|
+
const op = operation({
|
|
17
|
+
factor: retryOpts.factor ?? 10,
|
|
18
|
+
maxTimeout: retryOpts.maxTimeout ?? 60000,
|
|
19
|
+
minTimeout: retryOpts.minTimeout ?? 10000,
|
|
20
|
+
randomize: false,
|
|
21
|
+
retries: maxRetries,
|
|
22
|
+
});
|
|
23
|
+
try {
|
|
24
|
+
return await new Promise((resolve, reject) => {
|
|
25
|
+
op.attempt(async (attempt) => {
|
|
26
|
+
const urlString = typeof url === 'string' ? url : url.href ?? url.toString();
|
|
27
|
+
const { retry: _retry, timeout, dispatcher, ...fetchOpts } = opts;
|
|
28
|
+
const signal = timeout ? AbortSignal.timeout(timeout) : undefined;
|
|
29
|
+
try {
|
|
30
|
+
// undici's Response type differs slightly from globalThis.Response (iterator types),
|
|
31
|
+
// requiring the double cast. This is a known TypeScript/undici compatibility issue.
|
|
32
|
+
const res = await undiciFetch(urlString, { ...fetchOpts, signal, dispatcher });
|
|
33
|
+
// A retry on 409 sometimes helps when making requests to the Bit registry.
|
|
34
|
+
if ((res.status >= 500 && res.status < 600) || [408, 409, 420, 429].includes(res.status)) {
|
|
35
|
+
throw new ResponseError(res);
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
resolve(res);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
// Undici errors may not pass isNativeError check, so we handle them more carefully
|
|
43
|
+
const err = error;
|
|
44
|
+
// Check error code in both error.code and error.cause.code (undici wraps errors)
|
|
45
|
+
const errorCode = err?.code ?? err?.cause?.code;
|
|
46
|
+
if (typeof errorCode === 'string' &&
|
|
47
|
+
NO_RETRY_ERROR_CODES.has(errorCode)) {
|
|
48
|
+
reject(error);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
const retryTimeout = op.retry(err);
|
|
52
|
+
if (retryTimeout === false) {
|
|
53
|
+
reject(op.mainError());
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
// Extract error properties into a plain object because Error properties
|
|
57
|
+
// are non-enumerable and don't serialize well through the logging system
|
|
58
|
+
const errorInfo = {
|
|
59
|
+
name: err.name,
|
|
60
|
+
message: err.message,
|
|
61
|
+
code: err.code,
|
|
62
|
+
errno: err.errno,
|
|
63
|
+
// For HTTP errors from ResponseError class
|
|
64
|
+
status: err.status,
|
|
65
|
+
statusCode: err.statusCode,
|
|
66
|
+
// undici wraps the actual network error in a cause property
|
|
67
|
+
cause: err.cause ? {
|
|
68
|
+
code: err.cause.code,
|
|
69
|
+
errno: err.cause.errno,
|
|
70
|
+
} : undefined,
|
|
71
|
+
};
|
|
72
|
+
requestRetryLogger.debug({
|
|
73
|
+
attempt,
|
|
74
|
+
error: errorInfo,
|
|
75
|
+
maxRetries,
|
|
76
|
+
method: opts.method ?? 'GET',
|
|
77
|
+
timeout: retryTimeout,
|
|
78
|
+
url: urlString,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
if (err instanceof ResponseError) {
|
|
86
|
+
return err.res;
|
|
87
|
+
}
|
|
88
|
+
throw err;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
export class ResponseError extends Error {
|
|
92
|
+
res;
|
|
93
|
+
code;
|
|
94
|
+
status;
|
|
95
|
+
statusCode;
|
|
96
|
+
url;
|
|
97
|
+
constructor(res) {
|
|
98
|
+
super(res.statusText);
|
|
99
|
+
if (Error.captureStackTrace) {
|
|
100
|
+
Error.captureStackTrace(this, ResponseError);
|
|
101
|
+
}
|
|
102
|
+
this.name = this.constructor.name;
|
|
103
|
+
this.res = res;
|
|
104
|
+
// backward compat
|
|
105
|
+
this.code = this.status = this.statusCode = res.status;
|
|
106
|
+
this.url = res.url;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=fetch.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { URL } from 'node:url';
|
|
2
|
+
import type { FetchFromRegistry } from '@pnpm/fetching.types';
|
|
3
|
+
import type { RegistryConfig } from '@pnpm/types';
|
|
4
|
+
import { type DispatcherOptions } from './dispatcher.js';
|
|
5
|
+
import { type RequestInit } from './fetch.js';
|
|
6
|
+
export interface FetchWithDispatcherOptions extends RequestInit {
|
|
7
|
+
dispatcherOptions: DispatcherOptions;
|
|
8
|
+
}
|
|
9
|
+
export declare function fetchWithDispatcher(url: string | URL, opts: FetchWithDispatcherOptions): Promise<Response>;
|
|
10
|
+
export interface CreateDispatchedFetchOptions extends DispatcherOptions {
|
|
11
|
+
/**
|
|
12
|
+
* Per-registry config (TLS, auth, etc.). When set, the matching TLS entries
|
|
13
|
+
* are automatically extracted into `clientCertificates` so callers don't
|
|
14
|
+
* have to do it themselves.
|
|
15
|
+
*/
|
|
16
|
+
configByUri?: Record<string, RegistryConfig>;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Returns a {@link fetch} pre-bound to the given dispatcher options, so callers
|
|
20
|
+
* that need a fetch function (rather than a one-shot call) can route their
|
|
21
|
+
* requests through the configured proxy / TLS / local-address settings.
|
|
22
|
+
*/
|
|
23
|
+
export declare function createDispatchedFetch(opts: CreateDispatchedFetchOptions): (url: string | URL, opts?: RequestInit) => Promise<Response>;
|
|
24
|
+
export type { DispatcherOptions };
|
|
25
|
+
export interface CreateFetchFromRegistryOptions extends DispatcherOptions {
|
|
26
|
+
userAgent?: string;
|
|
27
|
+
configByUri?: Record<string, RegistryConfig>;
|
|
28
|
+
}
|
|
29
|
+
export declare function createFetchFromRegistry(defaultOpts: CreateFetchFromRegistryOptions): FetchFromRegistry;
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { URL } from 'node:url';
|
|
2
|
+
import { redactUrlCredentials } from '@pnpm/error';
|
|
3
|
+
import { getDispatcher } from './dispatcher.js';
|
|
4
|
+
import { fetch, isRedirect } from './fetch.js';
|
|
5
|
+
const USER_AGENT = 'pnpm'; // or maybe make it `${pkg.name}/${pkg.version} (+https://npm.im/${pkg.name})`
|
|
6
|
+
const FULL_DOC = 'application/json';
|
|
7
|
+
const ACCEPT_FULL_DOC = `${FULL_DOC}; q=1.0, */*`;
|
|
8
|
+
const ABBREVIATED_DOC = 'application/vnd.npm.install-v1+json';
|
|
9
|
+
const ACCEPT_ABBREVIATED_DOC = `${ABBREVIATED_DOC}; q=1.0, ${FULL_DOC}; q=0.8, */*`;
|
|
10
|
+
const MAX_FOLLOWED_REDIRECTS = 20;
|
|
11
|
+
export function fetchWithDispatcher(url, opts) {
|
|
12
|
+
const dispatcher = getDispatcher(url.toString(), {
|
|
13
|
+
...opts.dispatcherOptions,
|
|
14
|
+
strictSsl: opts.dispatcherOptions.strictSsl ?? true,
|
|
15
|
+
});
|
|
16
|
+
return fetch(url, {
|
|
17
|
+
...opts,
|
|
18
|
+
dispatcher,
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Returns a {@link fetch} pre-bound to the given dispatcher options, so callers
|
|
23
|
+
* that need a fetch function (rather than a one-shot call) can route their
|
|
24
|
+
* requests through the configured proxy / TLS / local-address settings.
|
|
25
|
+
*/
|
|
26
|
+
export function createDispatchedFetch(opts) {
|
|
27
|
+
const dispatcherOptions = {
|
|
28
|
+
...opts,
|
|
29
|
+
clientCertificates: opts.clientCertificates ?? extractTlsConfigs(opts.configByUri),
|
|
30
|
+
};
|
|
31
|
+
return (url, fetchOpts) => fetchWithDispatcher(url, { ...fetchOpts, dispatcherOptions });
|
|
32
|
+
}
|
|
33
|
+
export function createFetchFromRegistry(defaultOpts) {
|
|
34
|
+
const clientCertificates = extractTlsConfigs(defaultOpts.configByUri);
|
|
35
|
+
return async (url, opts) => {
|
|
36
|
+
const headers = {
|
|
37
|
+
'user-agent': USER_AGENT,
|
|
38
|
+
...getHeaders({
|
|
39
|
+
auth: opts?.authHeaderValue,
|
|
40
|
+
fullMetadata: opts?.fullMetadata,
|
|
41
|
+
method: opts?.method,
|
|
42
|
+
userAgent: defaultOpts.userAgent,
|
|
43
|
+
}),
|
|
44
|
+
};
|
|
45
|
+
if (opts?.ifNoneMatch) {
|
|
46
|
+
headers['if-none-match'] = opts.ifNoneMatch;
|
|
47
|
+
}
|
|
48
|
+
if (opts?.ifModifiedSince) {
|
|
49
|
+
headers['if-modified-since'] = opts.ifModifiedSince;
|
|
50
|
+
}
|
|
51
|
+
// Merge caller-provided headers (e.g. content-type, npm-otp) on top
|
|
52
|
+
if (opts?.headers) {
|
|
53
|
+
const optsHeaders = opts.headers instanceof Headers
|
|
54
|
+
? Object.fromEntries(opts.headers.entries())
|
|
55
|
+
: Array.isArray(opts.headers)
|
|
56
|
+
? Object.fromEntries(opts.headers)
|
|
57
|
+
: opts.headers;
|
|
58
|
+
Object.assign(headers, optsHeaders);
|
|
59
|
+
}
|
|
60
|
+
let redirects = 0;
|
|
61
|
+
let urlObject = new URL(url);
|
|
62
|
+
const originalHost = urlObject.host;
|
|
63
|
+
/* eslint-disable no-await-in-loop */
|
|
64
|
+
while (true) {
|
|
65
|
+
const dispatcherOptions = {
|
|
66
|
+
...defaultOpts,
|
|
67
|
+
...opts,
|
|
68
|
+
strictSsl: defaultOpts.strictSsl ?? true,
|
|
69
|
+
clientCertificates,
|
|
70
|
+
};
|
|
71
|
+
const response = await fetchWithDispatcher(urlObject, {
|
|
72
|
+
dispatcherOptions,
|
|
73
|
+
body: opts?.body,
|
|
74
|
+
// if verifying integrity, native fetch must not decompress
|
|
75
|
+
headers,
|
|
76
|
+
method: opts?.method,
|
|
77
|
+
redirect: 'manual',
|
|
78
|
+
retry: opts?.retry,
|
|
79
|
+
timeout: opts?.timeout ?? 60000,
|
|
80
|
+
});
|
|
81
|
+
if (!isRedirect(response.status) || redirects >= MAX_FOLLOWED_REDIRECTS) {
|
|
82
|
+
return response;
|
|
83
|
+
}
|
|
84
|
+
redirects++;
|
|
85
|
+
// This is a workaround to remove authorization headers on redirect.
|
|
86
|
+
// Related pnpm issue: https://github.com/pnpm/pnpm/issues/1815
|
|
87
|
+
urlObject = resolveRedirectUrl(response, urlObject);
|
|
88
|
+
if (originalHost === urlObject.host)
|
|
89
|
+
continue;
|
|
90
|
+
if (headers['authorization']) {
|
|
91
|
+
delete headers.authorization;
|
|
92
|
+
}
|
|
93
|
+
delete headers['npm-otp'];
|
|
94
|
+
}
|
|
95
|
+
/* eslint-enable no-await-in-loop */
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
function getHeaders(opts) {
|
|
99
|
+
const headers = {};
|
|
100
|
+
// The abbreviated/full-metadata Accept header is meaningful only on package
|
|
101
|
+
// metadata reads. Setting it on writes (PUT/POST/DELETE) breaks npmjs.org's
|
|
102
|
+
// dist-tag endpoint, which rejects the request with a generic 400.
|
|
103
|
+
if (!opts.method || opts.method === 'GET' || opts.method === 'HEAD') {
|
|
104
|
+
headers.accept = opts.fullMetadata === true ? ACCEPT_FULL_DOC : ACCEPT_ABBREVIATED_DOC;
|
|
105
|
+
}
|
|
106
|
+
if (opts.auth) {
|
|
107
|
+
headers['authorization'] = opts.auth;
|
|
108
|
+
}
|
|
109
|
+
if (opts.userAgent) {
|
|
110
|
+
headers['user-agent'] = opts.userAgent;
|
|
111
|
+
}
|
|
112
|
+
return headers;
|
|
113
|
+
}
|
|
114
|
+
function extractTlsConfigs(configByUri) {
|
|
115
|
+
if (!configByUri)
|
|
116
|
+
return undefined;
|
|
117
|
+
let result;
|
|
118
|
+
for (const [uri, config] of Object.entries(configByUri)) {
|
|
119
|
+
if (config.tls) {
|
|
120
|
+
result ??= {};
|
|
121
|
+
result[uri] = config.tls;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return result;
|
|
125
|
+
}
|
|
126
|
+
function resolveRedirectUrl(response, currentUrl) {
|
|
127
|
+
const location = response.headers.get('location');
|
|
128
|
+
if (!location) {
|
|
129
|
+
throw new Error(`Redirect location header missing for ${redactUrlCredentials(currentUrl.toString())}`);
|
|
130
|
+
}
|
|
131
|
+
return new URL(location, currentUrl);
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=fetchFromRegistry.js.map
|
package/lib/index.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { clearDispatcherCache, destroyDispatchers, getDispatcher } from './dispatcher.js';
|
|
2
|
+
export { fetch, isRedirect, type RetryTimeoutOptions } from './fetch.js';
|
|
3
|
+
export { createDispatchedFetch, type CreateDispatchedFetchOptions, createFetchFromRegistry, type CreateFetchFromRegistryOptions, type DispatcherOptions, fetchWithDispatcher } from './fetchFromRegistry.js';
|
|
4
|
+
export type { FetchFromRegistry } from '@pnpm/fetching.types';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pnpm/network.fetch",
|
|
3
|
-
"version": "1100.1.
|
|
3
|
+
"version": "1100.1.8",
|
|
4
4
|
"description": "Native fetch with retries",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pnpm",
|
|
@@ -30,10 +30,10 @@
|
|
|
30
30
|
],
|
|
31
31
|
"dependencies": {
|
|
32
32
|
"@pnpm/config.nerf-dart": "^2.0.1",
|
|
33
|
-
"@pnpm/core-loggers": "1100.2.
|
|
34
|
-
"@pnpm/error": "1100.0
|
|
35
|
-
"@pnpm/fetching.types": "1100.0.
|
|
36
|
-
"@pnpm/types": "1101.
|
|
33
|
+
"@pnpm/core-loggers": "1100.2.5",
|
|
34
|
+
"@pnpm/error": "1100.1.0",
|
|
35
|
+
"@pnpm/fetching.types": "1100.0.3",
|
|
36
|
+
"@pnpm/types": "1101.6.0",
|
|
37
37
|
"@zkochan/retry": "^0.2.0",
|
|
38
38
|
"lru-cache": "^11.5.0",
|
|
39
39
|
"socks": "^2.8.9",
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"devDependencies": {
|
|
46
46
|
"@jest/globals": "30.4.1",
|
|
47
47
|
"@pnpm/logger": "1100.0.0",
|
|
48
|
-
"@pnpm/network.fetch": "1100.1.
|
|
48
|
+
"@pnpm/network.fetch": "1100.1.8",
|
|
49
49
|
"https-proxy-server-express": "0.1.2"
|
|
50
50
|
},
|
|
51
51
|
"engines": {
|