@verdaccio/proxy 6.0.0-6-next.18 → 6.0.0-6-next.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/build/agent.d.ts +17 -0
- package/build/agent.js +63 -0
- package/build/agent.js.map +1 -0
- package/build/index.d.ts +1 -1
- package/build/index.js +4 -4
- package/build/index.js.map +1 -1
- package/build/proxy-utils.js.map +1 -1
- package/build/{up-storage.d.ts → proxy.d.ts} +29 -55
- package/build/proxy.js +565 -0
- package/build/proxy.js.map +1 -0
- package/jest.config.js +4 -5
- package/package.json +66 -68
- package/src/agent.ts +54 -0
- package/src/index.ts +1 -1
- package/src/{up-storage.ts → proxy.ts} +254 -357
- package/test/conf/proxy1.yaml +1 -1
- package/test/headers.auth.spec.ts +15 -18
- package/test/noProxy.spec.ts +215 -76
- package/test/partials/ssl-cert-snakeoil.key +15 -0
- package/test/partials/ssl-cert-snakeoil.pem +12 -0
- package/test/proxy.error.___.ts +126 -0
- package/test/proxy.metadata.spec.ts +354 -0
- package/test/proxy.search.spec.ts +7 -13
- package/test/proxy.tarball.spec.ts +142 -204
- package/tsconfig.json +0 -6
- package/build/up-storage.js +0 -693
- package/build/up-storage.js.map +0 -1
- package/test/proxy.error.spec.ts +0 -126
- package/test/proxy.metadata.ts +0 -133
|
@@ -1,33 +1,31 @@
|
|
|
1
|
-
/* global AbortController */
|
|
2
1
|
import JSONStream from 'JSONStream';
|
|
3
2
|
import buildDebug from 'debug';
|
|
3
|
+
import got, { RequiredRetryOptions, Headers as gotHeaders } from 'got';
|
|
4
|
+
import type { Agents, Options } from 'got';
|
|
4
5
|
import _ from 'lodash';
|
|
5
|
-
import request from 'request';
|
|
6
6
|
import Stream, { PassThrough, Readable } from 'stream';
|
|
7
|
-
import { Headers,
|
|
7
|
+
import { Headers, fetch as undiciFetch } from 'undici';
|
|
8
8
|
import { URL } from 'url';
|
|
9
9
|
|
|
10
10
|
import {
|
|
11
|
-
|
|
11
|
+
API_ERROR,
|
|
12
12
|
HEADERS,
|
|
13
|
-
HEADER_TYPE,
|
|
14
13
|
HTTP_STATUS,
|
|
15
14
|
TOKEN_BASIC,
|
|
16
15
|
TOKEN_BEARER,
|
|
17
16
|
constants,
|
|
18
17
|
errorUtils,
|
|
19
18
|
searchUtils,
|
|
20
|
-
validatioUtils,
|
|
21
19
|
} from '@verdaccio/core';
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
20
|
+
import { Manifest } from '@verdaccio/types';
|
|
21
|
+
import { Config, Logger, UpLinkConf } from '@verdaccio/types';
|
|
24
22
|
import { buildToken } from '@verdaccio/utils';
|
|
25
23
|
|
|
24
|
+
import CustomAgents, { AgentOptionsConf } from './agent';
|
|
26
25
|
import { parseInterval } from './proxy-utils';
|
|
27
26
|
|
|
28
27
|
const LoggerApi = require('@verdaccio/logger');
|
|
29
28
|
|
|
30
|
-
const fetch = require('undici-fetch');
|
|
31
29
|
const debug = buildDebug('verdaccio:proxy');
|
|
32
30
|
|
|
33
31
|
const encode = function (thing): string {
|
|
@@ -40,7 +38,7 @@ const contentTypeAccept = `${jsonContentType};`;
|
|
|
40
38
|
/**
|
|
41
39
|
* Just a helper (`config[key] || default` doesn't work because of zeroes)
|
|
42
40
|
*/
|
|
43
|
-
const setConfig = (config, key, def): string => {
|
|
41
|
+
const setConfig = (config: UpLinkConfLocal, key: string, def): string => {
|
|
44
42
|
return _.isNil(config[key]) === false ? config[key] : def;
|
|
45
43
|
};
|
|
46
44
|
|
|
@@ -71,9 +69,21 @@ export interface IProxy {
|
|
|
71
69
|
max_fails: number;
|
|
72
70
|
fail_timeout: number;
|
|
73
71
|
upname: string;
|
|
74
|
-
fetchTarball(url: string): IReadTarball;
|
|
75
72
|
search(options: ProxySearchParams): Promise<Stream.Readable>;
|
|
76
|
-
|
|
73
|
+
getRemoteMetadataNext(name: string, options: ISyncUplinksOptions): Promise<[Manifest, string]>;
|
|
74
|
+
fetchTarballNext(
|
|
75
|
+
url: string,
|
|
76
|
+
options: Pick<ISyncUplinksOptions, 'remoteAddress' | 'etag' | 'retry'>
|
|
77
|
+
): PassThrough;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// this type is need it by storage
|
|
81
|
+
export { Options as FetchOptions };
|
|
82
|
+
|
|
83
|
+
export interface ISyncUplinksOptions extends Options {
|
|
84
|
+
uplinksLook?: boolean;
|
|
85
|
+
etag?: string;
|
|
86
|
+
remoteAddress?: string;
|
|
77
87
|
}
|
|
78
88
|
|
|
79
89
|
/**
|
|
@@ -92,33 +102,33 @@ class ProxyStorage implements IProxy {
|
|
|
92
102
|
public timeout: number;
|
|
93
103
|
public max_fails: number;
|
|
94
104
|
public fail_timeout: number;
|
|
95
|
-
public agent_options:
|
|
105
|
+
public agent_options: AgentOptionsConf;
|
|
96
106
|
// FIXME: upname is assigned to each instance
|
|
97
107
|
// @ts-ignore
|
|
98
108
|
public upname: string;
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
public proxy: any;
|
|
109
|
+
public proxy: string | undefined;
|
|
110
|
+
private agent: Agents;
|
|
102
111
|
// @ts-ignore
|
|
103
112
|
public last_request_time: number | null;
|
|
104
113
|
public strict_ssl: boolean;
|
|
114
|
+
private retry: Partial<RequiredRetryOptions> | number;
|
|
105
115
|
|
|
106
|
-
|
|
107
|
-
* Constructor
|
|
108
|
-
* @param {*} config
|
|
109
|
-
* @param {*} mainConfig
|
|
110
|
-
*/
|
|
111
|
-
public constructor(config: UpLinkConfLocal, mainConfig: Config) {
|
|
116
|
+
public constructor(config: UpLinkConfLocal, mainConfig: Config, agent?: Agents) {
|
|
112
117
|
this.config = config;
|
|
113
118
|
this.failed_requests = 0;
|
|
114
119
|
this.userAgent = mainConfig.user_agent;
|
|
115
120
|
this.ca = config.ca;
|
|
116
121
|
this.logger = LoggerApi.logger.child({ sub: 'out' });
|
|
117
122
|
this.server_id = mainConfig.server_id;
|
|
118
|
-
|
|
123
|
+
this.agent_options = setConfig(this.config, 'agent_options', {
|
|
124
|
+
keepAlive: true,
|
|
125
|
+
maxSockets: 40,
|
|
126
|
+
maxFreeSockets: 10,
|
|
127
|
+
}) as AgentOptionsConf;
|
|
119
128
|
this.url = new URL(this.config.url);
|
|
120
|
-
|
|
121
|
-
|
|
129
|
+
const isHTTPS = this.url.protocol === 'https:';
|
|
130
|
+
this._setupProxy(this.url.hostname, config, mainConfig, isHTTPS);
|
|
131
|
+
this.agent = agent ?? this.getAgent();
|
|
122
132
|
this.config.url = this.config.url.replace(/\/$/, '');
|
|
123
133
|
|
|
124
134
|
if (this.config.timeout && Number(this.config.timeout) >= 1000) {
|
|
@@ -134,192 +144,25 @@ class ProxyStorage implements IProxy {
|
|
|
134
144
|
|
|
135
145
|
// a bunch of different configurable timers
|
|
136
146
|
this.maxage = parseInterval(setConfig(this.config, 'maxage', '2m'));
|
|
147
|
+
// https://github.com/sindresorhus/got/blob/main/documentation/6-timeout.md
|
|
137
148
|
this.timeout = parseInterval(setConfig(this.config, 'timeout', '30s'));
|
|
138
|
-
this.max_fails = Number(setConfig(this.config, 'max_fails', 2));
|
|
149
|
+
this.max_fails = Number(setConfig(this.config, 'max_fails', this.config.max_fails ?? 2));
|
|
139
150
|
this.fail_timeout = parseInterval(setConfig(this.config, 'fail_timeout', '5m'));
|
|
140
151
|
this.strict_ssl = Boolean(setConfig(this.config, 'strict_ssl', true));
|
|
141
|
-
this.
|
|
142
|
-
keepAlive: true,
|
|
143
|
-
maxSockets: 40,
|
|
144
|
-
maxFreeSockets: 10,
|
|
145
|
-
});
|
|
152
|
+
this.retry = { limit: this.max_fails ?? 2 };
|
|
146
153
|
}
|
|
147
154
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
let json;
|
|
156
|
-
|
|
157
|
-
if (this._statusCheck() === false) {
|
|
158
|
-
const streamRead = new Stream.Readable();
|
|
159
|
-
|
|
160
|
-
process.nextTick(function (): void {
|
|
161
|
-
if (cb) {
|
|
162
|
-
cb(errorUtils.getInternalError(errorUtils.API_ERROR.UPLINK_OFFLINE));
|
|
163
|
-
}
|
|
164
|
-
streamRead.emit('error', errorUtils.getInternalError(errorUtils.API_ERROR.UPLINK_OFFLINE));
|
|
165
|
-
});
|
|
166
|
-
streamRead._read = function (): void {};
|
|
167
|
-
// preventing 'Uncaught, unspecified "error" event'
|
|
168
|
-
streamRead.on('error', function (): void {});
|
|
169
|
-
return streamRead;
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
const self = this;
|
|
173
|
-
const headers: Headers = this._setHeaders(options);
|
|
174
|
-
|
|
175
|
-
this._addProxyHeaders(options.req, headers);
|
|
176
|
-
this._overrideWithUpLinkConfLocaligHeaders(headers);
|
|
177
|
-
|
|
178
|
-
const method = options.method || 'GET';
|
|
179
|
-
const uri = options.uri_full || this.config.url + options.uri;
|
|
180
|
-
|
|
181
|
-
self.logger.info(
|
|
182
|
-
{
|
|
183
|
-
method: method,
|
|
184
|
-
headers: headers,
|
|
185
|
-
uri: uri,
|
|
186
|
-
},
|
|
187
|
-
"making request: '@{method} @{uri}'"
|
|
188
|
-
);
|
|
189
|
-
|
|
190
|
-
if (validatioUtils.isObject(options.json)) {
|
|
191
|
-
json = JSON.stringify(options.json);
|
|
192
|
-
headers['Content-Type'] = headers['Content-Type'] || HEADERS.JSON;
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
const requestCallback = cb
|
|
196
|
-
? function (err, res, body): void {
|
|
197
|
-
let error;
|
|
198
|
-
const responseLength = err ? 0 : body.length;
|
|
199
|
-
// $FlowFixMe
|
|
200
|
-
processBody();
|
|
201
|
-
logActivity();
|
|
202
|
-
// $FlowFixMe
|
|
203
|
-
cb(err, res, body);
|
|
204
|
-
|
|
205
|
-
/**
|
|
206
|
-
* Perform a decode.
|
|
207
|
-
*/
|
|
208
|
-
function processBody(): void {
|
|
209
|
-
if (err) {
|
|
210
|
-
error = err.message;
|
|
211
|
-
return;
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
if (options.json && res.statusCode < 300) {
|
|
215
|
-
try {
|
|
216
|
-
// $FlowFixMe
|
|
217
|
-
body = JSON.parse(body.toString(CHARACTER_ENCODING.UTF8));
|
|
218
|
-
} catch (_err: any) {
|
|
219
|
-
body = {};
|
|
220
|
-
err = _err;
|
|
221
|
-
error = err.message;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
if (!err && validatioUtils.isObject(body)) {
|
|
226
|
-
if (_.isString(body.error)) {
|
|
227
|
-
error = body.error;
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
/**
|
|
232
|
-
* Perform a log.
|
|
233
|
-
*/
|
|
234
|
-
function logActivity(): void {
|
|
235
|
-
let message = "@{!status}, req: '@{request.method} @{request.url}'";
|
|
236
|
-
// FIXME: use LOG_VERDACCIO_BYTES
|
|
237
|
-
message += error ? ', error: @{!error}' : ', bytes: @{bytes.in}/@{bytes.out}';
|
|
238
|
-
self.logger.http(
|
|
239
|
-
{
|
|
240
|
-
// if error is null/false change this to undefined so it wont log
|
|
241
|
-
err: err || undefined,
|
|
242
|
-
request: { method: method, url: uri },
|
|
243
|
-
status: res != null ? res.statusCode : 'ERR',
|
|
244
|
-
error: error,
|
|
245
|
-
bytes: {
|
|
246
|
-
in: json ? json.length : 0,
|
|
247
|
-
out: responseLength || 0,
|
|
248
|
-
},
|
|
249
|
-
},
|
|
250
|
-
message
|
|
251
|
-
);
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
: undefined;
|
|
255
|
-
|
|
256
|
-
let requestOptions = {
|
|
257
|
-
url: uri,
|
|
258
|
-
method: method,
|
|
259
|
-
headers: headers,
|
|
260
|
-
body: json,
|
|
261
|
-
proxy: this.proxy,
|
|
262
|
-
encoding: null,
|
|
263
|
-
gzip: true,
|
|
264
|
-
timeout: this.timeout,
|
|
265
|
-
strictSSL: this.strict_ssl,
|
|
266
|
-
agentOptions: this.agent_options,
|
|
267
|
-
};
|
|
268
|
-
|
|
269
|
-
if (this.ca) {
|
|
270
|
-
requestOptions = Object.assign({}, requestOptions, {
|
|
271
|
-
ca: this.ca,
|
|
272
|
-
});
|
|
155
|
+
private getAgent() {
|
|
156
|
+
if (!this.agent) {
|
|
157
|
+
// TODO: the config.ca (certificates) is not yet injected here
|
|
158
|
+
const agentInstance = new CustomAgents(this.config.url, this.proxy, this.agent_options);
|
|
159
|
+
return agentInstance.get();
|
|
160
|
+
} else {
|
|
161
|
+
return this.agent;
|
|
273
162
|
}
|
|
274
|
-
|
|
275
|
-
const req = request(requestOptions, requestCallback);
|
|
276
|
-
|
|
277
|
-
let statusCalled = false;
|
|
278
|
-
req.on('response', function (res): void {
|
|
279
|
-
// FIXME: _verdaccio_aborted seems not used
|
|
280
|
-
// @ts-ignore
|
|
281
|
-
if (!req._verdaccio_aborted && !statusCalled) {
|
|
282
|
-
statusCalled = true;
|
|
283
|
-
self._statusCheck(true);
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
if (_.isNil(requestCallback) === false) {
|
|
287
|
-
(function do_log(): void {
|
|
288
|
-
const message = "@{!status}, req: '@{request.method} @{request.url}' (streaming)";
|
|
289
|
-
self.logger.http(
|
|
290
|
-
{
|
|
291
|
-
request: {
|
|
292
|
-
method: method,
|
|
293
|
-
url: uri,
|
|
294
|
-
},
|
|
295
|
-
status: _.isNull(res) === false ? res.statusCode : 'ERR',
|
|
296
|
-
},
|
|
297
|
-
message
|
|
298
|
-
);
|
|
299
|
-
})();
|
|
300
|
-
}
|
|
301
|
-
});
|
|
302
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
303
|
-
req.on('error', function (_err): void {
|
|
304
|
-
// FIXME: _verdaccio_aborted seems not used
|
|
305
|
-
// @ts-ignore
|
|
306
|
-
if (!req._verdaccio_aborted && !statusCalled) {
|
|
307
|
-
statusCalled = true;
|
|
308
|
-
self._statusCheck(false);
|
|
309
|
-
}
|
|
310
|
-
});
|
|
311
|
-
// @ts-ignore
|
|
312
|
-
return req;
|
|
313
163
|
}
|
|
314
164
|
|
|
315
|
-
|
|
316
|
-
* Set default headers.
|
|
317
|
-
* @param {Object} options
|
|
318
|
-
* @return {Object}
|
|
319
|
-
* @private
|
|
320
|
-
*/
|
|
321
|
-
private _setHeaders(options: any): Headers {
|
|
322
|
-
const headers = options.headers || {};
|
|
165
|
+
public getHeadersNext(headers = {}): gotHeaders {
|
|
323
166
|
const accept = HEADERS.ACCEPT;
|
|
324
167
|
const acceptEncoding = HEADERS.ACCEPT_ENCODING;
|
|
325
168
|
const userAgent = HEADERS.USER_AGENT;
|
|
@@ -328,8 +171,7 @@ class ProxyStorage implements IProxy {
|
|
|
328
171
|
headers[acceptEncoding] = headers[acceptEncoding] || 'gzip';
|
|
329
172
|
// registry.npmjs.org will only return search result if user-agent include string 'npm'
|
|
330
173
|
headers[userAgent] = headers[userAgent] || `npm (${this.userAgent})`;
|
|
331
|
-
|
|
332
|
-
return this._setAuth(headers);
|
|
174
|
+
return this.setAuthNext(headers);
|
|
333
175
|
}
|
|
334
176
|
|
|
335
177
|
/**
|
|
@@ -338,10 +180,9 @@ class ProxyStorage implements IProxy {
|
|
|
338
180
|
* @return {Object}
|
|
339
181
|
* @private
|
|
340
182
|
*/
|
|
341
|
-
private
|
|
183
|
+
private setAuthNext(headers: gotHeaders): gotHeaders {
|
|
342
184
|
const { auth } = this.config;
|
|
343
|
-
|
|
344
|
-
if (_.isNil(auth) || headers[HEADERS.AUTHORIZATION]) {
|
|
185
|
+
if (typeof auth === 'undefined' || typeof headers[HEADERS.AUTHORIZATION] === 'string') {
|
|
345
186
|
return headers;
|
|
346
187
|
}
|
|
347
188
|
|
|
@@ -354,13 +195,12 @@ class ProxyStorage implements IProxy {
|
|
|
354
195
|
// https://github.com/verdaccio/verdaccio/releases/tag/v2.5.0
|
|
355
196
|
let token: any;
|
|
356
197
|
const tokenConf: any = auth;
|
|
357
|
-
|
|
358
198
|
if (_.isNil(tokenConf.token) === false && _.isString(tokenConf.token)) {
|
|
359
199
|
token = tokenConf.token;
|
|
360
200
|
} else if (_.isNil(tokenConf.token_env) === false) {
|
|
361
|
-
if (
|
|
201
|
+
if (typeof tokenConf.token_env === 'string') {
|
|
362
202
|
token = process.env[tokenConf.token_env];
|
|
363
|
-
} else if (
|
|
203
|
+
} else if (typeof tokenConf.token_env === 'boolean' && tokenConf.token_env) {
|
|
364
204
|
token = process.env.NPM_TOKEN;
|
|
365
205
|
} else {
|
|
366
206
|
this.logger.error(constants.ERROR_CODE.token_required);
|
|
@@ -370,7 +210,7 @@ class ProxyStorage implements IProxy {
|
|
|
370
210
|
token = process.env.NPM_TOKEN;
|
|
371
211
|
}
|
|
372
212
|
|
|
373
|
-
if (
|
|
213
|
+
if (typeof token === 'undefined') {
|
|
374
214
|
this._throwErrorAuth(constants.ERROR_CODE.token_required);
|
|
375
215
|
}
|
|
376
216
|
|
|
@@ -427,6 +267,7 @@ class ProxyStorage implements IProxy {
|
|
|
427
267
|
|
|
428
268
|
* @param {Object} headers
|
|
429
269
|
* @private
|
|
270
|
+
* @deprecated use applyUplinkHeaders
|
|
430
271
|
*/
|
|
431
272
|
private _overrideWithUpLinkConfLocaligHeaders(headers: Headers): any {
|
|
432
273
|
if (!this.config.headers) {
|
|
@@ -440,98 +281,200 @@ class ProxyStorage implements IProxy {
|
|
|
440
281
|
}
|
|
441
282
|
}
|
|
442
283
|
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
const headers
|
|
284
|
+
private applyUplinkHeaders(headers: gotHeaders): gotHeaders {
|
|
285
|
+
if (!this.config.headers) {
|
|
286
|
+
return headers;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// add/override headers specified in the config
|
|
290
|
+
/* eslint guard-for-in: 0 */
|
|
291
|
+
for (const key in this.config.headers) {
|
|
292
|
+
headers[key] = this.config.headers[key];
|
|
293
|
+
}
|
|
294
|
+
return headers;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
public async getRemoteMetadataNext(
|
|
298
|
+
name: string,
|
|
299
|
+
options: ISyncUplinksOptions
|
|
300
|
+
): Promise<[Manifest, string]> {
|
|
301
|
+
if (this._ifRequestFailure()) {
|
|
302
|
+
throw errorUtils.getInternalError(API_ERROR.UPLINK_OFFLINE);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// FUTURE: allow mix headers that comes from the client
|
|
306
|
+
debug('get metadata for %s', name);
|
|
307
|
+
let headers = this.getHeadersNext(options?.headers);
|
|
308
|
+
headers = this.addProxyHeaders(headers, options.remoteAddress);
|
|
309
|
+
headers = this.applyUplinkHeaders(headers);
|
|
310
|
+
// the following headers cannot be overwritten
|
|
451
311
|
if (_.isNil(options.etag) === false) {
|
|
452
|
-
headers[
|
|
312
|
+
headers[HEADERS.NONE_MATCH] = options.etag;
|
|
453
313
|
headers[HEADERS.ACCEPT] = contentTypeAccept;
|
|
454
314
|
}
|
|
315
|
+
const method = options.method || 'GET';
|
|
316
|
+
const uri = this.config.url + `/${encode(name)}`;
|
|
317
|
+
debug('request uri for %s retry %s', uri);
|
|
318
|
+
let response;
|
|
319
|
+
let responseLength = 0;
|
|
320
|
+
try {
|
|
321
|
+
const retry = options?.retry ?? this.retry;
|
|
322
|
+
debug('retry times %s for %s', retry, uri);
|
|
323
|
+
response = await got(uri, {
|
|
324
|
+
headers,
|
|
325
|
+
responseType: 'json',
|
|
326
|
+
method,
|
|
327
|
+
agent: this.agent,
|
|
328
|
+
retry,
|
|
329
|
+
// @ts-ignore
|
|
330
|
+
timeout: { request: options?.timeout ?? this.timeout },
|
|
331
|
+
hooks: {
|
|
332
|
+
afterResponse: [
|
|
333
|
+
(afterResponse) => {
|
|
334
|
+
const code = afterResponse.statusCode;
|
|
335
|
+
debug('code response %s', code);
|
|
336
|
+
if (code >= HTTP_STATUS.OK && code < HTTP_STATUS.MULTIPLE_CHOICES) {
|
|
337
|
+
if (this.failed_requests >= this.max_fails) {
|
|
338
|
+
this.failed_requests = 0;
|
|
339
|
+
this.logger.warn(
|
|
340
|
+
{
|
|
341
|
+
host: this.url.host,
|
|
342
|
+
},
|
|
343
|
+
'host @{host} is now online'
|
|
344
|
+
);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
455
347
|
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
348
|
+
return afterResponse;
|
|
349
|
+
},
|
|
350
|
+
],
|
|
351
|
+
beforeRetry: [
|
|
352
|
+
// FUTURE: got 12.0.0, the option arg should be removed
|
|
353
|
+
(_options, error: any, count) => {
|
|
354
|
+
this.failed_requests = count ?? 0;
|
|
355
|
+
this.logger.info(
|
|
356
|
+
{
|
|
357
|
+
request: {
|
|
358
|
+
method: method,
|
|
359
|
+
url: uri,
|
|
360
|
+
},
|
|
361
|
+
error: error.message,
|
|
362
|
+
retryCount: this.failed_requests,
|
|
363
|
+
},
|
|
364
|
+
"retry @{retryCount} req: '@{request.method} @{request.url}'"
|
|
365
|
+
);
|
|
366
|
+
if (this.failed_requests >= this.max_fails) {
|
|
367
|
+
this.logger.warn(
|
|
368
|
+
{
|
|
369
|
+
host: this.url.host,
|
|
370
|
+
},
|
|
371
|
+
'host @{host} is now offline'
|
|
372
|
+
);
|
|
373
|
+
}
|
|
374
|
+
},
|
|
375
|
+
],
|
|
376
|
+
},
|
|
377
|
+
})
|
|
378
|
+
.on('request', () => {
|
|
379
|
+
this.last_request_time = Date.now();
|
|
380
|
+
})
|
|
381
|
+
.on('response', (eventResponse) => {
|
|
382
|
+
const message = "@{!status}, req: '@{request.method} @{request.url}' (streaming)";
|
|
383
|
+
this.logger.http(
|
|
384
|
+
{
|
|
385
|
+
request: {
|
|
386
|
+
method: method,
|
|
387
|
+
url: uri,
|
|
388
|
+
},
|
|
389
|
+
status: _.isNull(eventResponse) === false ? eventResponse.statusCode : 'ERR',
|
|
390
|
+
},
|
|
391
|
+
message
|
|
392
|
+
);
|
|
393
|
+
})
|
|
394
|
+
.on('downloadProgress', (progress) => {
|
|
395
|
+
if (progress.total) {
|
|
396
|
+
debug('responseLength %s', progress.total);
|
|
397
|
+
responseLength = progress.total;
|
|
398
|
+
}
|
|
399
|
+
});
|
|
400
|
+
const etag = response.headers.etag as string;
|
|
401
|
+
const data = response.body;
|
|
402
|
+
|
|
403
|
+
// not modified status (304) registry does not return any payload
|
|
404
|
+
// it is handled as an error
|
|
405
|
+
if (response?.statusCode === HTTP_STATUS.NOT_MODIFIED) {
|
|
406
|
+
throw errorUtils.getCode(HTTP_STATUS.NOT_MODIFIED, API_ERROR.NOT_MODIFIED_NO_DATA);
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
debug('uri %s success', uri);
|
|
410
|
+
const message = "@{!status}, req: '@{request.method} @{request.url}'";
|
|
411
|
+
this.logger.http(
|
|
412
|
+
{
|
|
413
|
+
// if error is null/false change this to undefined so it wont log
|
|
414
|
+
request: { method: method, url: uri },
|
|
415
|
+
status: response.statusCode,
|
|
416
|
+
bytes: {
|
|
417
|
+
in: options?.json ? JSON.stringify(options?.json).length : 0,
|
|
418
|
+
out: responseLength || 0,
|
|
419
|
+
},
|
|
420
|
+
},
|
|
421
|
+
message
|
|
422
|
+
);
|
|
423
|
+
return [data, etag];
|
|
424
|
+
} catch (err: any) {
|
|
425
|
+
debug('uri %s fail', uri);
|
|
426
|
+
if (err.code === 'ERR_NON_2XX_3XX_RESPONSE') {
|
|
427
|
+
const code = err.response.statusCode;
|
|
428
|
+
if (code === HTTP_STATUS.NOT_FOUND) {
|
|
429
|
+
throw errorUtils.getNotFound(errorUtils.API_ERROR.NOT_PACKAGE_UPLINK);
|
|
469
430
|
}
|
|
470
|
-
|
|
431
|
+
|
|
432
|
+
if (!(code >= HTTP_STATUS.OK && code < HTTP_STATUS.MULTIPLE_CHOICES)) {
|
|
471
433
|
const error = errorUtils.getInternalError(
|
|
472
|
-
`${errorUtils.API_ERROR.BAD_STATUS_CODE}: ${
|
|
434
|
+
`${errorUtils.API_ERROR.BAD_STATUS_CODE}: ${code}`
|
|
473
435
|
);
|
|
474
|
-
|
|
475
|
-
error.remoteStatus =
|
|
476
|
-
|
|
436
|
+
// we need this code to identify outside which status code triggered the error
|
|
437
|
+
error.remoteStatus = code;
|
|
438
|
+
throw error;
|
|
477
439
|
}
|
|
478
|
-
callback(null, body, res.headers.etag);
|
|
479
440
|
}
|
|
480
|
-
|
|
441
|
+
throw err;
|
|
442
|
+
}
|
|
481
443
|
}
|
|
482
444
|
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
const
|
|
490
|
-
let
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
readStream
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
stream.emit(HEADER_TYPE.CONTENT_LENGTH, res.headers[HEADER_TYPE.CONTENT_LENGTH]);
|
|
515
|
-
}
|
|
445
|
+
// FIXME: handle stream and retry
|
|
446
|
+
public fetchTarballNext(
|
|
447
|
+
url: string,
|
|
448
|
+
overrideOptions: Pick<ISyncUplinksOptions, 'remoteAddress' | 'etag' | 'retry'>
|
|
449
|
+
): any {
|
|
450
|
+
debug('fetching url for %s', url);
|
|
451
|
+
const options = { ...this.config, ...overrideOptions };
|
|
452
|
+
let headers = this.getHeadersNext(options?.headers);
|
|
453
|
+
headers = this.addProxyHeaders(headers, options.remoteAddress);
|
|
454
|
+
headers = this.applyUplinkHeaders(headers);
|
|
455
|
+
// the following headers cannot be overwritten
|
|
456
|
+
if (_.isNil(options.etag) === false) {
|
|
457
|
+
headers[HEADERS.NONE_MATCH] = options.etag;
|
|
458
|
+
headers[HEADERS.ACCEPT] = contentTypeAccept;
|
|
459
|
+
}
|
|
460
|
+
const method = 'GET';
|
|
461
|
+
// const uri = this.config.url + `/${encode(name)}`;
|
|
462
|
+
debug('request uri for %s', url);
|
|
463
|
+
|
|
464
|
+
const readStream = got
|
|
465
|
+
.stream(url, {
|
|
466
|
+
headers,
|
|
467
|
+
method,
|
|
468
|
+
agent: this.agent,
|
|
469
|
+
// FIXME: this should be taken from construtor as priority
|
|
470
|
+
retry: this.retry ?? options?.retry,
|
|
471
|
+
timeout: this.timeout,
|
|
472
|
+
})
|
|
473
|
+
.on('request', () => {
|
|
474
|
+
this.last_request_time = Date.now();
|
|
475
|
+
});
|
|
516
476
|
|
|
517
|
-
|
|
518
|
-
});
|
|
519
|
-
|
|
520
|
-
readStream.on('error', function (err) {
|
|
521
|
-
stream.emit('error', err);
|
|
522
|
-
});
|
|
523
|
-
readStream.on('data', function (data) {
|
|
524
|
-
current_length += data.length;
|
|
525
|
-
});
|
|
526
|
-
readStream.on('end', function (data) {
|
|
527
|
-
if (data) {
|
|
528
|
-
current_length += data.length;
|
|
529
|
-
}
|
|
530
|
-
if (expected_length && current_length != expected_length) {
|
|
531
|
-
stream.emit('error', errorUtils.getInternalError(errorUtils.API_ERROR.CONTENT_MISMATCH));
|
|
532
|
-
}
|
|
533
|
-
});
|
|
534
|
-
return stream;
|
|
477
|
+
return readStream;
|
|
535
478
|
}
|
|
536
479
|
|
|
537
480
|
/**
|
|
@@ -548,7 +491,7 @@ class ProxyStorage implements IProxy {
|
|
|
548
491
|
// FIXME: a better way to remove duplicate slashes?
|
|
549
492
|
const uri = fullURL.href.replace(/([^:]\/)\/+/g, '$1');
|
|
550
493
|
this.logger.http({ uri, uplink: this.upname }, 'search request to uplink @{uplink} - @{uri}');
|
|
551
|
-
|
|
494
|
+
response = await undiciFetch(uri, {
|
|
552
495
|
method: 'GET',
|
|
553
496
|
// FUTURE: whitelist domains what we are sending not need it headers, security check
|
|
554
497
|
// headers: new Headers({
|
|
@@ -557,7 +500,6 @@ class ProxyStorage implements IProxy {
|
|
|
557
500
|
// }),
|
|
558
501
|
signal: abort?.signal,
|
|
559
502
|
});
|
|
560
|
-
response = await fetch(request);
|
|
561
503
|
debug('response.status %o', response.status);
|
|
562
504
|
|
|
563
505
|
if (response.status >= HTTP_STATUS.BAD_REQUEST) {
|
|
@@ -571,70 +513,29 @@ class ProxyStorage implements IProxy {
|
|
|
571
513
|
streamResponse.pipe(JSONStream.parse('objects')).pipe(streamSearch, { end: true });
|
|
572
514
|
return streamSearch;
|
|
573
515
|
} catch (err: any) {
|
|
574
|
-
this.logger.error(
|
|
516
|
+
this.logger.error(
|
|
517
|
+
{ errorMessage: err?.message, name: this.upname },
|
|
518
|
+
'proxy uplink @{name} search error: @{errorMessage}'
|
|
519
|
+
);
|
|
575
520
|
throw err;
|
|
576
521
|
}
|
|
577
522
|
}
|
|
578
523
|
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
// Only submit X-Forwarded-For field if we don't have a proxy selected
|
|
588
|
-
// in the config file.
|
|
589
|
-
//
|
|
590
|
-
// Otherwise misconfigured proxy could return 407:
|
|
591
|
-
// https://github.com/rlidwka/sinopia/issues/254
|
|
592
|
-
// @ts-ignore
|
|
593
|
-
if (!this.proxy) {
|
|
594
|
-
headers[HEADERS.FORWARDED_FOR] =
|
|
595
|
-
(req.headers['x-forwarded-for'] ? req.headers['x-forwarded-for'] + ', ' : '') +
|
|
596
|
-
req.connection.remoteAddress;
|
|
597
|
-
}
|
|
524
|
+
private addProxyHeaders(headers: gotHeaders, remoteAddress?: string): gotHeaders {
|
|
525
|
+
// Only submit X-Forwarded-For field if we don't have a proxy selected
|
|
526
|
+
// in the config file.
|
|
527
|
+
//
|
|
528
|
+
// Otherwise misconfigured proxy could return 407
|
|
529
|
+
if (!this.proxy) {
|
|
530
|
+
headers[HEADERS.FORWARDED_FOR] =
|
|
531
|
+
(headers['x-forwarded-for'] ? headers['x-forwarded-for'] + ', ' : '') + remoteAddress;
|
|
598
532
|
}
|
|
599
533
|
|
|
600
534
|
// always attach Via header to avoid loops, even if we're not proxying
|
|
601
|
-
headers['
|
|
602
|
-
|
|
603
|
-
headers['Via'] += '1.1 ' + this.server_id + ' (Verdaccio)';
|
|
604
|
-
}
|
|
535
|
+
headers['via'] = headers['via'] ? headers['via'] + ', ' : '';
|
|
536
|
+
headers['via'] += '1.1 ' + this.server_id + ' (Verdaccio)';
|
|
605
537
|
|
|
606
|
-
|
|
607
|
-
* Check whether the remote host is available.
|
|
608
|
-
* @param {*} alive
|
|
609
|
-
* @return {Boolean}
|
|
610
|
-
*/
|
|
611
|
-
private _statusCheck(alive?: boolean): boolean | void {
|
|
612
|
-
if (arguments.length === 0) {
|
|
613
|
-
return this._ifRequestFailure() === false;
|
|
614
|
-
}
|
|
615
|
-
if (alive) {
|
|
616
|
-
if (this.failed_requests >= this.max_fails) {
|
|
617
|
-
this.logger.warn(
|
|
618
|
-
{
|
|
619
|
-
host: this.url.host,
|
|
620
|
-
},
|
|
621
|
-
'host @{host} is back online'
|
|
622
|
-
);
|
|
623
|
-
}
|
|
624
|
-
this.failed_requests = 0;
|
|
625
|
-
} else {
|
|
626
|
-
this.failed_requests++;
|
|
627
|
-
if (this.failed_requests === this.max_fails) {
|
|
628
|
-
this.logger.warn(
|
|
629
|
-
{
|
|
630
|
-
host: this.url.host,
|
|
631
|
-
},
|
|
632
|
-
'host @{host} is now offline'
|
|
633
|
-
);
|
|
634
|
-
}
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
this.last_request_time = Date.now();
|
|
538
|
+
return headers;
|
|
638
539
|
}
|
|
639
540
|
|
|
640
541
|
/**
|
|
@@ -692,24 +593,20 @@ class ProxyStorage implements IProxy {
|
|
|
692
593
|
if (noProxyItem[0] !== '.') {
|
|
693
594
|
noProxyItem = '.' + noProxyItem;
|
|
694
595
|
}
|
|
695
|
-
if (hostname.
|
|
596
|
+
if (hostname.endsWith(noProxyItem)) {
|
|
696
597
|
if (this.proxy) {
|
|
697
598
|
this.logger.debug(
|
|
698
599
|
{ url: this.url.href, rule: noProxyItem },
|
|
699
600
|
'not using proxy for @{url}, excluded by @{rule} rule'
|
|
700
601
|
);
|
|
701
|
-
|
|
702
|
-
this.proxy = false;
|
|
602
|
+
this.proxy = undefined;
|
|
703
603
|
}
|
|
704
604
|
break;
|
|
705
605
|
}
|
|
706
606
|
}
|
|
707
607
|
}
|
|
708
608
|
|
|
709
|
-
|
|
710
|
-
if (_.isString(this.proxy) === false) {
|
|
711
|
-
delete this.proxy;
|
|
712
|
-
} else {
|
|
609
|
+
if (typeof this.proxy === 'string') {
|
|
713
610
|
this.logger.debug(
|
|
714
611
|
{ url: this.url.href, proxy: this.proxy },
|
|
715
612
|
'using proxy @{proxy} for @{url}'
|