@verdaccio/proxy 6.0.0-6-next.52 → 6.0.0-6-next.53

package/src/proxy.ts

@@ -1,10 +1,15 @@
 import JSONStream from 'JSONStream';
 import buildDebug from 'debug';
-import got, { RequiredRetryOptions, Headers as gotHeaders } from 'got';
-import type { Agents, Options } from 'got';
+import got, {
+  Agents,
+  Delays,
+  Options,
+  RequestError,
+  RetryOptions,
+  Headers as gotHeaders,
+} from 'got-cjs';
 import _ from 'lodash';
 import Stream, { PassThrough, Readable } from 'stream';
-import { Headers, fetch as undiciFetch } from 'undici';
 import { URL } from 'url';
 
 import {
@@ -24,8 +29,6 @@ import { buildToken } from '@verdaccio/utils';
 import CustomAgents, { AgentOptionsConf } from './agent';
 import { parseInterval } from './proxy-utils';
 
-const LoggerApi = require('@verdaccio/logger');
-
 const debug = buildDebug('verdaccio:proxy');
 
 const encode = function (thing): string {
@@ -51,10 +54,11 @@ export interface ProxyList {
 }
 
 export type ProxySearchParams = {
-  headers?: Headers;
   url: string;
-  query?: searchUtils.SearchQuery;
   abort: AbortController;
+  query?: searchUtils.SearchQuery;
+  headers?: Headers;
+  retry?: Partial<RetryOptions>;
 };
 export interface IProxy {
   config: UpLinkConfLocal;
@@ -65,15 +69,18 @@ export interface IProxy {
   server_id: string;
   url: URL;
   maxage: number;
-  timeout: number;
+  timeout: Delays;
   max_fails: number;
   fail_timeout: number;
   upname: string;
   search(options: ProxySearchParams): Promise<Stream.Readable>;
-  getRemoteMetadata(name: string, options: ISyncUplinksOptions): Promise<[Manifest, string]>;
+  getRemoteMetadata(
+    name: string,
+    options: Partial<ISyncUplinksOptions>
+  ): Promise<[Manifest, string]>;
   fetchTarball(
     url: string,
-    options: Pick<ISyncUplinksOptions, 'remoteAddress' | 'etag' | 'retry'>
+    options: Partial<Pick<ISyncUplinksOptions, 'remoteAddress' | 'etag' | 'retry'>>
   ): PassThrough;
 }
 
@@ -99,7 +106,7 @@ class ProxyStorage implements IProxy {
   public server_id: string;
   public url: URL;
   public maxage: number;
-  public timeout: number;
+  public timeout: Delays;
   public max_fails: number;
   public fail_timeout: number;
   public agent_options: AgentOptionsConf;
@@ -111,14 +118,14 @@ class ProxyStorage implements IProxy {
   // @ts-ignore
   public last_request_time: number | null;
   public strict_ssl: boolean;
-  private retry: Partial<RequiredRetryOptions> | number;
+  private retry: Partial<RetryOptions>;
 
-  public constructor(config: UpLinkConfLocal, mainConfig: Config, agent?: Agents) {
+  public constructor(config: UpLinkConfLocal, mainConfig: Config, logger: Logger, agent?: Agents) {
     this.config = config;
     this.failed_requests = 0;
     this.userAgent = mainConfig.user_agent ?? 'hidden';
     this.ca = config.ca;
-    this.logger = LoggerApi.logger.child({ sub: 'out' });
+    this.logger = logger;
     this.server_id = mainConfig.server_id;
     this.agent_options = setConfig(this.config, 'agent_options', {
       keepAlive: true,
@@ -145,7 +152,10 @@ class ProxyStorage implements IProxy {
     // a bunch of different configurable timers
     this.maxage = parseInterval(setConfig(this.config, 'maxage', '2m'));
     // https://github.com/sindresorhus/got/blob/main/documentation/6-timeout.md
-    this.timeout = parseInterval(setConfig(this.config, 'timeout', '30s'));
+    this.timeout = {
+      request: parseInterval(setConfig(this.config, 'timeout', '30s')),
+    };
+    debug('set timeout %s', this.timeout);
     this.max_fails = Number(setConfig(this.config, 'max_fails', this.config.max_fails ?? 2));
     this.fail_timeout = parseInterval(setConfig(this.config, 'fail_timeout', '5m'));
     this.strict_ssl = Boolean(setConfig(this.config, 'strict_ssl', true));
@@ -162,7 +172,7 @@ class ProxyStorage implements IProxy {
     }
   }
 
-  public getHeadersNext(headers = {}): gotHeaders {
+  public getHeaders(headers = {}): gotHeaders {
     const accept = HEADERS.ACCEPT;
     const acceptEncoding = HEADERS.ACCEPT_ENCODING;
     const userAgent = HEADERS.USER_AGENT;
@@ -296,15 +306,15 @@ class ProxyStorage implements IProxy {
 
   public async getRemoteMetadata(
     name: string,
-    options: ISyncUplinksOptions
+    options: Partial<ISyncUplinksOptions>
   ): Promise<[Manifest, string]> {
     if (this._ifRequestFailure()) {
       throw errorUtils.getInternalError(API_ERROR.UPLINK_OFFLINE);
     }
 
     // FUTURE: allow mix headers that comes from the client
-    debug('get metadata for %s', name);
-    let headers = this.getHeadersNext(options?.headers);
+    debug('getting metadata for package %s', name);
+    let headers = this.getHeaders(options?.headers);
     headers = this.addProxyHeaders(headers, options.remoteAddress);
     headers = this.applyUplinkHeaders(headers);
     // the following headers cannot be overwritten
@@ -314,25 +324,24 @@ class ProxyStorage implements IProxy {
     }
     const method = options.method || 'GET';
     const uri = this.config.url + `/${encode(name)}`;
-    debug('request uri for %s retry %s', uri);
+    debug('set retry limit is %s', this.retry.limit);
     let response;
     let responseLength = 0;
     try {
       const retry = options?.retry ?? this.retry;
-      debug('retry times %s for %s', retry, uri);
+      debug('retry initial count %s', retry);
       response = await got(uri, {
         headers,
         responseType: 'json',
         method,
         agent: this.agent,
         retry,
-        // @ts-ignore
-        timeout: { request: options?.timeout ?? this.timeout },
+        timeout: this.timeout,
         hooks: {
           afterResponse: [
             (afterResponse) => {
               const code = afterResponse.statusCode;
-              debug('code response %s', code);
+              debug('after response code is %s', code);
               if (code >= HTTP_STATUS.OK && code < HTTP_STATUS.MULTIPLE_CHOICES) {
                 if (this.failed_requests >= this.max_fails) {
                   this.failed_requests = 0;
@@ -349,8 +358,8 @@ class ProxyStorage implements IProxy {
             },
           ],
           beforeRetry: [
-            // FUTURE: got 12.0.0, the option arg should be removed
-            (_options, error: any, count) => {
+            (error: RequestError, count: number) => {
+              debug('retry %s count: %s', uri, count);
               this.failed_requests = count ?? 0;
               this.logger.info(
                 {
@@ -378,7 +387,7 @@ class ProxyStorage implements IProxy {
         .on('request', () => {
           this.last_request_time = Date.now();
         })
-        .on('response', (eventResponse) => {
+        .on<any>('response', (eventResponse) => {
           const message = "@{!status}, req: '@{request.method} @{request.url}' (streaming)";
           this.logger.http(
             {
@@ -422,9 +431,10 @@ class ProxyStorage implements IProxy {
       );
       return [data, etag];
     } catch (err: any) {
-      debug('uri %s fail', uri);
+      debug('error %s on uri %s', err.code, uri);
       if (err.code === 'ERR_NON_2XX_3XX_RESPONSE') {
         const code = err.response.statusCode;
+        debug('error code %s', code);
         if (code === HTTP_STATUS.NOT_FOUND) {
           throw errorUtils.getNotFound(errorUtils.API_ERROR.NOT_PACKAGE_UPLINK);
         }
@@ -437,6 +447,15 @@ class ProxyStorage implements IProxy {
           error.remoteStatus = code;
           throw error;
         }
+      } else if (err.code === 'ETIMEDOUT') {
+        debug('error code timeout');
+        const code = err.code;
+        const error = errorUtils.getInternalError(
+          `${errorUtils.API_ERROR.SERVER_TIME_OUT}: ${code}`
+        );
+        // we need this code to identify outside which status code triggered the error
+        error.remoteStatus = code;
+        throw error;
       }
       throw err;
     }
@@ -449,7 +468,7 @@ class ProxyStorage implements IProxy {
   ): any {
     debug('fetching url for %s', url);
     const options = { ...this.config, ...overrideOptions };
-    let headers = this.getHeadersNext(options?.headers);
+    let headers = this.getHeaders(options?.headers);
     headers = this.addProxyHeaders(headers, options.remoteAddress);
     headers = this.applyUplinkHeaders(headers);
     // the following headers cannot be overwritten
@@ -482,37 +501,31 @@ class ProxyStorage implements IProxy {
    * @param {*} options request options
    * @return {Stream}
    */
-  public async search({ url, abort }: ProxySearchParams): Promise<Stream.Readable> {
-    debug('search url %o', url);
-
-    let response;
+  public async search({ url, abort, retry }: ProxySearchParams): Promise<Stream.Readable> {
     try {
       const fullURL = new URL(`${this.url}${url}`);
       // FIXME: a better way to remove duplicate slashes?
       const uri = fullURL.href.replace(/([^:]\/)\/+/g, '$1');
       this.logger.http({ uri, uplink: this.upname }, 'search request to uplink @{uplink} - @{uri}');
-      response = await undiciFetch(uri, {
-        method: 'GET',
-        // FUTURE: whitelist domains what we are sending not need it headers, security check
-        // headers: new Headers({
-        //   ...headers,
-        //   connection: 'keep-alive',
-        // }),
-        signal: abort?.signal,
+      debug('searching on %s', uri);
+      const response = got(uri, {
+        signal: abort ? abort.signal : {},
+        agent: this.agent,
+        timeout: this.timeout,
+        retry: retry ?? this.retry,
       });
-      debug('response.status  %o', response.status);
-
-      if (response.status >= HTTP_STATUS.BAD_REQUEST) {
-        throw errorUtils.getInternalError(`bad status code ${response.status} from uplink`);
-      }
 
-      const streamSearch = new PassThrough({ objectMode: true });
       const res = await response.text();
+      const streamSearch = new PassThrough({ objectMode: true });
       const streamResponse = Readable.from(res);
       // objects is one of the properties on the body, it ignores date and total
       streamResponse.pipe(JSONStream.parse('objects')).pipe(streamSearch, { end: true });
       return streamSearch;
     } catch (err: any) {
+      debug('search error %s', err);
+      if (err.response.statusCode === 409) {
+        throw errorUtils.getInternalError(`bad status code ${err.response.statusCode} from uplink`);
+      }
       this.logger.error(
         { errorMessage: err?.message, name: this.upname },
         'proxy uplink @{name} search error: @{errorMessage}'

package/test/headers.auth.spec.ts

@@ -1,11 +1,23 @@
 import { DEFAULT_REGISTRY } from '@verdaccio/config';
 import { HEADERS, TOKEN_BASIC, TOKEN_BEARER, constants } from '@verdaccio/core';
-import { setup } from '@verdaccio/logger';
+import { Logger } from '@verdaccio/types';
 import { buildToken } from '@verdaccio/utils';
 
 import { ProxyStorage } from '../src';
 
-setup();
+const mockDebug = jest.fn();
+const mockInfo = jest.fn();
+const mockHttp = jest.fn();
+const mockError = jest.fn();
+const mockWarn = jest.fn();
+
+const logger = {
+  debug: mockDebug,
+  info: mockInfo,
+  http: mockHttp,
+  error: mockError,
+  warn: mockWarn,
+} as unknown as Logger;
 
 function createUplink(config) {
   const defaultConfig = {
@@ -13,12 +25,12 @@ function createUplink(config) {
   };
   const mergeConfig = Object.assign({}, defaultConfig, config);
   // @ts-ignore
-  return new ProxyStorage(mergeConfig, {});
+  return new ProxyStorage(mergeConfig, {}, logger);
 }
 
 function setHeadersNext(config: unknown = {}, headers: any = {}) {
   const uplink = createUplink(config);
-  return uplink.getHeadersNext({ ...headers });
+  return uplink.getHeaders({ ...headers });
 }
 
 describe('setHeadersNext', () => {

package/test/noProxy.spec.ts

@@ -1,11 +1,13 @@
+import { logger, setup } from '@verdaccio/logger';
+
 import { ProxyStorage } from '../src';
 
-require('@verdaccio/logger').setup([]);
+setup({});
 
 function getProxyInstance(host, uplinkConf, appConfig) {
   uplinkConf.url = host;
 
-  return new ProxyStorage(uplinkConf, appConfig);
+  return new ProxyStorage(uplinkConf, appConfig, logger);
 }
 
 describe('Use proxy', () => {

package/test/proxy.metadata.spec.ts

@@ -4,6 +4,7 @@ import { setTimeout } from 'timers/promises';
 
 import { Config, parseConfigFile } from '@verdaccio/config';
 import { API_ERROR, errorUtils } from '@verdaccio/core';
+import { Logger } from '@verdaccio/types';
 
 import { ProxyStorage } from '../src';
 
@@ -15,6 +16,14 @@ const mockHttp = jest.fn();
 const mockError = jest.fn();
 const mockWarn = jest.fn();
 
+const logger = {
+  debug: mockDebug,
+  info: mockInfo,
+  http: mockHttp,
+  error: mockError,
+  warn: mockWarn,
+} as unknown as Logger;
+
 // mock to get the headers fixed value
 jest.mock('crypto', () => {
   return {
@@ -31,22 +40,6 @@ jest.mock('crypto', () => {
   };
 });
 
-jest.mock('@verdaccio/logger', () => {
-  const originalLogger = jest.requireActual('@verdaccio/logger');
-  return {
-    ...originalLogger,
-    logger: {
-      child: () => ({
-        debug: (arg, msg) => mockDebug(arg, msg),
-        info: (arg, msg) => mockInfo(arg, msg),
-        http: (arg, msg) => mockHttp(arg, msg),
-        error: (arg, msg) => mockError(arg, msg),
-        warn: (arg, msg) => mockWarn(arg, msg),
-      }),
-    },
-  };
-});
-
 const domain = 'https://registry.npmjs.org';
 
 describe('proxy', () => {
@@ -77,7 +70,7 @@ describe('proxy', () => {
         })
           .get('/jquery')
           .reply(200, { body: 'test' });
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         const [manifest] = await prox1.getRemoteMetadata('jquery', {
           remoteAddress: '127.0.0.1',
         });
@@ -103,7 +96,7 @@ describe('proxy', () => {
               etag: () => `_ref_4444`,
             }
           );
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         const [manifest, etag] = await prox1.getRemoteMetadata('jquery', {
           remoteAddress: '127.0.0.1',
         });
@@ -130,7 +123,7 @@ describe('proxy', () => {
               etag: () => `_ref_4444`,
             }
           );
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         const [manifest, etag] = await prox1.getRemoteMetadata('jquery', {
           etag: 'foo',
           remoteAddress: '127.0.0.1',
@@ -145,7 +138,7 @@ describe('proxy', () => {
         nock(domain)
           .get('/jquery')
           .reply(200, { body: { name: 'foo', version: '1.0.0' } }, {});
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         await prox1.getRemoteMetadata('jquery', {
           remoteAddress: '127.0.0.1',
         });
@@ -174,7 +167,7 @@ describe('proxy', () => {
     describe('error handling', () => {
       test('proxy call with 304', async () => {
         nock(domain).get('/jquery').reply(304);
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         await expect(prox1.getRemoteMetadata('jquery', { etag: 'rev_3333' })).rejects.toThrow(
           'no data'
         );
@@ -182,7 +175,7 @@ describe('proxy', () => {
 
       test('reply with error', async () => {
         nock(domain).get('/jquery').replyWithError('something awful happened');
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         await expect(
           prox1.getRemoteMetadata('jquery', {
             remoteAddress: '127.0.0.1',
@@ -192,15 +185,15 @@ describe('proxy', () => {
 
       test('reply with 409 error', async () => {
         nock(domain).get('/jquery').reply(409);
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
-        await expect(prox1.getRemoteMetadata('jquery', { retry: 0 })).rejects.toThrow(
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
+        await expect(prox1.getRemoteMetadata('jquery', { retry: { limit: 0 } })).rejects.toThrow(
           new Error('bad status code: 409')
         );
       });
 
       test('reply with bad body json format', async () => {
         nock(domain).get('/jquery').reply(200, 'some-text');
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         await expect(
           prox1.getRemoteMetadata('jquery', {
             remoteAddress: '127.0.0.1',
@@ -214,7 +207,7 @@ describe('proxy', () => {
 
       test('400 error proxy call', async () => {
         nock(domain).get('/jquery').reply(409);
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         await expect(
           prox1.getRemoteMetadata('jquery', {
             remoteAddress: '127.0.0.1',
@@ -226,7 +219,7 @@ describe('proxy', () => {
 
       test('proxy  not found', async () => {
         nock(domain).get('/jquery').reply(404);
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         await expect(
           prox1.getRemoteMetadata('jquery', {
             remoteAddress: '127.0.0.1',
@@ -253,7 +246,7 @@ describe('proxy', () => {
           .once()
           .reply(200, { body: { name: 'foo', version: '1.0.0' } });
 
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         const [manifest] = await prox1.getRemoteMetadata('jquery', {
           retry: { limit: 2 },
         });
@@ -269,10 +262,10 @@ describe('proxy', () => {
         );
       });
 
-      test('retry is exceded and uplink goes offline with logging activity', async () => {
+      test('retry count is exceded and uplink goes offline with logging activity', async () => {
         nock(domain).get('/jquery').times(10).reply(500);
 
-        const prox1 = new ProxyStorage(defaultRequestOptions, conf);
+        const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
         await expect(
           prox1.getRemoteMetadata('jquery', {
             remoteAddress: '127.0.0.1',
@@ -307,7 +300,8 @@ describe('proxy', () => {
 
         const prox1 = new ProxyStorage(
           { ...defaultRequestOptions, fail_timeout: '1s', max_fails: 1 },
-          conf
+          conf,
+          logger
         );
         // force retry
         await expect(
@@ -350,5 +344,127 @@ describe('proxy', () => {
         );
       }, 10000);
     });
+
+    describe('timeout', () => {
+      test('fail for timeout (2 seconds)', async () => {
+        nock(domain)
+          .get('/jquery')
+          .times(10)
+          .delayConnection(6000)
+          .reply(200, { body: { name: 'foo', version: '1.0.0' } });
+
+        const confTimeout = { ...defaultRequestOptions };
+        // @ts-expect-error
+        confTimeout.timeout = '2s';
+        const prox1 = new ProxyStorage(confTimeout, conf, logger);
+        await expect(
+          prox1.getRemoteMetadata('jquery', {
+            retry: { limit: 0 },
+          })
+        ).rejects.toThrow('ETIMEDOUT');
+      }, 10000);
+
+      test('fail for one failure and timeout (2 seconds)', async () => {
+        nock(domain)
+          .get('/jquery')
+          .times(1)
+          .reply(500)
+          .get('/jquery')
+          .delayConnection(4000)
+          .reply(200, { body: { name: 'foo', version: '1.0.0' } });
+
+        const confTimeout = { ...defaultRequestOptions };
+        // @ts-expect-error
+        confTimeout.timeout = '2s';
+        const prox1 = new ProxyStorage(confTimeout, conf, logger);
+        await expect(
+          prox1.getRemoteMetadata('jquery', {
+            retry: { limit: 1 },
+          })
+        ).rejects.toThrow('ETIMEDOUT');
+      }, 10000);
+
+      // test('retry count is exceded and uplink goes offline with logging activity', async () => {
+      //   nock(domain).get('/jquery').times(10).reply(500);
+
+      //   const prox1 = new ProxyStorage(defaultRequestOptions, conf, logger);
+      //   await expect(
+      //     prox1.getRemoteMetadata('jquery', {
+      //       remoteAddress: '127.0.0.1',
+      //       retry: { limit: 2 },
+      //     })
+      //   ).rejects.toThrow();
+      //   await expect(
+      //     prox1.getRemoteMetadata('jquery', {
+      //       remoteAddress: '127.0.0.1',
+      //       retry: { limit: 2 },
+      //     })
+      //   ).rejects.toThrow(errorUtils.getInternalError(errorUtils.API_ERROR.UPLINK_OFFLINE));
+      //   expect(mockWarn).toHaveBeenCalledTimes(1);
+      //   expect(mockWarn).toHaveBeenLastCalledWith(
+      //     {
+      //       host: 'registry.npmjs.org',
+      //     },
+      //     'host @{host} is now offline'
+      //   );
+      // });
+
+      // test('fails calls and recover with 200 with log online activity', async () => {
+      //   // This unit test is designed to verify if the uplink goes to offline
+      //   // and recover after the fail_timeout has expired.
+      //   nock(domain)
+      //     .get('/jquery')
+      //     .thrice()
+      //     .reply(500, 'some-text')
+      //     .get('/jquery')
+      //     .once()
+      //     .reply(200, { body: { name: 'foo', version: '1.0.0' } });
+
+      //   const prox1 = new ProxyStorage(
+      //     { ...defaultRequestOptions, fail_timeout: '1s', max_fails: 1 },
+      //     conf,
+      //     logger
+      //   );
+      //   // force retry
+      //   await expect(
+      //     prox1.getRemoteMetadata('jquery', {
+      //       remoteAddress: '127.0.0.1',
+      //       retry: { limit: 2 },
+      //     })
+      //   ).rejects.toThrow();
+      //   // display offline error on exausted retry
+      //   await expect(
+      //     prox1.getRemoteMetadata('jquery', {
+      //       remoteAddress: '127.0.0.1',
+      //       retry: { limit: 2 },
+      //     })
+      //   ).rejects.toThrow(errorUtils.getInternalError(errorUtils.API_ERROR.UPLINK_OFFLINE));
+      //   expect(mockWarn).toHaveBeenCalledTimes(2);
+      //   expect(mockWarn).toHaveBeenLastCalledWith(
+      //     {
+      //       host: 'registry.npmjs.org',
+      //     },
+      //     'host @{host} is now offline'
+      //   );
+      //   expect(mockWarn).toHaveBeenLastCalledWith(
+      //     {
+      //       host: 'registry.npmjs.org',
+      //     },
+      //     'host @{host} is now offline'
+      //   );
+      //   // this is based on max_fails, if change that also change here acordingly
+      //   await setTimeout(3000);
+      //   const [manifest] = await prox1.getRemoteMetadata('jquery', {
+      //     retry: { limit: 2 },
+      //   });
+      //   expect(manifest).toEqual({ body: { name: 'foo', version: '1.0.0' } });
+      //   expect(mockWarn).toHaveBeenLastCalledWith(
+      //     {
+      //       host: 'registry.npmjs.org',
+      //     },
+      //     'host @{host} is now online'
+      //   );
+      // }, 10000);
+    });
   });
 });