@verdaccio/store 7.0.0-next-7.11 → 7.0.0-next-7.13

package/build/storage.js

@@ -16,14 +16,12 @@ var _core = require("@verdaccio/core");
 var _loaders = require("@verdaccio/loaders");
 var _logger = require("@verdaccio/logger");
 var _proxy = require("@verdaccio/proxy");
+var _search = _interopRequireDefault(require("@verdaccio/search"));
 var _tarball = require("@verdaccio/tarball");
 var _utils = require("@verdaccio/utils");
 var _2 = require(".");
-var _TransFormResults = require("./lib/TransFormResults");
-var _searchUtils = require("./lib/search-utils");
 var _starUtils = require("./lib/star-utils");
 var _storageUtils = require("./lib/storage-utils");
-var _uplinkUtil = require("./lib/uplink-util");
 var _versionsUtils = require("./lib/versions-utils");
 var _localStorage = require("./local-storage");
 function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function (e) { return e ? t : r; })(e); }
@@ -36,10 +34,11 @@ const PROTO_NAME = exports.PROTO_NAME = '__proto__';
 class Storage {
   constructor(config) {
     this.config = config;
-    this.uplinks = (0, _uplinkUtil.setupUpLinks)(config);
     this.logger = _logger.logger.child({
       module: 'storage'
     });
+    this.uplinks = (0, _proxy.setupUpLinks)(config, this.logger);
+    this.searchService = new _search.default(config, this.logger);
     this.filters = null;
     // @ts-ignore
     this.localStorage = null;
@@ -164,74 +163,19 @@ class Storage {
   /**
    * Handle search on packages and proxies.
    * Iterate all proxies configured and search in all endpoints in v2 and pipe all responses
-   * to a stream, once the proxies request has finished search in local storage for all packages
+   *  once the proxies request has finished search in local storage for all packages
    * (privated and cached).
    */
   async search(options) {
-    const transformResults = new _TransFormResults.TransFormResults({
-      objectMode: true
-    });
-    const streamPassThrough = new _stream.PassThrough({
-      objectMode: true
-    });
-    const upLinkList = this.getProxyList();
-    debug('uplinks found %s', upLinkList.length);
-    const searchUplinksStreams = upLinkList.map(uplinkId => {
-      const uplink = this.uplinks[uplinkId];
-      if (!uplink) {
-        // this line should never happens
-        this.logger.error({
-          uplinkId
-        }, 'uplink @upLinkId not found');
-      }
-      return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough);
-    });
-    try {
-      debug('searching on %s uplinks...', searchUplinksStreams === null || searchUplinksStreams === void 0 ? void 0 : searchUplinksStreams.length);
-      // only process those streams end successfully, if all request fails
-      // just include local storage results (if local fails then return 500)
-      await Promise.allSettled([...searchUplinksStreams]);
-      debug('searching all uplinks done');
-    } catch (err) {
-      this.logger.error({
-        err: err === null || err === void 0 ? void 0 : err.message
-      }, ' error on uplinks search @{err}');
-      streamPassThrough.emit('error', err);
-    }
-    debug('search local');
-    try {
-      await this.searchCachedPackages(streamPassThrough, options.query);
-    } catch (err) {
-      this.logger.error({
-        err: err === null || err === void 0 ? void 0 : err.message
-      }, ' error on local search @{err}');
-      streamPassThrough.emit('error', err);
-    }
-    const data = [];
-    const outPutStream = new _stream.PassThrough({
-      objectMode: true
-    });
-    (0, _stream.pipeline)(streamPassThrough, transformResults, outPutStream, err => {
-      if (err) {
-        this.logger.error({
-          err: err === null || err === void 0 ? void 0 : err.message
-        }, ' error on search @{err}');
-        throw _core.errorUtils.getInternalError(err ? err.message : 'unknown search error');
-      } else {
-        debug('pipeline succeeded');
-      }
-    });
-    outPutStream.on('data', chunk => {
-      data.push(chunk);
-    });
-    return new Promise(resolve => {
-      outPutStream.on('finish', async () => {
-        const searchFinalResults = (0, _searchUtils.removeDuplicates)(data);
-        debug('search stream total results: %o', searchFinalResults.length);
-        return resolve(searchFinalResults);
-      });
-      debug('search done');
-    });
+    debug('search on cache packages');
+    const cachePackages = await this.getCachedPackages(options.query);
+    debug('search found on cache packages %o', cachePackages.length);
+    const remotePackages = await this.searchService.search(options);
+    debug('search found on remote packages %o', remotePackages.length);
+    const totalResults = [...cachePackages, ...remotePackages];
+    const uniqueResults = (0, _versionsUtils.removeLowerVersions)(totalResults);
+    debug('unique results %o', uniqueResults.length);
+    return uniqueResults;
   }
   async getTarballFromUpstream(name, filename, {
     signal
@@ -313,7 +257,7 @@ class Storage {
       // should not be the case
       const passThroughRemoteStream = new _stream.PassThrough();
       // ensure get the latest data
-      const [updatedManifest] = await this.syncUplinksMetadataNext(name, cachedManifest, {
+      const [updatedManifest] = await this.syncUplinksMetadata(name, cachedManifest, {
         uplinksLook: true
       });
       const distFile = updatedManifest._distfiles[filename];
@@ -356,7 +300,7 @@ class Storage {
    * @param param2
    * @returns
    */
-  async getTarballNext(name, filename, {
+  async getTarball(name, filename, {
     signal
   }) {
     debug('get tarball for package %o filename %o', name, filename);
@@ -568,32 +512,11 @@ class Storage {
       }, plugin => {
         return typeof plugin.filter_metadata !== 'undefined';
       }, (_this$config = this.config) === null || _this$config === void 0 ? void 0 : (_this$config$serverSe = _this$config.serverSettings) === null || _this$config$serverSe === void 0 ? void 0 : _this$config$serverSe.pluginPrefix);
-      debug('filters available %o', this.filters);
+      debug('filters available %o', this.filters.length);
     }
     return;
   }
 
-  /**
-   * Consume the upstream and pipe it to a transformable stream.
-   */
-  consumeSearchStream(uplinkId, uplink, options, searchPassThrough) {
-    return uplink.search({
-      ...options
-    }).then(bodyStream => {
-      bodyStream.pipe(searchPassThrough, {
-        end: false
-      });
-      bodyStream.on('error', err => {
-        _logger.logger.error({
-          uplinkId,
-          err: err
-        }, 'search error for uplink @{uplinkId}: @{err?.message}');
-        searchPassThrough.end();
-      });
-      return new Promise(resolve => bodyStream.on('end', resolve));
-    });
-  }
-
   /**
    * Retrieve a wrapper that provide access to the package location.
    * @param {Object} pkgName package name.
@@ -623,9 +546,14 @@ class Storage {
       signal
     });
   }
-  async searchCachedPackages(searchStream, query) {
-    debug('search on each package');
-    this.logger.info({
+  async getCachedPackages(query) {
+    debug('search on each package', query);
+    const results = [];
+    if (typeof query === 'undefined' || typeof (query === null || query === void 0 ? void 0 : query.text) === 'undefined') {
+      debug('search query for cached not found');
+      return results;
+    }
+    _logger.logger.info({
       t: query.text,
       q: query.quality,
       p: query.popularity,
@@ -633,16 +561,16 @@ class Storage {
       s: query.size
     }, 'search by text @{t}| maintenance @{m}| quality @{q}| popularity @{p}');
     if (typeof this.localStorage.getStoragePlugin().search === 'undefined') {
-      this.logger.info('plugin search not implemented yet');
-      searchStream.end();
+      _logger.logger.info('plugin search not implemented yet');
     } else {
-      debug('search on each package by plugin');
+      debug('search on each package by plugin query');
       const items = await this.localStorage.getStoragePlugin().search(query);
       try {
         for (const searchItem of items) {
           const manifest = await this.getPackageLocalMetadata(searchItem.package.name);
           if (_lodash.default.isEmpty(manifest === null || manifest === void 0 ? void 0 : manifest.versions) === false) {
-            const searchPackage = (0, _2.mapManifestToSearchPackageBody)(manifest, searchItem);
+            const searchPackage = (0, _storageUtils.mapManifestToSearchPackageBody)(manifest, searchItem);
+            debug('search local stream found %o', searchPackage.name);
             const searchPackageItem = {
               package: searchPackage,
               score: searchItem.score,
@@ -652,19 +580,21 @@ class Storage {
               // FUTURE: find a better way to calculate the score
               searchScore: 1
             };
-            searchStream.write(searchPackageItem);
+            results.push(searchPackageItem);
+          } else {
+            debug('local item without versions detected %s', searchItem.package.name);
           }
         }
         debug('search local stream end');
-        searchStream.end();
       } catch (err) {
         this.logger.error({
           err,
           query
         }, 'error on search by plugin @{err.message}');
-        searchStream.emit('error', err);
+        throw err;
       }
     }
+    return results;
   }
   async removePackageByRevision(pkgName, revision) {
     const storage = this.getPrivatePackageStorage(pkgName);
@@ -1314,7 +1244,7 @@ class Storage {
   async checkPackageRemote(name, uplinksLook) {
     try {
       // we provide a null manifest, thus the manifest returned will be the remote one
-      const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadataNext(name, null, {
+      const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadata(name, null, {
         uplinksLook
       });
 
@@ -1419,7 +1349,7 @@ class Storage {
     // if we can't get the local metadata, we try to get the remote metadata
     // if we do to have local metadata, we try to update it with the upstream registry
     debug('sync uplinks for %o', name);
-    const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadataNext(name, data, {
+    const [remoteManifest, upLinksErrors] = await this.syncUplinksMetadata(name, data, {
       uplinksLook: options.uplinksLook,
       retry: options.retry,
       remoteAddress: options.requestOptions.remoteAddress
@@ -1465,7 +1395,7 @@ class Storage {
     in that case the request returns empty body and we want ask next on the list if has fresh
     updates.
    */
-  async syncUplinksMetadataNext(name, localManifest, options = {}) {
+  async syncUplinksMetadata(name, localManifest, options = {}) {
     let found = localManifest !== null;
     let syncManifest = null;
     const upLinks = [];
@@ -1581,7 +1511,7 @@ class Storage {
     // merge time field cache and remote
     _cacheManifest = (0, _storageUtils.mergeUplinkTimeIntoLocalNext)(_cacheManifest, remoteManifest);
     // update the _uplinks field in the cache
-    _cacheManifest = (0, _uplinkUtil.updateVersionsHiddenUpLinkNext)(_cacheManifest, uplink);
+    _cacheManifest = (0, _proxy.updateVersionsHiddenUpLinkNext)(_cacheManifest, uplink);
     try {
       // merge versions from remote into the cache
       _cacheManifest = (0, _storageUtils.mergeVersions)(_cacheManifest, remoteManifest);