@verdaccio/store 6.0.0-6-next.18 → 6.0.0-6-next.21

package/src/storage.ts

@@ -2,16 +2,24 @@ import assert from 'assert';
 import async, { AsyncResultArrayCallback } from 'async';
 import buildDebug from 'debug';
 import _ from 'lodash';
-import semver from 'semver';
 
 import { hasProxyTo } from '@verdaccio/config';
-import { API_ERROR, DIST_TAGS, HTTP_STATUS, errorUtils } from '@verdaccio/core';
-import { pkgUtils, validatioUtils } from '@verdaccio/core';
+import {
+  API_ERROR,
+  DIST_TAGS,
+  HTTP_STATUS,
+  errorUtils,
+  pkgUtils,
+  validatioUtils,
+} from '@verdaccio/core';
 import { logger } from '@verdaccio/logger';
 import { ProxyStorage } from '@verdaccio/proxy';
 import { IProxy, ProxyList } from '@verdaccio/proxy';
 import { ReadTarball } from '@verdaccio/streams';
-import { convertDistRemoteToLocalTarballUrls } from '@verdaccio/tarball';
+import {
+  convertDistRemoteToLocalTarballUrls,
+  convertDistVersionToLocalTarballsUrl,
+} from '@verdaccio/tarball';
 import {
   Callback,
   CallbackAction,
@@ -32,7 +40,8 @@ import {
 import { getVersion, normalizeDistTags } from '@verdaccio/utils';
 
 import { LocalStorage } from './local-storage';
-import { SearchInstance, SearchManager } from './search';
+import { SearchManager } from './search';
+// import { isPublishablePackage, validateInputs } from './star-utils';
 import {
   checkPackageLocal,
   checkPackageRemote,
@@ -42,12 +51,9 @@ import {
   publishPackage,
 } from './storage-utils';
 import { IGetPackageOptions, IGetPackageOptionsNext, IPluginFilters, ISyncUplinks } from './type';
+// import { StarBody, Users } from './type';
 import { setupUpLinks, updateVersionsHiddenUpLink } from './uplink-util';
 
-if (semver.lte(process.version, 'v15.0.0')) {
-  global.AbortController = require('abortcontroller-polyfill/dist/cjs-ponyfill').AbortController;
-}
-
 const debug = buildDebug('verdaccio:storage');
 class Storage {
   public localStorage: LocalStorage;
@@ -110,6 +116,33 @@ class Storage {
     }
   }
 
+  /**
+   * Add a {name} package to a system
+   Function checks if package with the same name is available from uplinks.
+   If it isn't, we create package locally
+   Used storages: local (write) && uplinks
+   */
+  public async addPackageNext(name: string, metadata: Package): Promise<Package> {
+    try {
+      debug('add package for %o', name);
+      await checkPackageLocal(name, this.localStorage);
+      debug('look up remote for %o', name);
+      await checkPackageRemote(
+        name,
+        this._isAllowPublishOffline(),
+        this._syncUplinksMetadata.bind(this)
+      );
+      debug('publishing a package for %o', name);
+      // FIXME: publishPackage should return fresh metadata from backend
+      // instead return metadata
+      await publishPackage(name, metadata, this.localStorage as LocalStorage);
+      return metadata;
+    } catch (err: any) {
+      debug('error on add a package for %o with error %o', name, err);
+      throw err;
+    }
+  }
+
   private _isAllowPublishOffline(): boolean {
     return (
       typeof this.config.publish !== 'undefined' &&
@@ -145,6 +178,16 @@ class Storage {
     this.localStorage.addVersion(name, version, metadata, tag, callback);
   }
 
+  public async addVersionNext(
+    name: string,
+    version: string,
+    metadata: Version,
+    tag: StringValue
+  ): Promise<void> {
+    debug('add the version %o for package %o', version, name);
+    return this.localStorage.addVersionNext(name, version, metadata, tag);
+  }
+
   /**
    * Tags a package version with a provided tag
    Used storages: local (write)
@@ -169,6 +212,16 @@ class Storage {
     this.localStorage.changePackage(name, metadata, revision, callback);
   }
 
+  /**
+   * Change an existing package (i.e. unpublish one version)
+   Function changes a package info from local storage and all uplinks with write access./
+   Used storages: local (write)
+   */
+  public async changePackageNext(name: string, metadata: Package, revision: string): Promise<void> {
+    debug('change existing package for package %o revision %o', name, revision);
+    this.localStorage.changePackageNext(name, metadata, revision);
+  }
+
   /**
    * Remove a package from a system
    Function removes a package from local storage
@@ -177,8 +230,6 @@ class Storage {
   public async removePackage(name: string): Promise<void> {
     debug('remove packagefor package %o', name);
     await this.localStorage.removePackage(name);
-    // update the indexer
-    SearchInstance.remove(name);
   }
 
   /**
@@ -346,7 +397,130 @@ class Storage {
     }
   }
 
-  public async getPackageNext(options: IGetPackageOptionsNext): Promise<Package | Version> {
+  // public async starPackage(body: StarBody, options: IGetPackageOptionsNext): Promise<void> {
+  //   debug('star package');
+  //   const manifest = await this.getPackageNext(options);
+  //   const newStarUser = body[constants.USERS];
+  //   const remoteUsername: string = options.remoteUser.name as string;
+  //   const localStarUsers = manifest[constants.USERS];
+  //   // Check is star or unstar
+  //   const isStar = Object.keys(newStarUser).includes(remoteUsername);
+  //   debug('is start? %o', isStar);
+  //   if (
+  //     _.isNil(localStarUsers) === false &&
+  //     validateInputs(localStarUsers, remoteUsername, isStar)
+  //   ) {
+  //     // return afterChangePackage();
+  //   }
+  //   const users: Users = isStar
+  //     ? {
+  //         ...localStarUsers,
+  //         [remoteUsername]: true,
+  //       }
+  //     : _.reduce(
+  //         localStarUsers,
+  //         (users, value, key) => {
+  //           if (key !== remoteUsername) {
+  //             users[key] = value;
+  //           }
+  //           return users;
+  //         },
+  //         {}
+  //       );
+  //   debug('update package for  %o', name);
+  // }
+
+  // public async publish(body: any, options: IGetPackageOptionsNext): Promise<any> {
+  //   const { name } = options;
+  //   debug('publishing or updating a new version for %o', name);
+  //   // we check if the request is npm star
+  //   if (!isPublishablePackage(body) && isObject(body.users)) {
+  //     debug('starting star a package');
+  //     await this.starPackage(body as StarBody, options);
+  //   }
+
+  //   return { ok: API_MESSAGE.PKG_CHANGED, success: true };
+  // }
+
+  public async getPackageByVersion(options: IGetPackageOptionsNext): Promise<Version> {
+    const queryVersion = options.version as string;
+    if (_.isNil(queryVersion)) {
+      throw errorUtils.getNotFound(`${API_ERROR.VERSION_NOT_EXIST}: ${queryVersion}`);
+    }
+
+    // we have version, so we need to return specific version
+    const [convertedManifest] = await this.getPackageNext(options);
+
+    const version: Version | undefined = getVersion(convertedManifest.versions, queryVersion);
+
+    debug('query by latest version %o and result %o', queryVersion, version);
+    if (typeof version !== 'undefined') {
+      debug('latest version found %o', version);
+      return convertDistVersionToLocalTarballsUrl(
+        convertedManifest.name,
+        version,
+        options.requestOptions,
+        this.config.url_prefix
+      );
+    }
+
+    // the version could be a dist-tag eg: beta, alpha, so we find the matched version
+    // on disg-tag list
+    if (_.isNil(convertedManifest[DIST_TAGS]) === false) {
+      if (_.isNil(convertedManifest[DIST_TAGS][queryVersion]) === false) {
+        // the version found as a distag
+        const matchedDisTagVersion: string = convertedManifest[DIST_TAGS][queryVersion];
+        debug('dist-tag version found %o', matchedDisTagVersion);
+        const disTagVersion: Version | undefined = getVersion(
+          convertedManifest.versions,
+          matchedDisTagVersion
+        );
+        if (typeof disTagVersion !== 'undefined') {
+          debug('dist-tag found %o', disTagVersion);
+          return convertDistVersionToLocalTarballsUrl(
+            convertedManifest.name,
+            disTagVersion,
+            options.requestOptions,
+            this.config.url_prefix
+          );
+        }
+      }
+    } else {
+      debug('dist tag not detected');
+    }
+
+    // we didn't find the version, not found error
+    debug('package version not found %o', queryVersion);
+    throw errorUtils.getNotFound(`${API_ERROR.VERSION_NOT_EXIST}: ${queryVersion}`);
+  }
+
+  public async getPackageManifest(options: IGetPackageOptionsNext): Promise<Package> {
+    // convert dist remotes to local bars
+    const [manifest] = await this.getPackageNext(options);
+    const convertedManifest = convertDistRemoteToLocalTarballUrls(
+      manifest,
+      options.requestOptions,
+      this.config.url_prefix
+    );
+
+    return convertedManifest;
+  }
+
+  /**
+   * Return a manifest or version based on the options.
+   * @param options {Object}
+   * @returns A package manifest or specific version
+   */
+  public async getPackageByOptions(options: IGetPackageOptionsNext): Promise<Package | Version> {
+    // if no version we return the whole manifest
+    if (_.isNil(options.version) === false) {
+      return this.getPackageByVersion(options);
+    } else {
+      return this.getPackageManifest(options);
+    }
+  }
+
+  public async getPackageNext(options: IGetPackageOptionsNext): Promise<[Package, any[]]> {
     const { name } = options;
     debug('get package for %o', name);
     try {
@@ -363,59 +537,11 @@ class Storage {
       // time to sync with uplinks if we have any
       debug('sync uplinks for %o', name);
       // @ts-expect-error
-      const [manifest, errors] = await this._syncUplinksMetadataNext(name, data, {
+      return this._syncUplinksMetadataNext(name, data, {
         req: options.req,
         uplinksLook: options.uplinksLook,
         keepUpLinkData: options.keepUpLinkData,
       });
-      debug('no. sync uplinks errors %o', errors?.length);
-
-      // TODO: we could improve performance here, if a specific version is requested
-      // we just convert that version and return it otherwise we convert
-      // the whole manifest, bonus points for contribution :)
-      // convert dist remotes to local bars
-      const convertedManifest = convertDistRemoteToLocalTarballUrls(
-        manifest,
-        options.requestOptions,
-        this.config.url_prefix
-      );
-      // if no version we return the whole manifest
-      if (_.isNil(options.version)) {
-        return convertedManifest;
-      }
-
-      // we have version, so we need to return specific version
-      const queryVersion = options.version as string;
-      const version: Version | undefined = getVersion(convertedManifest.versions, options.version);
-      debug('query by latest version %o and result %o', options.version, version);
-      if (typeof version !== 'undefined') {
-        debug('latest version found %o', version);
-        return version;
-      }
-
-      // the version could be a dist-tag eg: beta, alpha, so we find the matched version
-      // on disg-tag list
-      if (_.isNil(convertedManifest[DIST_TAGS]) === false) {
-        if (_.isNil(convertedManifest[DIST_TAGS][queryVersion]) === false) {
-          // the version found as a distag
-          const matchedDisTagVersion: string = convertedManifest[DIST_TAGS][queryVersion];
-          debug('dist-tag version found %o', matchedDisTagVersion);
-          const disTagVersion: Version | undefined = getVersion(
-            convertedManifest.versions,
-            matchedDisTagVersion
-          );
-          if (typeof disTagVersion !== 'undefined') {
-            debug('dist-tag found %o', disTagVersion);
-            return disTagVersion;
-          }
-        }
-      } else {
-        debug('dist tag not detected');
-      }
-
-      // we didn't find the version, not found error
-      debug('package version not found %o', queryVersion);
-      throw errorUtils.getNotFound(`${API_ERROR.VERSION_NOT_EXIST}: ${queryVersion}`);
     } catch (err: any) {
       this.logger.error(
         { name, err: err.message },
@@ -446,7 +572,7 @@ class Storage {
             _attachments: {},
           });
 
-          debug('no. sync uplinks errors %o', uplinkErrors?.length);
+          debug('no. sync uplinks errors %o for %s', uplinkErrors?.length, name);
           resolve([normalizedPkg, uplinkErrors]);
         }
       );

package/src/type.ts

@@ -1,4 +1,4 @@
-import { Callback, Config, IPluginStorageFilter } from '@verdaccio/types';
+import { Callback, Config, IPluginStorageFilter, RemoteUser } from '@verdaccio/types';
 
 // @deprecated
 export interface IGetPackageOptions {
@@ -15,6 +15,7 @@ export type IGetPackageOptionsNext = {
   name: string;
   version?: string;
   keepUpLinkData?: boolean;
+  remoteUser?: RemoteUser;
   uplinksLook: boolean;
   requestOptions: {
     // RequestOptions from url package
@@ -30,4 +31,13 @@ export interface ISyncUplinks {
   req?: Request;
 }
 
+export type Users = {
+  [key: string]: string;
+};
+export interface StarBody {
+  _id: string;
+  _rev: string;
+  users: Users;
+}
+
 export type IPluginFilters = IPluginStorageFilter<Config>[];

package/test/search.spec.ts

@@ -1,10 +1,11 @@
+import { setGlobalDispatcher } from 'undici';
+
 import { Config } from '@verdaccio/config';
 import { searchUtils } from '@verdaccio/core';
 import { setup } from '@verdaccio/logger';
 import { configExample } from '@verdaccio/mock';
 
 import { Storage, removeDuplicates } from '../src';
-import { SearchInstance } from '../src/search';
 
 setup([]);
 
@@ -30,7 +31,6 @@ describe('search', () => {
     test('search items', async () => {
       const { MockAgent } = require('undici');
       // FIXME: fetch is already part of undici
-      const { setGlobalDispatcher } = require('undici-fetch');
       const domain = 'http://localhost:4873';
       const url = '/-/v1/search?maintenance=1&popularity=1&quality=1&size=10&text=verdaccio';
       const response = require('./fixtures/search.json');
@@ -52,68 +52,4 @@ describe('search', () => {
       expect(results).toHaveLength(4);
     });
   });
-
-  describe('search index', () => {
-    const packages = [
-      {
-        name: 'test1',
-        description: 'description',
-        _npmUser: {
-          name: 'test_user',
-        },
-      },
-      {
-        name: 'test2',
-        description: 'description',
-        _npmUser: {
-          name: 'test_user',
-        },
-      },
-      {
-        name: 'test3',
-        description: 'description',
-        _npmUser: {
-          name: 'test_user',
-        },
-      },
-    ];
-
-    test('search query item', async () => {
-      const config = new Config(configExample());
-      const storage = new Storage(config);
-      await storage.init(config);
-      SearchInstance.configureStorage(storage);
-      packages.map(function (item) {
-        // @ts-ignore
-        SearchInstance.add(item);
-      });
-      const result = SearchInstance.query('t');
-      expect(result).toHaveLength(3);
-    });
-
-    test('search remove item', async () => {
-      const config = new Config(configExample());
-      const storage = new Storage(config);
-      await storage.init(config);
-      SearchInstance.configureStorage(storage);
-      packages.map(function (item) {
-        // @ts-ignore
-        SearchInstance.add(item);
-      });
-      const item = {
-        name: 'test6',
-        description: 'description',
-        _npmUser: {
-          name: 'test_user',
-        },
-      };
-      // @ts-ignore
-      SearchInstance.add(item);
-      let result = SearchInstance.query('test6');
-      expect(result).toHaveLength(1);
-      SearchInstance.remove(item.name);
-      result = SearchInstance.query('test6');
-      expect(result).toHaveLength(0);
-    });
-  });
 });

package/test/storage-utils.spec.ts

@@ -3,7 +3,13 @@ import assert from 'assert';
 import { DIST_TAGS } from '@verdaccio/core';
 import { Package } from '@verdaccio/types';
 
-import { STORAGE, mergeUplinkTimeIntoLocal, normalizePackage } from '../src/storage-utils';
+import {
+  STORAGE,
+  hasInvalidPublishBody,
+  isDifferentThanOne,
+  mergeUplinkTimeIntoLocal,
+  normalizePackage,
+} from '../src/storage-utils';
 import { tagVersion } from '../src/storage-utils';
 import { readFile } from './fixtures/test.utils';
 
@@ -172,4 +178,92 @@ describe('Storage Utils', () => {
       });
     });
   });
+
+  describe('isDifferentThanOne', () => {
+    test('isDifferentThanOne is true', () => {
+      expect(isDifferentThanOne({})).toBeTruthy();
+    });
+    test('isDifferentThanOne is false', () => {
+      expect(
+        isDifferentThanOne({
+          foo: 'bar',
+        })
+      ).toBeFalsy();
+    });
+    test('isDifferentThanOne with two items is true', () => {
+      expect(
+        isDifferentThanOne({
+          foo: 'bar',
+          foo1: 'bar',
+        })
+      ).toBeTruthy();
+    });
+  });
+
+  describe('hasInvalidPublishBody', () => {
+    test('should be valid', () => {
+      expect(
+        hasInvalidPublishBody({
+          _attachments: {
+            'forbidden-place-1.0.6.tgz': {
+              content_type: 'application/octet-stream',
+              data: 'foo',
+              length: 512,
+            },
+          },
+          versions: {
+            // @ts-expect-error
+            '1.0.0': {},
+          },
+        })
+      ).toBeFalsy();
+    });
+
+    test('should be invalid due missing versions', () => {
+      expect(
+        hasInvalidPublishBody({
+          _attachments: {},
+          versions: {},
+        })
+      ).toBeTruthy();
+    });
+
+    test('should be invalid due missing _attachments', () => {
+      expect(
+        hasInvalidPublishBody({
+          _attachments: {},
+          versions: {},
+        })
+      ).toBeTruthy();
+    });
+
+    test('should be invalid due invalid empty versions  object', () => {
+      expect(
+        hasInvalidPublishBody({
+          _attachments: {
+            'forbidden-place-1.0.6.tgz': {
+              content_type: 'application/octet-stream',
+              data: 'foo',
+              length: 512,
+            },
+          },
+          versions: {},
+        })
+      ).toBeTruthy();
+    });
+
+    test('should be invalid due empty _attachments object', () => {
+      expect(
+        hasInvalidPublishBody({
+          _attachments: {},
+          versions: {
+            // @ts-expect-error
+            '1.0.0': {},
+            // @ts-expect-error
+            '1.0.1': {},
+          },
+        })
+      ).toBeTruthy();
+    });
+  });
 });

package/test/storage.spec.ts

@@ -3,9 +3,9 @@ import * as httpMocks from 'node-mocks-http';
 
 import { Config } from '@verdaccio/config';
 import { HEADERS, errorUtils } from '@verdaccio/core';
-import { generatePackageMetadata } from '@verdaccio/helper';
 import { setup } from '@verdaccio/logger';
 import { configExample, generateRamdonStorage } from '@verdaccio/mock';
+import { generatePackageMetadata } from '@verdaccio/test-helper';
 
 import { Storage } from '../src';
 
@@ -60,7 +60,7 @@ describe('storage', () => {
         const storage = new Storage(config);
         await storage.init(config);
         await expect(
-          storage.getPackageNext({
+          storage.getPackageByOptions({
             name: 'foo',
             uplinksLook: true,
             req,
@@ -92,7 +92,7 @@ describe('storage', () => {
         const storage = new Storage(config);
         await storage.init(config);
         await expect(
-          storage.getPackageNext({
+          storage.getPackageByOptions({
             name: 'foo',
             version: '1.0.0',
             uplinksLook: true,
@@ -125,7 +125,7 @@ describe('storage', () => {
         const storage = new Storage(config);
         await storage.init(config);
         await expect(
-          storage.getPackageNext({
+          storage.getPackageByOptions({
             name: 'foo',
             version: 'latest',
             uplinksLook: true,
@@ -158,7 +158,7 @@ describe('storage', () => {
         const storage = new Storage(config);
         await storage.init(config);
         await expect(
-          storage.getPackageNext({
+          storage.getPackageByOptions({
             name: 'foo',
             version: '1.0.0-does-not-exist',
             uplinksLook: true,
@@ -193,7 +193,7 @@ describe('storage', () => {
         const storage = new Storage(config);
         await storage.init(config);
         await expect(
-          storage.getPackageNext({
+          storage.getPackageByOptions({
             name: 'foo2',
             uplinksLook: true,
             req,
@@ -228,7 +228,7 @@ describe('storage', () => {
         const storage = new Storage(config);
         await storage.init(config);
         await expect(
-          storage.getPackageNext({
+          storage.getPackageByOptions({
             name: 'foo2',
             uplinksLook: true,
             req,