@verdaccio/store 6.0.0-6-next.18 → 6.0.0-6-next.21

package/src/local-storage.ts

@@ -43,6 +43,8 @@ import {
 
 const debug = buildDebug('verdaccio:storage:local');
 
+export const noSuchFile = 'ENOENT';
+export const resourceNotAvailable = 'EAGAIN';
 export type IPluginStorage = pluginUtils.IPluginStorage<Config>;
 
 export function normalizeSearchPackage(
@@ -365,6 +367,144 @@ class LocalStorage {
     );
   }
 
+  public async addVersionNext(
+    name: string,
+    version: string,
+    metadata: Version,
+    tag: StringValue
+  ): Promise<void> {
+    debug(`add version %s package for %s`, version, name);
+    await this.updatePackageNext(name, async (data: Package): Promise<Package> => {
+      debug('%s package is being updated', name);
+      // keep only one readme per package
+      data.readme = metadata.readme;
+      debug('%s` readme mutated', name);
+      // TODO: lodash remove
+      metadata = cleanUpReadme(metadata);
+      metadata.contributors = normalizeContributors(metadata.contributors as Author[]);
+      debug('%s` contributors normalized', name);
+      const hasVersion = data.versions[version] != null;
+      if (hasVersion) {
+        debug('%s version %s already exists', name, version);
+        throw errorUtils.getConflict();
+      }
+
+      // if uploaded tarball has a different shasum, it's very likely that we
+      // have some kind of error
+      if (validatioUtils.isObject(metadata.dist) && _.isString(metadata.dist.tarball)) {
+        const tarball = metadata.dist.tarball.replace(/.*\//, '');
+
+        if (validatioUtils.isObject(data._attachments[tarball])) {
+          if (
+            _.isNil(data._attachments[tarball].shasum) === false &&
+            _.isNil(metadata.dist.shasum) === false
+          ) {
+            if (data._attachments[tarball].shasum != metadata.dist.shasum) {
+              const errorMessage =
+                `shasum error, ` +
+                `${data._attachments[tarball].shasum} != ${metadata.dist.shasum}`;
+              throw errorUtils.getBadRequest(errorMessage);
+            }
+          }
+
+          const currentDate = new Date().toISOString();
+
+          // some old storage do not have this field #740
+          if (_.isNil(data.time)) {
+            data.time = {};
+          }
+
+          data.time['modified'] = currentDate;
+
+          if ('created' in data.time === false) {
+            data.time.created = currentDate;
+          }
+
+          data.time[version] = currentDate;
+          data._attachments[tarball].version = version;
+        }
+      }
+
+      data.versions[version] = metadata;
+      tagVersion(data, version, tag);
+
+      try {
+        debug('%s` add on database', name);
+        await this.storagePlugin.add(name);
+      } catch (err: any) {
+        throw errorUtils.getBadData(err.message);
+      }
+      return data;
+    });
+
+    // this._updatePackage(
+    //   name,
+    //   async (data, cb: Callback): Promise<void> => {
+    //     debug('%s package is being updated', name);
+    //     // keep only one readme per package
+    //     data.readme = metadata.readme;
+    //     debug('%s` readme mutated', name);
+    //     // TODO: lodash remove
+    //     metadata = cleanUpReadme(metadata);
+    //     metadata.contributors = normalizeContributors(metadata.contributors as Author[]);
+    //     debug('%s` contributors normalized', name);
+    //     const hasVersion = data.versions[version] != null;
+    //     if (hasVersion) {
+    //       debug('%s version %s already exists', name, version);
+    //       return cb(errorUtils.getConflict());
+    //     }
+
+    //     // if uploaded tarball has a different shasum, it's very likely that we
+    //     // have some kind of error
+    //     if (validatioUtils.isObject(metadata.dist) && _.isString(metadata.dist.tarball)) {
+    //       const tarball = metadata.dist.tarball.replace(/.*\//, '');
+
+    //       if (validatioUtils.isObject(data._attachments[tarball])) {
+    //         if (
+    //           _.isNil(data._attachments[tarball].shasum) === false &&
+    //           _.isNil(metadata.dist.shasum) === false
+    //         ) {
+    //           if (data._attachments[tarball].shasum != metadata.dist.shasum) {
+    //             const errorMessage =
+    //               `shasum error, ` +
+    //               `${data._attachments[tarball].shasum} != ${metadata.dist.shasum}`;
+    //             return cb(errorUtils.getBadRequest(errorMessage));
+    //           }
+    //         }
+
+    //         const currentDate = new Date().toISOString();
+
+    //         // some old storage do not have this field #740
+    //         if (_.isNil(data.time)) {
+    //           data.time = {};
+    //         }
+
+    //         data.time['modified'] = currentDate;
+
+    //         if ('created' in data.time === false) {
+    //           data.time.created = currentDate;
+    //         }
+
+    //         data.time[version] = currentDate;
+    //         data._attachments[tarball].version = version;
+    //       }
+    //     }
+
+    //     data.versions[version] = metadata;
+    //     tagVersion(data, version, tag);
+
+    //     try {
+    //       debug('%s` add on database', name);
+    //       await this.storagePlugin.add(name);
+    //       cb();
+    //     } catch (err: any) {
+    //       cb(errorUtils.getBadData(err.message));
+    //     }
+    //   },
+    //   callback
+    // );
+  }
+
   /**
    * Merge a new list of tags for a local packages with the existing one.
    * @param {*} pkgName
@@ -408,7 +548,7 @@ class LocalStorage {
   public changePackage(
     name: string,
     incomingPkg: Package,
-    revision: string | void,
+    revision: string | undefined,
     callback: Callback
   ): void {
     debug(`change package tags for %o revision %s`, name, revision);
@@ -471,6 +611,70 @@ class LocalStorage {
       }
     );
   }
+
+  /**
+   * Update the package metadata, tags and attachments (tarballs).
+   * Note: Currently supports unpublishing and deprecation.
+   * @param {*} name
+   * @param {*} incomingPkg
+   * @param {*} revision
+   * @param {*} callback
+   * @return {Function}
+   */
+  public async changePackageNext(
+    name: string,
+    incomingPkg: Package,
+    revision: string | undefined
+  ): Promise<void> {
+    debug(`change package tags for %o revision %s`, name, revision);
+    if (
+      !validatioUtils.isObject(incomingPkg.versions) ||
+      !validatioUtils.isObject(incomingPkg[DIST_TAGS])
+    ) {
+      debug(`change package bad data for %o`, name);
+      throw errorUtils.getBadData();
+    }
+
+    debug(`change package udapting package for %o`, name);
+    await this.updatePackageNext(name, async (localData: Package): Promise<Package> => {
+      for (const version in localData.versions) {
+        const incomingVersion = incomingPkg.versions[version];
+        if (_.isNil(incomingVersion)) {
+          this.logger.info({ name: name, version: version }, 'unpublishing @{name}@@{version}');
+
+          // FIXME: I prefer return a new object rather mutate the metadata
+          delete localData.versions[version];
+          delete localData.time![version];
+
+          for (const file in localData._attachments) {
+            if (localData._attachments[file].version === version) {
+              delete localData._attachments[file].version;
+            }
+          }
+        } else if (Object.prototype.hasOwnProperty.call(incomingVersion, 'deprecated')) {
+          const incomingDeprecated = incomingVersion.deprecated;
+          if (incomingDeprecated != localData.versions[version].deprecated) {
+            if (!incomingDeprecated) {
+              this.logger.info(
+                { name: name, version: version },
+                'undeprecating @{name}@@{version}'
+              );
+              delete localData.versions[version].deprecated;
+            } else {
+              this.logger.info({ name: name, version: version }, 'deprecating @{name}@@{version}');
+              localData.versions[version].deprecated = incomingDeprecated;
+            }
+            localData.time!.modified = new Date().toISOString();
+          }
+        }
+      }
+
+      localData[USERS] = incomingPkg[USERS];
+      localData[DIST_TAGS] = incomingPkg[DIST_TAGS];
+      return localData;
+    });
+  }
+
   /**
    * Remove a tarball.
    * @param {*} name
@@ -754,10 +958,16 @@ class LocalStorage {
               reject(err);
             }
 
+            if (_.isEmpty(pkg?.versions)) {
+              return resolve({});
+            }
+
             const searchPackage = normalizeSearchPackage(pkg, searchItem);
             const searchPackageItem: searchUtils.SearchPackageItem = {
               package: searchPackage,
               score: searchItem.score,
+              verdaccioPkgCached: searchItem.verdaccioPkgCached,
+              verdaccioPrivate: searchItem.verdaccioPrivate,
               flags: searchItem?.flags,
               // FUTURE: find a better way to calculate the score
               searchScore: 1,
@@ -890,12 +1100,12 @@ class LocalStorage {
   private _updatePackage(
     name: string,
     updateHandler: StorageUpdateCallback,
-    callback: Callback
+    onEnd: Callback
   ): void {
     const storage: IPackageStorage = this._getLocalStorage(name);
 
     if (!storage) {
-      return callback(errorUtils.getNotFound());
+      return onEnd(errorUtils.getNotFound());
     }
 
     storage.updatePackage(
@@ -903,10 +1113,42 @@ class LocalStorage {
       updateHandler,
       this._writePackage.bind(this),
       normalizePackage,
-      callback
+      onEnd
     );
   }
 
+  /**
+   * @param {*} name package name
+   * @param {*} updateHandler function(package, cb) - update function
+   * @param {*} callback callback that gets invoked after it's all updated
+   * @return {Function}
+   */
+  private async updatePackageNext(
+    name: string,
+    updateHandler: (manifest: Package) => Promise<Package>
+  ): Promise<void> {
+    const storage: IPackageStorage = this._getLocalStorage(name);
+
+    if (!storage) {
+      throw errorUtils.getNotFound();
+    }
+
+    // we update the package on the local storage
+    const updatedManifest: Package = await storage.updatePackageNext(name, updateHandler);
+    // after correctly updated write to the storage
+    try {
+      await this.writePackageNext(name, normalizePackage(updatedManifest));
+    } catch (err: any) {
+      if (err.code === resourceNotAvailable) {
+        throw errorUtils.getInternalError('resource temporarily unavailable');
+      } else if (err.code === noSuchFile) {
+        throw errorUtils.getNotFound();
+      } else {
+        throw err;
+      }
+    }
+  }
+
   /**
    * Update the revision (_rev) string for a package.
    * @param {*} name
@@ -922,6 +1164,15 @@ class LocalStorage {
     storage.savePackage(name, this._setDefaultRevision(json), callback);
   }
 
+  private async writePackageNext(name: string, json: Package): Promise<void> {
+    const storage: any = this._getLocalStorage(name);
+    if (_.isNil(storage)) {
+      // TODO: replace here 500 error
+      throw errorUtils.getBadData();
+    }
+    await storage.savePackageNext(name, this._setDefaultRevision(json));
+  }
+
   private _setDefaultRevision(json: Package): Package {
     // calculate revision from couch db
     if (_.isString(json._rev) === false) {

package/src/search.ts

@@ -2,8 +2,6 @@
 // eslint-disable no-invalid-this
 import buildDebug from 'debug';
 import _ from 'lodash';
-import lunr from 'lunr';
-import lunrMutable from 'lunr-mutable-indexes';
 import { PassThrough, Transform, pipeline } from 'stream';
 
 import { VerdaccioError } from '@verdaccio/core';
@@ -20,8 +18,8 @@ export interface ISearchResult {
   ref: string;
   score: number;
 }
+// @deprecated not longer used
 export interface IWebSearch {
-  index: lunrMutable.index;
   storage: Storage;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   query(query: string): ISearchResult[];
@@ -33,7 +31,8 @@ export interface IWebSearch {
 
 export function removeDuplicates(results: searchUtils.SearchPackageItem[]) {
   const pkgNames: any[] = [];
-  return results.filter((pkg) => {
+  const orderByResults = _.orderBy(results, ['verdaccioPrivate', 'asc']);
+  return orderByResults.filter((pkg) => {
     if (pkgNames.includes(pkg?.package?.name)) {
       return false;
     }
@@ -58,16 +57,18 @@ class TransFormResults extends Transform {
    */
   public _transform(chunk, _encoding, callback) {
     if (_.isArray(chunk)) {
+      // from remotes we should expect chunks as arrays
       (chunk as searchUtils.SearchItem[])
         .filter((pkgItem) => {
           debug(`streaming remote pkg name ${pkgItem?.package?.name}`);
           return true;
         })
         .forEach((pkgItem) => {
-          this.push(pkgItem);
+          this.push({ ...pkgItem, verdaccioPkgCached: false, verdaccioPrivate: false });
         });
       return callback();
     } else {
+      // local we expect objects
       debug(`streaming local pkg name ${chunk?.package?.name}`);
       this.push(chunk);
       return callback();
@@ -105,30 +106,34 @@ export class SearchManager {
       if (!uplink) {
         // this should never tecnically happens
         logger.fatal({ uplinkId }, 'uplink @upLinkId not found');
-        throw new Error(`uplink ${uplinkId} not found`);
       }
       return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough);
     });
 
     try {
       debug('search uplinks');
-      await Promise.all([...searchUplinksStreams]);
+      // we only process those streams end successfully, if all fails
+      // we just include local storage
+      await Promise.allSettled([...searchUplinksStreams]);
       debug('search uplinks done');
-    } catch (err) {
-      logger.error({ err }, ' error on uplinks search @{err}');
+    } catch (err: any) {
+      logger.error({ err: err?.message }, ' error on uplinks search @{err}');
       streamPassThrough.emit('error', err);
-      throw err;
     }
     debug('search local');
-    await this.localStorage.search(streamPassThrough, options.query as searchUtils.SearchQuery);
-
+    try {
+      await this.localStorage.search(streamPassThrough, options.query as searchUtils.SearchQuery);
+    } catch (err: any) {
+      logger.error({ err: err?.message }, ' error on local search @{err}');
+      streamPassThrough.emit('error', err);
+    }
     const data: searchUtils.SearchPackageItem[] = [];
     const outPutStream = new PassThrough({ objectMode: true });
     pipeline(streamPassThrough, transformResults, outPutStream, (err) => {
       if (err) {
         throw errorUtils.getInternalError(err ? err.message : 'unknown error');
       } else {
-        debug('Pipeline succeeded.');
+        debug('pipeline succeeded');
       }
     });
 
@@ -138,9 +143,9 @@ export class SearchManager {
 
     return new Promise((resolve) => {
       outPutStream.on('finish', async () => {
-        const checkAccessPromises: searchUtils.SearchPackageItem[] = removeDuplicates(data);
-        debug('stream finish event %s', checkAccessPromises.length);
-        return resolve(checkAccessPromises);
+        const searchFinalResults: searchUtils.SearchPackageItem[] = removeDuplicates(data);
+        debug('search stream total results: %o', searchFinalResults.length);
+        return resolve(searchFinalResults);
       });
       debug('search done');
     });
@@ -168,111 +173,3 @@ export class SearchManager {
     });
   }
 }
-
-/**
- * Handle the search Indexer.
- */
-class Search implements IWebSearch {
-  public readonly index: lunrMutable.index;
-  // @ts-ignore
-  public storage: Storage;
-
-  /**
-   * Constructor.
-   */
-  public constructor() {
-    this.index = lunrMutable(function (): void {
-      // FIXME: there is no types for this library
-      /* eslint no-invalid-this:off */
-      // @ts-ignore
-      this.field('name', { boost: 10 });
-      // @ts-ignore
-      this.field('description', { boost: 4 });
-      // @ts-ignore
-      this.field('author', { boost: 6 });
-      // @ts-ignore
-      this.field('keywords', { boost: 7 });
-      // @ts-ignore
-      this.field('version');
-      // @ts-ignore
-      this.field('readme');
-    });
-
-    this.index.builder.pipeline.remove(lunr.stemmer);
-  }
-
-  public init() {
-    return Promise.resolve();
-  }
-
-  /**
-   * Performs a query to the indexer.
-   * If the keyword is a * it returns all local elements
-   * otherwise performs a search
-   * @param {*} q the keyword
-   * @return {Array} list of results.
-   */
-  public query(query: string): ISearchResult[] {
-    const localStorage = this.storage.localStorage as LocalStorage;
-
-    return query === '*'
-      ? (localStorage.storagePlugin as any).get((items): any => {
-          items.map(function (pkg): any {
-            return { ref: pkg, score: 1 };
-          });
-        })
-      : this.index.search(`*${query}*`);
-  }
-
-  /**
-   * Add a new element to index
-   * @param {*} pkg the package
-   */
-  public add(pkg: Version): void {
-    this.index.add({
-      id: pkg.name,
-      name: pkg.name,
-      description: pkg.description,
-      version: `v${pkg.version}`,
-      keywords: pkg.keywords,
-      author: pkg._npmUser ? pkg._npmUser.name : '???',
-    });
-  }
-
-  /**
-   * Remove an element from the index.
-   * @param {*} name the id element
-   */
-  public remove(name: string): void {
-    this.index.remove({ id: name });
-  }
-
-  /**
-   * Force a re-index.
-   */
-  public reindex(): void {
-    this.storage.getLocalDatabase((error, packages): void => {
-      if (error) {
-        // that function shouldn't produce any
-        throw error;
-      }
-      let i = packages.length;
-      while (i--) {
-        this.add(packages[i]);
-      }
-    });
-  }
-
-  /**
-   * Set up the {Storage}
-   * @param {*} storage An storage reference.
-   */
-  public configureStorage(storage: Storage): void {
-    this.storage = storage;
-    this.reindex();
-  }
-}
-
-const SearchInstance = new Search();
-
-export { SearchInstance };

package/src/star-utils.ts

@@ -0,0 +1,38 @@
+import _ from 'lodash';
+
+import { Package } from '@verdaccio/types';
+
+import { Users } from '.';
+
+/**
+ * Check whether the package metadta has enough data to be published
+ * @param pkg metadata
+ */
+
+export function isPublishablePackage(pkg: Package): boolean {
+  const keys: string[] = Object.keys(pkg);
+
+  return _.includes(keys, 'versions');
+}
+
+export function isRelatedToDeprecation(pkgInfo: Package): boolean {
+  const { versions } = pkgInfo;
+  for (const version in versions) {
+    if (Object.prototype.hasOwnProperty.call(versions[version], 'deprecated')) {
+      return true;
+    }
+  }
+  return false;
+}
+
+export function validateInputs(localUsers: Users, username: string, isStar: boolean): boolean {
+  const isExistlocalUsers = _.isNil(localUsers[username]) === false;
+  if (isStar && isExistlocalUsers && localUsers[username]) {
+    return true;
+  } else if (!isStar && isExistlocalUsers) {
+    return false;
+  } else if (!isStar && !isExistlocalUsers) {
+    return true;
+  }
+  return false;
+}

package/src/storage-utils.ts

@@ -3,11 +3,10 @@ import semver from 'semver';
 
 import { errorUtils, pkgUtils, validatioUtils } from '@verdaccio/core';
 import { API_ERROR, DIST_TAGS, HTTP_STATUS, USERS } from '@verdaccio/core';
-import { Package, StringValue, Version } from '@verdaccio/types';
-import { generateRandomHexString, isNil, normalizeDistTags } from '@verdaccio/utils';
+import { AttachMents, Package, StringValue, Version, Versions } from '@verdaccio/types';
+import { generateRandomHexString, isNil, isObject, normalizeDistTags } from '@verdaccio/utils';
 
 import { LocalStorage } from './local-storage';
-import { SearchInstance } from './search';
 
 export const STORAGE = {
   PACKAGE_FILE_NAME: 'package.json',
@@ -148,11 +147,9 @@ export function publishPackage(
   localStorage: LocalStorage
 ): Promise<any> {
   return new Promise<void>((resolve, reject): void => {
-    localStorage.addPackage(name, metadata, (err, latest): void => {
+    localStorage.addPackage(name, metadata, (err): void => {
       if (!_.isNull(err)) {
         return reject(err);
-      } else if (!_.isUndefined(latest)) {
-        SearchInstance.add(latest);
       }
       return resolve();
     });
@@ -246,3 +243,21 @@ export function tagVersion(data: Package, version: string, tag: StringValue): bo
   }
   return false;
 }
+
+export function isDifferentThanOne(versions: Versions | AttachMents): boolean {
+  return Object.keys(versions).length !== 1;
+}
+
+export function hasInvalidPublishBody(manifest: Pick<Package, '_attachments' | 'versions'>) {
+  if (!manifest) {
+    return false;
+  }
+
+  const { _attachments, versions } = manifest;
+  const res =
+    isObject(_attachments) === false ||
+    isDifferentThanOne(_attachments) ||
+    isObject(versions) === false ||
+    isDifferentThanOne(versions);
+  return res;
+}