@verdaccio/store 6.0.0-6-next.21 → 6.0.0-6-next.24

package/src/search.ts

@@ -1,175 +0,0 @@
-/* eslint-disable @typescript-eslint/no-unused-vars */
-// eslint-disable no-invalid-this
-import buildDebug from 'debug';
-import _ from 'lodash';
-import { PassThrough, Transform, pipeline } from 'stream';
-
-import { VerdaccioError } from '@verdaccio/core';
-import { errorUtils, searchUtils } from '@verdaccio/core';
-import { logger } from '@verdaccio/logger';
-import { IProxy, ProxyList, ProxySearchParams } from '@verdaccio/proxy';
-import { Version } from '@verdaccio/types';
-
-import { LocalStorage } from './local-storage';
-import { Storage } from './storage';
-
-const debug = buildDebug('verdaccio:storage:search');
-export interface ISearchResult {
-  ref: string;
-  score: number;
-}
-// @deprecated not longer used
-export interface IWebSearch {
-  storage: Storage;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  query(query: string): ISearchResult[];
-  add(pkg: Version): void;
-  remove(name: string): void;
-  reindex(): void;
-  configureStorage(storage: Storage): void;
-}
-
-export function removeDuplicates(results: searchUtils.SearchPackageItem[]) {
-  const pkgNames: any[] = [];
-  const orderByResults = _.orderBy(results, ['verdaccioPrivate', 'asc']);
-  return orderByResults.filter((pkg) => {
-    if (pkgNames.includes(pkg?.package?.name)) {
-      return false;
-    }
-    pkgNames.push(pkg?.package?.name);
-    return true;
-  });
-}
-
-class TransFormResults extends Transform {
-  public constructor(options) {
-    super(options);
-  }
-
-  /**
-   * Transform either array of packages or a single package into a stream of packages.
-   * From uplinks the chunks are array but from local packages are objects.
-   * @param {string} chunk
-   * @param {string} encoding
-   * @param {function} done
-   * @returns {void}
-   * @override
-   */
-  public _transform(chunk, _encoding, callback) {
-    if (_.isArray(chunk)) {
-      // from remotes we should expect chunks as arrays
-      (chunk as searchUtils.SearchItem[])
-        .filter((pkgItem) => {
-          debug(`streaming remote pkg name ${pkgItem?.package?.name}`);
-          return true;
-        })
-        .forEach((pkgItem) => {
-          this.push({ ...pkgItem, verdaccioPkgCached: false, verdaccioPrivate: false });
-        });
-      return callback();
-    } else {
-      // local we expect objects
-      debug(`streaming local pkg name ${chunk?.package?.name}`);
-      this.push(chunk);
-      return callback();
-    }
-  }
-}
-
-export class SearchManager {
-  public readonly uplinks: ProxyList;
-  public readonly localStorage: LocalStorage;
-  constructor(uplinks: ProxyList, storage: LocalStorage) {
-    this.uplinks = uplinks;
-    this.localStorage = storage;
-  }
-
-  public get proxyList() {
-    const uplinksList = Object.keys(this.uplinks);
-
-    return uplinksList;
-  }
-
-  /**
-   * Handle search on packages and proxies.
-   * Iterate all proxies configured and search in all endpoints in v2 and pipe all responses
-   * to a stream, once the proxies request has finished search in local storage for all packages
-   * (privated and cached).
-   */
-  public async search(options: ProxySearchParams): Promise<searchUtils.SearchPackageItem[]> {
-    const transformResults = new TransFormResults({ objectMode: true });
-    const streamPassThrough = new PassThrough({ objectMode: true });
-    const upLinkList = this.proxyList;
-
-    const searchUplinksStreams = upLinkList.map((uplinkId) => {
-      const uplink = this.uplinks[uplinkId];
-      if (!uplink) {
-        // this should never tecnically happens
-        logger.fatal({ uplinkId }, 'uplink @upLinkId not found');
-      }
-      return this.consumeSearchStream(uplinkId, uplink, options, streamPassThrough);
-    });
-
-    try {
-      debug('search uplinks');
-      // we only process those streams end successfully, if all fails
-      // we just include local storage
-      await Promise.allSettled([...searchUplinksStreams]);
-      debug('search uplinks done');
-    } catch (err: any) {
-      logger.error({ err: err?.message }, ' error on uplinks search @{err}');
-      streamPassThrough.emit('error', err);
-    }
-    debug('search local');
-    try {
-      await this.localStorage.search(streamPassThrough, options.query as searchUtils.SearchQuery);
-    } catch (err: any) {
-      logger.error({ err: err?.message }, ' error on local search @{err}');
-      streamPassThrough.emit('error', err);
-    }
-    const data: searchUtils.SearchPackageItem[] = [];
-    const outPutStream = new PassThrough({ objectMode: true });
-    pipeline(streamPassThrough, transformResults, outPutStream, (err) => {
-      if (err) {
-        throw errorUtils.getInternalError(err ? err.message : 'unknown error');
-      } else {
-        debug('pipeline succeeded');
-      }
-    });
-
-    outPutStream.on('data', (chunk) => {
-      data.push(chunk);
-    });
-
-    return new Promise((resolve) => {
-      outPutStream.on('finish', async () => {
-        const searchFinalResults: searchUtils.SearchPackageItem[] = removeDuplicates(data);
-        debug('search stream total results: %o', searchFinalResults.length);
-        return resolve(searchFinalResults);
-      });
-      debug('search done');
-    });
-  }
-
-  /**
-   * Consume the upstream and pipe it to a transformable stream.
-   */
-  private consumeSearchStream(
-    uplinkId: string,
-    uplink: IProxy,
-    options: ProxySearchParams,
-    searchPassThrough: PassThrough
-  ): Promise<any> {
-    return uplink.search({ ...options }).then((bodyStream) => {
-      bodyStream.pipe(searchPassThrough, { end: false });
-      bodyStream.on('error', (err: VerdaccioError): void => {
-        logger.error(
-          { uplinkId, err: err },
-          'search error for uplink @{uplinkId}: @{err?.message}'
-        );
-        searchPassThrough.end();
-      });
-      return new Promise((resolve) => bodyStream.on('end', resolve));
-    });
-  }
-}

package/src/storage-utils.ts

@@ -1,263 +0,0 @@
-import _ from 'lodash';
-import semver from 'semver';
-
-import { errorUtils, pkgUtils, validatioUtils } from '@verdaccio/core';
-import { API_ERROR, DIST_TAGS, HTTP_STATUS, USERS } from '@verdaccio/core';
-import { AttachMents, Package, StringValue, Version, Versions } from '@verdaccio/types';
-import { generateRandomHexString, isNil, isObject, normalizeDistTags } from '@verdaccio/utils';
-
-import { LocalStorage } from './local-storage';
-
-export const STORAGE = {
-  PACKAGE_FILE_NAME: 'package.json',
-  FILE_EXIST_ERROR: 'EEXISTS',
-  NO_SUCH_FILE_ERROR: 'ENOENT',
-  DEFAULT_REVISION: '0-0000000000000000',
-};
-
-export function generatePackageTemplate(name: string): Package {
-  return {
-    // standard things
-    name,
-    versions: {},
-    time: {},
-    [USERS]: {},
-    [DIST_TAGS]: {},
-    _uplinks: {},
-    _distfiles: {},
-    _attachments: {},
-    _rev: '',
-  };
-}
-
-/**
- * Normalize package properties, tags, revision id.
- * @param {Object} pkg package reference.
- */
-export function normalizePackage(pkg: Package): Package {
-  const pkgProperties = ['versions', 'dist-tags', '_distfiles', '_attachments', '_uplinks', 'time'];
-
-  pkgProperties.forEach((key): void => {
-    const pkgProp = pkg[key];
-
-    if (isNil(pkgProp) || validatioUtils.isObject(pkgProp) === false) {
-      pkg[key] = {};
-    }
-  });
-
-  if (_.isString(pkg._rev) === false) {
-    pkg._rev = STORAGE.DEFAULT_REVISION;
-  }
-
-  if (_.isString(pkg._id) === false) {
-    pkg._id = pkg.name;
-  }
-
-  // normalize dist-tags
-  return normalizeDistTags(pkg);
-}
-
-export function generateRevision(rev: string): string {
-  const _rev = rev.split('-');
-
-  return (+_rev[0] || 0) + 1 + '-' + generateRandomHexString();
-}
-
-export function getLatestReadme(pkg: Package): string {
-  const versions = pkg['versions'] || {};
-  const distTags = pkg[DIST_TAGS] || {};
-  // FIXME: here is a bit tricky add the types
-  const latestVersion: Version | any = distTags['latest'] ? versions[distTags['latest']] || {} : {};
-  let readme = _.trim(pkg.readme || latestVersion.readme || '');
-  if (readme) {
-    return readme;
-  }
-
-  // In case of empty readme - trying to get ANY readme in the following order:
-  // 'next','beta','alpha','test','dev','canary'
-  const readmeDistTagsPriority = ['next', 'beta', 'alpha', 'test', 'dev', 'canary'];
-  readmeDistTagsPriority.forEach(function (tag): string | void {
-    if (readme) {
-      return readme;
-    }
-    const version: Version | any = distTags[tag] ? versions[distTags[tag]] || {} : {};
-    readme = _.trim(version.readme || readme);
-  });
-  return readme;
-}
-
-// FIXME: type any due this
-export function cleanUpReadme(version: any): Version {
-  if (isNil(version) === false) {
-    delete version.readme;
-  }
-
-  return version;
-}
-
-export const WHITELIST = [
-  '_rev',
-  'name',
-  'versions',
-  'dist-tags',
-  'readme',
-  'time',
-  '_id',
-  'users',
-];
-
-export function cleanUpLinksRef(result: Package, keepUpLinkData?: boolean): Package {
-  const propertyToKeep = [...WHITELIST];
-  if (keepUpLinkData === true) {
-    propertyToKeep.push('_uplinks');
-  }
-
-  for (const i in result) {
-    if (propertyToKeep.indexOf(i) === -1) {
-      // Remove sections like '_uplinks' from response
-      delete result[i];
-    }
-  }
-
-  return result;
-}
-
-/**
- * Check whether a package it is already a local package
- * @param {*} name
- * @param {*} localStorage
- */
-export function checkPackageLocal(name: string, localStorage: LocalStorage): Promise<any> {
-  return new Promise<void>((resolve, reject): void => {
-    localStorage.getPackageMetadata(name, (err, results): void => {
-      if (!isNil(err) && err.status !== HTTP_STATUS.NOT_FOUND) {
-        return reject(err);
-      }
-      if (results) {
-        return reject(errorUtils.getConflict(API_ERROR.PACKAGE_EXIST));
-      }
-      return resolve();
-    });
-  });
-}
-
-export function publishPackage(
-  name: string,
-  metadata: any,
-  localStorage: LocalStorage
-): Promise<any> {
-  return new Promise<void>((resolve, reject): void => {
-    localStorage.addPackage(name, metadata, (err): void => {
-      if (!_.isNull(err)) {
-        return reject(err);
-      }
-      return resolve();
-    });
-  });
-}
-
-export function checkPackageRemote(
-  name: string,
-  isAllowPublishOffline: boolean,
-  syncMetadata: Function
-): Promise<any> {
-  return new Promise<void>((resolve, reject): void => {
-    syncMetadata(name, null, {}, (err, packageJsonLocal, upLinksErrors): void => {
-      // something weird
-      if (err && err.status !== HTTP_STATUS.NOT_FOUND) {
-        return reject(err);
-      }
-
-      // checking package exist already
-      if (isNil(packageJsonLocal) === false) {
-        return reject(errorUtils.getConflict(API_ERROR.PACKAGE_EXIST));
-      }
-
-      for (let errorItem = 0; errorItem < upLinksErrors.length; errorItem++) {
-        // checking error
-        // if uplink fails with a status other than 404, we report failure
-        if (isNil(upLinksErrors[errorItem][0]) === false) {
-          if (upLinksErrors[errorItem][0].status !== HTTP_STATUS.NOT_FOUND) {
-            if (isAllowPublishOffline) {
-              return resolve();
-            }
-
-            return reject(errorUtils.getServiceUnavailable(API_ERROR.UPLINK_OFFLINE_PUBLISH));
-          }
-        }
-      }
-
-      return resolve();
-    });
-  });
-}
-
-export function mergeUplinkTimeIntoLocal(localMetadata: Package, remoteMetadata: Package): any {
-  if ('time' in remoteMetadata) {
-    return Object.assign({}, localMetadata.time, remoteMetadata.time);
-  }
-
-  return localMetadata.time;
-}
-
-export function prepareSearchPackage(data: Package): any {
-  const latest = pkgUtils.getLatest(data);
-
-  if (latest && data.versions[latest]) {
-    const version: Version = data.versions[latest];
-    const versions: any = { [latest]: 'latest' };
-    const pkg: any = {
-      name: version.name,
-      description: version.description,
-      [DIST_TAGS]: { latest },
-      maintainers: version.maintainers || [version.author].filter(Boolean),
-      author: version.author,
-      repository: version.repository,
-      readmeFilename: version.readmeFilename || '',
-      homepage: version.homepage,
-      keywords: version.keywords,
-      bugs: version.bugs,
-      license: version.license,
-      // time: {
-      //   modified: time,
-      // },
-      versions,
-    };
-
-    return pkg;
-  }
-}
-
-/**
- * Create a tag for a package
- * @param {*} data
- * @param {*} version
- * @param {*} tag
- * @return {Boolean} whether a package has been tagged
- */
-export function tagVersion(data: Package, version: string, tag: StringValue): boolean {
-  if (tag && data[DIST_TAGS][tag] !== version && semver.parse(version, true)) {
-    // valid version - store
-    data[DIST_TAGS][tag] = version;
-    return true;
-  }
-  return false;
-}
-
-export function isDifferentThanOne(versions: Versions | AttachMents): boolean {
-  return Object.keys(versions).length !== 1;
-}
-
-export function hasInvalidPublishBody(manifest: Pick<Package, '_attachments' | 'versions'>) {
-  if (!manifest) {
-    return false;
-  }
-
-  const { _attachments, versions } = manifest;
-  const res =
-    isObject(_attachments) === false ||
-    isDifferentThanOne(_attachments) ||
-    isObject(versions) === false ||
-    isDifferentThanOne(versions);
-  return res;
-}

package/src/uplink-util.ts

@@ -1,32 +0,0 @@
-import { IProxy, ProxyList, ProxyStorage } from '@verdaccio/proxy';
-import { Config, Versions } from '@verdaccio/types';
-
-/**
- * Set up the Up Storage for each link.
- */
-export function setupUpLinks(config: Config): ProxyList {
-  const uplinks: ProxyList = {};
-
-  for (const uplinkName in config.uplinks) {
-    if (Object.prototype.hasOwnProperty.call(config.uplinks, uplinkName)) {
-      // instance for each up-link definition
-      const proxy: IProxy = new ProxyStorage(config.uplinks[uplinkName], config);
-      proxy.upname = uplinkName;
-
-      uplinks[uplinkName] = proxy;
-    }
-  }
-
-  return uplinks;
-}
-
-export function updateVersionsHiddenUpLink(versions: Versions, upLink: IProxy): void {
-  for (const i in versions) {
-    if (Object.prototype.hasOwnProperty.call(versions, i)) {
-      const version = versions[i];
-
-      // holds a "hidden" value to be used by the package storage.
-      version[Symbol.for('__verdaccio_uplink')] = upLink.upname;
-    }
-  }
-}