@splitsoftware/splitio-commons 2.5.0-rc.0 → 11.5.0-rc.0

package/src/storages/pluggable/TelemetryCachePluggable.ts

@@ -1,159 +0,0 @@
-import { ILogger } from '../../logger/types';
-import { Method, MultiConfigs, MultiMethodExceptions, MultiMethodLatencies } from '../../sync/submitters/types';
-import { KeyBuilderSS } from '../KeyBuilderSS';
-import { IPluggableStorageWrapper, ITelemetryCacheAsync } from '../types';
-import { findLatencyIndex } from '../findLatencyIndex';
-import { getTelemetryConfigStats } from '../../sync/submitters/telemetrySubmitter';
-import { CONSUMER_MODE, STORAGE_PLUGGABLE } from '../../utils/constants';
-import { isString, isNaNNumber } from '../../utils/lang';
-import { MAX_LATENCY_BUCKET_COUNT, newBuckets } from '../inMemory/TelemetryCacheInMemory';
-import { parseLatencyField, parseExceptionField, parseMetadata } from '../utils';
-
-export class TelemetryCachePluggable implements ITelemetryCacheAsync {
-
-  /**
-   * Create a Telemetry cache that uses a storage wrapper.
-   * @param log -  Logger instance.
-   * @param keys -  Key builder.
-   * @param wrapper -  Adapted wrapper storage.
-   */
-  constructor(private readonly log: ILogger, private readonly keys: KeyBuilderSS, private readonly wrapper: IPluggableStorageWrapper) { }
-
-  recordLatency(method: Method, latencyMs: number) {
-    return this.wrapper.incr(this.keys.buildLatencyKey(method, findLatencyIndex(latencyMs)))
-      .catch(() => { /* Handle rejections for telemetry */ });
-  }
-  recordException(method: Method) {
-    return this.wrapper.incr(this.keys.buildExceptionKey(method))
-      .catch(() => { /* Handle rejections for telemetry */ });
-  }
-
-  recordConfig() {
-    const value = JSON.stringify(getTelemetryConfigStats(CONSUMER_MODE, STORAGE_PLUGGABLE));
-    return this.wrapper.set(this.keys.buildInitKey(), value).catch(() => { /* Handle rejections for telemetry */ });
-  }
-
-  /**
-   * Pop telemetry latencies.
-   * The returned promise rejects if wrapper operations fail.
-   */
-  popLatencies(): Promise<MultiMethodLatencies> {
-    return this.wrapper.getKeysByPrefix(this.keys.latencyPrefix).then(latencyKeys => {
-      return latencyKeys.length ?
-        this.wrapper.getMany(latencyKeys).then(latencies => {
-
-          const result: MultiMethodLatencies = new Map();
-
-          for (let i = 0; i < latencyKeys.length; i++) {
-            const field = latencyKeys[i].split('::')[1];
-
-            const parsedField = parseLatencyField(field);
-            if (isString(parsedField)) {
-              this.log.error(`Ignoring invalid latency field: ${field}: ${parsedField}`);
-              continue;
-            }
-
-            // @ts-ignore
-            const count = parseInt(latencies[i]);
-            if (isNaNNumber(count)) {
-              this.log.error(`Ignoring latency with invalid count: ${latencies[i]}`);
-              continue;
-            }
-
-            const [metadata, method, bucket] = parsedField;
-
-            if (bucket >= MAX_LATENCY_BUCKET_COUNT) {
-              this.log.error(`Ignoring latency with invalid bucket: ${bucket}`);
-              continue;
-            }
-
-            const methodLatencies = result.get(metadata) || {};
-            methodLatencies[method] = methodLatencies[method] || newBuckets();
-            methodLatencies[method]![bucket] = count;
-            result.set(metadata, methodLatencies);
-          }
-
-          return Promise.all(latencyKeys.map((latencyKey) => this.wrapper.del(latencyKey))).then(() => result);
-        }) :
-        // If latencyKeys is empty, return an empty map.
-        new Map();
-    });
-  }
-
-  /**
-   * Pop telemetry exceptions.
-   * The returned promise rejects if wrapper operations fail.
-   */
-  popExceptions(): Promise<MultiMethodExceptions> {
-    return this.wrapper.getKeysByPrefix(this.keys.exceptionPrefix).then(exceptionKeys => {
-      return exceptionKeys.length ?
-        this.wrapper.getMany(exceptionKeys).then(exceptions => {
-
-          const result: MultiMethodExceptions = new Map();
-
-          for (let i = 0; i < exceptionKeys.length; i++) {
-            const field = exceptionKeys[i].split('::')[1];
-
-            const parsedField = parseExceptionField(field);
-            if (isString(parsedField)) {
-              this.log.error(`Ignoring invalid exception field: ${field}: ${parsedField}`);
-              continue;
-            }
-
-            // @ts-ignore
-            const count = parseInt(exceptions[i]);
-            if (isNaNNumber(count)) {
-              this.log.error(`Ignoring exception with invalid count: ${exceptions[i]}`);
-              continue;
-            }
-
-            const [metadata, method] = parsedField;
-
-            if (!result.has(metadata)) result.set(metadata, {});
-            result.get(metadata)![method] = count;
-          }
-
-          return Promise.all(exceptionKeys.map((exceptionKey) => this.wrapper.del(exceptionKey))).then(() => result);
-        }) :
-        // If exceptionKeys is empty, return an empty map.
-        new Map();
-    });
-  }
-
-  /**
-   * Pop telemetry configs.
-   * The returned promise rejects if wrapper operations fail.
-   */
-  popConfigs(): Promise<MultiConfigs> {
-    return this.wrapper.getKeysByPrefix(this.keys.initPrefix).then(configKeys => {
-      return configKeys.length ?
-        this.wrapper.getMany(configKeys).then(configs => {
-
-          const result: MultiConfigs = new Map();
-
-          for (let i = 0; i < configKeys.length; i++) {
-            const field = configKeys[i].split('::')[1];
-
-            const parsedField = parseMetadata(field);
-            if (isString(parsedField)) {
-              this.log.error(`Ignoring invalid config field: ${field}: ${parsedField}`);
-              continue;
-            }
-
-            const [metadata] = parsedField;
-
-            try { // @ts-ignore
-              const config = JSON.parse(configs[i]);
-              result.set(metadata, config);
-            } catch (e) {
-              this.log.error(`Ignoring invalid config: ${configs[i]}`);
-            }
-          }
-
-          return Promise.all(configKeys.map((configKey) => this.wrapper.del(configKey))).then(() => result);
-        }) :
-        // If configKeys is empty, return an empty map.
-        new Map();
-    });
-  }
-}

package/src/storages/pluggable/UniqueKeysCachePluggable.ts

@@ -1,67 +0,0 @@
-import { IPluggableStorageWrapper, IUniqueKeysCacheBase } from '../types';
-import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
-import { DEFAULT_CACHE_SIZE, REFRESH_RATE } from '../inRedis/constants';
-import { LOG_PREFIX } from './constants';
-import { ILogger } from '../../logger/types';
-import { UniqueKeysItemSs } from '../../sync/submitters/types';
-import { setToArray } from '../../utils/lang/sets';
-
-export class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
-
-  private readonly log: ILogger;
-  private readonly key: string;
-  private readonly wrapper: IPluggableStorageWrapper;
-  private readonly refreshRate: number;
-  private intervalId: any;
-
-  constructor(log: ILogger, key: string, wrapper: IPluggableStorageWrapper, uniqueKeysQueueSize = DEFAULT_CACHE_SIZE, refreshRate = REFRESH_RATE) {
-    super(uniqueKeysQueueSize);
-    this.log = log;
-    this.key = key;
-    this.wrapper = wrapper;
-    this.refreshRate = refreshRate;
-    this.onFullQueue = () => { this.storeUniqueKeys(); };
-  }
-
-  storeUniqueKeys() {
-    const featureNames = Object.keys(this.uniqueKeysTracker);
-    if (!featureNames.length) return Promise.resolve(false);
-
-    const uniqueKeysArray = featureNames.map((featureName) => {
-      const featureKeys = setToArray(this.uniqueKeysTracker[featureName]);
-      const uniqueKeysPayload = {
-        f: featureName,
-        ks: featureKeys
-      };
-      return JSON.stringify(uniqueKeysPayload);
-    });
-
-    this.clear();
-    return this.wrapper.pushItems(this.key, uniqueKeysArray)
-      .catch(err => {
-        this.log.error(`${LOG_PREFIX}Error in uniqueKeys pipeline: ${err}.`);
-        return false;
-      });
-  }
-
-
-  start() {
-    this.intervalId = setInterval(this.storeUniqueKeys.bind(this), this.refreshRate);
-  }
-
-  stop() {
-    clearInterval(this.intervalId);
-    return this.storeUniqueKeys();
-  }
-
-  /**
-   * Async consumer API, used by synchronizer.
-   * @param count - number of items to pop from the queue. If not provided or equal 0, all items will be popped.
-   */
-  popNRaw(count = 0): Promise<UniqueKeysItemSs[]> {
-    return Promise.resolve(count || this.wrapper.getItemsCount(this.key))
-      .then(count => this.wrapper.popItems(this.key, count))
-      .then((uniqueKeyItems) => uniqueKeyItems.map(uniqueKeyItem => JSON.parse(uniqueKeyItem)));
-  }
-
-}

package/src/storages/pluggable/constants.ts

@@ -1 +0,0 @@
-export const LOG_PREFIX = 'storage:pluggable: ';

package/src/storages/pluggable/inMemoryWrapper.ts

@@ -1,130 +0,0 @@
-import { IPluggableStorageWrapper } from '../types';
-import { startsWith, toNumber } from '../../utils/lang';
-import { setToArray } from '../../utils/lang/sets';
-
-/**
- * Creates a IPluggableStorageWrapper implementation that stores items in memory.
- * The `_cache` property is the object were items are stored.
- * Intended for testing purposes.
- *
- * @param connDelay - delay in millis for `connect` resolve. If not provided, `connect` resolves immediately.
- */
-export function inMemoryWrapperFactory(connDelay?: number): IPluggableStorageWrapper & { _cache: Record<string, string | string[] | Set<string>>, _setConnDelay(connDelay: number): void } {
-
-  let _cache: Record<string, string | string[] | Set<string>> = {};
-  let _connDelay = connDelay;
-
-  return {
-    /** Holds items (for key-value operations), list of items (for list operations) and set of items (for set operations) */
-    _cache,
-
-    get(key) {
-      return Promise.resolve(key in _cache ? _cache[key] as string : null);
-    },
-    set(key: string, value: string) {
-      const result = key in _cache;
-      _cache[key] = value;
-      return Promise.resolve(result);
-    },
-    getAndSet(key: string, value: string) {
-      const result = key in _cache ? _cache[key] as string : null;
-      _cache[key] = value;
-      return Promise.resolve(result);
-    },
-    del(key: string) {
-      const result = key in _cache;
-      delete _cache[key];
-      return Promise.resolve(result);
-    },
-    getKeysByPrefix(prefix: string) {
-      return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix)));
-    },
-    incr(key: string, increment = 1) {
-      if (key in _cache) {
-        const count = toNumber(_cache[key]) + increment;
-        if (isNaN(count)) return Promise.reject('Given key is not a number');
-        _cache[key] = count + '';
-        return Promise.resolve(count);
-      } else {
-        _cache[key] = '' + increment;
-        return Promise.resolve(1);
-      }
-    },
-    decr(key: string, decrement = 1) {
-      if (key in _cache) {
-        const count = toNumber(_cache[key]) - decrement;
-        if (isNaN(count)) return Promise.reject('Given key is not a number');
-        _cache[key] = count + '';
-        return Promise.resolve(count);
-      } else {
-        _cache[key] = '-' + decrement;
-        return Promise.resolve(-1);
-      }
-    },
-    getMany(keys: string[]) {
-      return Promise.resolve(keys.map(key => _cache[key] ? _cache[key] as string : null));
-    },
-    pushItems(key: string, items: string[]) { // @ts-ignore
-      if (!(key in _cache)) _cache[key] = [];
-      const list = _cache[key];
-      if (Array.isArray(list)) {
-        list.push(...items);
-        return Promise.resolve();
-      }
-      return Promise.reject('key is not a list');
-    },
-    popItems(key: string, count: number) {
-      const list = _cache[key];
-      return Promise.resolve(Array.isArray(list) ? list.splice(0, count) : []);
-    },
-    getItemsCount(key: string) {
-      const list = _cache[key];
-      return Promise.resolve(Array.isArray(list) ? list.length : 0);
-    },
-    itemContains(key: string, item: string) {
-      const set = _cache[key];
-      if (!set) return Promise.resolve(false);
-      if (set instanceof Set) return Promise.resolve(set.has(item));
-      return Promise.reject('key is not a set');
-    },
-    addItems(key: string, items: string[]) {
-      if (!(key in _cache)) _cache[key] = new Set();
-      const set = _cache[key];
-      if (set instanceof Set) {
-        items.forEach(item => set.add(item));
-        return Promise.resolve();
-      }
-      return Promise.reject('key is not a set');
-    },
-    removeItems(key: string, items: string[]) {
-      if (!(key in _cache)) _cache[key] = new Set();
-      const set = _cache[key];
-      if (set instanceof Set) {
-        items.forEach(item => set.delete(item));
-        return Promise.resolve();
-      }
-      return Promise.reject('key is not a set');
-    },
-    getItems(key: string) {
-      const set = _cache[key];
-      if (!set) return Promise.resolve([]);
-      if (set instanceof Set) return Promise.resolve(setToArray(set));
-      return Promise.reject('key is not a set');
-    },
-
-    // always connects and disconnects
-    connect() {
-      if (typeof _connDelay === 'number') {
-        return new Promise(res => setTimeout(res, _connDelay));
-      } else {
-        return Promise.resolve();
-      }
-    },
-    disconnect() { return Promise.resolve(); },
-
-    // for testing
-    _setConnDelay(connDelay: number) {
-      _connDelay = connDelay;
-    }
-  };
-}

package/src/storages/pluggable/index.ts

@@ -1,152 +0,0 @@
-import { IPluggableStorageWrapper, IStorageAsync, IStorageAsyncFactory, IStorageFactoryParams, ITelemetryCacheAsync } from '../types';
-
-import { KeyBuilderSS } from '../KeyBuilderSS';
-import { SplitsCachePluggable } from './SplitsCachePluggable';
-import { SegmentsCachePluggable } from './SegmentsCachePluggable';
-import { ImpressionsCachePluggable } from './ImpressionsCachePluggable';
-import { EventsCachePluggable } from './EventsCachePluggable';
-import { wrapperAdapter, METHODS_TO_PROMISE_WRAP } from './wrapperAdapter';
-import { isObject } from '../../utils/lang';
-import { getStorageHash, validatePrefix } from '../KeyBuilder';
-import { CONSUMER_PARTIAL_MODE, STORAGE_PLUGGABLE } from '../../utils/constants';
-import { ImpressionsCacheInMemory } from '../inMemory/ImpressionsCacheInMemory';
-import { EventsCacheInMemory } from '../inMemory/EventsCacheInMemory';
-import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
-import { shouldRecordTelemetry, TelemetryCacheInMemory } from '../inMemory/TelemetryCacheInMemory';
-import { TelemetryCachePluggable } from './TelemetryCachePluggable';
-import { ImpressionCountsCachePluggable } from './ImpressionCountsCachePluggable';
-import { UniqueKeysCachePluggable } from './UniqueKeysCachePluggable';
-import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
-import { UniqueKeysCacheInMemoryCS } from '../inMemory/UniqueKeysCacheInMemoryCS';
-import { metadataBuilder } from '../utils';
-import { LOG_PREFIX } from '../pluggable/constants';
-import { RBSegmentsCachePluggable } from './RBSegmentsCachePluggable';
-
-const NO_VALID_WRAPPER = 'Expecting pluggable storage `wrapper` in options, but no valid wrapper instance was provided.';
-const NO_VALID_WRAPPER_INTERFACE = 'The provided wrapper instance doesn’t follow the expected interface. Check our docs.';
-
-export interface PluggableStorageOptions {
-  prefix?: string
-  wrapper: IPluggableStorageWrapper
-}
-
-/**
- * Validate pluggable storage factory options.
- *
- * @param options - user options
- * @throws Will throw an error if the options are invalid. Example: wrapper is not provided or doesn't have some methods.
- */
-function validatePluggableStorageOptions(options: any) {
-  if (!isObject(options) || !isObject(options.wrapper)) throw new Error(NO_VALID_WRAPPER);
-
-  const wrapper = options.wrapper;
-  const missingMethods = METHODS_TO_PROMISE_WRAP.filter(method => typeof wrapper[method] !== 'function');
-  if (missingMethods.length) throw new Error(`${NO_VALID_WRAPPER_INTERFACE} The following methods are missing or invalid: ${missingMethods}`);
-}
-
-// Async return type in `client.track` method on consumer partial mode
-// No need to promisify impressions cache
-function promisifyEventsTrack(events: any) {
-  const origTrack = events.track;
-  events.track = function () {
-    return Promise.resolve(origTrack.apply(this, arguments));
-  };
-  return events;
-}
-
-/**
- * Pluggable storage factory for consumer server-side & client-side SplitFactory.
- */
-export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyncFactory {
-
-  validatePluggableStorageOptions(options);
-
-  const prefix = validatePrefix(options.prefix);
-
-  function PluggableStorageFactory(params: IStorageFactoryParams): IStorageAsync {
-    const { onReadyCb, settings, settings: { log, mode, scheduler: { impressionsQueueSize, eventsQueueSize } } } = params;
-    const metadata = metadataBuilder(settings);
-    const keys = new KeyBuilderSS(prefix, metadata);
-    const wrapper = wrapperAdapter(log, options.wrapper);
-
-    const isSynchronizer = mode === undefined; // If mode is not defined, the synchronizer is running
-    const isPartialConsumer = mode === CONSUMER_PARTIAL_MODE;
-
-    const telemetry = shouldRecordTelemetry(params) || isSynchronizer ?
-      isPartialConsumer ?
-        new TelemetryCacheInMemory() :
-        new TelemetryCachePluggable(log, keys, wrapper) :
-      undefined;
-
-    const impressionCountsCache = isPartialConsumer ?
-      new ImpressionCountsCacheInMemory() :
-      new ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper);
-
-    const uniqueKeysCache = isPartialConsumer ?
-      settings.core.key === undefined ? new UniqueKeysCacheInMemory() : new UniqueKeysCacheInMemoryCS() :
-      new UniqueKeysCachePluggable(log, keys.buildUniqueKeysKey(), wrapper);
-
-    // Connects to wrapper and emits SDK_READY event on main client
-    const connectPromise = wrapper.connect().then(() => {
-      if (isSynchronizer) {
-        // @TODO reuse InLocalStorage::validateCache logic
-        // In standalone or producer mode, clear storage if SDK key, flags filter criteria or flags spec version was modified
-        return wrapper.get(keys.buildHashKey()).then((hash) => {
-          const currentHash = getStorageHash(settings);
-          if (hash !== currentHash) {
-            log.info(LOG_PREFIX + 'Storage HASH has changed (SDK key, flags filter criteria or flags spec version was modified). Clearing cache');
-            return wrapper.getKeysByPrefix(`${keys.prefix}.`).then(storageKeys => {
-              return Promise.all(storageKeys.map(storageKey => wrapper.del(storageKey)));
-            }).then(() => wrapper.set(keys.buildHashKey(), currentHash));
-          }
-        }).then(() => {
-          onReadyCb();
-        });
-      } else {
-        // Start periodic flush of async storages if not running synchronizer (producer mode)
-        if ((impressionCountsCache as ImpressionCountsCachePluggable).start) (impressionCountsCache as ImpressionCountsCachePluggable).start();
-        if ((uniqueKeysCache as UniqueKeysCachePluggable).start) (uniqueKeysCache as UniqueKeysCachePluggable).start();
-        if (telemetry && (telemetry as ITelemetryCacheAsync).recordConfig) (telemetry as ITelemetryCacheAsync).recordConfig();
-
-        onReadyCb();
-      }
-    }).catch((e) => {
-      e = e || new Error('Error connecting wrapper');
-      onReadyCb(e);
-      return e; // Propagate error for shared clients
-    });
-
-    return {
-      splits: new SplitsCachePluggable(log, keys, wrapper, settings.sync.__splitFiltersValidation),
-      rbSegments: new RBSegmentsCachePluggable(log, keys, wrapper),
-      segments: new SegmentsCachePluggable(log, keys, wrapper),
-      impressions: isPartialConsumer ? new ImpressionsCacheInMemory(impressionsQueueSize) : new ImpressionsCachePluggable(log, keys.buildImpressionsKey(), wrapper, metadata),
-      impressionCounts: impressionCountsCache,
-      events: isPartialConsumer ? promisifyEventsTrack(new EventsCacheInMemory(eventsQueueSize)) : new EventsCachePluggable(log, keys.buildEventsKey(), wrapper, metadata),
-      telemetry,
-      uniqueKeys: uniqueKeysCache,
-
-      // Stop periodic flush and disconnect the underlying storage
-      destroy() {
-        return Promise.all(isSynchronizer ? [] : [
-          (impressionCountsCache as ImpressionCountsCachePluggable).stop && (impressionCountsCache as ImpressionCountsCachePluggable).stop(),
-          (uniqueKeysCache as UniqueKeysCachePluggable).stop && (uniqueKeysCache as UniqueKeysCachePluggable).stop(),
-        ]).then(() => wrapper.disconnect());
-      },
-
-      // emits SDK_READY event on shared clients and returns a reference to the storage
-      shared(_, onReadyCb) {
-        connectPromise.then(onReadyCb);
-
-        return {
-          ...this,
-          // no-op destroy, to disconnect the wrapper only when the main client is destroyed
-          destroy() { }
-        };
-      }
-    };
-  }
-
-  PluggableStorageFactory.type = STORAGE_PLUGGABLE;
-  return PluggableStorageFactory;
-}

package/src/storages/pluggable/wrapperAdapter.ts

@@ -1,58 +0,0 @@
-import { ILogger } from '../../logger/types';
-import { IPluggableStorageWrapper } from '../types';
-import { LOG_PREFIX } from './constants';
-
-export const METHODS_TO_PROMISE_WRAP: string[] = [
-  'get',
-  'set',
-  'getAndSet',
-  'del',
-  'getKeysByPrefix',
-  'incr',
-  'decr',
-  'getMany',
-  'pushItems',
-  'popItems',
-  'getItemsCount',
-  'itemContains',
-  'addItems',
-  'removeItems',
-  'getItems',
-  'connect',
-  'disconnect'
-];
-
-/**
- * Adapter of the Pluggable Storage Wrapper.
- * Used to handle exceptions as rejected promises, in order to simplify the error handling on storages.
- *
- * @param log - logger instance
- * @param wrapper - storage wrapper to adapt
- * @returns an adapted version of the given storage wrapper
- */
-export function wrapperAdapter(log: ILogger, wrapper: IPluggableStorageWrapper): IPluggableStorageWrapper {
-
-  const wrapperAdapter: Record<string, Function> = {};
-
-  METHODS_TO_PROMISE_WRAP.forEach((method) => {
-
-    // Logs error and wraps it into a rejected promise.
-    function handleError(e: any) {
-      log.error(`${LOG_PREFIX} Wrapper '${method}' operation threw an error. Message: ${e}`);
-      return Promise.reject(e);
-    }
-
-    wrapperAdapter[method] = function () {
-      try {
-        // @ts-ignore
-        return wrapper[method].apply(wrapper, arguments).then(value => value).catch(handleError);
-      } catch (e) {
-        return handleError(e);
-      }
-    };
-
-  });
-
-  // @ts-ignore
-  return wrapperAdapter;
-}