@splitsoftware/splitio-commons 2.5.0-rc.0 → 11.5.0-rc.0

package/src/storages/inRedis/ImpressionCountsCacheInRedis.ts

@@ -1,95 +0,0 @@
-import { ILogger } from '../../logger/types';
-import { ImpressionCountsPayload } from '../../sync/submitters/types';
-import { forOwn } from '../../utils/lang';
-import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
-import { LOG_PREFIX, REFRESH_RATE, TTL_REFRESH } from './constants';
-import type { RedisAdapter } from './RedisAdapter';
-
-export class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory {
-
-  private readonly log: ILogger;
-  private readonly key: string;
-  private readonly redis: RedisAdapter;
-  private readonly refreshRate: number;
-  private intervalId: any;
-
-  constructor(log: ILogger, key: string, redis: RedisAdapter, impressionCountsCacheSize?: number, refreshRate = REFRESH_RATE) {
-    super(impressionCountsCacheSize);
-    this.log = log;
-    this.key = key;
-    this.redis = redis;
-    this.refreshRate = refreshRate;
-    this.onFullQueue = () => { this.postImpressionCountsInRedis(); };
-  }
-
-  private postImpressionCountsInRedis() {
-    const counts = this.pop();
-    const keys = Object.keys(counts);
-    if (!keys.length) return Promise.resolve(false);
-
-    const pipeline = this.redis.pipeline();
-    keys.forEach(key => {
-      pipeline.hincrby(this.key, key, counts[key]);
-    });
-    return pipeline.exec()
-      .then(data => {
-        // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
-        if (data.length && data.length === keys.length) {
-          return this.redis.expire(this.key, TTL_REFRESH);
-        }
-      })
-      .catch(err => {
-        this.log.error(`${LOG_PREFIX}Error in impression counts pipeline: ${err}.`);
-        return false;
-      });
-  }
-
-  start() {
-    this.intervalId = setInterval(this.postImpressionCountsInRedis.bind(this), this.refreshRate);
-  }
-
-  stop() {
-    clearInterval(this.intervalId);
-    return this.postImpressionCountsInRedis();
-  }
-
-  // Async consumer API, used by synchronizer
-  getImpressionsCount(): Promise<ImpressionCountsPayload | undefined> {
-    return this.redis.hgetall(this.key)
-      .then(counts => {
-        if (!Object.keys(counts).length) return undefined;
-
-        this.redis.del(this.key).catch(() => { /* no-op */ });
-
-        const pf: ImpressionCountsPayload['pf'] = [];
-
-        forOwn(counts, (count, key) => {
-          const nameAndTime = key.split('::');
-          if (nameAndTime.length !== 2) {
-            this.log.error(`${LOG_PREFIX}Error spliting key ${key}`);
-            return;
-          }
-
-          const timeFrame = parseInt(nameAndTime[1]);
-          if (isNaN(timeFrame)) {
-            this.log.error(`${LOG_PREFIX}Error parsing time frame ${nameAndTime[1]}`);
-            return;
-          }
-
-          const rawCount = parseInt(count);
-          if (isNaN(rawCount)) {
-            this.log.error(`${LOG_PREFIX}Error parsing raw count ${count}`);
-            return;
-          }
-
-          pf.push({
-            f: nameAndTime[0],
-            m: timeFrame,
-            rc: rawCount,
-          });
-        });
-
-        return { pf };
-      });
-  }
-}

package/src/storages/inRedis/ImpressionsCacheInRedis.ts

@@ -1,61 +0,0 @@
-import { IImpressionsCacheAsync } from '../types';
-import { IMetadata } from '../../dtos/types';
-import SplitIO from '../../../types/splitio';
-import { StoredImpressionWithMetadata } from '../../sync/submitters/types';
-import { ILogger } from '../../logger/types';
-import { impressionsToJSON } from '../utils';
-import type { RedisAdapter } from './RedisAdapter';
-
-const IMPRESSIONS_TTL_REFRESH = 3600; // 1 hr
-
-export class ImpressionsCacheInRedis implements IImpressionsCacheAsync {
-
-  private readonly log: ILogger;
-  private readonly key: string;
-  private readonly redis: RedisAdapter;
-  private readonly metadata: IMetadata;
-
-  constructor(log: ILogger, key: string, redis: RedisAdapter, metadata: IMetadata) {
-    this.log = log;
-    this.key = key;
-    this.redis = redis;
-    this.metadata = metadata;
-  }
-
-  track(impressions: SplitIO.ImpressionDTO[]): Promise<void> { // @ts-ignore
-    return this.redis.rpush(
-      this.key,
-      impressionsToJSON(impressions, this.metadata),
-    ).then(queuedCount => {
-      // If this is the creation of the key on Redis, set the expiration for it in 1hr.
-      if (queuedCount === impressions.length) {
-        return this.redis.expire(this.key, IMPRESSIONS_TTL_REFRESH);
-      }
-    });
-  }
-
-  count(): Promise<number> {
-    return this.redis.llen(this.key).catch(() => 0);
-  }
-
-  drop(count?: number): Promise<any> {
-    if (!count) return this.redis.del(this.key);
-
-    return this.redis.ltrim(this.key, count, -1);
-  }
-
-  popNWithMetadata(count: number): Promise<StoredImpressionWithMetadata[]> {
-    return this.redis.lrange(this.key, 0, count - 1).then(items => {
-      return this.redis.ltrim(this.key, items.length, -1).then(() => {
-        // This operation will simply do nothing if the key no longer exists (queue is empty)
-        // It's only done in the "successful" exit path so that the TTL is not overriden if impressons weren't
-        // popped correctly. This will result in impressions getting lost but will prevent the queue from taking
-        // a huge amount of memory.
-        this.redis.expire(this.key, IMPRESSIONS_TTL_REFRESH).catch(() => { }); // noop catch handler
-
-        return items.map(item => JSON.parse(item) as StoredImpressionWithMetadata);
-      });
-    });
-  }
-
-}

package/src/storages/inRedis/RBSegmentsCacheInRedis.ts

@@ -1,79 +0,0 @@
-import { isNaNNumber } from '../../utils/lang';
-import { IRBSegmentsCacheAsync } from '../types';
-import { ILogger } from '../../logger/types';
-import { IRBSegment } from '../../dtos/types';
-import { LOG_PREFIX } from './constants';
-import { setToArray } from '../../utils/lang/sets';
-import { RedisAdapter } from './RedisAdapter';
-import { KeyBuilderSS } from '../KeyBuilderSS';
-
-export class RBSegmentsCacheInRedis implements IRBSegmentsCacheAsync {
-
-  private readonly log: ILogger;
-  private readonly keys: KeyBuilderSS;
-  private readonly redis: RedisAdapter;
-
-  constructor(log: ILogger, keys: KeyBuilderSS, redis: RedisAdapter) {
-    this.log = log;
-    this.keys = keys;
-    this.redis = redis;
-  }
-
-  get(name: string): Promise<IRBSegment | null> {
-    return this.redis.get(this.keys.buildRBSegmentKey(name))
-      .then(maybeRBSegment => maybeRBSegment && JSON.parse(maybeRBSegment));
-  }
-
-  private getNames(): Promise<string[]> {
-    return this.redis.keys(this.keys.searchPatternForRBSegmentKeys()).then(
-      (listOfKeys) => listOfKeys.map(this.keys.extractKey)
-    );
-  }
-
-  contains(names: Set<string>): Promise<boolean> {
-    const namesArray = setToArray(names);
-    return this.getNames().then(namesInStorage => {
-      return namesArray.every(name => namesInStorage.includes(name));
-    });
-  }
-
-  update(toAdd: IRBSegment[], toRemove: IRBSegment[], changeNumber: number): Promise<boolean> {
-    return Promise.all([
-      this.setChangeNumber(changeNumber),
-      Promise.all(toAdd.map(toAdd => {
-        const key = this.keys.buildRBSegmentKey(toAdd.name);
-        const stringifiedNewRBSegment = JSON.stringify(toAdd);
-        return this.redis.set(key, stringifiedNewRBSegment).then(() => true);
-      })),
-      Promise.all(toRemove.map(toRemove => {
-        const key = this.keys.buildRBSegmentKey(toRemove.name);
-        return this.redis.del(key).then(status => status === 1);
-      }))
-    ]).then(([, added, removed]) => {
-      return added.some(result => result) || removed.some(result => result);
-    });
-  }
-
-  setChangeNumber(changeNumber: number) {
-    return this.redis.set(this.keys.buildRBSegmentsTillKey(), changeNumber + '').then(
-      status => status === 'OK'
-    );
-  }
-
-  getChangeNumber(): Promise<number> {
-    return this.redis.get(this.keys.buildRBSegmentsTillKey()).then((value: string | null) => {
-      const i = parseInt(value as string, 10);
-
-      return isNaNNumber(i) ? -1 : i;
-    }).catch((e) => {
-      this.log.error(LOG_PREFIX + 'Could not retrieve changeNumber from storage. Error: ' + e);
-      return -1;
-    });
-  }
-
-  // @TODO implement if required by DataLoader or producer mode
-  clear() {
-    return Promise.resolve();
-  }
-
-}

package/src/storages/inRedis/RedisAdapter.ts

@@ -1,209 +0,0 @@
-import ioredis, { Pipeline } from 'ioredis';
-import { ILogger } from '../../logger/types';
-import { merge, isString } from '../../utils/lang';
-import { thenable } from '../../utils/promise/thenable';
-import { timeout } from '../../utils/promise/timeout';
-import { setToArray } from '../../utils/lang/sets';
-
-const LOG_PREFIX = 'storage:redis-adapter: ';
-
-// If we ever decide to fully wrap every method, there's a Commander.getBuiltinCommands from ioredis.
-const METHODS_TO_PROMISE_WRAP = ['set', 'exec', 'del', 'get', 'keys', 'sadd', 'srem', 'sismember', 'smembers', 'incr', 'rpush', 'expire', 'mget', 'lrange', 'ltrim', 'hset', 'hincrby', 'popNRaw'];
-const METHODS_TO_PROMISE_WRAP_EXEC = ['pipeline'];
-
-// Not part of the settings since it'll vary on each storage. We should be removing storage specific logic from elsewhere.
-const DEFAULT_OPTIONS = {
-  connectionTimeout: 10000,
-  operationTimeout: 5000
-};
-// Library specifics.
-const DEFAULT_LIBRARY_OPTIONS = {
-  enableOfflineQueue: false,
-  connectTimeout: DEFAULT_OPTIONS.connectionTimeout,
-  lazyConnect: false
-};
-
-interface IRedisCommand {
-  resolve: () => void,
-  reject: (err?: any) => void,
-  command: () => Promise<void>,
-  name: string
-}
-
-/**
- * Redis adapter on top of the library of choice (written with ioredis) for some extra control.
- */
-export class RedisAdapter extends ioredis {
-  private readonly log: ILogger;
-  private _options: object;
-  private _notReadyCommandsQueue?: IRedisCommand[];
-  private _runningCommands: Set<Promise<any>>;
-
-  constructor(log: ILogger, storageSettings: Record<string, any> = {}) {
-    const options = RedisAdapter._defineOptions(storageSettings);
-    // Call the ioredis constructor
-    super(...RedisAdapter._defineLibrarySettings(options));
-
-    this.log = log;
-    this._options = options;
-    this._notReadyCommandsQueue = [];
-    this._runningCommands = new Set();
-    this._listenToEvents();
-    this._setTimeoutWrappers();
-    this._setDisconnectWrapper();
-  }
-
-  _listenToEvents() {
-    this.once('ready', () => {
-      const commandsCount = this._notReadyCommandsQueue ? this._notReadyCommandsQueue.length : 0;
-      this.log.info(LOG_PREFIX + `Redis connection established. Queued commands: ${commandsCount}.`);
-
-      this._notReadyCommandsQueue && this._notReadyCommandsQueue.forEach(queued => {
-        this.log.info(LOG_PREFIX + `Executing queued ${queued.name} command.`);
-        queued.command().then(queued.resolve).catch(queued.reject);
-      });
-      // After the SDK is ready for the first time we'll stop queueing commands. This is just so we can keep handling BUR for them.
-      this._notReadyCommandsQueue = undefined;
-    });
-    this.once('close', () => {
-      this.log.info(LOG_PREFIX + 'Redis connection closed.');
-    });
-  }
-
-  _setTimeoutWrappers() {
-    const instance: Record<string, any> = this;
-
-    const wrapCommand = (originalMethod: Function, methodName: string) => {
-      // The value of "this" in this function should be the instance actually executing the method. It might be the instance referred (the base one)
-      // or it can be the instance of a Pipeline object.
-      return function (this: RedisAdapter | Pipeline) {
-        const params = arguments;
-        const caller = this;
-
-        function commandWrapper() {
-          instance.log.debug(`${LOG_PREFIX}Executing ${methodName}.`);
-          const result = originalMethod.apply(caller, params);
-
-          if (thenable(result)) {
-            // For handling pending commands on disconnect, add to the set and remove once finished.
-            // On sync commands there's no need, only thenables.
-            instance._runningCommands.add(result);
-            const cleanUpRunningCommandsCb = function () {
-              instance._runningCommands.delete(result);
-            };
-            // Both success and error remove from queue.
-            result.then(cleanUpRunningCommandsCb, cleanUpRunningCommandsCb);
-
-            return timeout(instance._options.operationTimeout, result).catch(err => {
-              instance.log.error(`${LOG_PREFIX}${methodName} operation threw an error or exceeded configured timeout of ${instance._options.operationTimeout}ms. Message: ${err}`);
-              // Handling is not the adapter responsibility.
-              throw err;
-            });
-          }
-
-          return result;
-        }
-
-        if (instance._notReadyCommandsQueue) {
-          return new Promise((resolve, reject) => {
-            instance._notReadyCommandsQueue.unshift({
-              resolve,
-              reject,
-              command: commandWrapper,
-              name: methodName.toUpperCase()
-            });
-          });
-        } else {
-          return commandWrapper();
-        }
-      };
-    };
-
-    // Wrap regular async methods to track timeouts and queue when Redis is not yet executing commands.
-    METHODS_TO_PROMISE_WRAP.forEach(methodName => {
-      const originalFn = instance[methodName];
-      instance[methodName] = wrapCommand(originalFn, methodName);
-    });
-
-    // Special handling for pipeline~like methods. We need to wrap the async trigger, which is exec, but return the Pipeline right away.
-    METHODS_TO_PROMISE_WRAP_EXEC.forEach(methodName => {
-      const originalFn = instance[methodName];
-      // "First level wrapper" to handle the sync execution and wrap async, queueing later if applicable.
-      instance[methodName] = function () {
-        const res = originalFn.apply(instance, arguments);
-        const originalExec = res.exec;
-
-        res.exec = wrapCommand(originalExec, methodName + '.exec').bind(res);
-
-        return res;
-      };
-    });
-  }
-
-  _setDisconnectWrapper() {
-    const instance = this;
-    const originalMethod = instance.disconnect;
-
-    instance.disconnect = function disconnect(...params: []) {
-
-      setTimeout(function deferredDisconnect() {
-        if (instance._runningCommands.size > 0) {
-          instance.log.info(LOG_PREFIX + `Attempting to disconnect but there are ${instance._runningCommands.size} commands still waiting for resolution. Defering disconnection until those finish.`);
-
-          Promise.all(setToArray(instance._runningCommands))
-            .then(() => {
-              instance.log.debug(LOG_PREFIX + 'Pending commands finished successfully, disconnecting.');
-              originalMethod.apply(instance, params);
-            })
-            .catch(e => {
-              instance.log.warn(LOG_PREFIX + `Pending commands finished with error: ${e}. Proceeding with disconnection.`);
-              originalMethod.apply(instance, params);
-            });
-        } else {
-          instance.log.debug(LOG_PREFIX + 'No commands pending execution, disconnect.');
-          // Nothing pending, just proceed.
-          originalMethod.apply(instance, params);
-        }
-      }, 10);
-    };
-  }
-
-  /**
-   * Receives the options and returns an array of parameters for the ioredis constructor.
-   * Keeping both redis setup options for backwards compatibility.
-   */
-  static _defineLibrarySettings(options: Record<string, any>) {
-    const opts = merge({}, DEFAULT_LIBRARY_OPTIONS);
-    const result: any[] = [opts];
-
-    if (!isString(options.url)) {
-      merge(opts, { // If it's not the string URL, merge the params separately.
-        host: options.host,
-        port: options.port,
-        db: options.db,
-        password: options.pass
-      });
-    } else { // If it IS the string URL, that'll be the first param for ioredis.
-      result.unshift(options.url);
-    }
-    if (options.connectionTimeout) {
-      merge(opts, { connectTimeout: options.connectionTimeout });
-    }
-    if (options.tls) {
-      merge(opts, { tls: options.tls });
-    }
-
-    return result;
-  }
-
-  /**
-   * Parses the options into what we care about.
-   */
-  static _defineOptions({ connectionTimeout, operationTimeout, url, host, port, db, pass, tls }: Record<string, any>) {
-    const parsedOptions = {
-      connectionTimeout, operationTimeout, url, host, port, db, pass, tls
-    };
-
-    return merge({}, DEFAULT_OPTIONS, parsedOptions);
-  }
-}

package/src/storages/inRedis/SegmentsCacheInRedis.ts

@@ -1,70 +0,0 @@
-import { ILogger } from '../../logger/types';
-import { isNaNNumber } from '../../utils/lang';
-import { LOG_PREFIX } from '../inLocalStorage/constants';
-import { KeyBuilderSS } from '../KeyBuilderSS';
-import { ISegmentsCacheAsync } from '../types';
-import type { RedisAdapter } from './RedisAdapter';
-
-export class SegmentsCacheInRedis implements ISegmentsCacheAsync {
-
-  private readonly log: ILogger;
-  private readonly redis: RedisAdapter;
-  private readonly keys: KeyBuilderSS;
-
-  constructor(log: ILogger, keys: KeyBuilderSS, redis: RedisAdapter) {
-    this.log = log;
-    this.redis = redis;
-    this.keys = keys;
-  }
-
-  /**
-   * Update the given segment `name` with the lists of `addedKeys`, `removedKeys` and `changeNumber`.
-   * The returned promise is resolved if the operation success, with `true` if the segment was updated (i.e., some key was added or removed),
-   * or rejected if it fails (e.g., Redis operation fails).
-   */
-  update(name: string, addedKeys: string[], removedKeys: string[], changeNumber: number) {
-    const segmentKey = this.keys.buildSegmentNameKey(name);
-
-    return Promise.all([
-      addedKeys.length && this.redis.sadd(segmentKey, addedKeys),
-      removedKeys.length && this.redis.srem(segmentKey, removedKeys),
-      this.redis.set(this.keys.buildSegmentTillKey(name), changeNumber + '')
-    ]).then(() => {
-      return addedKeys.length > 0 || removedKeys.length > 0;
-    });
-  }
-
-  isInSegment(name: string, key: string) {
-    return this.redis.sismember(
-      this.keys.buildSegmentNameKey(name), key
-    ).then(matches => matches !== 0);
-  }
-
-  getChangeNumber(name: string) {
-    return this.redis.get(this.keys.buildSegmentTillKey(name)).then((value: string | null) => {
-      const i = parseInt(value as string, 10);
-
-      return isNaNNumber(i) ? undefined : i;
-    }).catch((e) => {
-      this.log.error(LOG_PREFIX + 'Could not retrieve changeNumber from segments storage. Error: ' + e);
-      return undefined;
-    });
-  }
-
-  registerSegments(segments: string[]) {
-    if (segments.length) {
-      return this.redis.sadd(this.keys.buildRegisteredSegmentsKey(), segments).then(() => true);
-    } else {
-      return Promise.resolve(true);
-    }
-  }
-
-  getRegisteredSegments() {
-    return this.redis.smembers(this.keys.buildRegisteredSegmentsKey());
-  }
-
-  // @TODO remove or implement. It is not being used.
-  clear() {
-    return Promise.resolve();
-  }
-}