@splitsoftware/splitio-commons 1.6.2-rc.9 → 1.7.0

package/src/storages/inMemory/UniqueKeysCacheInMemoryCS.ts

@@ -0,0 +1,86 @@
+import { IUniqueKeysCacheBase } from '../types';
+import { ISet, setToArray, _Set } from '../../utils/lang/sets';
+import { UniqueKeysPayloadCs } from '../../sync/submitters/types';
+import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
+
+export class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
+
+  private onFullQueue?: () => void;
+  private readonly maxStorage: number;
+  private uniqueTrackerSize = 0;
+  private uniqueKeysTracker: { [userKey: string]: ISet<string> };
+
+  /**
+   *
+   * @param impressionsQueueSize number of queued impressions to call onFullQueueCb.
+   * Default value is 0, that means no maximum value, in case we want to avoid this being triggered.
+   */
+  constructor(uniqueKeysQueueSize = DEFAULT_CACHE_SIZE) {
+    this.maxStorage = uniqueKeysQueueSize;
+    this.uniqueKeysTracker = {};
+  }
+
+  setOnFullQueueCb(cb: () => void) {
+    this.onFullQueue = cb;
+  }
+
+  /**
+   * Store unique keys per feature.
+   */
+  track(userKey: string, featureName: string) {
+
+    if (!this.uniqueKeysTracker[userKey]) this.uniqueKeysTracker[userKey] = new _Set();
+    const tracker = this.uniqueKeysTracker[userKey];
+    if (!tracker.has(featureName)) {
+      tracker.add(featureName);
+      this.uniqueTrackerSize++;
+    }
+    if (this.uniqueTrackerSize >= this.maxStorage && this.onFullQueue) {
+      this.uniqueTrackerSize = 0;
+      this.onFullQueue();
+    }
+  }
+
+  /**
+   * Clear the data stored on the cache.
+   */
+  clear() {
+    this.uniqueKeysTracker = {};
+  }
+
+  /**
+   * Pop the collected data, used as payload for posting.
+   */
+  pop() {
+    const data = this.uniqueKeysTracker;
+    this.uniqueKeysTracker = {};
+    return this.fromUniqueKeysCollector(data);
+  }
+
+  /**
+   * Check if the cache is empty.
+   */
+  isEmpty() {
+    return Object.keys(this.uniqueKeysTracker).length === 0;
+  }
+
+  /**
+   * Converts `uniqueKeys` data from cache into request payload.
+   */
+  private fromUniqueKeysCollector(uniqueKeys: { [userKey: string]: ISet<string> }): UniqueKeysPayloadCs {
+    const payload = [];
+    const userKeys = Object.keys(uniqueKeys);
+    for (let k = 0; k < userKeys.length; k++) {
+      const userKey = userKeys[k];
+      const featureNames = setToArray(uniqueKeys[userKey]);
+      const uniqueKeysPayload = {
+        k: userKey,
+        fs: featureNames
+      };
+
+      payload.push(uniqueKeysPayload);
+    }
+    return { keys: payload };
+  }
+
+}

package/src/storages/inRedis/ImpressionCountsCacheInRedis.ts

@@ -0,0 +1,95 @@
+import { Redis } from 'ioredis';
+import { ILogger } from '../../logger/types';
+import { ImpressionCountsPayload } from '../../sync/submitters/types';
+import { forOwn } from '../../utils/lang';
+import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
+import { LOG_PREFIX, REFRESH_RATE, TTL_REFRESH } from './constants';
+
+export class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory {
+
+  private readonly log: ILogger;
+  private readonly key: string;
+  private readonly redis: Redis;
+  private readonly refreshRate: number;
+  private intervalId: any;
+
+  constructor(log: ILogger, key: string, redis: Redis, impressionCountsCacheSize?: number, refreshRate = REFRESH_RATE) {
+    super(impressionCountsCacheSize);
+    this.log = log;
+    this.key = key;
+    this.redis = redis;
+    this.refreshRate = refreshRate;
+    this.onFullQueue = () => { this.postImpressionCountsInRedis(); };
+  }
+
+  private postImpressionCountsInRedis() {
+    const counts = this.pop();
+    const keys = Object.keys(counts);
+    if (!keys.length) return Promise.resolve(false);
+
+    const pipeline = this.redis.pipeline();
+    keys.forEach(key => {
+      pipeline.hincrby(this.key, key, counts[key]);
+    });
+    return pipeline.exec()
+      .then(data => {
+        // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
+        if (data.length && data.length === keys.length) {
+          return this.redis.expire(this.key, TTL_REFRESH);
+        }
+      })
+      .catch(err => {
+        this.log.error(`${LOG_PREFIX}Error in impression counts pipeline: ${err}.`);
+        return false;
+      });
+  }
+
+  start() {
+    this.intervalId = setInterval(this.postImpressionCountsInRedis.bind(this), this.refreshRate);
+  }
+
+  stop() {
+    clearInterval(this.intervalId);
+    return this.postImpressionCountsInRedis();
+  }
+
+  // Async consumer API, used by synchronizer
+  getImpressionsCount(): Promise<ImpressionCountsPayload | undefined> {
+    return this.redis.hgetall(this.key)
+      .then(counts => {
+        if (!Object.keys(counts).length) return undefined;
+
+        this.redis.del(this.key).catch(() => { /* no-op */ });
+
+        const pf: ImpressionCountsPayload['pf'] = [];
+
+        forOwn(counts, (count, key) => {
+          const nameAndTime = key.split('::');
+          if (nameAndTime.length !== 2) {
+            this.log.error(`${LOG_PREFIX}Error spliting key ${key}`);
+            return;
+          }
+
+          const timeFrame = parseInt(nameAndTime[1]);
+          if (isNaN(timeFrame)) {
+            this.log.error(`${LOG_PREFIX}Error parsing time frame ${nameAndTime[1]}`);
+            return;
+          }
+
+          const rawCount = parseInt(count);
+          if (isNaN(rawCount)) {
+            this.log.error(`${LOG_PREFIX}Error parsing raw count ${count}`);
+            return;
+          }
+
+          pf.push({
+            f: nameAndTime[0],
+            m: timeFrame,
+            rc: rawCount,
+          });
+        });
+
+        return { pf };
+      });
+  }
+}

package/src/storages/inRedis/ImpressionsCacheInRedis.ts

@@ -4,6 +4,7 @@ import { ImpressionDTO } from '../../types';
 import { Redis } from 'ioredis';
 import { StoredImpressionWithMetadata } from '../../sync/submitters/types';
 import { ILogger } from '../../logger/types';
+import { impressionsToJSON } from '../utils';
 
 const IMPRESSIONS_TTL_REFRESH = 3600; // 1 hr
 
@@ -24,7 +25,7 @@ export class ImpressionsCacheInRedis implements IImpressionsCacheAsync {
   track(impressions: ImpressionDTO[]): Promise<void> { // @ts-ignore
     return this.redis.rpush(
       this.key,
-      this._toJSON(impressions)
+      impressionsToJSON(impressions, this.metadata),
     ).then(queuedCount => {
       // If this is the creation of the key on Redis, set the expiration for it in 1hr.
       if (queuedCount === impressions.length) {
@@ -33,27 +34,6 @@ export class ImpressionsCacheInRedis implements IImpressionsCacheAsync {
     });
   }
 
-  private _toJSON(impressions: ImpressionDTO[]): string[] {
-    return impressions.map(impression => {
-      const {
-        keyName, bucketingKey, feature, treatment, label, time, changeNumber
-      } = impression;
-
-      return JSON.stringify({
-        m: this.metadata,
-        i: {
-          k: keyName,
-          b: bucketingKey,
-          f: feature,
-          t: treatment,
-          r: label,
-          c: changeNumber,
-          m: time
-        }
-      });
-    });
-  }
-
   count(): Promise<number> {
     return this.redis.llen(this.key).catch(() => 0);
   }

package/src/storages/inRedis/TelemetryCacheInRedis.ts

@@ -1,6 +1,6 @@
 import { ILogger } from '../../logger/types';
 import { Method, MultiConfigs, MultiMethodExceptions, MultiMethodLatencies } from '../../sync/submitters/types';
-import { KeyBuilderSS, parseExceptionField, parseLatencyField, parseMetadata } from '../KeyBuilderSS';
+import { KeyBuilderSS } from '../KeyBuilderSS';
 import { ITelemetryCacheAsync } from '../types';
 import { findLatencyIndex } from '../findLatencyIndex';
 import { Redis } from 'ioredis';
@@ -9,6 +9,7 @@ import { CONSUMER_MODE, STORAGE_REDIS } from '../../utils/constants';
 import { isNaNNumber, isString } from '../../utils/lang';
 import { _Map } from '../../utils/lang/maps';
 import { MAX_LATENCY_BUCKET_COUNT, newBuckets } from '../inMemory/TelemetryCacheInMemory';
+import { parseLatencyField, parseExceptionField, parseMetadata } from '../utils';
 
 export class TelemetryCacheInRedis implements ITelemetryCacheAsync {
 
@@ -76,7 +77,7 @@ export class TelemetryCacheInRedis implements ITelemetryCacheAsync {
           tr: newBuckets(),
         });
 
-        result.get(metadata)![method][bucket] = count;
+        result.get(metadata)![method]![bucket] = count;
       });
 
       return this.redis.del(this.keys.latencyPrefix).then(() => result);

package/src/storages/inRedis/UniqueKeysCacheInRedis.ts

@@ -0,0 +1,77 @@
+import { IUniqueKeysCacheBase } from '../types';
+import { Redis } from 'ioredis';
+import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
+import { setToArray } from '../../utils/lang/sets';
+import { DEFAULT_CACHE_SIZE, REFRESH_RATE, TTL_REFRESH } from './constants';
+import { LOG_PREFIX } from './constants';
+import { ILogger } from '../../logger/types';
+import { UniqueKeysItemSs } from '../../sync/submitters/types';
+
+export class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
+
+  private readonly log: ILogger;
+  private readonly key: string;
+  private readonly redis: Redis;
+  private readonly refreshRate: number;
+  private intervalId: any;
+
+  constructor(log: ILogger, key: string, redis: Redis, uniqueKeysQueueSize = DEFAULT_CACHE_SIZE, refreshRate = REFRESH_RATE) {
+    super(uniqueKeysQueueSize);
+    this.log = log;
+    this.key = key;
+    this.redis = redis;
+    this.refreshRate = refreshRate;
+    this.onFullQueue = () => { this.postUniqueKeysInRedis(); };
+  }
+
+  private postUniqueKeysInRedis() {
+    const featureNames = Object.keys(this.uniqueKeysTracker);
+    if (!featureNames.length) return Promise.resolve(false);
+
+    const pipeline = this.redis.pipeline();
+    for (let i = 0; i < featureNames.length; i++) {
+      const featureName = featureNames[i];
+      const featureKeys = setToArray(this.uniqueKeysTracker[featureName]);
+      const uniqueKeysPayload = {
+        f: featureName,
+        ks: featureKeys
+      };
+
+      pipeline.rpush(this.key, JSON.stringify(uniqueKeysPayload));
+    }
+    this.clear();
+    return pipeline.exec()
+      .then(data => {
+        // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
+        if (data.length && data.length === featureNames.length) {
+          return this.redis.expire(this.key, TTL_REFRESH);
+        }
+      })
+      .catch(err => {
+        this.log.error(`${LOG_PREFIX}Error in uniqueKeys pipeline: ${err}.`);
+        return false;
+      });
+  }
+
+
+  start() {
+    this.intervalId = setInterval(this.postUniqueKeysInRedis.bind(this), this.refreshRate);
+  }
+
+  stop() {
+    clearInterval(this.intervalId);
+    return this.postUniqueKeysInRedis();
+  }
+
+  /**
+   * Async consumer API, used by synchronizer.
+   * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
+   */
+  popNRaw(count = 0): Promise<UniqueKeysItemSs[]> {
+    return this.redis.lrange(this.key, 0, count - 1).then(uniqueKeyItems => {
+      return this.redis.ltrim(this.key, uniqueKeyItems.length, -1)
+        .then(() => uniqueKeyItems.map(uniqueKeyItem => JSON.parse(uniqueKeyItem)));
+    });
+  }
+
+}

package/src/storages/inRedis/constants.ts

@@ -1 +1,4 @@
 export const LOG_PREFIX = 'storage:redis: ';
+export const DEFAULT_CACHE_SIZE = 30000;
+export const REFRESH_RATE = 300000; // 300.000 ms = start after 5 mins
+export const TTL_REFRESH = 3600; // 1hr

package/src/storages/inRedis/index.ts

@@ -6,8 +6,11 @@ import { SplitsCacheInRedis } from './SplitsCacheInRedis';
 import { SegmentsCacheInRedis } from './SegmentsCacheInRedis';
 import { ImpressionsCacheInRedis } from './ImpressionsCacheInRedis';
 import { EventsCacheInRedis } from './EventsCacheInRedis';
-import { STORAGE_REDIS } from '../../utils/constants';
+import { DEBUG, NONE, STORAGE_REDIS } from '../../utils/constants';
 import { TelemetryCacheInRedis } from './TelemetryCacheInRedis';
+import { UniqueKeysCacheInRedis } from './UniqueKeysCacheInRedis';
+import { ImpressionCountsCacheInRedis } from './ImpressionCountsCacheInRedis';
+import { metadataBuilder } from '../utils';
 
 export interface InRedisStorageOptions {
   prefix?: string
@@ -22,15 +25,20 @@ export function InRedisStorage(options: InRedisStorageOptions = {}): IStorageAsy
 
   const prefix = validatePrefix(options.prefix);
 
-  function InRedisStorageFactory({ log, metadata, onReadyCb }: IStorageFactoryParams): IStorageAsync {
-
+  function InRedisStorageFactory(params: IStorageFactoryParams): IStorageAsync {
+    const { onReadyCb, settings, settings: { log, sync: { impressionsMode } } } = params;
+    const metadata = metadataBuilder(settings);
     const keys = new KeyBuilderSS(prefix, metadata);
     const redisClient = new RedisAdapter(log, options.options || {});
     const telemetry = new TelemetryCacheInRedis(log, keys, redisClient);
+    const impressionCountsCache = impressionsMode !== DEBUG ? new ImpressionCountsCacheInRedis(log, keys.buildImpressionsCountKey(), redisClient) : undefined;
+    const uniqueKeysCache = impressionsMode === NONE ? new UniqueKeysCacheInRedis(log, keys.buildUniqueKeysKey(), redisClient) : undefined;
 
     // subscription to Redis connect event in order to emit SDK_READY event on consumer mode
     redisClient.on('connect', () => {
       onReadyCb();
+      if (impressionCountsCache) impressionCountsCache.start();
+      if (uniqueKeysCache) uniqueKeysCache.start();
 
       // Synchronize config
       telemetry.recordConfig();
@@ -40,13 +48,18 @@ export function InRedisStorage(options: InRedisStorageOptions = {}): IStorageAsy
       splits: new SplitsCacheInRedis(log, keys, redisClient),
       segments: new SegmentsCacheInRedis(log, keys, redisClient),
       impressions: new ImpressionsCacheInRedis(log, keys.buildImpressionsKey(), redisClient, metadata),
+      impressionCounts: impressionCountsCache,
       events: new EventsCacheInRedis(log, keys.buildEventsKey(), redisClient, metadata),
       telemetry,
+      uniqueKeys: uniqueKeysCache,
 
       // When using REDIS we should:
       // 1- Disconnect from the storage
-      destroy() {
-        redisClient.disconnect();
+      destroy(): Promise<void> {
+        let promises = [];
+        if (impressionCountsCache) promises.push(impressionCountsCache.stop());
+        if (uniqueKeysCache) promises.push(uniqueKeysCache.stop());
+        return Promise.all(promises).then(() => { redisClient.disconnect(); });
         // @TODO check that caches works as expected when redisClient is disconnected
       }
     };

package/src/storages/pluggable/ImpressionCountsCachePluggable.ts

@@ -0,0 +1,92 @@
+import { ILogger } from '../../logger/types';
+import { ImpressionCountsPayload } from '../../sync/submitters/types';
+import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
+import { REFRESH_RATE } from '../inRedis/constants';
+import { IPluggableStorageWrapper } from '../types';
+import { LOG_PREFIX } from './constants';
+
+export class ImpressionCountsCachePluggable extends ImpressionCountsCacheInMemory {
+
+  private readonly log: ILogger;
+  private readonly key: string;
+  private readonly wrapper: IPluggableStorageWrapper;
+  private readonly refreshRate: number;
+  private intervalId: any;
+
+  constructor(log: ILogger, key: string, wrapper: IPluggableStorageWrapper, impressionCountsCacheSize?: number, refreshRate = REFRESH_RATE) {
+    super(impressionCountsCacheSize);
+    this.log = log;
+    this.key = key;
+    this.wrapper = wrapper;
+    this.refreshRate = refreshRate;
+    this.onFullQueue = () => { this.storeImpressionCounts(); };
+  }
+
+  private storeImpressionCounts() {
+    const counts = this.pop();
+    const keys = Object.keys(counts);
+    if (!keys.length) return Promise.resolve(false);
+
+    const pipeline = keys.reduce<Promise<any>>((pipeline, key) => {
+      return pipeline.then(() => this.wrapper.incr(`${this.key}::${key}`, counts[key]));
+    }, Promise.resolve());
+
+    return pipeline.catch(err => {
+      this.log.error(`${LOG_PREFIX}Error in impression counts pipeline: ${err}.`);
+      return false;
+    });
+  }
+
+  start() {
+    this.intervalId = setInterval(this.storeImpressionCounts.bind(this), this.refreshRate);
+  }
+
+  stop() {
+    clearInterval(this.intervalId);
+    return this.storeImpressionCounts();
+  }
+
+  // Async consumer API, used by synchronizer
+  getImpressionsCount(): Promise<ImpressionCountsPayload | undefined> {
+    return this.wrapper.getKeysByPrefix(this.key)
+      .then(keys => {
+        return keys.length ? Promise.all(keys.map(key => this.wrapper.get(key)))
+          .then(counts => {
+            keys.forEach(key => this.wrapper.del(key).catch(() => { /* noop */ }));
+
+            const pf = [];
+
+            for (let i = 0; i < keys.length; i++) {
+              const key = keys[i];
+              const count = counts[i];
+
+              const keyFeatureNameAndTime = key.split('::');
+              if (keyFeatureNameAndTime.length !== 3) {
+                this.log.error(`${LOG_PREFIX}Error spliting key ${key}`);
+                continue;
+              }
+
+              const timeFrame = parseInt(keyFeatureNameAndTime[2]);
+              if (isNaN(timeFrame)) {
+                this.log.error(`${LOG_PREFIX}Error parsing time frame ${keyFeatureNameAndTime[2]}`);
+                continue;
+              }
+              // @ts-ignore
+              const rawCount = parseInt(count);
+              if (isNaN(rawCount)) {
+                this.log.error(`${LOG_PREFIX}Error parsing raw count ${count}`);
+                continue;
+              }
+
+              pf.push({
+                f: keyFeatureNameAndTime[1],
+                m: timeFrame,
+                rc: rawCount,
+              });
+            }
+
+            return { pf };
+          }) : undefined;
+      });
+  }
+}

package/src/storages/pluggable/ImpressionsCachePluggable.ts

@@ -1,8 +1,9 @@
 import { IPluggableStorageWrapper, IImpressionsCacheAsync } from '../types';
 import { IMetadata } from '../../dtos/types';
 import { ImpressionDTO } from '../../types';
-import { ILogger } from '../../logger/types';
 import { StoredImpressionWithMetadata } from '../../sync/submitters/types';
+import { ILogger } from '../../logger/types';
+import { impressionsToJSON } from '../utils';
 
 export class ImpressionsCachePluggable implements IImpressionsCacheAsync {
 
@@ -27,31 +28,10 @@ export class ImpressionsCachePluggable implements IImpressionsCacheAsync {
   track(impressions: ImpressionDTO[]): Promise<void> {
     return this.wrapper.pushItems(
       this.key,
-      this._toJSON(impressions)
+      impressionsToJSON(impressions, this.metadata)
     );
   }
 
-  private _toJSON(impressions: ImpressionDTO[]): string[] {
-    return impressions.map(impression => {
-      const {
-        keyName, bucketingKey, feature, treatment, label, time, changeNumber
-      } = impression;
-
-      return JSON.stringify({
-        m: this.metadata,
-        i: {
-          k: keyName,
-          b: bucketingKey,
-          f: feature,
-          t: treatment,
-          r: label,
-          c: changeNumber,
-          m: time
-        }
-      } as StoredImpressionWithMetadata);
-    });
-  }
-
   /**
    * Returns a promise that resolves with the count of stored impressions, or 0 if there was some error.
    * The promise will never be rejected.

package/src/storages/pluggable/TelemetryCachePluggable.ts

@@ -1,6 +1,6 @@
 import { ILogger } from '../../logger/types';
 import { Method, MultiConfigs, MultiMethodExceptions, MultiMethodLatencies } from '../../sync/submitters/types';
-import { KeyBuilderSS, parseExceptionField, parseLatencyField, parseMetadata } from '../KeyBuilderSS';
+import { KeyBuilderSS } from '../KeyBuilderSS';
 import { IPluggableStorageWrapper, ITelemetryCacheAsync } from '../types';
 import { findLatencyIndex } from '../findLatencyIndex';
 import { getTelemetryConfigStats } from '../../sync/submitters/telemetrySubmitter';
@@ -8,6 +8,7 @@ import { CONSUMER_MODE, STORAGE_PLUGGABLE } from '../../utils/constants';
 import { isString, isNaNNumber } from '../../utils/lang';
 import { _Map } from '../../utils/lang/maps';
 import { MAX_LATENCY_BUCKET_COUNT, newBuckets } from '../inMemory/TelemetryCacheInMemory';
+import { parseLatencyField, parseExceptionField, parseMetadata } from '../utils';
 
 export class TelemetryCachePluggable implements ITelemetryCacheAsync {
 
@@ -75,7 +76,7 @@ export class TelemetryCachePluggable implements ITelemetryCacheAsync {
               tr: newBuckets(),
             });
 
-            result.get(metadata)![method][bucket] = count;
+            result.get(metadata)![method]![bucket] = count;
           }
 
           return Promise.all(latencyKeys.map((latencyKey) => this.wrapper.del(latencyKey))).then(() => result);

package/src/storages/pluggable/UniqueKeysCachePluggable.ts

@@ -0,0 +1,67 @@
+import { IPluggableStorageWrapper, IUniqueKeysCacheBase } from '../types';
+import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
+import { setToArray } from '../../utils/lang/sets';
+import { DEFAULT_CACHE_SIZE, REFRESH_RATE } from '../inRedis/constants';
+import { LOG_PREFIX } from './constants';
+import { ILogger } from '../../logger/types';
+import { UniqueKeysItemSs } from '../../sync/submitters/types';
+
+export class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
+
+  private readonly log: ILogger;
+  private readonly key: string;
+  private readonly wrapper: IPluggableStorageWrapper;
+  private readonly refreshRate: number;
+  private intervalId: any;
+
+  constructor(log: ILogger, key: string, wrapper: IPluggableStorageWrapper, uniqueKeysQueueSize = DEFAULT_CACHE_SIZE, refreshRate = REFRESH_RATE) {
+    super(uniqueKeysQueueSize);
+    this.log = log;
+    this.key = key;
+    this.wrapper = wrapper;
+    this.refreshRate = refreshRate;
+    this.onFullQueue = () => { this.storeUniqueKeys(); };
+  }
+
+  storeUniqueKeys() {
+    const featureNames = Object.keys(this.uniqueKeysTracker);
+    if (!featureNames.length) return Promise.resolve(false);
+
+    const pipeline = featureNames.reduce<Promise<any>>((pipeline, featureName) => {
+      const featureKeys = setToArray(this.uniqueKeysTracker[featureName]);
+      const uniqueKeysPayload = {
+        f: featureName,
+        ks: featureKeys
+      };
+
+      return pipeline.then(() => this.wrapper.pushItems(this.key, [JSON.stringify(uniqueKeysPayload)]));
+    }, Promise.resolve());
+
+    this.clear();
+    return pipeline.catch(err => {
+      this.log.error(`${LOG_PREFIX}Error in uniqueKeys pipeline: ${err}.`);
+      return false;
+    });
+  }
+
+
+  start() {
+    this.intervalId = setInterval(this.storeUniqueKeys.bind(this), this.refreshRate);
+  }
+
+  stop() {
+    clearInterval(this.intervalId);
+    return this.storeUniqueKeys();
+  }
+
+  /**
+   * Async consumer API, used by synchronizer.
+   * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
+   */
+  popNRaw(count = 0): Promise<UniqueKeysItemSs[]> {
+    return Promise.resolve(count || this.wrapper.getItemsCount(this.key))
+      .then(count => this.wrapper.popItems(this.key, count))
+      .then((uniqueKeyItems) => uniqueKeyItems.map(uniqueKeyItem => JSON.parse(uniqueKeyItem)));
+  }
+
+}

package/src/storages/pluggable/inMemoryWrapper.ts

@@ -39,25 +39,25 @@ export function inMemoryWrapperFactory(connDelay?: number): IPluggableStorageWra
     getKeysByPrefix(prefix: string) {
       return Promise.resolve(Object.keys(_cache).filter(key => startsWith(key, prefix)));
     },
-    incr(key: string) {
+    incr(key: string, increment = 1) {
       if (key in _cache) {
-        const count = toNumber(_cache[key]) + 1;
+        const count = toNumber(_cache[key]) + increment;
         if (isNaN(count)) return Promise.reject('Given key is not a number');
         _cache[key] = count + '';
         return Promise.resolve(count);
       } else {
-        _cache[key] = '1';
+        _cache[key] = '' + increment;
         return Promise.resolve(1);
       }
     },
-    decr(key: string) {
+    decr(key: string, decrement = 1) {
       if (key in _cache) {
-        const count = toNumber(_cache[key]) - 1;
+        const count = toNumber(_cache[key]) - decrement;
         if (isNaN(count)) return Promise.reject('Given key is not a number');
         _cache[key] = count + '';
         return Promise.resolve(count);
       } else {
-        _cache[key] = '-1';
+        _cache[key] = '-' + decrement;
         return Promise.resolve(-1);
       }
     },