@splitsoftware/splitio-commons 1.6.2-rc.2 → 1.6.2-rc.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/cjs/consent/sdkUserConsent.js +2 -2
  2. package/cjs/sdkFactory/index.js +11 -2
  3. package/cjs/storages/KeyBuilderSS.js +6 -0
  4. package/cjs/storages/inMemory/ImpressionCountsCacheInMemory.js +12 -1
  5. package/cjs/storages/inMemory/uniqueKeysCacheInMemory.js +2 -2
  6. package/cjs/storages/inMemory/uniqueKeysCacheInMemoryCS.js +2 -2
  7. package/cjs/storages/inRedis/ImpressionCountsCacheInRedis.js +46 -0
  8. package/cjs/storages/inRedis/constants.js +4 -1
  9. package/cjs/storages/inRedis/index.js +15 -1
  10. package/cjs/storages/inRedis/uniqueKeysCacheInRedis.js +55 -0
  11. package/cjs/trackers/impressionsTracker.js +17 -15
  12. package/esm/consent/sdkUserConsent.js +2 -2
  13. package/esm/sdkFactory/index.js +12 -3
  14. package/esm/storages/KeyBuilderSS.js +6 -0
  15. package/esm/storages/inMemory/ImpressionCountsCacheInMemory.js +12 -1
  16. package/esm/storages/inMemory/uniqueKeysCacheInMemory.js +1 -1
  17. package/esm/storages/inMemory/uniqueKeysCacheInMemoryCS.js +1 -1
  18. package/esm/storages/inRedis/ImpressionCountsCacheInRedis.js +43 -0
  19. package/esm/storages/inRedis/constants.js +3 -0
  20. package/esm/storages/inRedis/index.js +16 -2
  21. package/esm/storages/inRedis/uniqueKeysCacheInRedis.js +52 -0
  22. package/esm/trackers/impressionsTracker.js +17 -15
  23. package/package.json +1 -1
  24. package/src/consent/sdkUserConsent.ts +2 -2
  25. package/src/sdkFactory/index.ts +13 -3
  26. package/src/storages/KeyBuilderSS.ts +8 -0
  27. package/src/storages/inMemory/ImpressionCountsCacheInMemory.ts +16 -1
  28. package/src/storages/inMemory/uniqueKeysCacheInMemory.ts +3 -4
  29. package/src/storages/inMemory/uniqueKeysCacheInMemoryCS.ts +1 -2
  30. package/src/storages/inRedis/ImpressionCountsCacheInRedis.ts +48 -0
  31. package/src/storages/inRedis/constants.ts +3 -0
  32. package/src/storages/inRedis/index.ts +12 -3
  33. package/src/storages/inRedis/uniqueKeysCacheInRedis.ts +61 -0
  34. package/src/storages/types.ts +5 -2
  35. package/src/trackers/impressionsTracker.ts +16 -14
  36. package/types/storages/KeyBuilderSS.d.ts +2 -0
  37. package/types/storages/inMemory/ImpressionCountsCacheInMemory.d.ts +5 -1
  38. package/types/storages/inMemory/uniqueKeysCacheInMemory.d.ts +5 -2
  39. package/types/storages/inRedis/ImpressionCountsCacheInRedis.d.ts +13 -0
  40. package/types/storages/inRedis/constants.d.ts +3 -0
  41. package/types/storages/inRedis/uniqueKeysCacheInRedis.d.ts +14 -0
  42. package/types/storages/types.d.ts +4 -4
  43. package/types/sync/submitters/impressionCountsSubmitterInRedis.d.ts +5 -0
  44. package/types/sync/submitters/uniqueKeysSubmitterInRedis.d.ts +5 -0
@@ -39,8 +39,8 @@ function createUserConsentAPI(params) {
39
39
  if (events.clear)
40
40
  events.clear(); // @ts-ignore
41
41
  if (impressions.clear)
42
- impressions.clear();
43
- if (impressionCounts)
42
+ impressions.clear(); // @ts-ignore
43
+ if (impressionCounts && impressionCounts.clear)
44
44
  impressionCounts.clear();
45
45
  }
46
46
  }
@@ -58,8 +58,17 @@ function sdkFactory(params) {
58
58
  var integrationsManager = integrationsManagerFactory && integrationsManagerFactory({ settings: settings, storage: storage });
59
59
  var observer = impressionsObserverFactory();
60
60
  var uniqueKeysTracker = storageFactoryParams.impressionsMode === constants_3.NONE ? (0, uniqueKeysTracker_1.uniqueKeysTrackerFactory)(log, storage.uniqueKeys, filterAdapterFactory && filterAdapterFactory()) : undefined;
61
- var strategy = (storageFactoryParams.optimize) ? (0, strategyOptimized_1.strategyOptimizedFactory)(observer, storage.impressionCounts) :
62
- (storageFactoryParams.impressionsMode === constants_3.NONE) ? (0, strategyNone_1.strategyNoneFactory)(storage.impressionCounts, uniqueKeysTracker) : (0, strategyDebug_1.strategyDebugFactory)(observer);
61
+ var strategy;
62
+ switch (storageFactoryParams.impressionsMode) {
63
+ case constants_3.OPTIMIZED:
64
+ strategy = (0, strategyOptimized_1.strategyOptimizedFactory)(observer, storage.impressionCounts);
65
+ break;
66
+ case constants_3.NONE:
67
+ strategy = (0, strategyNone_1.strategyNoneFactory)(storage.impressionCounts, uniqueKeysTracker);
68
+ break;
69
+ default:
70
+ strategy = (0, strategyDebug_1.strategyDebugFactory)(observer);
71
+ }
63
72
  var impressionsTracker = (0, impressionsTracker_1.impressionsTrackerFactory)(settings, storage.impressions, strategy, integrationsManager, storage.telemetry);
64
73
  var eventTracker = (0, eventTracker_1.eventTrackerFactory)(settings, storage.events, integrationsManager, storage.telemetry);
65
74
  var telemetryTracker = (0, telemetryTracker_1.telemetryTrackerFactory)(storage.telemetry, platform.now);
@@ -23,6 +23,12 @@ var KeyBuilderSS = /** @class */ (function (_super) {
23
23
  KeyBuilderSS.prototype.buildImpressionsKey = function () {
24
24
  return this.prefix + ".impressions";
25
25
  };
26
+ KeyBuilderSS.prototype.buildImpressionsCountKey = function () {
27
+ return this.prefix + ".impressions.count";
28
+ };
29
+ KeyBuilderSS.prototype.buildUniqueKeysKey = function () {
30
+ return this.prefix + ".uniquekeys";
31
+ };
26
32
  KeyBuilderSS.prototype.buildEventsKey = function () {
27
33
  return this.prefix + ".events";
28
34
  };
@@ -2,9 +2,13 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.ImpressionCountsCacheInMemory = void 0;
4
4
  var time_1 = require("../../utils/time");
5
+ var constants_1 = require("../inRedis/constants");
5
6
  var ImpressionCountsCacheInMemory = /** @class */ (function () {
6
- function ImpressionCountsCacheInMemory() {
7
+ function ImpressionCountsCacheInMemory(impressionCountsCacheSize) {
8
+ if (impressionCountsCacheSize === void 0) { impressionCountsCacheSize = constants_1.DEFAULT_CACHE_SIZE; }
7
9
  this.cache = {};
10
+ this.cacheSize = 0;
11
+ this.maxStorage = impressionCountsCacheSize;
8
12
  }
9
13
  /**
10
14
  * Builds key to be stored in the cache with the featureName and the timeFrame truncated.
@@ -19,6 +23,13 @@ var ImpressionCountsCacheInMemory = /** @class */ (function () {
19
23
  var key = this._makeKey(featureName, timeFrame);
20
24
  var currentAmount = this.cache[key];
21
25
  this.cache[key] = currentAmount ? currentAmount + amount : amount;
26
+ if (this.onFullQueue) {
27
+ this.cacheSize = this.cacheSize + amount;
28
+ if (this.cacheSize >= this.maxStorage) {
29
+ this.onFullQueue();
30
+ this.cacheSize = 0;
31
+ }
32
+ }
22
33
  };
23
34
  /**
24
35
  * Pop the collected data, used as payload for posting.
@@ -2,10 +2,10 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.UniqueKeysCacheInMemory = void 0;
4
4
  var sets_1 = require("../../utils/lang/sets");
5
- var DEFAULT_CACHE_SIZE = 30000;
5
+ var constants_1 = require("../inRedis/constants");
6
6
  var UniqueKeysCacheInMemory = /** @class */ (function () {
7
7
  function UniqueKeysCacheInMemory(uniqueKeysQueueSize) {
8
- if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = DEFAULT_CACHE_SIZE; }
8
+ if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = constants_1.DEFAULT_CACHE_SIZE; }
9
9
  this.uniqueTrackerSize = 0;
10
10
  this.maxStorage = uniqueKeysQueueSize;
11
11
  this.uniqueKeysTracker = {};
@@ -2,7 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.UniqueKeysCacheInMemoryCS = void 0;
4
4
  var sets_1 = require("../../utils/lang/sets");
5
- var DEFAULT_CACHE_SIZE = 30000;
5
+ var constants_1 = require("../inRedis/constants");
6
6
  var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
7
7
  /**
8
8
  *
@@ -10,7 +10,7 @@ var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
10
10
  * Default value is 0, that means no maximum value, in case we want to avoid this being triggered.
11
11
  */
12
12
  function UniqueKeysCacheInMemoryCS(uniqueKeysQueueSize) {
13
- if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = DEFAULT_CACHE_SIZE; }
13
+ if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = constants_1.DEFAULT_CACHE_SIZE; }
14
14
  this.uniqueTrackerSize = 0;
15
15
  this.maxStorage = uniqueKeysQueueSize;
16
16
  this.uniqueKeysTracker = {};
@@ -0,0 +1,46 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ImpressionCountsCacheInRedis = void 0;
4
+ var tslib_1 = require("tslib");
5
+ var ImpressionCountsCacheInMemory_1 = require("../inMemory/ImpressionCountsCacheInMemory");
6
+ var constants_1 = require("./constants");
7
+ var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
8
+ (0, tslib_1.__extends)(ImpressionCountsCacheInRedis, _super);
9
+ function ImpressionCountsCacheInRedis(log, key, redis, impressionCountsCacheSize) {
10
+ var _this = _super.call(this, impressionCountsCacheSize) || this;
11
+ _this.log = log;
12
+ _this.key = key;
13
+ _this.redis = redis;
14
+ _this.onFullQueue = function () { _this.postImpressionCountsInRedis(); };
15
+ return _this;
16
+ }
17
+ ImpressionCountsCacheInRedis.prototype.postImpressionCountsInRedis = function () {
18
+ var _this = this;
19
+ var counts = this.pop();
20
+ var keys = Object.keys(counts);
21
+ var pipeline = this.redis.pipeline();
22
+ keys.forEach(function (key) {
23
+ pipeline.hincrby(_this.key, key, counts[key]);
24
+ });
25
+ return pipeline.exec()
26
+ .then(function (data) {
27
+ // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
28
+ if (data.length && data.length === keys.length) {
29
+ return _this.redis.expire(_this.key, constants_1.TTL_REFRESH);
30
+ }
31
+ })
32
+ .catch(function (err) {
33
+ _this.log.error(constants_1.LOG_PREFIX + "Error in impression counts pipeline: " + err + ".");
34
+ return false;
35
+ });
36
+ };
37
+ ImpressionCountsCacheInRedis.prototype.start = function (refreshRate) {
38
+ if (refreshRate === void 0) { refreshRate = constants_1.REFRESH_RATE; }
39
+ this.handle = setInterval(this.postImpressionCountsInRedis.bind(this), refreshRate);
40
+ };
41
+ ImpressionCountsCacheInRedis.prototype.stop = function () {
42
+ clearInterval(this.handle);
43
+ };
44
+ return ImpressionCountsCacheInRedis;
45
+ }(ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory));
46
+ exports.ImpressionCountsCacheInRedis = ImpressionCountsCacheInRedis;
@@ -1,4 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.LOG_PREFIX = void 0;
3
+ exports.TTL_REFRESH = exports.REFRESH_RATE = exports.DEFAULT_CACHE_SIZE = exports.LOG_PREFIX = void 0;
4
4
  exports.LOG_PREFIX = 'storage:redis: ';
5
+ exports.DEFAULT_CACHE_SIZE = 30000;
6
+ exports.REFRESH_RATE = 300000; // 300.000 ms = start after 5 mins
7
+ exports.TTL_REFRESH = 3600; // 1hr
@@ -10,6 +10,8 @@ var ImpressionsCacheInRedis_1 = require("./ImpressionsCacheInRedis");
10
10
  var EventsCacheInRedis_1 = require("./EventsCacheInRedis");
11
11
  var constants_1 = require("../../utils/constants");
12
12
  var TelemetryCacheInRedis_1 = require("./TelemetryCacheInRedis");
13
+ var uniqueKeysCacheInRedis_1 = require("./uniqueKeysCacheInRedis");
14
+ var ImpressionCountsCacheInRedis_1 = require("./ImpressionCountsCacheInRedis");
13
15
  /**
14
16
  * InRedis storage factory for consumer server-side SplitFactory, that uses `Ioredis` Redis client for Node.
15
17
  * @see {@link https://www.npmjs.com/package/ioredis}
@@ -18,13 +20,19 @@ function InRedisStorage(options) {
18
20
  if (options === void 0) { options = {}; }
19
21
  var prefix = (0, KeyBuilder_1.validatePrefix)(options.prefix);
20
22
  function InRedisStorageFactory(_a) {
21
- var log = _a.log, metadata = _a.metadata, onReadyCb = _a.onReadyCb;
23
+ var log = _a.log, metadata = _a.metadata, onReadyCb = _a.onReadyCb, impressionsMode = _a.impressionsMode;
22
24
  var keys = new KeyBuilderSS_1.KeyBuilderSS(prefix, metadata);
23
25
  var redisClient = new RedisAdapter_1.RedisAdapter(log, options.options || {});
24
26
  var telemetry = new TelemetryCacheInRedis_1.TelemetryCacheInRedis(log, keys, redisClient);
27
+ var impressionCountsCache = impressionsMode !== constants_1.DEBUG ? new ImpressionCountsCacheInRedis_1.ImpressionCountsCacheInRedis(log, keys.buildImpressionsCountKey(), redisClient) : undefined;
28
+ var uniqueKeysCache = impressionsMode === constants_1.NONE ? new uniqueKeysCacheInRedis_1.UniqueKeysCacheInRedis(log, keys.buildUniqueKeysKey(), redisClient) : undefined;
25
29
  // subscription to Redis connect event in order to emit SDK_READY event on consumer mode
26
30
  redisClient.on('connect', function () {
27
31
  onReadyCb();
32
+ if (impressionCountsCache)
33
+ impressionCountsCache.start();
34
+ if (uniqueKeysCache)
35
+ uniqueKeysCache.start();
28
36
  // Synchronize config
29
37
  telemetry.recordConfig();
30
38
  });
@@ -32,12 +40,18 @@ function InRedisStorage(options) {
32
40
  splits: new SplitsCacheInRedis_1.SplitsCacheInRedis(log, keys, redisClient),
33
41
  segments: new SegmentsCacheInRedis_1.SegmentsCacheInRedis(log, keys, redisClient),
34
42
  impressions: new ImpressionsCacheInRedis_1.ImpressionsCacheInRedis(log, keys.buildImpressionsKey(), redisClient, metadata),
43
+ impressionCounts: impressionCountsCache,
35
44
  events: new EventsCacheInRedis_1.EventsCacheInRedis(log, keys.buildEventsKey(), redisClient, metadata),
36
45
  telemetry: telemetry,
46
+ uniqueKeys: uniqueKeysCache,
37
47
  // When using REDIS we should:
38
48
  // 1- Disconnect from the storage
39
49
  destroy: function () {
40
50
  redisClient.disconnect();
51
+ if (impressionCountsCache)
52
+ impressionCountsCache.stop();
53
+ if (uniqueKeysCache)
54
+ uniqueKeysCache.stop();
41
55
  // @TODO check that caches works as expected when redisClient is disconnected
42
56
  }
43
57
  };
@@ -0,0 +1,55 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.UniqueKeysCacheInRedis = void 0;
4
+ var tslib_1 = require("tslib");
5
+ var uniqueKeysCacheInMemory_1 = require("../inMemory/uniqueKeysCacheInMemory");
6
+ var sets_1 = require("../../utils/lang/sets");
7
+ var constants_1 = require("./constants");
8
+ var constants_2 = require("./constants");
9
+ var UniqueKeysCacheInRedis = /** @class */ (function (_super) {
10
+ (0, tslib_1.__extends)(UniqueKeysCacheInRedis, _super);
11
+ function UniqueKeysCacheInRedis(log, key, redis, uniqueKeysQueueSize) {
12
+ if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = constants_1.DEFAULT_CACHE_SIZE; }
13
+ var _this = _super.call(this, uniqueKeysQueueSize) || this;
14
+ _this.log = log;
15
+ _this.key = key;
16
+ _this.redis = redis;
17
+ _this.onFullQueue = function () { _this.postUniqueKeysInRedis(); };
18
+ return _this;
19
+ }
20
+ UniqueKeysCacheInRedis.prototype.postUniqueKeysInRedis = function () {
21
+ var _this = this;
22
+ var pipeline = this.redis.pipeline();
23
+ var featureNames = Object.keys(this.uniqueKeysTracker);
24
+ for (var i = 0; i < featureNames.length; i++) {
25
+ var featureName = featureNames[i];
26
+ var featureKeys = (0, sets_1.setToArray)(this.uniqueKeysTracker[featureName]);
27
+ var uniqueKeysPayload = {
28
+ f: featureName,
29
+ ks: featureKeys
30
+ };
31
+ pipeline.rpush(this.key, JSON.stringify(uniqueKeysPayload));
32
+ }
33
+ this.clear();
34
+ return pipeline.exec()
35
+ .then(function (data) {
36
+ // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
37
+ if (data.length && data.length === featureNames.length) {
38
+ return _this.redis.expire(_this.key, constants_1.TTL_REFRESH);
39
+ }
40
+ })
41
+ .catch(function (err) {
42
+ _this.log.error(constants_2.LOG_PREFIX + "Error in uniqueKeys pipeline: " + err + ".");
43
+ return false;
44
+ });
45
+ };
46
+ UniqueKeysCacheInRedis.prototype.start = function (refreshRate) {
47
+ if (refreshRate === void 0) { refreshRate = constants_1.REFRESH_RATE; }
48
+ this.handle = setInterval(this.postUniqueKeysInRedis.bind(this), refreshRate);
49
+ };
50
+ UniqueKeysCacheInRedis.prototype.stop = function () {
51
+ clearInterval(this.handle);
52
+ };
53
+ return UniqueKeysCacheInRedis;
54
+ }(uniqueKeysCacheInMemory_1.UniqueKeysCacheInMemory));
55
+ exports.UniqueKeysCacheInRedis = UniqueKeysCacheInRedis;
@@ -23,21 +23,23 @@ function impressionsTrackerFactory(settings, impressionsCache, strategy, integra
23
23
  var impressionsCount = impressions.length;
24
24
  var _a = strategy.process(impressions), impressionsToStore = _a.impressionsToStore, impressionsToListener = _a.impressionsToListener, deduped = _a.deduped;
25
25
  var impressionsToListenerCount = impressionsToListener.length;
26
- var res = impressionsCache.track(impressionsToStore);
27
- // If we're on an async storage, handle error and log it.
28
- if ((0, thenable_1.thenable)(res)) {
29
- res.then(function () {
30
- log.info(constants_1.IMPRESSIONS_TRACKER_SUCCESS, [impressionsCount]);
31
- }).catch(function (err) {
32
- log.error(constants_1.ERROR_IMPRESSIONS_TRACKER, [impressionsCount, err]);
33
- });
34
- }
35
- else {
36
- // Record when impressionsCache is sync only (standalone mode)
37
- // @TODO we are not dropping impressions on full queue yet, so DROPPED stats are not recorded
38
- if (telemetryCache) {
39
- telemetryCache.recordImpressionStats(constants_2.QUEUED, impressionsToStore.length);
40
- telemetryCache.recordImpressionStats(constants_2.DEDUPED, deduped);
26
+ if (impressionsToStore.length > 0) {
27
+ var res = impressionsCache.track(impressionsToStore);
28
+ // If we're on an async storage, handle error and log it.
29
+ if ((0, thenable_1.thenable)(res)) {
30
+ res.then(function () {
31
+ log.info(constants_1.IMPRESSIONS_TRACKER_SUCCESS, [impressionsCount]);
32
+ }).catch(function (err) {
33
+ log.error(constants_1.ERROR_IMPRESSIONS_TRACKER, [impressionsCount, err]);
34
+ });
35
+ }
36
+ else {
37
+ // Record when impressionsCache is sync only (standalone mode)
38
+ // @TODO we are not dropping impressions on full queue yet, so DROPPED stats are not recorded
39
+ if (telemetryCache) {
40
+ telemetryCache.recordImpressionStats(constants_2.QUEUED, impressionsToStore.length);
41
+ telemetryCache.recordImpressionStats(constants_2.DEDUPED, deduped);
42
+ }
41
43
  }
42
44
  }
43
45
  // @TODO next block might be handled by the integration manager. In that case, the metadata object doesn't need to be passed in the constructor
@@ -36,8 +36,8 @@ export function createUserConsentAPI(params) {
36
36
  if (events.clear)
37
37
  events.clear(); // @ts-ignore
38
38
  if (impressions.clear)
39
- impressions.clear();
40
- if (impressionCounts)
39
+ impressions.clear(); // @ts-ignore
40
+ if (impressionCounts && impressionCounts.clear)
41
41
  impressionCounts.clear();
42
42
  }
43
43
  }
@@ -14,7 +14,7 @@ import { strategyDebugFactory } from '../trackers/strategy/strategyDebug';
14
14
  import { strategyOptimizedFactory } from '../trackers/strategy/strategyOptimized';
15
15
  import { strategyNoneFactory } from '../trackers/strategy/strategyNone';
16
16
  import { uniqueKeysTrackerFactory } from '../trackers/uniqueKeysTracker';
17
- import { NONE } from '../utils/constants';
17
+ import { NONE, OPTIMIZED } from '../utils/constants';
18
18
  /**
19
19
  * Modular SDK factory
20
20
  */
@@ -55,8 +55,17 @@ export function sdkFactory(params) {
55
55
  var integrationsManager = integrationsManagerFactory && integrationsManagerFactory({ settings: settings, storage: storage });
56
56
  var observer = impressionsObserverFactory();
57
57
  var uniqueKeysTracker = storageFactoryParams.impressionsMode === NONE ? uniqueKeysTrackerFactory(log, storage.uniqueKeys, filterAdapterFactory && filterAdapterFactory()) : undefined;
58
- var strategy = (storageFactoryParams.optimize) ? strategyOptimizedFactory(observer, storage.impressionCounts) :
59
- (storageFactoryParams.impressionsMode === NONE) ? strategyNoneFactory(storage.impressionCounts, uniqueKeysTracker) : strategyDebugFactory(observer);
58
+ var strategy;
59
+ switch (storageFactoryParams.impressionsMode) {
60
+ case OPTIMIZED:
61
+ strategy = strategyOptimizedFactory(observer, storage.impressionCounts);
62
+ break;
63
+ case NONE:
64
+ strategy = strategyNoneFactory(storage.impressionCounts, uniqueKeysTracker);
65
+ break;
66
+ default:
67
+ strategy = strategyDebugFactory(observer);
68
+ }
60
69
  var impressionsTracker = impressionsTrackerFactory(settings, storage.impressions, strategy, integrationsManager, storage.telemetry);
61
70
  var eventTracker = eventTrackerFactory(settings, storage.events, integrationsManager, storage.telemetry);
62
71
  var telemetryTracker = telemetryTrackerFactory(storage.telemetry, platform.now);
@@ -20,6 +20,12 @@ var KeyBuilderSS = /** @class */ (function (_super) {
20
20
  KeyBuilderSS.prototype.buildImpressionsKey = function () {
21
21
  return this.prefix + ".impressions";
22
22
  };
23
+ KeyBuilderSS.prototype.buildImpressionsCountKey = function () {
24
+ return this.prefix + ".impressions.count";
25
+ };
26
+ KeyBuilderSS.prototype.buildUniqueKeysKey = function () {
27
+ return this.prefix + ".uniquekeys";
28
+ };
23
29
  KeyBuilderSS.prototype.buildEventsKey = function () {
24
30
  return this.prefix + ".events";
25
31
  };
@@ -1,7 +1,11 @@
1
1
  import { truncateTimeFrame } from '../../utils/time';
2
+ import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
2
3
  var ImpressionCountsCacheInMemory = /** @class */ (function () {
3
- function ImpressionCountsCacheInMemory() {
4
+ function ImpressionCountsCacheInMemory(impressionCountsCacheSize) {
5
+ if (impressionCountsCacheSize === void 0) { impressionCountsCacheSize = DEFAULT_CACHE_SIZE; }
4
6
  this.cache = {};
7
+ this.cacheSize = 0;
8
+ this.maxStorage = impressionCountsCacheSize;
5
9
  }
6
10
  /**
7
11
  * Builds key to be stored in the cache with the featureName and the timeFrame truncated.
@@ -16,6 +20,13 @@ var ImpressionCountsCacheInMemory = /** @class */ (function () {
16
20
  var key = this._makeKey(featureName, timeFrame);
17
21
  var currentAmount = this.cache[key];
18
22
  this.cache[key] = currentAmount ? currentAmount + amount : amount;
23
+ if (this.onFullQueue) {
24
+ this.cacheSize = this.cacheSize + amount;
25
+ if (this.cacheSize >= this.maxStorage) {
26
+ this.onFullQueue();
27
+ this.cacheSize = 0;
28
+ }
29
+ }
19
30
  };
20
31
  /**
21
32
  * Pop the collected data, used as payload for posting.
@@ -1,5 +1,5 @@
1
1
  import { setToArray, _Set } from '../../utils/lang/sets';
2
- var DEFAULT_CACHE_SIZE = 30000;
2
+ import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
3
3
  var UniqueKeysCacheInMemory = /** @class */ (function () {
4
4
  function UniqueKeysCacheInMemory(uniqueKeysQueueSize) {
5
5
  if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = DEFAULT_CACHE_SIZE; }
@@ -1,5 +1,5 @@
1
1
  import { setToArray, _Set } from '../../utils/lang/sets';
2
- var DEFAULT_CACHE_SIZE = 30000;
2
+ import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
3
3
  var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
4
4
  /**
5
5
  *
@@ -0,0 +1,43 @@
1
+ import { __extends } from "tslib";
2
+ import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
3
+ import { LOG_PREFIX, REFRESH_RATE, TTL_REFRESH } from './constants';
4
+ var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
5
+ __extends(ImpressionCountsCacheInRedis, _super);
6
+ function ImpressionCountsCacheInRedis(log, key, redis, impressionCountsCacheSize) {
7
+ var _this = _super.call(this, impressionCountsCacheSize) || this;
8
+ _this.log = log;
9
+ _this.key = key;
10
+ _this.redis = redis;
11
+ _this.onFullQueue = function () { _this.postImpressionCountsInRedis(); };
12
+ return _this;
13
+ }
14
+ ImpressionCountsCacheInRedis.prototype.postImpressionCountsInRedis = function () {
15
+ var _this = this;
16
+ var counts = this.pop();
17
+ var keys = Object.keys(counts);
18
+ var pipeline = this.redis.pipeline();
19
+ keys.forEach(function (key) {
20
+ pipeline.hincrby(_this.key, key, counts[key]);
21
+ });
22
+ return pipeline.exec()
23
+ .then(function (data) {
24
+ // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
25
+ if (data.length && data.length === keys.length) {
26
+ return _this.redis.expire(_this.key, TTL_REFRESH);
27
+ }
28
+ })
29
+ .catch(function (err) {
30
+ _this.log.error(LOG_PREFIX + "Error in impression counts pipeline: " + err + ".");
31
+ return false;
32
+ });
33
+ };
34
+ ImpressionCountsCacheInRedis.prototype.start = function (refreshRate) {
35
+ if (refreshRate === void 0) { refreshRate = REFRESH_RATE; }
36
+ this.handle = setInterval(this.postImpressionCountsInRedis.bind(this), refreshRate);
37
+ };
38
+ ImpressionCountsCacheInRedis.prototype.stop = function () {
39
+ clearInterval(this.handle);
40
+ };
41
+ return ImpressionCountsCacheInRedis;
42
+ }(ImpressionCountsCacheInMemory));
43
+ export { ImpressionCountsCacheInRedis };
@@ -1 +1,4 @@
1
1
  export var LOG_PREFIX = 'storage:redis: ';
2
+ export var DEFAULT_CACHE_SIZE = 30000;
3
+ export var REFRESH_RATE = 300000; // 300.000 ms = start after 5 mins
4
+ export var TTL_REFRESH = 3600; // 1hr
@@ -5,8 +5,10 @@ import { SplitsCacheInRedis } from './SplitsCacheInRedis';
5
5
  import { SegmentsCacheInRedis } from './SegmentsCacheInRedis';
6
6
  import { ImpressionsCacheInRedis } from './ImpressionsCacheInRedis';
7
7
  import { EventsCacheInRedis } from './EventsCacheInRedis';
8
- import { STORAGE_REDIS } from '../../utils/constants';
8
+ import { DEBUG, NONE, STORAGE_REDIS } from '../../utils/constants';
9
9
  import { TelemetryCacheInRedis } from './TelemetryCacheInRedis';
10
+ import { UniqueKeysCacheInRedis } from './uniqueKeysCacheInRedis';
11
+ import { ImpressionCountsCacheInRedis } from './ImpressionCountsCacheInRedis';
10
12
  /**
11
13
  * InRedis storage factory for consumer server-side SplitFactory, that uses `Ioredis` Redis client for Node.
12
14
  * @see {@link https://www.npmjs.com/package/ioredis}
@@ -15,13 +17,19 @@ export function InRedisStorage(options) {
15
17
  if (options === void 0) { options = {}; }
16
18
  var prefix = validatePrefix(options.prefix);
17
19
  function InRedisStorageFactory(_a) {
18
- var log = _a.log, metadata = _a.metadata, onReadyCb = _a.onReadyCb;
20
+ var log = _a.log, metadata = _a.metadata, onReadyCb = _a.onReadyCb, impressionsMode = _a.impressionsMode;
19
21
  var keys = new KeyBuilderSS(prefix, metadata);
20
22
  var redisClient = new RedisAdapter(log, options.options || {});
21
23
  var telemetry = new TelemetryCacheInRedis(log, keys, redisClient);
24
+ var impressionCountsCache = impressionsMode !== DEBUG ? new ImpressionCountsCacheInRedis(log, keys.buildImpressionsCountKey(), redisClient) : undefined;
25
+ var uniqueKeysCache = impressionsMode === NONE ? new UniqueKeysCacheInRedis(log, keys.buildUniqueKeysKey(), redisClient) : undefined;
22
26
  // subscription to Redis connect event in order to emit SDK_READY event on consumer mode
23
27
  redisClient.on('connect', function () {
24
28
  onReadyCb();
29
+ if (impressionCountsCache)
30
+ impressionCountsCache.start();
31
+ if (uniqueKeysCache)
32
+ uniqueKeysCache.start();
25
33
  // Synchronize config
26
34
  telemetry.recordConfig();
27
35
  });
@@ -29,12 +37,18 @@ export function InRedisStorage(options) {
29
37
  splits: new SplitsCacheInRedis(log, keys, redisClient),
30
38
  segments: new SegmentsCacheInRedis(log, keys, redisClient),
31
39
  impressions: new ImpressionsCacheInRedis(log, keys.buildImpressionsKey(), redisClient, metadata),
40
+ impressionCounts: impressionCountsCache,
32
41
  events: new EventsCacheInRedis(log, keys.buildEventsKey(), redisClient, metadata),
33
42
  telemetry: telemetry,
43
+ uniqueKeys: uniqueKeysCache,
34
44
  // When using REDIS we should:
35
45
  // 1- Disconnect from the storage
36
46
  destroy: function () {
37
47
  redisClient.disconnect();
48
+ if (impressionCountsCache)
49
+ impressionCountsCache.stop();
50
+ if (uniqueKeysCache)
51
+ uniqueKeysCache.stop();
38
52
  // @TODO check that caches works as expected when redisClient is disconnected
39
53
  }
40
54
  };
@@ -0,0 +1,52 @@
1
+ import { __extends } from "tslib";
2
+ import { UniqueKeysCacheInMemory } from '../inMemory/uniqueKeysCacheInMemory';
3
+ import { setToArray } from '../../utils/lang/sets';
4
+ import { DEFAULT_CACHE_SIZE, REFRESH_RATE, TTL_REFRESH } from './constants';
5
+ import { LOG_PREFIX } from './constants';
6
+ var UniqueKeysCacheInRedis = /** @class */ (function (_super) {
7
+ __extends(UniqueKeysCacheInRedis, _super);
8
+ function UniqueKeysCacheInRedis(log, key, redis, uniqueKeysQueueSize) {
9
+ if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = DEFAULT_CACHE_SIZE; }
10
+ var _this = _super.call(this, uniqueKeysQueueSize) || this;
11
+ _this.log = log;
12
+ _this.key = key;
13
+ _this.redis = redis;
14
+ _this.onFullQueue = function () { _this.postUniqueKeysInRedis(); };
15
+ return _this;
16
+ }
17
+ UniqueKeysCacheInRedis.prototype.postUniqueKeysInRedis = function () {
18
+ var _this = this;
19
+ var pipeline = this.redis.pipeline();
20
+ var featureNames = Object.keys(this.uniqueKeysTracker);
21
+ for (var i = 0; i < featureNames.length; i++) {
22
+ var featureName = featureNames[i];
23
+ var featureKeys = setToArray(this.uniqueKeysTracker[featureName]);
24
+ var uniqueKeysPayload = {
25
+ f: featureName,
26
+ ks: featureKeys
27
+ };
28
+ pipeline.rpush(this.key, JSON.stringify(uniqueKeysPayload));
29
+ }
30
+ this.clear();
31
+ return pipeline.exec()
32
+ .then(function (data) {
33
+ // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
34
+ if (data.length && data.length === featureNames.length) {
35
+ return _this.redis.expire(_this.key, TTL_REFRESH);
36
+ }
37
+ })
38
+ .catch(function (err) {
39
+ _this.log.error(LOG_PREFIX + "Error in uniqueKeys pipeline: " + err + ".");
40
+ return false;
41
+ });
42
+ };
43
+ UniqueKeysCacheInRedis.prototype.start = function (refreshRate) {
44
+ if (refreshRate === void 0) { refreshRate = REFRESH_RATE; }
45
+ this.handle = setInterval(this.postUniqueKeysInRedis.bind(this), refreshRate);
46
+ };
47
+ UniqueKeysCacheInRedis.prototype.stop = function () {
48
+ clearInterval(this.handle);
49
+ };
50
+ return UniqueKeysCacheInRedis;
51
+ }(UniqueKeysCacheInMemory));
52
+ export { UniqueKeysCacheInRedis };
@@ -20,21 +20,23 @@ export function impressionsTrackerFactory(settings, impressionsCache, strategy,
20
20
  var impressionsCount = impressions.length;
21
21
  var _a = strategy.process(impressions), impressionsToStore = _a.impressionsToStore, impressionsToListener = _a.impressionsToListener, deduped = _a.deduped;
22
22
  var impressionsToListenerCount = impressionsToListener.length;
23
- var res = impressionsCache.track(impressionsToStore);
24
- // If we're on an async storage, handle error and log it.
25
- if (thenable(res)) {
26
- res.then(function () {
27
- log.info(IMPRESSIONS_TRACKER_SUCCESS, [impressionsCount]);
28
- }).catch(function (err) {
29
- log.error(ERROR_IMPRESSIONS_TRACKER, [impressionsCount, err]);
30
- });
31
- }
32
- else {
33
- // Record when impressionsCache is sync only (standalone mode)
34
- // @TODO we are not dropping impressions on full queue yet, so DROPPED stats are not recorded
35
- if (telemetryCache) {
36
- telemetryCache.recordImpressionStats(QUEUED, impressionsToStore.length);
37
- telemetryCache.recordImpressionStats(DEDUPED, deduped);
23
+ if (impressionsToStore.length > 0) {
24
+ var res = impressionsCache.track(impressionsToStore);
25
+ // If we're on an async storage, handle error and log it.
26
+ if (thenable(res)) {
27
+ res.then(function () {
28
+ log.info(IMPRESSIONS_TRACKER_SUCCESS, [impressionsCount]);
29
+ }).catch(function (err) {
30
+ log.error(ERROR_IMPRESSIONS_TRACKER, [impressionsCount, err]);
31
+ });
32
+ }
33
+ else {
34
+ // Record when impressionsCache is sync only (standalone mode)
35
+ // @TODO we are not dropping impressions on full queue yet, so DROPPED stats are not recorded
36
+ if (telemetryCache) {
37
+ telemetryCache.recordImpressionStats(QUEUED, impressionsToStore.length);
38
+ telemetryCache.recordImpressionStats(DEDUPED, deduped);
39
+ }
38
40
  }
39
41
  }
40
42
  // @TODO next block might be handled by the integration manager. In that case, the metadata object doesn't need to be passed in the constructor
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@splitsoftware/splitio-commons",
3
- "version": "1.6.2-rc.2",
3
+ "version": "1.6.2-rc.5",
4
4
  "description": "Split Javascript SDK common components",
5
5
  "main": "cjs/index.js",
6
6
  "module": "esm/index.js",
@@ -40,8 +40,8 @@ export function createUserConsentAPI(params: ISdkFactoryContext) {
40
40
 
41
41
  // @ts-ignore, clear method is present in storage for standalone and partial consumer mode
42
42
  if (events.clear) events.clear(); // @ts-ignore
43
- if (impressions.clear) impressions.clear();
44
- if (impressionCounts) impressionCounts.clear();
43
+ if (impressions.clear) impressions.clear();// @ts-ignore
44
+ if (impressionCounts && impressionCounts.clear) impressionCounts.clear();
45
45
  }
46
46
  } else {
47
47
  log.info(USER_CONSENT_NOT_UPDATED, [newConsentStatus]);
@@ -17,7 +17,7 @@ import { strategyDebugFactory } from '../trackers/strategy/strategyDebug';
17
17
  import { strategyOptimizedFactory } from '../trackers/strategy/strategyOptimized';
18
18
  import { strategyNoneFactory } from '../trackers/strategy/strategyNone';
19
19
  import { uniqueKeysTrackerFactory } from '../trackers/uniqueKeysTracker';
20
- import { NONE } from '../utils/constants';
20
+ import { NONE, OPTIMIZED } from '../utils/constants';
21
21
 
22
22
  /**
23
23
  * Modular SDK factory
@@ -72,8 +72,18 @@ export function sdkFactory(params: ISdkFactoryParams): SplitIO.ICsSDK | SplitIO.
72
72
 
73
73
  const observer = impressionsObserverFactory();
74
74
  const uniqueKeysTracker = storageFactoryParams.impressionsMode === NONE ? uniqueKeysTrackerFactory(log, storage.uniqueKeys!, filterAdapterFactory && filterAdapterFactory()) : undefined;
75
- const strategy = (storageFactoryParams.optimize) ? strategyOptimizedFactory(observer, storage.impressionCounts!) :
76
- (storageFactoryParams.impressionsMode === NONE) ? strategyNoneFactory(storage.impressionCounts!, uniqueKeysTracker!) : strategyDebugFactory(observer);
75
+
76
+ let strategy;
77
+ switch (storageFactoryParams.impressionsMode) {
78
+ case OPTIMIZED:
79
+ strategy = strategyOptimizedFactory(observer, storage.impressionCounts!);
80
+ break;
81
+ case NONE:
82
+ strategy = strategyNoneFactory(storage.impressionCounts!, uniqueKeysTracker!);
83
+ break;
84
+ default:
85
+ strategy = strategyDebugFactory(observer);
86
+ }
77
87
 
78
88
  const impressionsTracker = impressionsTrackerFactory(settings, storage.impressions, strategy, integrationsManager, storage.telemetry);
79
89
  const eventTracker = eventTrackerFactory(settings, storage.events, integrationsManager, storage.telemetry);
@@ -27,6 +27,14 @@ export class KeyBuilderSS extends KeyBuilder {
27
27
  return `${this.prefix}.impressions`;
28
28
  }
29
29
 
30
+ buildImpressionsCountKey() {
31
+ return `${this.prefix}.impressions.count`;
32
+ }
33
+
34
+ buildUniqueKeysKey() {
35
+ return `${this.prefix}.uniquekeys`;
36
+ }
37
+
30
38
  buildEventsKey() {
31
39
  return `${this.prefix}.events`;
32
40
  }
@@ -1,8 +1,16 @@
1
1
  import { truncateTimeFrame } from '../../utils/time';
2
+ import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
2
3
  import { IImpressionCountsCacheSync } from '../types';
3
4
 
4
5
  export class ImpressionCountsCacheInMemory implements IImpressionCountsCacheSync {
5
- private cache: Record<string, number> = {};
6
+ protected cache: Record<string, number> = {};
7
+ private readonly maxStorage: number;
8
+ protected onFullQueue?: () => void;
9
+ private cacheSize = 0;
10
+
11
+ constructor(impressionCountsCacheSize: number = DEFAULT_CACHE_SIZE) {
12
+ this.maxStorage = impressionCountsCacheSize;
13
+ }
6
14
 
7
15
  /**
8
16
  * Builds key to be stored in the cache with the featureName and the timeFrame truncated.
@@ -18,6 +26,13 @@ export class ImpressionCountsCacheInMemory implements IImpressionCountsCacheSync
18
26
  const key = this._makeKey(featureName, timeFrame);
19
27
  const currentAmount = this.cache[key];
20
28
  this.cache[key] = currentAmount ? currentAmount + amount : amount;
29
+ if (this.onFullQueue) {
30
+ this.cacheSize = this.cacheSize + amount;
31
+ if (this.cacheSize >= this.maxStorage) {
32
+ this.onFullQueue();
33
+ this.cacheSize = 0;
34
+ }
35
+ }
21
36
  }
22
37
 
23
38
 
@@ -1,15 +1,14 @@
1
1
  import { IUniqueKeysCacheBase } from '../types';
2
2
  import { ISet, setToArray, _Set } from '../../utils/lang/sets';
3
3
  import { UniqueKeysPayloadSs } from '../../sync/submitters/types';
4
-
5
- const DEFAULT_CACHE_SIZE = 30000;
4
+ import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
6
5
 
7
6
  export class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
8
7
 
9
- private onFullQueue?: () => void;
8
+ protected onFullQueue?: () => void;
10
9
  private readonly maxStorage: number;
11
10
  private uniqueTrackerSize = 0;
12
- private uniqueKeysTracker: { [keys: string]: ISet<string> };
11
+ protected uniqueKeysTracker: { [keys: string]: ISet<string> };
13
12
 
14
13
  constructor(uniqueKeysQueueSize: number = DEFAULT_CACHE_SIZE) {
15
14
  this.maxStorage = uniqueKeysQueueSize;
@@ -1,8 +1,7 @@
1
1
  import { IUniqueKeysCacheBase } from '../types';
2
2
  import { ISet, setToArray, _Set } from '../../utils/lang/sets';
3
3
  import { UniqueKeysPayloadCs } from '../../sync/submitters/types';
4
-
5
- const DEFAULT_CACHE_SIZE = 30000;
4
+ import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
6
5
 
7
6
  export class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
8
7
 
@@ -0,0 +1,48 @@
1
+ import { Redis } from 'ioredis';
2
+ import { ILogger } from '../../logger/types';
3
+ import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
4
+ import { LOG_PREFIX, REFRESH_RATE, TTL_REFRESH } from './constants';
5
+
6
+ export class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory {
7
+
8
+ private readonly log: ILogger;
9
+ private readonly key: string;
10
+ private readonly redis: Redis;
11
+ private handle: any;
12
+
13
+ constructor(log: ILogger, key: string, redis: Redis, impressionCountsCacheSize?: number) {
14
+ super(impressionCountsCacheSize);
15
+ this.log = log;
16
+ this.key = key;
17
+ this.redis = redis;
18
+ this.onFullQueue = () => { this.postImpressionCountsInRedis(); };
19
+ }
20
+
21
+ postImpressionCountsInRedis(){
22
+ const counts = this.pop();
23
+ const keys = Object.keys(counts);
24
+ const pipeline = this.redis.pipeline();
25
+ keys.forEach(key => {
26
+ pipeline.hincrby(this.key, key, counts[key]);
27
+ });
28
+ return pipeline.exec()
29
+ .then(data => {
30
+ // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
31
+ if (data.length && data.length === keys.length) {
32
+ return this.redis.expire(this.key, TTL_REFRESH);
33
+ }
34
+ })
35
+ .catch(err => {
36
+ this.log.error(`${LOG_PREFIX}Error in impression counts pipeline: ${err}.`);
37
+ return false;
38
+ });
39
+ }
40
+
41
+ start(refreshRate: number = REFRESH_RATE) {
42
+ this.handle = setInterval(this.postImpressionCountsInRedis.bind(this), refreshRate);
43
+ }
44
+
45
+ stop() {
46
+ clearInterval(this.handle);
47
+ }
48
+ }
@@ -1 +1,4 @@
1
1
  export const LOG_PREFIX = 'storage:redis: ';
2
+ export const DEFAULT_CACHE_SIZE = 30000;
3
+ export const REFRESH_RATE = 300000; // 300.000 ms = start after 5 mins
4
+ export const TTL_REFRESH = 3600; // 1hr
@@ -6,8 +6,10 @@ import { SplitsCacheInRedis } from './SplitsCacheInRedis';
6
6
  import { SegmentsCacheInRedis } from './SegmentsCacheInRedis';
7
7
  import { ImpressionsCacheInRedis } from './ImpressionsCacheInRedis';
8
8
  import { EventsCacheInRedis } from './EventsCacheInRedis';
9
- import { STORAGE_REDIS } from '../../utils/constants';
9
+ import { DEBUG, NONE, STORAGE_REDIS } from '../../utils/constants';
10
10
  import { TelemetryCacheInRedis } from './TelemetryCacheInRedis';
11
+ import { UniqueKeysCacheInRedis } from './uniqueKeysCacheInRedis';
12
+ import { ImpressionCountsCacheInRedis } from './ImpressionCountsCacheInRedis';
11
13
 
12
14
  export interface InRedisStorageOptions {
13
15
  prefix?: string
@@ -22,15 +24,18 @@ export function InRedisStorage(options: InRedisStorageOptions = {}): IStorageAsy
22
24
 
23
25
  const prefix = validatePrefix(options.prefix);
24
26
 
25
- function InRedisStorageFactory({ log, metadata, onReadyCb }: IStorageFactoryParams): IStorageAsync {
26
-
27
+ function InRedisStorageFactory({ log, metadata, onReadyCb, impressionsMode }: IStorageFactoryParams): IStorageAsync {
27
28
  const keys = new KeyBuilderSS(prefix, metadata);
28
29
  const redisClient = new RedisAdapter(log, options.options || {});
29
30
  const telemetry = new TelemetryCacheInRedis(log, keys, redisClient);
31
+ const impressionCountsCache = impressionsMode !== DEBUG ? new ImpressionCountsCacheInRedis(log, keys.buildImpressionsCountKey(), redisClient) : undefined;
32
+ const uniqueKeysCache = impressionsMode === NONE ? new UniqueKeysCacheInRedis(log, keys.buildUniqueKeysKey(), redisClient) : undefined;
30
33
 
31
34
  // subscription to Redis connect event in order to emit SDK_READY event on consumer mode
32
35
  redisClient.on('connect', () => {
33
36
  onReadyCb();
37
+ if (impressionCountsCache) impressionCountsCache.start();
38
+ if (uniqueKeysCache) uniqueKeysCache.start();
34
39
 
35
40
  // Synchronize config
36
41
  telemetry.recordConfig();
@@ -40,13 +45,17 @@ export function InRedisStorage(options: InRedisStorageOptions = {}): IStorageAsy
40
45
  splits: new SplitsCacheInRedis(log, keys, redisClient),
41
46
  segments: new SegmentsCacheInRedis(log, keys, redisClient),
42
47
  impressions: new ImpressionsCacheInRedis(log, keys.buildImpressionsKey(), redisClient, metadata),
48
+ impressionCounts: impressionCountsCache,
43
49
  events: new EventsCacheInRedis(log, keys.buildEventsKey(), redisClient, metadata),
44
50
  telemetry,
51
+ uniqueKeys: uniqueKeysCache,
45
52
 
46
53
  // When using REDIS we should:
47
54
  // 1- Disconnect from the storage
48
55
  destroy() {
49
56
  redisClient.disconnect();
57
+ if (impressionCountsCache) impressionCountsCache.stop();
58
+ if (uniqueKeysCache) uniqueKeysCache.stop();
50
59
  // @TODO check that caches works as expected when redisClient is disconnected
51
60
  }
52
61
  };
@@ -0,0 +1,61 @@
1
+ import { IUniqueKeysCacheBase } from '../types';
2
+ import { Redis } from 'ioredis';
3
+ import { UniqueKeysCacheInMemory } from '../inMemory/uniqueKeysCacheInMemory';
4
+ import { setToArray } from '../../utils/lang/sets';
5
+ import { DEFAULT_CACHE_SIZE, REFRESH_RATE, TTL_REFRESH } from './constants';
6
+ import { LOG_PREFIX } from './constants';
7
+ import { ILogger } from '../../logger/types';
8
+
9
+ export class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
10
+
11
+ private readonly log: ILogger;
12
+ private readonly key: string;
13
+ private readonly redis: Redis;
14
+ private handle: any;
15
+
16
+ constructor(log: ILogger, key: string, redis: Redis, uniqueKeysQueueSize: number = DEFAULT_CACHE_SIZE) {
17
+ super(uniqueKeysQueueSize);
18
+ this.log = log;
19
+ this.key = key;
20
+ this.redis = redis;
21
+ this.onFullQueue = () => {this.postUniqueKeysInRedis();};
22
+ }
23
+
24
+ postUniqueKeysInRedis() {
25
+ const pipeline = this.redis.pipeline();
26
+
27
+ const featureNames = Object.keys(this.uniqueKeysTracker);
28
+ for (let i = 0; i < featureNames.length; i++) {
29
+ const featureName = featureNames[i];
30
+ const featureKeys = setToArray(this.uniqueKeysTracker[featureName]);
31
+ const uniqueKeysPayload = {
32
+ f: featureName,
33
+ ks: featureKeys
34
+ };
35
+
36
+ pipeline.rpush(this.key, JSON.stringify(uniqueKeysPayload));
37
+ }
38
+ this.clear();
39
+ return pipeline.exec()
40
+ .then(data => {
41
+ // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
42
+ if (data.length && data.length === featureNames.length) {
43
+ return this.redis.expire(this.key, TTL_REFRESH);
44
+ }
45
+ })
46
+ .catch(err => {
47
+ this.log.error(`${LOG_PREFIX}Error in uniqueKeys pipeline: ${err}.`);
48
+ return false;
49
+ });
50
+ }
51
+
52
+
53
+ start(refreshRate: number = REFRESH_RATE) {
54
+ this.handle = setInterval(this.postUniqueKeysInRedis.bind(this), refreshRate);
55
+ }
56
+
57
+ stop() {
58
+ clearInterval(this.handle);
59
+ }
60
+
61
+ }
@@ -347,7 +347,7 @@ export interface IEventsCacheAsync extends IEventsCacheBase, IRecorderCacheProdu
347
347
  * Only in memory. Named `ImpressionsCounter` in spec.
348
348
  */
349
349
  export interface IImpressionCountsCacheSync extends IRecorderCacheProducerSync<Record<string, number>> {
350
- // Used by impressions tracker
350
+ // Used by impressions tracker
351
351
  track(featureName: string, timeFrame: number, amount: number): void
352
352
 
353
353
  // Used by impressions count submitter in standalone and producer mode
@@ -455,6 +455,7 @@ export interface IStorageBase<
455
455
  TSplitsCache extends ISplitsCacheBase,
456
456
  TSegmentsCache extends ISegmentsCacheBase,
457
457
  TImpressionsCache extends IImpressionsCacheBase,
458
+ TImpressionsCountCache extends IImpressionCountsCacheSync,
458
459
  TEventsCache extends IEventsCacheBase,
459
460
  TTelemetryCache extends ITelemetryCacheSync | ITelemetryCacheAsync,
460
461
  TUniqueKeysCache extends IUniqueKeysCacheBase
@@ -462,7 +463,7 @@ export interface IStorageBase<
462
463
  splits: TSplitsCache,
463
464
  segments: TSegmentsCache,
464
465
  impressions: TImpressionsCache,
465
- impressionCounts?: IImpressionCountsCacheSync,
466
+ impressionCounts?: TImpressionsCountCache,
466
467
  events: TEventsCache,
467
468
  telemetry?: TTelemetryCache,
468
469
  uniqueKeys?: TUniqueKeysCache,
@@ -474,6 +475,7 @@ export interface IStorageSync extends IStorageBase<
474
475
  ISplitsCacheSync,
475
476
  ISegmentsCacheSync,
476
477
  IImpressionsCacheSync,
478
+ IImpressionCountsCacheSync,
477
479
  IEventsCacheSync,
478
480
  ITelemetryCacheSync,
479
481
  IUniqueKeysCacheBase
@@ -483,6 +485,7 @@ export interface IStorageAsync extends IStorageBase<
483
485
  ISplitsCacheAsync,
484
486
  ISegmentsCacheAsync,
485
487
  IImpressionsCacheAsync | IImpressionsCacheSync,
488
+ IImpressionCountsCacheSync,
486
489
  IEventsCacheAsync | IEventsCacheSync,
487
490
  ITelemetryCacheAsync,
488
491
  IUniqueKeysCacheBase
@@ -34,21 +34,23 @@ export function impressionsTrackerFactory(
34
34
 
35
35
  const impressionsToListenerCount = impressionsToListener.length;
36
36
 
37
- const res = impressionsCache.track(impressionsToStore);
37
+ if ( impressionsToStore.length>0 ){
38
+ const res = impressionsCache.track(impressionsToStore);
38
39
 
39
- // If we're on an async storage, handle error and log it.
40
- if (thenable(res)) {
41
- res.then(() => {
42
- log.info(IMPRESSIONS_TRACKER_SUCCESS, [impressionsCount]);
43
- }).catch(err => {
44
- log.error(ERROR_IMPRESSIONS_TRACKER, [impressionsCount, err]);
45
- });
46
- } else {
47
- // Record when impressionsCache is sync only (standalone mode)
48
- // @TODO we are not dropping impressions on full queue yet, so DROPPED stats are not recorded
49
- if (telemetryCache) {
50
- (telemetryCache as ITelemetryCacheSync).recordImpressionStats(QUEUED, impressionsToStore.length);
51
- (telemetryCache as ITelemetryCacheSync).recordImpressionStats(DEDUPED, deduped);
40
+ // If we're on an async storage, handle error and log it.
41
+ if (thenable(res)) {
42
+ res.then(() => {
43
+ log.info(IMPRESSIONS_TRACKER_SUCCESS, [impressionsCount]);
44
+ }).catch(err => {
45
+ log.error(ERROR_IMPRESSIONS_TRACKER, [impressionsCount, err]);
46
+ });
47
+ } else {
48
+ // Record when impressionsCache is sync only (standalone mode)
49
+ // @TODO we are not dropping impressions on full queue yet, so DROPPED stats are not recorded
50
+ if (telemetryCache) {
51
+ (telemetryCache as ITelemetryCacheSync).recordImpressionStats(QUEUED, impressionsToStore.length);
52
+ (telemetryCache as ITelemetryCacheSync).recordImpressionStats(DEDUPED, deduped);
53
+ }
52
54
  }
53
55
  }
54
56
 
@@ -6,6 +6,8 @@ export declare class KeyBuilderSS extends KeyBuilder {
6
6
  constructor(prefix: string, metadata: IMetadata);
7
7
  buildRegisteredSegmentsKey(): string;
8
8
  buildImpressionsKey(): string;
9
+ buildImpressionsCountKey(): string;
10
+ buildUniqueKeysKey(): string;
9
11
  buildEventsKey(): string;
10
12
  searchPatternForSplitKeys(): string;
11
13
  buildLatencyKey(method: Method, bucket: number): string;
@@ -1,6 +1,10 @@
1
1
  import { IImpressionCountsCacheSync } from '../types';
2
2
  export declare class ImpressionCountsCacheInMemory implements IImpressionCountsCacheSync {
3
- private cache;
3
+ protected cache: Record<string, number>;
4
+ private readonly maxStorage;
5
+ protected onFullQueue?: () => void;
6
+ private cacheSize;
7
+ constructor(impressionCountsCacheSize?: number);
4
8
  /**
5
9
  * Builds key to be stored in the cache with the featureName and the timeFrame truncated.
6
10
  */
@@ -1,10 +1,13 @@
1
1
  import { IUniqueKeysCacheBase } from '../types';
2
+ import { ISet } from '../../utils/lang/sets';
2
3
  import { UniqueKeysPayloadSs } from '../../sync/submitters/types';
3
4
  export declare class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
4
- private onFullQueue?;
5
+ protected onFullQueue?: () => void;
5
6
  private readonly maxStorage;
6
7
  private uniqueTrackerSize;
7
- private uniqueKeysTracker;
8
+ protected uniqueKeysTracker: {
9
+ [keys: string]: ISet<string>;
10
+ };
8
11
  constructor(uniqueKeysQueueSize?: number);
9
12
  setOnFullQueueCb(cb: () => void): void;
10
13
  /**
@@ -0,0 +1,13 @@
1
+ import { Redis } from 'ioredis';
2
+ import { ILogger } from '../../logger/types';
3
+ import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
4
+ export declare class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory {
5
+ private readonly log;
6
+ private readonly key;
7
+ private readonly redis;
8
+ private handle;
9
+ constructor(log: ILogger, key: string, redis: Redis, impressionCountsCacheSize?: number);
10
+ postImpressionCountsInRedis(): Promise<boolean | import("ioredis").BooleanResponse | undefined>;
11
+ start(refreshRate?: number): void;
12
+ stop(): void;
13
+ }
@@ -1 +1,4 @@
1
1
  export declare const LOG_PREFIX = "storage:redis: ";
2
+ export declare const DEFAULT_CACHE_SIZE = 30000;
3
+ export declare const REFRESH_RATE = 300000;
4
+ export declare const TTL_REFRESH = 3600;
@@ -0,0 +1,14 @@
1
+ import { IUniqueKeysCacheBase } from '../types';
2
+ import { Redis } from 'ioredis';
3
+ import { UniqueKeysCacheInMemory } from '../inMemory/uniqueKeysCacheInMemory';
4
+ import { ILogger } from '../../logger/types';
5
+ export declare class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
6
+ private readonly log;
7
+ private readonly key;
8
+ private readonly redis;
9
+ private handle;
10
+ constructor(log: ILogger, key: string, redis: Redis, uniqueKeysQueueSize?: number);
11
+ postUniqueKeysInRedis(): Promise<boolean | import("ioredis").BooleanResponse | undefined>;
12
+ start(refreshRate?: number): void;
13
+ stop(): void;
14
+ }
@@ -373,20 +373,20 @@ export interface ITelemetryCacheAsync extends ITelemetryEvaluationProducerAsync
373
373
  /**
374
374
  * Storages
375
375
  */
376
- export interface IStorageBase<TSplitsCache extends ISplitsCacheBase, TSegmentsCache extends ISegmentsCacheBase, TImpressionsCache extends IImpressionsCacheBase, TEventsCache extends IEventsCacheBase, TTelemetryCache extends ITelemetryCacheSync | ITelemetryCacheAsync, TUniqueKeysCache extends IUniqueKeysCacheBase> {
376
+ export interface IStorageBase<TSplitsCache extends ISplitsCacheBase, TSegmentsCache extends ISegmentsCacheBase, TImpressionsCache extends IImpressionsCacheBase, TImpressionsCountCache extends IImpressionCountsCacheSync, TEventsCache extends IEventsCacheBase, TTelemetryCache extends ITelemetryCacheSync | ITelemetryCacheAsync, TUniqueKeysCache extends IUniqueKeysCacheBase> {
377
377
  splits: TSplitsCache;
378
378
  segments: TSegmentsCache;
379
379
  impressions: TImpressionsCache;
380
- impressionCounts?: IImpressionCountsCacheSync;
380
+ impressionCounts?: TImpressionsCountCache;
381
381
  events: TEventsCache;
382
382
  telemetry?: TTelemetryCache;
383
383
  uniqueKeys?: TUniqueKeysCache;
384
384
  destroy(): void | Promise<void>;
385
385
  shared?: (matchingKey: string, onReadyCb: (error?: any) => void) => this;
386
386
  }
387
- export interface IStorageSync extends IStorageBase<ISplitsCacheSync, ISegmentsCacheSync, IImpressionsCacheSync, IEventsCacheSync, ITelemetryCacheSync, IUniqueKeysCacheBase> {
387
+ export interface IStorageSync extends IStorageBase<ISplitsCacheSync, ISegmentsCacheSync, IImpressionsCacheSync, IImpressionCountsCacheSync, IEventsCacheSync, ITelemetryCacheSync, IUniqueKeysCacheBase> {
388
388
  }
389
- export interface IStorageAsync extends IStorageBase<ISplitsCacheAsync, ISegmentsCacheAsync, IImpressionsCacheAsync | IImpressionsCacheSync, IEventsCacheAsync | IEventsCacheSync, ITelemetryCacheAsync, IUniqueKeysCacheBase> {
389
+ export interface IStorageAsync extends IStorageBase<ISplitsCacheAsync, ISegmentsCacheAsync, IImpressionsCacheAsync | IImpressionsCacheSync, IImpressionCountsCacheSync, IEventsCacheAsync | IEventsCacheSync, ITelemetryCacheAsync, IUniqueKeysCacheBase> {
390
390
  }
391
391
  /** StorageFactory */
392
392
  export declare type DataLoader = (storage: IStorageSync, matchingKey: string) => void;
@@ -0,0 +1,5 @@
1
+ import { ISdkFactoryContextSync } from '../../sdkFactory/types';
2
+ /**
3
+ * Submitter that periodically posts impression counts
4
+ */
5
+ export declare function impressionCountsSubmitterInRedisFactory(params: ISdkFactoryContextSync): import("../types").ISyncTask<[], void>;
@@ -0,0 +1,5 @@
1
+ import { ISdkFactoryContextSync } from '../../sdkFactory/types';
2
+ /**
3
+ * Submitter that periodically posts impression counts
4
+ */
5
+ export declare function uniqueKeysSubmitterInRedisFactory(params: ISdkFactoryContextSync): import("../types").ISyncTask<[], void>;