@splitsoftware/splitio-commons 1.6.2-rc.10 → 1.6.2-rc.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/cjs/storages/inMemory/UniqueKeysCacheInMemory.js +24 -25
  2. package/cjs/storages/inMemory/UniqueKeysCacheInMemoryCS.js +10 -12
  3. package/cjs/storages/inRedis/ImpressionCountsCacheInRedis.js +35 -0
  4. package/cjs/storages/inRedis/UniqueKeysCacheInRedis.js +12 -0
  5. package/cjs/storages/pluggable/ImpressionCountsCachePluggable.js +38 -0
  6. package/cjs/storages/pluggable/UniqueKeysCachePluggable.js +11 -0
  7. package/cjs/storages/pluggable/index.js +21 -15
  8. package/esm/storages/inMemory/UniqueKeysCacheInMemory.js +22 -24
  9. package/esm/storages/inMemory/UniqueKeysCacheInMemoryCS.js +10 -12
  10. package/esm/storages/inRedis/ImpressionCountsCacheInRedis.js +35 -0
  11. package/esm/storages/inRedis/UniqueKeysCacheInRedis.js +12 -0
  12. package/esm/storages/pluggable/ImpressionCountsCachePluggable.js +38 -0
  13. package/esm/storages/pluggable/UniqueKeysCachePluggable.js +11 -0
  14. package/esm/storages/pluggable/index.js +21 -15
  15. package/package.json +1 -1
  16. package/src/storages/inMemory/UniqueKeysCacheInMemory.ts +25 -27
  17. package/src/storages/inMemory/UniqueKeysCacheInMemoryCS.ts +11 -13
  18. package/src/storages/inRedis/ImpressionCountsCacheInRedis.ts +43 -1
  19. package/src/storages/inRedis/UniqueKeysCacheInRedis.ts +13 -1
  20. package/src/storages/inRedis/index.ts +1 -1
  21. package/src/storages/pluggable/ImpressionCountsCachePluggable.ts +45 -0
  22. package/src/storages/pluggable/UniqueKeysCachePluggable.ts +11 -0
  23. package/src/storages/pluggable/index.ts +22 -13
  24. package/src/storages/types.ts +2 -2
  25. package/src/sync/submitters/types.ts +8 -6
  26. package/types/storages/inMemory/uniqueKeysCacheInMemory.d.ts +9 -9
  27. package/types/storages/inMemory/uniqueKeysCacheInMemoryCS.d.ts +2 -4
  28. package/types/storages/inRedis/ImpressionCountsCacheInRedis.d.ts +2 -0
  29. package/types/storages/inRedis/uniqueKeysCacheInRedis.d.ts +6 -0
  30. package/types/storages/pluggable/ImpressionCountsCachePluggable.d.ts +2 -0
  31. package/types/storages/pluggable/UniqueKeysCachePluggable.d.ts +6 -0
  32. package/types/sync/submitters/types.d.ts +7 -6
@@ -1,8 +1,26 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.UniqueKeysCacheInMemory = void 0;
3
+ exports.UniqueKeysCacheInMemory = exports.fromUniqueKeysCollector = void 0;
4
4
  var sets_1 = require("../../utils/lang/sets");
5
5
  var constants_1 = require("../inRedis/constants");
6
+ /**
7
+ * Converts `uniqueKeys` data from cache into request payload for SS.
8
+ */
9
+ function fromUniqueKeysCollector(uniqueKeys) {
10
+ var payload = [];
11
+ var featureNames = Object.keys(uniqueKeys);
12
+ for (var i = 0; i < featureNames.length; i++) {
13
+ var featureName = featureNames[i];
14
+ var userKeys = (0, sets_1.setToArray)(uniqueKeys[featureName]);
15
+ var uniqueKeysPayload = {
16
+ f: featureName,
17
+ ks: userKeys
18
+ };
19
+ payload.push(uniqueKeysPayload);
20
+ }
21
+ return { keys: payload };
22
+ }
23
+ exports.fromUniqueKeysCollector = fromUniqueKeysCollector;
6
24
  var UniqueKeysCacheInMemory = /** @class */ (function () {
7
25
  function UniqueKeysCacheInMemory(uniqueKeysQueueSize) {
8
26
  if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = constants_1.DEFAULT_CACHE_SIZE; }
@@ -14,16 +32,14 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
14
32
  this.onFullQueue = cb;
15
33
  };
16
34
  /**
17
- * Store unique keys in sequential order
18
- * key: string = feature name.
19
- * value: Set<string> = set of unique keys.
35
+ * Store unique keys per feature.
20
36
  */
21
- UniqueKeysCacheInMemory.prototype.track = function (key, featureName) {
37
+ UniqueKeysCacheInMemory.prototype.track = function (userKey, featureName) {
22
38
  if (!this.uniqueKeysTracker[featureName])
23
39
  this.uniqueKeysTracker[featureName] = new sets_1._Set();
24
40
  var tracker = this.uniqueKeysTracker[featureName];
25
- if (!tracker.has(key)) {
26
- tracker.add(key);
41
+ if (!tracker.has(userKey)) {
42
+ tracker.add(userKey);
27
43
  this.uniqueTrackerSize++;
28
44
  }
29
45
  if (this.uniqueTrackerSize >= this.maxStorage && this.onFullQueue) {
@@ -43,7 +59,7 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
43
59
  UniqueKeysCacheInMemory.prototype.pop = function () {
44
60
  var data = this.uniqueKeysTracker;
45
61
  this.uniqueKeysTracker = {};
46
- return this.fromUniqueKeysCollector(data);
62
+ return fromUniqueKeysCollector(data);
47
63
  };
48
64
  /**
49
65
  * Check if the cache is empty.
@@ -51,23 +67,6 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
51
67
  UniqueKeysCacheInMemory.prototype.isEmpty = function () {
52
68
  return Object.keys(this.uniqueKeysTracker).length === 0;
53
69
  };
54
- /**
55
- * Converts `uniqueKeys` data from cache into request payload for SS.
56
- */
57
- UniqueKeysCacheInMemory.prototype.fromUniqueKeysCollector = function (uniqueKeys) {
58
- var payload = [];
59
- var featureNames = Object.keys(uniqueKeys);
60
- for (var i = 0; i < featureNames.length; i++) {
61
- var featureName = featureNames[i];
62
- var featureKeys = (0, sets_1.setToArray)(uniqueKeys[featureName]);
63
- var uniqueKeysPayload = {
64
- f: featureName,
65
- ks: featureKeys
66
- };
67
- payload.push(uniqueKeysPayload);
68
- }
69
- return { keys: payload };
70
- };
71
70
  return UniqueKeysCacheInMemory;
72
71
  }());
73
72
  exports.UniqueKeysCacheInMemory = UniqueKeysCacheInMemory;
@@ -19,14 +19,12 @@ var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
19
19
  this.onFullQueue = cb;
20
20
  };
21
21
  /**
22
- * Store unique keys in sequential order
23
- * key: string = key.
24
- * value: HashSet<string> = set of split names.
22
+ * Store unique keys per feature.
25
23
  */
26
- UniqueKeysCacheInMemoryCS.prototype.track = function (key, featureName) {
27
- if (!this.uniqueKeysTracker[key])
28
- this.uniqueKeysTracker[key] = new sets_1._Set();
29
- var tracker = this.uniqueKeysTracker[key];
24
+ UniqueKeysCacheInMemoryCS.prototype.track = function (userKey, featureName) {
25
+ if (!this.uniqueKeysTracker[userKey])
26
+ this.uniqueKeysTracker[userKey] = new sets_1._Set();
27
+ var tracker = this.uniqueKeysTracker[userKey];
30
28
  if (!tracker.has(featureName)) {
31
29
  tracker.add(featureName);
32
30
  this.uniqueTrackerSize++;
@@ -61,12 +59,12 @@ var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
61
59
  */
62
60
  UniqueKeysCacheInMemoryCS.prototype.fromUniqueKeysCollector = function (uniqueKeys) {
63
61
  var payload = [];
64
- var featureKeys = Object.keys(uniqueKeys);
65
- for (var k = 0; k < featureKeys.length; k++) {
66
- var featureKey = featureKeys[k];
67
- var featureNames = (0, sets_1.setToArray)(uniqueKeys[featureKey]);
62
+ var userKeys = Object.keys(uniqueKeys);
63
+ for (var k = 0; k < userKeys.length; k++) {
64
+ var userKey = userKeys[k];
65
+ var featureNames = (0, sets_1.setToArray)(uniqueKeys[userKey]);
68
66
  var uniqueKeysPayload = {
69
- k: featureKey,
67
+ k: userKey,
70
68
  fs: featureNames
71
69
  };
72
70
  payload.push(uniqueKeysPayload);
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.ImpressionCountsCacheInRedis = void 0;
4
4
  var tslib_1 = require("tslib");
5
+ var lang_1 = require("../../utils/lang");
5
6
  var ImpressionCountsCacheInMemory_1 = require("../inMemory/ImpressionCountsCacheInMemory");
6
7
  var constants_1 = require("./constants");
7
8
  var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
@@ -45,6 +46,40 @@ var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
45
46
  clearInterval(this.intervalId);
46
47
  return this.postImpressionCountsInRedis();
47
48
  };
49
+ // Async consumer API, used by synchronizer
50
+ ImpressionCountsCacheInRedis.prototype.getImpressionsCount = function () {
51
+ var _this = this;
52
+ return this.redis.hgetall(this.key)
53
+ .then(function (counts) {
54
+ if (!Object.keys(counts).length)
55
+ return undefined;
56
+ _this.redis.del(_this.key).catch(function () { });
57
+ var impressionsCount = { pf: [] };
58
+ (0, lang_1.forOwn)(counts, function (count, key) {
59
+ var nameAndTime = key.split('::');
60
+ if (nameAndTime.length !== 2) {
61
+ _this.log.error(constants_1.LOG_PREFIX + "Error spliting key " + key);
62
+ return;
63
+ }
64
+ var timeFrame = parseInt(nameAndTime[1]);
65
+ if (isNaN(timeFrame)) {
66
+ _this.log.error(constants_1.LOG_PREFIX + "Error parsing time frame " + nameAndTime[1]);
67
+ return;
68
+ }
69
+ var rawCount = parseInt(count);
70
+ if (isNaN(rawCount)) {
71
+ _this.log.error(constants_1.LOG_PREFIX + "Error parsing raw count " + count);
72
+ return;
73
+ }
74
+ impressionsCount.pf.push({
75
+ f: nameAndTime[0],
76
+ m: timeFrame,
77
+ rc: rawCount,
78
+ });
79
+ });
80
+ return impressionsCount;
81
+ });
82
+ };
48
83
  return ImpressionCountsCacheInRedis;
49
84
  }(ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory));
50
85
  exports.ImpressionCountsCacheInRedis = ImpressionCountsCacheInRedis;
@@ -54,6 +54,18 @@ var UniqueKeysCacheInRedis = /** @class */ (function (_super) {
54
54
  clearInterval(this.intervalId);
55
55
  return this.postUniqueKeysInRedis();
56
56
  };
57
+ /**
58
+ * Async consumer API, used by synchronizer.
59
+ * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
60
+ */
61
+ UniqueKeysCacheInRedis.prototype.popNRaw = function (count) {
62
+ var _this = this;
63
+ if (count === void 0) { count = 0; }
64
+ return this.redis.lrange(this.key, 0, count - 1).then(function (uniqueKeyItems) {
65
+ return _this.redis.ltrim(_this.key, uniqueKeyItems.length, -1)
66
+ .then(function () { return uniqueKeyItems.map(function (uniqueKeyItem) { return JSON.parse(uniqueKeyItem); }); });
67
+ });
68
+ };
57
69
  return UniqueKeysCacheInRedis;
58
70
  }(UniqueKeysCacheInMemory_1.UniqueKeysCacheInMemory));
59
71
  exports.UniqueKeysCacheInRedis = UniqueKeysCacheInRedis;
@@ -38,6 +38,44 @@ var ImpressionCountsCachePluggable = /** @class */ (function (_super) {
38
38
  clearInterval(this.intervalId);
39
39
  return this.storeImpressionCounts();
40
40
  };
41
+ // Async consumer API, used by synchronizer
42
+ ImpressionCountsCachePluggable.prototype.getImpressionsCount = function () {
43
+ var _this = this;
44
+ return this.wrapper.getKeysByPrefix(this.key)
45
+ .then(function (keys) {
46
+ return keys.length ? Promise.all(keys.map(function (key) { return _this.wrapper.get(key); }))
47
+ .then(function (counts) {
48
+ keys.forEach(function (key) { return _this.wrapper.del(key).catch(function () { }); });
49
+ var impressionsCount = { pf: [] };
50
+ for (var i = 0; i < keys.length; i++) {
51
+ var key = keys[i];
52
+ var count = counts[i];
53
+ var keyFeatureNameAndTime = key.split('::');
54
+ if (keyFeatureNameAndTime.length !== 3) {
55
+ _this.log.error(constants_2.LOG_PREFIX + "Error spliting key " + key);
56
+ continue;
57
+ }
58
+ var timeFrame = parseInt(keyFeatureNameAndTime[2]);
59
+ if (isNaN(timeFrame)) {
60
+ _this.log.error(constants_2.LOG_PREFIX + "Error parsing time frame " + keyFeatureNameAndTime[2]);
61
+ continue;
62
+ }
63
+ // @ts-ignore
64
+ var rawCount = parseInt(count);
65
+ if (isNaN(rawCount)) {
66
+ _this.log.error(constants_2.LOG_PREFIX + "Error parsing raw count " + count);
67
+ continue;
68
+ }
69
+ impressionsCount.pf.push({
70
+ f: keyFeatureNameAndTime[1],
71
+ m: timeFrame,
72
+ rc: rawCount,
73
+ });
74
+ }
75
+ return impressionsCount;
76
+ }) : undefined;
77
+ });
78
+ };
41
79
  return ImpressionCountsCachePluggable;
42
80
  }(ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory));
43
81
  exports.ImpressionCountsCachePluggable = ImpressionCountsCachePluggable;
@@ -45,6 +45,17 @@ var UniqueKeysCachePluggable = /** @class */ (function (_super) {
45
45
  clearInterval(this.intervalId);
46
46
  return this.storeUniqueKeys();
47
47
  };
48
+ /**
49
+ * Async consumer API, used by synchronizer.
50
+ * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
51
+ */
52
+ UniqueKeysCachePluggable.prototype.popNRaw = function (count) {
53
+ var _this = this;
54
+ if (count === void 0) { count = 0; }
55
+ return Promise.resolve(count || this.wrapper.getItemsCount(this.key))
56
+ .then(function (count) { return _this.wrapper.popItems(_this.key, count); })
57
+ .then(function (uniqueKeyItems) { return uniqueKeyItems.map(function (uniqueKeyItem) { return JSON.parse(uniqueKeyItem); }); });
58
+ };
48
59
  return UniqueKeysCachePluggable;
49
60
  }(UniqueKeysCacheInMemory_1.UniqueKeysCacheInMemory));
50
61
  exports.UniqueKeysCachePluggable = UniqueKeysCachePluggable;
@@ -55,14 +55,19 @@ function PluggableStorage(options) {
55
55
  var log = params.log, metadata = params.metadata, onReadyCb = params.onReadyCb, mode = params.mode, eventsQueueSize = params.eventsQueueSize, impressionsQueueSize = params.impressionsQueueSize, impressionsMode = params.impressionsMode, matchingKey = params.matchingKey;
56
56
  var keys = new KeyBuilderSS_1.KeyBuilderSS(prefix, metadata);
57
57
  var wrapper = (0, wrapperAdapter_1.wrapperAdapter)(log, options.wrapper);
58
+ var isSyncronizer = mode === undefined; // If mode is not defined, the synchronizer is running
58
59
  var isPartialConsumer = mode === constants_1.CONSUMER_PARTIAL_MODE;
59
- var telemetry = (0, TelemetryCacheInMemory_1.shouldRecordTelemetry)(params) ?
60
- isPartialConsumer ? new TelemetryCacheInMemory_1.TelemetryCacheInMemory() : new TelemetryCachePluggable_1.TelemetryCachePluggable(log, keys, wrapper) :
60
+ var telemetry = (0, TelemetryCacheInMemory_1.shouldRecordTelemetry)(params) || isSyncronizer ?
61
+ isPartialConsumer ?
62
+ new TelemetryCacheInMemory_1.TelemetryCacheInMemory() :
63
+ new TelemetryCachePluggable_1.TelemetryCachePluggable(log, keys, wrapper) :
61
64
  undefined;
62
- var impressionCountsCache = impressionsMode !== constants_1.DEBUG ?
63
- isPartialConsumer ? new ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory() : new ImpressionCountsCachePluggable_1.ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
65
+ var impressionCountsCache = impressionsMode !== constants_1.DEBUG || isSyncronizer ?
66
+ isPartialConsumer ?
67
+ new ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory() :
68
+ new ImpressionCountsCachePluggable_1.ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
64
69
  undefined;
65
- var uniqueKeysCache = impressionsMode === constants_1.NONE ?
70
+ var uniqueKeysCache = impressionsMode === constants_1.NONE || isSyncronizer ?
66
71
  isPartialConsumer ?
67
72
  matchingKey === undefined ? new UniqueKeysCacheInMemory_1.UniqueKeysCacheInMemory() : new UniqueKeysCacheInMemoryCS_1.UniqueKeysCacheInMemoryCS() :
68
73
  new UniqueKeysCachePluggable_1.UniqueKeysCachePluggable(log, keys.buildUniqueKeysKey(), wrapper) :
@@ -70,14 +75,15 @@ function PluggableStorage(options) {
70
75
  // Connects to wrapper and emits SDK_READY event on main client
71
76
  var connectPromise = wrapper.connect().then(function () {
72
77
  onReadyCb();
73
- // Start periodic flush of async storages
74
- if (impressionCountsCache && impressionCountsCache.start)
75
- impressionCountsCache.start();
76
- if (uniqueKeysCache && uniqueKeysCache.start)
77
- uniqueKeysCache.start();
78
- // If mode is not defined, it means that the synchronizer is running and so we don't have to record telemetry
79
- if (telemetry && telemetry.recordConfig && mode)
80
- telemetry.recordConfig();
78
+ // Start periodic flush of async storages if not running synchronizer (producer mode)
79
+ if (!isSyncronizer) {
80
+ if (impressionCountsCache && impressionCountsCache.start)
81
+ impressionCountsCache.start();
82
+ if (uniqueKeysCache && uniqueKeysCache.start)
83
+ uniqueKeysCache.start();
84
+ if (telemetry && telemetry.recordConfig)
85
+ telemetry.recordConfig();
86
+ }
81
87
  }).catch(function (e) {
82
88
  e = e || new Error('Error connecting wrapper');
83
89
  onReadyCb(e);
@@ -91,9 +97,9 @@ function PluggableStorage(options) {
91
97
  events: isPartialConsumer ? promisifyEventsTrack(new EventsCacheInMemory_1.EventsCacheInMemory(eventsQueueSize)) : new EventsCachePluggable_1.EventsCachePluggable(log, keys.buildEventsKey(), wrapper, metadata),
92
98
  telemetry: telemetry,
93
99
  uniqueKeys: uniqueKeysCache,
94
- // Disconnect the underlying storage
100
+ // Stop periodic flush and disconnect the underlying storage
95
101
  destroy: function () {
96
- return Promise.all([
102
+ return Promise.all(isSyncronizer ? [] : [
97
103
  impressionCountsCache && impressionCountsCache.stop && impressionCountsCache.stop(),
98
104
  uniqueKeysCache && uniqueKeysCache.stop && uniqueKeysCache.stop(),
99
105
  ]).then(function () { return wrapper.disconnect(); });
@@ -1,5 +1,22 @@
1
1
  import { setToArray, _Set } from '../../utils/lang/sets';
2
2
  import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
3
+ /**
4
+ * Converts `uniqueKeys` data from cache into request payload for SS.
5
+ */
6
+ export function fromUniqueKeysCollector(uniqueKeys) {
7
+ var payload = [];
8
+ var featureNames = Object.keys(uniqueKeys);
9
+ for (var i = 0; i < featureNames.length; i++) {
10
+ var featureName = featureNames[i];
11
+ var userKeys = setToArray(uniqueKeys[featureName]);
12
+ var uniqueKeysPayload = {
13
+ f: featureName,
14
+ ks: userKeys
15
+ };
16
+ payload.push(uniqueKeysPayload);
17
+ }
18
+ return { keys: payload };
19
+ }
3
20
  var UniqueKeysCacheInMemory = /** @class */ (function () {
4
21
  function UniqueKeysCacheInMemory(uniqueKeysQueueSize) {
5
22
  if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = DEFAULT_CACHE_SIZE; }
@@ -11,16 +28,14 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
11
28
  this.onFullQueue = cb;
12
29
  };
13
30
  /**
14
- * Store unique keys in sequential order
15
- * key: string = feature name.
16
- * value: Set<string> = set of unique keys.
31
+ * Store unique keys per feature.
17
32
  */
18
- UniqueKeysCacheInMemory.prototype.track = function (key, featureName) {
33
+ UniqueKeysCacheInMemory.prototype.track = function (userKey, featureName) {
19
34
  if (!this.uniqueKeysTracker[featureName])
20
35
  this.uniqueKeysTracker[featureName] = new _Set();
21
36
  var tracker = this.uniqueKeysTracker[featureName];
22
- if (!tracker.has(key)) {
23
- tracker.add(key);
37
+ if (!tracker.has(userKey)) {
38
+ tracker.add(userKey);
24
39
  this.uniqueTrackerSize++;
25
40
  }
26
41
  if (this.uniqueTrackerSize >= this.maxStorage && this.onFullQueue) {
@@ -40,7 +55,7 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
40
55
  UniqueKeysCacheInMemory.prototype.pop = function () {
41
56
  var data = this.uniqueKeysTracker;
42
57
  this.uniqueKeysTracker = {};
43
- return this.fromUniqueKeysCollector(data);
58
+ return fromUniqueKeysCollector(data);
44
59
  };
45
60
  /**
46
61
  * Check if the cache is empty.
@@ -48,23 +63,6 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
48
63
  UniqueKeysCacheInMemory.prototype.isEmpty = function () {
49
64
  return Object.keys(this.uniqueKeysTracker).length === 0;
50
65
  };
51
- /**
52
- * Converts `uniqueKeys` data from cache into request payload for SS.
53
- */
54
- UniqueKeysCacheInMemory.prototype.fromUniqueKeysCollector = function (uniqueKeys) {
55
- var payload = [];
56
- var featureNames = Object.keys(uniqueKeys);
57
- for (var i = 0; i < featureNames.length; i++) {
58
- var featureName = featureNames[i];
59
- var featureKeys = setToArray(uniqueKeys[featureName]);
60
- var uniqueKeysPayload = {
61
- f: featureName,
62
- ks: featureKeys
63
- };
64
- payload.push(uniqueKeysPayload);
65
- }
66
- return { keys: payload };
67
- };
68
66
  return UniqueKeysCacheInMemory;
69
67
  }());
70
68
  export { UniqueKeysCacheInMemory };
@@ -16,14 +16,12 @@ var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
16
16
  this.onFullQueue = cb;
17
17
  };
18
18
  /**
19
- * Store unique keys in sequential order
20
- * key: string = key.
21
- * value: HashSet<string> = set of split names.
19
+ * Store unique keys per feature.
22
20
  */
23
- UniqueKeysCacheInMemoryCS.prototype.track = function (key, featureName) {
24
- if (!this.uniqueKeysTracker[key])
25
- this.uniqueKeysTracker[key] = new _Set();
26
- var tracker = this.uniqueKeysTracker[key];
21
+ UniqueKeysCacheInMemoryCS.prototype.track = function (userKey, featureName) {
22
+ if (!this.uniqueKeysTracker[userKey])
23
+ this.uniqueKeysTracker[userKey] = new _Set();
24
+ var tracker = this.uniqueKeysTracker[userKey];
27
25
  if (!tracker.has(featureName)) {
28
26
  tracker.add(featureName);
29
27
  this.uniqueTrackerSize++;
@@ -58,12 +56,12 @@ var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
58
56
  */
59
57
  UniqueKeysCacheInMemoryCS.prototype.fromUniqueKeysCollector = function (uniqueKeys) {
60
58
  var payload = [];
61
- var featureKeys = Object.keys(uniqueKeys);
62
- for (var k = 0; k < featureKeys.length; k++) {
63
- var featureKey = featureKeys[k];
64
- var featureNames = setToArray(uniqueKeys[featureKey]);
59
+ var userKeys = Object.keys(uniqueKeys);
60
+ for (var k = 0; k < userKeys.length; k++) {
61
+ var userKey = userKeys[k];
62
+ var featureNames = setToArray(uniqueKeys[userKey]);
65
63
  var uniqueKeysPayload = {
66
- k: featureKey,
64
+ k: userKey,
67
65
  fs: featureNames
68
66
  };
69
67
  payload.push(uniqueKeysPayload);
@@ -1,4 +1,5 @@
1
1
  import { __extends } from "tslib";
2
+ import { forOwn } from '../../utils/lang';
2
3
  import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
3
4
  import { LOG_PREFIX, REFRESH_RATE, TTL_REFRESH } from './constants';
4
5
  var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
@@ -42,6 +43,40 @@ var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
42
43
  clearInterval(this.intervalId);
43
44
  return this.postImpressionCountsInRedis();
44
45
  };
46
+ // Async consumer API, used by synchronizer
47
+ ImpressionCountsCacheInRedis.prototype.getImpressionsCount = function () {
48
+ var _this = this;
49
+ return this.redis.hgetall(this.key)
50
+ .then(function (counts) {
51
+ if (!Object.keys(counts).length)
52
+ return undefined;
53
+ _this.redis.del(_this.key).catch(function () { });
54
+ var impressionsCount = { pf: [] };
55
+ forOwn(counts, function (count, key) {
56
+ var nameAndTime = key.split('::');
57
+ if (nameAndTime.length !== 2) {
58
+ _this.log.error(LOG_PREFIX + "Error spliting key " + key);
59
+ return;
60
+ }
61
+ var timeFrame = parseInt(nameAndTime[1]);
62
+ if (isNaN(timeFrame)) {
63
+ _this.log.error(LOG_PREFIX + "Error parsing time frame " + nameAndTime[1]);
64
+ return;
65
+ }
66
+ var rawCount = parseInt(count);
67
+ if (isNaN(rawCount)) {
68
+ _this.log.error(LOG_PREFIX + "Error parsing raw count " + count);
69
+ return;
70
+ }
71
+ impressionsCount.pf.push({
72
+ f: nameAndTime[0],
73
+ m: timeFrame,
74
+ rc: rawCount,
75
+ });
76
+ });
77
+ return impressionsCount;
78
+ });
79
+ };
45
80
  return ImpressionCountsCacheInRedis;
46
81
  }(ImpressionCountsCacheInMemory));
47
82
  export { ImpressionCountsCacheInRedis };
@@ -51,6 +51,18 @@ var UniqueKeysCacheInRedis = /** @class */ (function (_super) {
51
51
  clearInterval(this.intervalId);
52
52
  return this.postUniqueKeysInRedis();
53
53
  };
54
+ /**
55
+ * Async consumer API, used by synchronizer.
56
+ * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
57
+ */
58
+ UniqueKeysCacheInRedis.prototype.popNRaw = function (count) {
59
+ var _this = this;
60
+ if (count === void 0) { count = 0; }
61
+ return this.redis.lrange(this.key, 0, count - 1).then(function (uniqueKeyItems) {
62
+ return _this.redis.ltrim(_this.key, uniqueKeyItems.length, -1)
63
+ .then(function () { return uniqueKeyItems.map(function (uniqueKeyItem) { return JSON.parse(uniqueKeyItem); }); });
64
+ });
65
+ };
54
66
  return UniqueKeysCacheInRedis;
55
67
  }(UniqueKeysCacheInMemory));
56
68
  export { UniqueKeysCacheInRedis };
@@ -35,6 +35,44 @@ var ImpressionCountsCachePluggable = /** @class */ (function (_super) {
35
35
  clearInterval(this.intervalId);
36
36
  return this.storeImpressionCounts();
37
37
  };
38
+ // Async consumer API, used by synchronizer
39
+ ImpressionCountsCachePluggable.prototype.getImpressionsCount = function () {
40
+ var _this = this;
41
+ return this.wrapper.getKeysByPrefix(this.key)
42
+ .then(function (keys) {
43
+ return keys.length ? Promise.all(keys.map(function (key) { return _this.wrapper.get(key); }))
44
+ .then(function (counts) {
45
+ keys.forEach(function (key) { return _this.wrapper.del(key).catch(function () { }); });
46
+ var impressionsCount = { pf: [] };
47
+ for (var i = 0; i < keys.length; i++) {
48
+ var key = keys[i];
49
+ var count = counts[i];
50
+ var keyFeatureNameAndTime = key.split('::');
51
+ if (keyFeatureNameAndTime.length !== 3) {
52
+ _this.log.error(LOG_PREFIX + "Error spliting key " + key);
53
+ continue;
54
+ }
55
+ var timeFrame = parseInt(keyFeatureNameAndTime[2]);
56
+ if (isNaN(timeFrame)) {
57
+ _this.log.error(LOG_PREFIX + "Error parsing time frame " + keyFeatureNameAndTime[2]);
58
+ continue;
59
+ }
60
+ // @ts-ignore
61
+ var rawCount = parseInt(count);
62
+ if (isNaN(rawCount)) {
63
+ _this.log.error(LOG_PREFIX + "Error parsing raw count " + count);
64
+ continue;
65
+ }
66
+ impressionsCount.pf.push({
67
+ f: keyFeatureNameAndTime[1],
68
+ m: timeFrame,
69
+ rc: rawCount,
70
+ });
71
+ }
72
+ return impressionsCount;
73
+ }) : undefined;
74
+ });
75
+ };
38
76
  return ImpressionCountsCachePluggable;
39
77
  }(ImpressionCountsCacheInMemory));
40
78
  export { ImpressionCountsCachePluggable };
@@ -42,6 +42,17 @@ var UniqueKeysCachePluggable = /** @class */ (function (_super) {
42
42
  clearInterval(this.intervalId);
43
43
  return this.storeUniqueKeys();
44
44
  };
45
+ /**
46
+ * Async consumer API, used by synchronizer.
47
+ * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
48
+ */
49
+ UniqueKeysCachePluggable.prototype.popNRaw = function (count) {
50
+ var _this = this;
51
+ if (count === void 0) { count = 0; }
52
+ return Promise.resolve(count || this.wrapper.getItemsCount(this.key))
53
+ .then(function (count) { return _this.wrapper.popItems(_this.key, count); })
54
+ .then(function (uniqueKeyItems) { return uniqueKeyItems.map(function (uniqueKeyItem) { return JSON.parse(uniqueKeyItem); }); });
55
+ };
45
56
  return UniqueKeysCachePluggable;
46
57
  }(UniqueKeysCacheInMemory));
47
58
  export { UniqueKeysCachePluggable };
@@ -52,14 +52,19 @@ export function PluggableStorage(options) {
52
52
  var log = params.log, metadata = params.metadata, onReadyCb = params.onReadyCb, mode = params.mode, eventsQueueSize = params.eventsQueueSize, impressionsQueueSize = params.impressionsQueueSize, impressionsMode = params.impressionsMode, matchingKey = params.matchingKey;
53
53
  var keys = new KeyBuilderSS(prefix, metadata);
54
54
  var wrapper = wrapperAdapter(log, options.wrapper);
55
+ var isSyncronizer = mode === undefined; // If mode is not defined, the synchronizer is running
55
56
  var isPartialConsumer = mode === CONSUMER_PARTIAL_MODE;
56
- var telemetry = shouldRecordTelemetry(params) ?
57
- isPartialConsumer ? new TelemetryCacheInMemory() : new TelemetryCachePluggable(log, keys, wrapper) :
57
+ var telemetry = shouldRecordTelemetry(params) || isSyncronizer ?
58
+ isPartialConsumer ?
59
+ new TelemetryCacheInMemory() :
60
+ new TelemetryCachePluggable(log, keys, wrapper) :
58
61
  undefined;
59
- var impressionCountsCache = impressionsMode !== DEBUG ?
60
- isPartialConsumer ? new ImpressionCountsCacheInMemory() : new ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
62
+ var impressionCountsCache = impressionsMode !== DEBUG || isSyncronizer ?
63
+ isPartialConsumer ?
64
+ new ImpressionCountsCacheInMemory() :
65
+ new ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
61
66
  undefined;
62
- var uniqueKeysCache = impressionsMode === NONE ?
67
+ var uniqueKeysCache = impressionsMode === NONE || isSyncronizer ?
63
68
  isPartialConsumer ?
64
69
  matchingKey === undefined ? new UniqueKeysCacheInMemory() : new UniqueKeysCacheInMemoryCS() :
65
70
  new UniqueKeysCachePluggable(log, keys.buildUniqueKeysKey(), wrapper) :
@@ -67,14 +72,15 @@ export function PluggableStorage(options) {
67
72
  // Connects to wrapper and emits SDK_READY event on main client
68
73
  var connectPromise = wrapper.connect().then(function () {
69
74
  onReadyCb();
70
- // Start periodic flush of async storages
71
- if (impressionCountsCache && impressionCountsCache.start)
72
- impressionCountsCache.start();
73
- if (uniqueKeysCache && uniqueKeysCache.start)
74
- uniqueKeysCache.start();
75
- // If mode is not defined, it means that the synchronizer is running and so we don't have to record telemetry
76
- if (telemetry && telemetry.recordConfig && mode)
77
- telemetry.recordConfig();
75
+ // Start periodic flush of async storages if not running synchronizer (producer mode)
76
+ if (!isSyncronizer) {
77
+ if (impressionCountsCache && impressionCountsCache.start)
78
+ impressionCountsCache.start();
79
+ if (uniqueKeysCache && uniqueKeysCache.start)
80
+ uniqueKeysCache.start();
81
+ if (telemetry && telemetry.recordConfig)
82
+ telemetry.recordConfig();
83
+ }
78
84
  }).catch(function (e) {
79
85
  e = e || new Error('Error connecting wrapper');
80
86
  onReadyCb(e);
@@ -88,9 +94,9 @@ export function PluggableStorage(options) {
88
94
  events: isPartialConsumer ? promisifyEventsTrack(new EventsCacheInMemory(eventsQueueSize)) : new EventsCachePluggable(log, keys.buildEventsKey(), wrapper, metadata),
89
95
  telemetry: telemetry,
90
96
  uniqueKeys: uniqueKeysCache,
91
- // Disconnect the underlying storage
97
+ // Stop periodic flush and disconnect the underlying storage
92
98
  destroy: function () {
93
- return Promise.all([
99
+ return Promise.all(isSyncronizer ? [] : [
94
100
  impressionCountsCache && impressionCountsCache.stop && impressionCountsCache.stop(),
95
101
  uniqueKeysCache && uniqueKeysCache.stop && uniqueKeysCache.stop(),
96
102
  ]).then(function () { return wrapper.disconnect(); });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@splitsoftware/splitio-commons",
3
- "version": "1.6.2-rc.10",
3
+ "version": "1.6.2-rc.12",
4
4
  "description": "Split Javascript SDK common components",
5
5
  "main": "cjs/index.js",
6
6
  "module": "esm/index.js",
@@ -3,12 +3,31 @@ import { ISet, setToArray, _Set } from '../../utils/lang/sets';
3
3
  import { UniqueKeysPayloadSs } from '../../sync/submitters/types';
4
4
  import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
5
5
 
6
+ /**
7
+ * Converts `uniqueKeys` data from cache into request payload for SS.
8
+ */
9
+ export function fromUniqueKeysCollector(uniqueKeys: { [featureName: string]: ISet<string> }): UniqueKeysPayloadSs {
10
+ const payload = [];
11
+ const featureNames = Object.keys(uniqueKeys);
12
+ for (let i = 0; i < featureNames.length; i++) {
13
+ const featureName = featureNames[i];
14
+ const userKeys = setToArray(uniqueKeys[featureName]);
15
+ const uniqueKeysPayload = {
16
+ f: featureName,
17
+ ks: userKeys
18
+ };
19
+
20
+ payload.push(uniqueKeysPayload);
21
+ }
22
+ return { keys: payload };
23
+ }
24
+
6
25
  export class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
7
26
 
8
27
  protected onFullQueue?: () => void;
9
28
  private readonly maxStorage: number;
10
29
  private uniqueTrackerSize = 0;
11
- protected uniqueKeysTracker: { [keys: string]: ISet<string> };
30
+ protected uniqueKeysTracker: { [featureName: string]: ISet<string> };
12
31
 
13
32
  constructor(uniqueKeysQueueSize = DEFAULT_CACHE_SIZE) {
14
33
  this.maxStorage = uniqueKeysQueueSize;
@@ -20,15 +39,13 @@ export class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
20
39
  }
21
40
 
22
41
  /**
23
- * Store unique keys in sequential order
24
- * key: string = feature name.
25
- * value: Set<string> = set of unique keys.
42
+ * Store unique keys per feature.
26
43
  */
27
- track(key: string, featureName: string) {
44
+ track(userKey: string, featureName: string) {
28
45
  if (!this.uniqueKeysTracker[featureName]) this.uniqueKeysTracker[featureName] = new _Set();
29
46
  const tracker = this.uniqueKeysTracker[featureName];
30
- if (!tracker.has(key)) {
31
- tracker.add(key);
47
+ if (!tracker.has(userKey)) {
48
+ tracker.add(userKey);
32
49
  this.uniqueTrackerSize++;
33
50
  }
34
51
  if (this.uniqueTrackerSize >= this.maxStorage && this.onFullQueue) {
@@ -50,7 +67,7 @@ export class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
50
67
  pop() {
51
68
  const data = this.uniqueKeysTracker;
52
69
  this.uniqueKeysTracker = {};
53
- return this.fromUniqueKeysCollector(data);
70
+ return fromUniqueKeysCollector(data);
54
71
  }
55
72
 
56
73
  /**
@@ -60,23 +77,4 @@ export class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
60
77
  return Object.keys(this.uniqueKeysTracker).length === 0;
61
78
  }
62
79
 
63
- /**
64
- * Converts `uniqueKeys` data from cache into request payload for SS.
65
- */
66
- private fromUniqueKeysCollector(uniqueKeys: { [featureName: string]: ISet<string> }): UniqueKeysPayloadSs {
67
- const payload = [];
68
- const featureNames = Object.keys(uniqueKeys);
69
- for (let i = 0; i < featureNames.length; i++) {
70
- const featureName = featureNames[i];
71
- const featureKeys = setToArray(uniqueKeys[featureName]);
72
- const uniqueKeysPayload = {
73
- f: featureName,
74
- ks: featureKeys
75
- };
76
-
77
- payload.push(uniqueKeysPayload);
78
- }
79
- return { keys: payload };
80
- }
81
-
82
80
  }
@@ -8,7 +8,7 @@ export class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
8
8
  private onFullQueue?: () => void;
9
9
  private readonly maxStorage: number;
10
10
  private uniqueTrackerSize = 0;
11
- private uniqueKeysTracker: { [keys: string]: ISet<string> };
11
+ private uniqueKeysTracker: { [userKey: string]: ISet<string> };
12
12
 
13
13
  /**
14
14
  *
@@ -25,14 +25,12 @@ export class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
25
25
  }
26
26
 
27
27
  /**
28
- * Store unique keys in sequential order
29
- * key: string = key.
30
- * value: HashSet<string> = set of split names.
28
+ * Store unique keys per feature.
31
29
  */
32
- track(key: string, featureName: string) {
30
+ track(userKey: string, featureName: string) {
33
31
 
34
- if (!this.uniqueKeysTracker[key]) this.uniqueKeysTracker[key] = new _Set();
35
- const tracker = this.uniqueKeysTracker[key];
32
+ if (!this.uniqueKeysTracker[userKey]) this.uniqueKeysTracker[userKey] = new _Set();
33
+ const tracker = this.uniqueKeysTracker[userKey];
36
34
  if (!tracker.has(featureName)) {
37
35
  tracker.add(featureName);
38
36
  this.uniqueTrackerSize++;
@@ -69,14 +67,14 @@ export class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
69
67
  /**
70
68
  * Converts `uniqueKeys` data from cache into request payload.
71
69
  */
72
- private fromUniqueKeysCollector(uniqueKeys: { [featureName: string]: ISet<string> }): UniqueKeysPayloadCs {
70
+ private fromUniqueKeysCollector(uniqueKeys: { [userKey: string]: ISet<string> }): UniqueKeysPayloadCs {
73
71
  const payload = [];
74
- const featureKeys = Object.keys(uniqueKeys);
75
- for (let k = 0; k < featureKeys.length; k++) {
76
- const featureKey = featureKeys[k];
77
- const featureNames = setToArray(uniqueKeys[featureKey]);
72
+ const userKeys = Object.keys(uniqueKeys);
73
+ for (let k = 0; k < userKeys.length; k++) {
74
+ const userKey = userKeys[k];
75
+ const featureNames = setToArray(uniqueKeys[userKey]);
78
76
  const uniqueKeysPayload = {
79
- k: featureKey,
77
+ k: userKey,
80
78
  fs: featureNames
81
79
  };
82
80
 
@@ -1,5 +1,7 @@
1
1
  import { Redis } from 'ioredis';
2
2
  import { ILogger } from '../../logger/types';
3
+ import { ImpressionCountsPayload } from '../../sync/submitters/types';
4
+ import { forOwn } from '../../utils/lang';
3
5
  import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
4
6
  import { LOG_PREFIX, REFRESH_RATE, TTL_REFRESH } from './constants';
5
7
 
@@ -20,7 +22,7 @@ export class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory
20
22
  this.onFullQueue = () => { this.postImpressionCountsInRedis(); };
21
23
  }
22
24
 
23
- private postImpressionCountsInRedis(){
25
+ private postImpressionCountsInRedis() {
24
26
  const counts = this.pop();
25
27
  const keys = Object.keys(counts);
26
28
  if (!keys.length) return Promise.resolve(false);
@@ -50,4 +52,44 @@ export class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory
50
52
  clearInterval(this.intervalId);
51
53
  return this.postImpressionCountsInRedis();
52
54
  }
55
+
56
+ // Async consumer API, used by synchronizer
57
+ getImpressionsCount(): Promise<ImpressionCountsPayload | undefined> {
58
+ return this.redis.hgetall(this.key)
59
+ .then(counts => {
60
+ if (!Object.keys(counts).length) return undefined;
61
+
62
+ this.redis.del(this.key).catch(() => { /* no-op */ });
63
+
64
+ const impressionsCount: ImpressionCountsPayload = { pf: [] };
65
+
66
+ forOwn(counts, (count, key) => {
67
+ const nameAndTime = key.split('::');
68
+ if (nameAndTime.length !== 2) {
69
+ this.log.error(`${LOG_PREFIX}Error spliting key ${key}`);
70
+ return;
71
+ }
72
+
73
+ const timeFrame = parseInt(nameAndTime[1]);
74
+ if (isNaN(timeFrame)) {
75
+ this.log.error(`${LOG_PREFIX}Error parsing time frame ${nameAndTime[1]}`);
76
+ return;
77
+ }
78
+
79
+ const rawCount = parseInt(count);
80
+ if (isNaN(rawCount)) {
81
+ this.log.error(`${LOG_PREFIX}Error parsing raw count ${count}`);
82
+ return;
83
+ }
84
+
85
+ impressionsCount.pf.push({
86
+ f: nameAndTime[0],
87
+ m: timeFrame,
88
+ rc: rawCount,
89
+ });
90
+ });
91
+
92
+ return impressionsCount;
93
+ });
94
+ }
53
95
  }
@@ -5,6 +5,7 @@ import { setToArray } from '../../utils/lang/sets';
5
5
  import { DEFAULT_CACHE_SIZE, REFRESH_RATE, TTL_REFRESH } from './constants';
6
6
  import { LOG_PREFIX } from './constants';
7
7
  import { ILogger } from '../../logger/types';
8
+ import { UniqueKeysItemSs } from '../../sync/submitters/types';
8
9
 
9
10
  export class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
10
11
 
@@ -20,7 +21,7 @@ export class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements I
20
21
  this.key = key;
21
22
  this.redis = redis;
22
23
  this.refreshRate = refreshRate;
23
- this.onFullQueue = () => {this.postUniqueKeysInRedis();};
24
+ this.onFullQueue = () => { this.postUniqueKeysInRedis(); };
24
25
  }
25
26
 
26
27
  private postUniqueKeysInRedis() {
@@ -62,4 +63,15 @@ export class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements I
62
63
  return this.postUniqueKeysInRedis();
63
64
  }
64
65
 
66
+ /**
67
+ * Async consumer API, used by synchronizer.
68
+ * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
69
+ */
70
+ popNRaw(count = 0): Promise<UniqueKeysItemSs[]> {
71
+ return this.redis.lrange(this.key, 0, count - 1).then(uniqueKeyItems => {
72
+ return this.redis.ltrim(this.key, uniqueKeyItems.length, -1)
73
+ .then(() => uniqueKeyItems.map(uniqueKeyItem => JSON.parse(uniqueKeyItem)));
74
+ });
75
+ }
76
+
65
77
  }
@@ -52,7 +52,7 @@ export function InRedisStorage(options: InRedisStorageOptions = {}): IStorageAsy
52
52
 
53
53
  // When using REDIS we should:
54
54
  // 1- Disconnect from the storage
55
- destroy(): Promise<void>{
55
+ destroy(): Promise<void> {
56
56
  let promises = [];
57
57
  if (impressionCountsCache) promises.push(impressionCountsCache.stop());
58
58
  if (uniqueKeysCache) promises.push(uniqueKeysCache.stop());
@@ -1,4 +1,5 @@
1
1
  import { ILogger } from '../../logger/types';
2
+ import { ImpressionCountsPayload } from '../../sync/submitters/types';
2
3
  import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
3
4
  import { REFRESH_RATE } from '../inRedis/constants';
4
5
  import { IPluggableStorageWrapper } from '../types';
@@ -44,4 +45,48 @@ export class ImpressionCountsCachePluggable extends ImpressionCountsCacheInMemor
44
45
  clearInterval(this.intervalId);
45
46
  return this.storeImpressionCounts();
46
47
  }
48
+
49
+ // Async consumer API, used by synchronizer
50
+ getImpressionsCount(): Promise<ImpressionCountsPayload | undefined> {
51
+ return this.wrapper.getKeysByPrefix(this.key)
52
+ .then(keys => {
53
+ return keys.length ? Promise.all(keys.map(key => this.wrapper.get(key)))
54
+ .then(counts => {
55
+ keys.forEach(key => this.wrapper.del(key).catch(() => { /* noop */ }));
56
+
57
+ const impressionsCount: ImpressionCountsPayload = { pf: [] };
58
+
59
+ for (let i = 0; i < keys.length; i++) {
60
+ const key = keys[i];
61
+ const count = counts[i];
62
+
63
+ const keyFeatureNameAndTime = key.split('::');
64
+ if (keyFeatureNameAndTime.length !== 3) {
65
+ this.log.error(`${LOG_PREFIX}Error spliting key ${key}`);
66
+ continue;
67
+ }
68
+
69
+ const timeFrame = parseInt(keyFeatureNameAndTime[2]);
70
+ if (isNaN(timeFrame)) {
71
+ this.log.error(`${LOG_PREFIX}Error parsing time frame ${keyFeatureNameAndTime[2]}`);
72
+ continue;
73
+ }
74
+ // @ts-ignore
75
+ const rawCount = parseInt(count);
76
+ if (isNaN(rawCount)) {
77
+ this.log.error(`${LOG_PREFIX}Error parsing raw count ${count}`);
78
+ continue;
79
+ }
80
+
81
+ impressionsCount.pf.push({
82
+ f: keyFeatureNameAndTime[1],
83
+ m: timeFrame,
84
+ rc: rawCount,
85
+ });
86
+ }
87
+
88
+ return impressionsCount;
89
+ }) : undefined;
90
+ });
91
+ }
47
92
  }
@@ -4,6 +4,7 @@ import { setToArray } from '../../utils/lang/sets';
4
4
  import { DEFAULT_CACHE_SIZE, REFRESH_RATE } from '../inRedis/constants';
5
5
  import { LOG_PREFIX } from './constants';
6
6
  import { ILogger } from '../../logger/types';
7
+ import { UniqueKeysItemSs } from '../../sync/submitters/types';
7
8
 
8
9
  export class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
9
10
 
@@ -53,4 +54,14 @@ export class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory implements
53
54
  return this.storeUniqueKeys();
54
55
  }
55
56
 
57
+ /**
58
+ * Async consumer API, used by synchronizer.
59
+ * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
60
+ */
61
+ popNRaw(count = 0): Promise<UniqueKeysItemSs[]> {
62
+ return Promise.resolve(count || this.wrapper.getItemsCount(this.key))
63
+ .then(count => this.wrapper.popItems(this.key, count))
64
+ .then((uniqueKeyItems) => uniqueKeyItems.map(uniqueKeyItem => JSON.parse(uniqueKeyItem)));
65
+ }
66
+
56
67
  }
@@ -64,14 +64,23 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
64
64
  const { log, metadata, onReadyCb, mode, eventsQueueSize, impressionsQueueSize, impressionsMode, matchingKey } = params;
65
65
  const keys = new KeyBuilderSS(prefix, metadata);
66
66
  const wrapper = wrapperAdapter(log, options.wrapper);
67
+
68
+ const isSyncronizer = mode === undefined; // If mode is not defined, the synchronizer is running
67
69
  const isPartialConsumer = mode === CONSUMER_PARTIAL_MODE;
68
- const telemetry = shouldRecordTelemetry(params) ?
69
- isPartialConsumer ? new TelemetryCacheInMemory() : new TelemetryCachePluggable(log, keys, wrapper) :
70
+
71
+ const telemetry = shouldRecordTelemetry(params) || isSyncronizer ?
72
+ isPartialConsumer ?
73
+ new TelemetryCacheInMemory() :
74
+ new TelemetryCachePluggable(log, keys, wrapper) :
70
75
  undefined;
71
- const impressionCountsCache = impressionsMode !== DEBUG ?
72
- isPartialConsumer ? new ImpressionCountsCacheInMemory() : new ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
76
+
77
+ const impressionCountsCache = impressionsMode !== DEBUG || isSyncronizer ?
78
+ isPartialConsumer ?
79
+ new ImpressionCountsCacheInMemory() :
80
+ new ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
73
81
  undefined;
74
- const uniqueKeysCache = impressionsMode === NONE ?
82
+
83
+ const uniqueKeysCache = impressionsMode === NONE || isSyncronizer ?
75
84
  isPartialConsumer ?
76
85
  matchingKey === undefined ? new UniqueKeysCacheInMemory() : new UniqueKeysCacheInMemoryCS() :
77
86
  new UniqueKeysCachePluggable(log, keys.buildUniqueKeysKey(), wrapper) :
@@ -81,12 +90,12 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
81
90
  const connectPromise = wrapper.connect().then(() => {
82
91
  onReadyCb();
83
92
 
84
- // Start periodic flush of async storages
85
- if (impressionCountsCache && (impressionCountsCache as ImpressionCountsCachePluggable).start) (impressionCountsCache as ImpressionCountsCachePluggable).start();
86
- if (uniqueKeysCache && (uniqueKeysCache as UniqueKeysCachePluggable).start) (uniqueKeysCache as UniqueKeysCachePluggable).start();
87
-
88
- // If mode is not defined, it means that the synchronizer is running and so we don't have to record telemetry
89
- if (telemetry && (telemetry as ITelemetryCacheAsync).recordConfig && mode) (telemetry as ITelemetryCacheAsync).recordConfig();
93
+ // Start periodic flush of async storages if not running synchronizer (producer mode)
94
+ if (!isSyncronizer) {
95
+ if (impressionCountsCache && (impressionCountsCache as ImpressionCountsCachePluggable).start) (impressionCountsCache as ImpressionCountsCachePluggable).start();
96
+ if (uniqueKeysCache && (uniqueKeysCache as UniqueKeysCachePluggable).start) (uniqueKeysCache as UniqueKeysCachePluggable).start();
97
+ if (telemetry && (telemetry as ITelemetryCacheAsync).recordConfig) (telemetry as ITelemetryCacheAsync).recordConfig();
98
+ }
90
99
  }).catch((e) => {
91
100
  e = e || new Error('Error connecting wrapper');
92
101
  onReadyCb(e);
@@ -102,9 +111,9 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
102
111
  telemetry,
103
112
  uniqueKeys: uniqueKeysCache,
104
113
 
105
- // Disconnect the underlying storage
114
+ // Stop periodic flush and disconnect the underlying storage
106
115
  destroy() {
107
- return Promise.all([
116
+ return Promise.all(isSyncronizer ? [] : [
108
117
  impressionCountsCache && (impressionCountsCache as ImpressionCountsCachePluggable).stop && (impressionCountsCache as ImpressionCountsCachePluggable).stop(),
109
118
  uniqueKeysCache && (uniqueKeysCache as UniqueKeysCachePluggable).stop && (uniqueKeysCache as UniqueKeysCachePluggable).stop(),
110
119
  ]).then(() => wrapper.disconnect());
@@ -349,12 +349,12 @@ export interface IEventsCacheAsync extends IEventsCacheBase, IRecorderCacheProdu
349
349
  * Only in memory. Named `ImpressionsCounter` in spec.
350
350
  */
351
351
  export interface IImpressionCountsCacheSync extends IRecorderCacheProducerSync<Record<string, number>> {
352
- // Used by impressions tracker
352
+ // Used by impressions tracker
353
353
  track(featureName: string, timeFrame: number, amount: number): void
354
354
 
355
355
  // Used by impressions count submitter in standalone and producer mode
356
356
  isEmpty(): boolean // check if cache is empty. Return true if the cache was just created or cleared.
357
- pop(toMerge?: Record<string, number> ): Record<string, number> // pop cache data
357
+ pop(toMerge?: Record<string, number>): Record<string, number> // pop cache data
358
358
  }
359
359
 
360
360
  export interface IUniqueKeysCacheBase {
@@ -37,13 +37,15 @@ export type ImpressionCountsPayload = {
37
37
  }[]
38
38
  }
39
39
 
40
+ export type UniqueKeysItemSs = {
41
+ /** Split name */
42
+ f: string
43
+ /** keyNames */
44
+ ks: string[]
45
+ }
46
+
40
47
  export type UniqueKeysPayloadSs = {
41
- keys: {
42
- /** Split name */
43
- f: string
44
- /** keyNames */
45
- ks: string[]
46
- }[]
48
+ keys: UniqueKeysItemSs[]
47
49
  }
48
50
 
49
51
  export type UniqueKeysPayloadCs = {
@@ -1,21 +1,25 @@
1
1
  import { IUniqueKeysCacheBase } from '../types';
2
2
  import { ISet } from '../../utils/lang/sets';
3
3
  import { UniqueKeysPayloadSs } from '../../sync/submitters/types';
4
+ /**
5
+ * Converts `uniqueKeys` data from cache into request payload for SS.
6
+ */
7
+ export declare function fromUniqueKeysCollector(uniqueKeys: {
8
+ [featureName: string]: ISet<string>;
9
+ }): UniqueKeysPayloadSs;
4
10
  export declare class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
5
11
  protected onFullQueue?: () => void;
6
12
  private readonly maxStorage;
7
13
  private uniqueTrackerSize;
8
14
  protected uniqueKeysTracker: {
9
- [keys: string]: ISet<string>;
15
+ [featureName: string]: ISet<string>;
10
16
  };
11
17
  constructor(uniqueKeysQueueSize?: number);
12
18
  setOnFullQueueCb(cb: () => void): void;
13
19
  /**
14
- * Store unique keys in sequential order
15
- * key: string = feature name.
16
- * value: Set<string> = set of unique keys.
20
+ * Store unique keys per feature.
17
21
  */
18
- track(key: string, featureName: string): void;
22
+ track(userKey: string, featureName: string): void;
19
23
  /**
20
24
  * Clear the data stored on the cache.
21
25
  */
@@ -28,8 +32,4 @@ export declare class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
28
32
  * Check if the cache is empty.
29
33
  */
30
34
  isEmpty(): boolean;
31
- /**
32
- * Converts `uniqueKeys` data from cache into request payload for SS.
33
- */
34
- private fromUniqueKeysCollector;
35
35
  }
@@ -13,11 +13,9 @@ export declare class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
13
13
  constructor(uniqueKeysQueueSize?: number);
14
14
  setOnFullQueueCb(cb: () => void): void;
15
15
  /**
16
- * Store unique keys in sequential order
17
- * key: string = key.
18
- * value: HashSet<string> = set of split names.
16
+ * Store unique keys per feature.
19
17
  */
20
- track(key: string, featureName: string): void;
18
+ track(userKey: string, featureName: string): void;
21
19
  /**
22
20
  * Clear the data stored on the cache.
23
21
  */
@@ -1,5 +1,6 @@
1
1
  import { Redis } from 'ioredis';
2
2
  import { ILogger } from '../../logger/types';
3
+ import { ImpressionCountsPayload } from '../../sync/submitters/types';
3
4
  import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
4
5
  export declare class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory {
5
6
  private readonly log;
@@ -11,4 +12,5 @@ export declare class ImpressionCountsCacheInRedis extends ImpressionCountsCacheI
11
12
  private postImpressionCountsInRedis;
12
13
  start(): void;
13
14
  stop(): Promise<boolean | import("ioredis").BooleanResponse | undefined>;
15
+ getImpressionsCount(): Promise<ImpressionCountsPayload | undefined>;
14
16
  }
@@ -2,6 +2,7 @@ import { IUniqueKeysCacheBase } from '../types';
2
2
  import { Redis } from 'ioredis';
3
3
  import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
4
4
  import { ILogger } from '../../logger/types';
5
+ import { UniqueKeysItemSs } from '../../sync/submitters/types';
5
6
  export declare class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
6
7
  private readonly log;
7
8
  private readonly key;
@@ -12,4 +13,9 @@ export declare class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory impl
12
13
  private postUniqueKeysInRedis;
13
14
  start(): void;
14
15
  stop(): Promise<boolean | import("ioredis").BooleanResponse | undefined>;
16
+ /**
17
+ * Async consumer API, used by synchronizer.
18
+ * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
19
+ */
20
+ popNRaw(count?: number): Promise<UniqueKeysItemSs[]>;
15
21
  }
@@ -1,4 +1,5 @@
1
1
  import { ILogger } from '../../logger/types';
2
+ import { ImpressionCountsPayload } from '../../sync/submitters/types';
2
3
  import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
3
4
  import { IPluggableStorageWrapper } from '../types';
4
5
  export declare class ImpressionCountsCachePluggable extends ImpressionCountsCacheInMemory {
@@ -11,4 +12,5 @@ export declare class ImpressionCountsCachePluggable extends ImpressionCountsCach
11
12
  private storeImpressionCounts;
12
13
  start(): void;
13
14
  stop(): Promise<any>;
15
+ getImpressionsCount(): Promise<ImpressionCountsPayload | undefined>;
14
16
  }
@@ -1,6 +1,7 @@
1
1
  import { IPluggableStorageWrapper, IUniqueKeysCacheBase } from '../types';
2
2
  import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
3
3
  import { ILogger } from '../../logger/types';
4
+ import { UniqueKeysItemSs } from '../../sync/submitters/types';
4
5
  export declare class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
5
6
  private readonly log;
6
7
  private readonly key;
@@ -11,4 +12,9 @@ export declare class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory im
11
12
  storeUniqueKeys(): Promise<any>;
12
13
  start(): void;
13
14
  stop(): Promise<any>;
15
+ /**
16
+ * Async consumer API, used by synchronizer.
17
+ * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
18
+ */
19
+ popNRaw(count?: number): Promise<UniqueKeysItemSs[]>;
14
20
  }
@@ -33,13 +33,14 @@ export declare type ImpressionCountsPayload = {
33
33
  rc: number;
34
34
  }[];
35
35
  };
36
+ export declare type UniqueKeysItemSs = {
37
+ /** Split name */
38
+ f: string;
39
+ /** keyNames */
40
+ ks: string[];
41
+ };
36
42
  export declare type UniqueKeysPayloadSs = {
37
- keys: {
38
- /** Split name */
39
- f: string;
40
- /** keyNames */
41
- ks: string[];
42
- }[];
43
+ keys: UniqueKeysItemSs[];
43
44
  };
44
45
  export declare type UniqueKeysPayloadCs = {
45
46
  keys: {