@splitsoftware/splitio-commons 1.6.2-rc.10 → 1.6.2-rc.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cjs/storages/inMemory/UniqueKeysCacheInMemory.js +24 -25
- package/cjs/storages/inMemory/UniqueKeysCacheInMemoryCS.js +10 -12
- package/cjs/storages/inRedis/ImpressionCountsCacheInRedis.js +35 -0
- package/cjs/storages/inRedis/UniqueKeysCacheInRedis.js +12 -0
- package/cjs/storages/pluggable/ImpressionCountsCachePluggable.js +38 -0
- package/cjs/storages/pluggable/UniqueKeysCachePluggable.js +11 -0
- package/cjs/storages/pluggable/index.js +11 -7
- package/esm/storages/inMemory/UniqueKeysCacheInMemory.js +22 -24
- package/esm/storages/inMemory/UniqueKeysCacheInMemoryCS.js +10 -12
- package/esm/storages/inRedis/ImpressionCountsCacheInRedis.js +35 -0
- package/esm/storages/inRedis/UniqueKeysCacheInRedis.js +12 -0
- package/esm/storages/pluggable/ImpressionCountsCachePluggable.js +38 -0
- package/esm/storages/pluggable/UniqueKeysCachePluggable.js +11 -0
- package/esm/storages/pluggable/index.js +11 -7
- package/package.json +1 -1
- package/src/storages/inMemory/UniqueKeysCacheInMemory.ts +25 -27
- package/src/storages/inMemory/UniqueKeysCacheInMemoryCS.ts +11 -13
- package/src/storages/inRedis/ImpressionCountsCacheInRedis.ts +43 -1
- package/src/storages/inRedis/UniqueKeysCacheInRedis.ts +13 -1
- package/src/storages/inRedis/index.ts +1 -1
- package/src/storages/pluggable/ImpressionCountsCachePluggable.ts +45 -0
- package/src/storages/pluggable/UniqueKeysCachePluggable.ts +11 -0
- package/src/storages/pluggable/index.ts +15 -8
- package/src/storages/types.ts +2 -2
- package/src/sync/submitters/types.ts +8 -6
- package/types/storages/inMemory/uniqueKeysCacheInMemory.d.ts +9 -9
- package/types/storages/inMemory/uniqueKeysCacheInMemoryCS.d.ts +2 -4
- package/types/storages/inRedis/ImpressionCountsCacheInRedis.d.ts +2 -0
- package/types/storages/inRedis/uniqueKeysCacheInRedis.d.ts +6 -0
- package/types/storages/pluggable/ImpressionCountsCachePluggable.d.ts +2 -0
- package/types/storages/pluggable/UniqueKeysCachePluggable.d.ts +6 -0
- package/types/sync/submitters/types.d.ts +7 -6
|
@@ -1,8 +1,26 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.UniqueKeysCacheInMemory = void 0;
|
|
3
|
+
exports.UniqueKeysCacheInMemory = exports.fromUniqueKeysCollector = void 0;
|
|
4
4
|
var sets_1 = require("../../utils/lang/sets");
|
|
5
5
|
var constants_1 = require("../inRedis/constants");
|
|
6
|
+
/**
|
|
7
|
+
* Converts `uniqueKeys` data from cache into request payload for SS.
|
|
8
|
+
*/
|
|
9
|
+
function fromUniqueKeysCollector(uniqueKeys) {
|
|
10
|
+
var payload = [];
|
|
11
|
+
var featureNames = Object.keys(uniqueKeys);
|
|
12
|
+
for (var i = 0; i < featureNames.length; i++) {
|
|
13
|
+
var featureName = featureNames[i];
|
|
14
|
+
var userKeys = (0, sets_1.setToArray)(uniqueKeys[featureName]);
|
|
15
|
+
var uniqueKeysPayload = {
|
|
16
|
+
f: featureName,
|
|
17
|
+
ks: userKeys
|
|
18
|
+
};
|
|
19
|
+
payload.push(uniqueKeysPayload);
|
|
20
|
+
}
|
|
21
|
+
return { keys: payload };
|
|
22
|
+
}
|
|
23
|
+
exports.fromUniqueKeysCollector = fromUniqueKeysCollector;
|
|
6
24
|
var UniqueKeysCacheInMemory = /** @class */ (function () {
|
|
7
25
|
function UniqueKeysCacheInMemory(uniqueKeysQueueSize) {
|
|
8
26
|
if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = constants_1.DEFAULT_CACHE_SIZE; }
|
|
@@ -14,16 +32,14 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
|
|
|
14
32
|
this.onFullQueue = cb;
|
|
15
33
|
};
|
|
16
34
|
/**
|
|
17
|
-
* Store unique keys
|
|
18
|
-
* key: string = feature name.
|
|
19
|
-
* value: Set<string> = set of unique keys.
|
|
35
|
+
* Store unique keys per feature.
|
|
20
36
|
*/
|
|
21
|
-
UniqueKeysCacheInMemory.prototype.track = function (
|
|
37
|
+
UniqueKeysCacheInMemory.prototype.track = function (userKey, featureName) {
|
|
22
38
|
if (!this.uniqueKeysTracker[featureName])
|
|
23
39
|
this.uniqueKeysTracker[featureName] = new sets_1._Set();
|
|
24
40
|
var tracker = this.uniqueKeysTracker[featureName];
|
|
25
|
-
if (!tracker.has(
|
|
26
|
-
tracker.add(
|
|
41
|
+
if (!tracker.has(userKey)) {
|
|
42
|
+
tracker.add(userKey);
|
|
27
43
|
this.uniqueTrackerSize++;
|
|
28
44
|
}
|
|
29
45
|
if (this.uniqueTrackerSize >= this.maxStorage && this.onFullQueue) {
|
|
@@ -43,7 +59,7 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
|
|
|
43
59
|
UniqueKeysCacheInMemory.prototype.pop = function () {
|
|
44
60
|
var data = this.uniqueKeysTracker;
|
|
45
61
|
this.uniqueKeysTracker = {};
|
|
46
|
-
return
|
|
62
|
+
return fromUniqueKeysCollector(data);
|
|
47
63
|
};
|
|
48
64
|
/**
|
|
49
65
|
* Check if the cache is empty.
|
|
@@ -51,23 +67,6 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
|
|
|
51
67
|
UniqueKeysCacheInMemory.prototype.isEmpty = function () {
|
|
52
68
|
return Object.keys(this.uniqueKeysTracker).length === 0;
|
|
53
69
|
};
|
|
54
|
-
/**
|
|
55
|
-
* Converts `uniqueKeys` data from cache into request payload for SS.
|
|
56
|
-
*/
|
|
57
|
-
UniqueKeysCacheInMemory.prototype.fromUniqueKeysCollector = function (uniqueKeys) {
|
|
58
|
-
var payload = [];
|
|
59
|
-
var featureNames = Object.keys(uniqueKeys);
|
|
60
|
-
for (var i = 0; i < featureNames.length; i++) {
|
|
61
|
-
var featureName = featureNames[i];
|
|
62
|
-
var featureKeys = (0, sets_1.setToArray)(uniqueKeys[featureName]);
|
|
63
|
-
var uniqueKeysPayload = {
|
|
64
|
-
f: featureName,
|
|
65
|
-
ks: featureKeys
|
|
66
|
-
};
|
|
67
|
-
payload.push(uniqueKeysPayload);
|
|
68
|
-
}
|
|
69
|
-
return { keys: payload };
|
|
70
|
-
};
|
|
71
70
|
return UniqueKeysCacheInMemory;
|
|
72
71
|
}());
|
|
73
72
|
exports.UniqueKeysCacheInMemory = UniqueKeysCacheInMemory;
|
|
@@ -19,14 +19,12 @@ var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
|
|
|
19
19
|
this.onFullQueue = cb;
|
|
20
20
|
};
|
|
21
21
|
/**
|
|
22
|
-
* Store unique keys
|
|
23
|
-
* key: string = key.
|
|
24
|
-
* value: HashSet<string> = set of split names.
|
|
22
|
+
* Store unique keys per feature.
|
|
25
23
|
*/
|
|
26
|
-
UniqueKeysCacheInMemoryCS.prototype.track = function (
|
|
27
|
-
if (!this.uniqueKeysTracker[
|
|
28
|
-
this.uniqueKeysTracker[
|
|
29
|
-
var tracker = this.uniqueKeysTracker[
|
|
24
|
+
UniqueKeysCacheInMemoryCS.prototype.track = function (userKey, featureName) {
|
|
25
|
+
if (!this.uniqueKeysTracker[userKey])
|
|
26
|
+
this.uniqueKeysTracker[userKey] = new sets_1._Set();
|
|
27
|
+
var tracker = this.uniqueKeysTracker[userKey];
|
|
30
28
|
if (!tracker.has(featureName)) {
|
|
31
29
|
tracker.add(featureName);
|
|
32
30
|
this.uniqueTrackerSize++;
|
|
@@ -61,12 +59,12 @@ var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
|
|
|
61
59
|
*/
|
|
62
60
|
UniqueKeysCacheInMemoryCS.prototype.fromUniqueKeysCollector = function (uniqueKeys) {
|
|
63
61
|
var payload = [];
|
|
64
|
-
var
|
|
65
|
-
for (var k = 0; k <
|
|
66
|
-
var
|
|
67
|
-
var featureNames = (0, sets_1.setToArray)(uniqueKeys[
|
|
62
|
+
var userKeys = Object.keys(uniqueKeys);
|
|
63
|
+
for (var k = 0; k < userKeys.length; k++) {
|
|
64
|
+
var userKey = userKeys[k];
|
|
65
|
+
var featureNames = (0, sets_1.setToArray)(uniqueKeys[userKey]);
|
|
68
66
|
var uniqueKeysPayload = {
|
|
69
|
-
k:
|
|
67
|
+
k: userKey,
|
|
70
68
|
fs: featureNames
|
|
71
69
|
};
|
|
72
70
|
payload.push(uniqueKeysPayload);
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ImpressionCountsCacheInRedis = void 0;
|
|
4
4
|
var tslib_1 = require("tslib");
|
|
5
|
+
var lang_1 = require("../../utils/lang");
|
|
5
6
|
var ImpressionCountsCacheInMemory_1 = require("../inMemory/ImpressionCountsCacheInMemory");
|
|
6
7
|
var constants_1 = require("./constants");
|
|
7
8
|
var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
|
|
@@ -45,6 +46,40 @@ var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
|
|
|
45
46
|
clearInterval(this.intervalId);
|
|
46
47
|
return this.postImpressionCountsInRedis();
|
|
47
48
|
};
|
|
49
|
+
// Async consumer API, used by synchronizer
|
|
50
|
+
ImpressionCountsCacheInRedis.prototype.getImpressionsCount = function () {
|
|
51
|
+
var _this = this;
|
|
52
|
+
return this.redis.hgetall(this.key)
|
|
53
|
+
.then(function (counts) {
|
|
54
|
+
if (!Object.keys(counts).length)
|
|
55
|
+
return undefined;
|
|
56
|
+
_this.redis.del(_this.key).catch(function () { });
|
|
57
|
+
var impressionsCount = { pf: [] };
|
|
58
|
+
(0, lang_1.forOwn)(counts, function (count, key) {
|
|
59
|
+
var nameAndTime = key.split('::');
|
|
60
|
+
if (nameAndTime.length !== 2) {
|
|
61
|
+
_this.log.error(constants_1.LOG_PREFIX + "Error spliting key " + key);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
var timeFrame = parseInt(nameAndTime[1]);
|
|
65
|
+
if (isNaN(timeFrame)) {
|
|
66
|
+
_this.log.error(constants_1.LOG_PREFIX + "Error parsing time frame " + nameAndTime[1]);
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
var rawCount = parseInt(count);
|
|
70
|
+
if (isNaN(rawCount)) {
|
|
71
|
+
_this.log.error(constants_1.LOG_PREFIX + "Error parsing raw count " + count);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
impressionsCount.pf.push({
|
|
75
|
+
f: nameAndTime[0],
|
|
76
|
+
m: timeFrame,
|
|
77
|
+
rc: rawCount,
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
return impressionsCount;
|
|
81
|
+
});
|
|
82
|
+
};
|
|
48
83
|
return ImpressionCountsCacheInRedis;
|
|
49
84
|
}(ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory));
|
|
50
85
|
exports.ImpressionCountsCacheInRedis = ImpressionCountsCacheInRedis;
|
|
@@ -54,6 +54,18 @@ var UniqueKeysCacheInRedis = /** @class */ (function (_super) {
|
|
|
54
54
|
clearInterval(this.intervalId);
|
|
55
55
|
return this.postUniqueKeysInRedis();
|
|
56
56
|
};
|
|
57
|
+
/**
|
|
58
|
+
* Async consumer API, used by synchronizer.
|
|
59
|
+
* @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
|
|
60
|
+
*/
|
|
61
|
+
UniqueKeysCacheInRedis.prototype.popNRaw = function (count) {
|
|
62
|
+
var _this = this;
|
|
63
|
+
if (count === void 0) { count = 0; }
|
|
64
|
+
return this.redis.lrange(this.key, 0, count - 1).then(function (uniqueKeyItems) {
|
|
65
|
+
return _this.redis.ltrim(_this.key, uniqueKeyItems.length, -1)
|
|
66
|
+
.then(function () { return uniqueKeyItems.map(function (uniqueKeyItem) { return JSON.parse(uniqueKeyItem); }); });
|
|
67
|
+
});
|
|
68
|
+
};
|
|
57
69
|
return UniqueKeysCacheInRedis;
|
|
58
70
|
}(UniqueKeysCacheInMemory_1.UniqueKeysCacheInMemory));
|
|
59
71
|
exports.UniqueKeysCacheInRedis = UniqueKeysCacheInRedis;
|
|
@@ -38,6 +38,44 @@ var ImpressionCountsCachePluggable = /** @class */ (function (_super) {
|
|
|
38
38
|
clearInterval(this.intervalId);
|
|
39
39
|
return this.storeImpressionCounts();
|
|
40
40
|
};
|
|
41
|
+
// Async consumer API, used by synchronizer
|
|
42
|
+
ImpressionCountsCachePluggable.prototype.getImpressionsCount = function () {
|
|
43
|
+
var _this = this;
|
|
44
|
+
return this.wrapper.getKeysByPrefix(this.key)
|
|
45
|
+
.then(function (keys) {
|
|
46
|
+
return keys.length ? Promise.all(keys.map(function (key) { return _this.wrapper.get(key); }))
|
|
47
|
+
.then(function (counts) {
|
|
48
|
+
keys.forEach(function (key) { return _this.wrapper.del(key).catch(function () { }); });
|
|
49
|
+
var impressionsCount = { pf: [] };
|
|
50
|
+
for (var i = 0; i < keys.length; i++) {
|
|
51
|
+
var key = keys[i];
|
|
52
|
+
var count = counts[i];
|
|
53
|
+
var keyFeatureNameAndTime = key.split('::');
|
|
54
|
+
if (keyFeatureNameAndTime.length !== 3) {
|
|
55
|
+
_this.log.error(constants_2.LOG_PREFIX + "Error spliting key " + key);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
var timeFrame = parseInt(keyFeatureNameAndTime[2]);
|
|
59
|
+
if (isNaN(timeFrame)) {
|
|
60
|
+
_this.log.error(constants_2.LOG_PREFIX + "Error parsing time frame " + keyFeatureNameAndTime[2]);
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
// @ts-ignore
|
|
64
|
+
var rawCount = parseInt(count);
|
|
65
|
+
if (isNaN(rawCount)) {
|
|
66
|
+
_this.log.error(constants_2.LOG_PREFIX + "Error parsing raw count " + count);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
impressionsCount.pf.push({
|
|
70
|
+
f: keyFeatureNameAndTime[1],
|
|
71
|
+
m: timeFrame,
|
|
72
|
+
rc: rawCount,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
return impressionsCount;
|
|
76
|
+
}) : undefined;
|
|
77
|
+
});
|
|
78
|
+
};
|
|
41
79
|
return ImpressionCountsCachePluggable;
|
|
42
80
|
}(ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory));
|
|
43
81
|
exports.ImpressionCountsCachePluggable = ImpressionCountsCachePluggable;
|
|
@@ -45,6 +45,17 @@ var UniqueKeysCachePluggable = /** @class */ (function (_super) {
|
|
|
45
45
|
clearInterval(this.intervalId);
|
|
46
46
|
return this.storeUniqueKeys();
|
|
47
47
|
};
|
|
48
|
+
/**
|
|
49
|
+
* Async consumer API, used by synchronizer.
|
|
50
|
+
* @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
|
|
51
|
+
*/
|
|
52
|
+
UniqueKeysCachePluggable.prototype.popNRaw = function (count) {
|
|
53
|
+
var _this = this;
|
|
54
|
+
if (count === void 0) { count = 0; }
|
|
55
|
+
return Promise.resolve(count || this.wrapper.getItemsCount(this.key))
|
|
56
|
+
.then(function (count) { return _this.wrapper.popItems(_this.key, count); })
|
|
57
|
+
.then(function (uniqueKeyItems) { return uniqueKeyItems.map(function (uniqueKeyItem) { return JSON.parse(uniqueKeyItem); }); });
|
|
58
|
+
};
|
|
48
59
|
return UniqueKeysCachePluggable;
|
|
49
60
|
}(UniqueKeysCacheInMemory_1.UniqueKeysCacheInMemory));
|
|
50
61
|
exports.UniqueKeysCachePluggable = UniqueKeysCachePluggable;
|
|
@@ -55,14 +55,19 @@ function PluggableStorage(options) {
|
|
|
55
55
|
var log = params.log, metadata = params.metadata, onReadyCb = params.onReadyCb, mode = params.mode, eventsQueueSize = params.eventsQueueSize, impressionsQueueSize = params.impressionsQueueSize, impressionsMode = params.impressionsMode, matchingKey = params.matchingKey;
|
|
56
56
|
var keys = new KeyBuilderSS_1.KeyBuilderSS(prefix, metadata);
|
|
57
57
|
var wrapper = (0, wrapperAdapter_1.wrapperAdapter)(log, options.wrapper);
|
|
58
|
+
var isSyncronizer = mode === undefined; // If mode is not defined, the synchronizer is running
|
|
58
59
|
var isPartialConsumer = mode === constants_1.CONSUMER_PARTIAL_MODE;
|
|
59
|
-
var telemetry = (0, TelemetryCacheInMemory_1.shouldRecordTelemetry)(params) ?
|
|
60
|
-
isPartialConsumer ?
|
|
60
|
+
var telemetry = (0, TelemetryCacheInMemory_1.shouldRecordTelemetry)(params) || isSyncronizer ?
|
|
61
|
+
isPartialConsumer ?
|
|
62
|
+
new TelemetryCacheInMemory_1.TelemetryCacheInMemory() :
|
|
63
|
+
new TelemetryCachePluggable_1.TelemetryCachePluggable(log, keys, wrapper) :
|
|
61
64
|
undefined;
|
|
62
|
-
var impressionCountsCache = impressionsMode !== constants_1.DEBUG ?
|
|
63
|
-
isPartialConsumer ?
|
|
65
|
+
var impressionCountsCache = impressionsMode !== constants_1.DEBUG || isSyncronizer ?
|
|
66
|
+
isPartialConsumer ?
|
|
67
|
+
new ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory() :
|
|
68
|
+
new ImpressionCountsCachePluggable_1.ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
|
|
64
69
|
undefined;
|
|
65
|
-
var uniqueKeysCache = impressionsMode === constants_1.NONE ?
|
|
70
|
+
var uniqueKeysCache = impressionsMode === constants_1.NONE || isSyncronizer ?
|
|
66
71
|
isPartialConsumer ?
|
|
67
72
|
matchingKey === undefined ? new UniqueKeysCacheInMemory_1.UniqueKeysCacheInMemory() : new UniqueKeysCacheInMemoryCS_1.UniqueKeysCacheInMemoryCS() :
|
|
68
73
|
new UniqueKeysCachePluggable_1.UniqueKeysCachePluggable(log, keys.buildUniqueKeysKey(), wrapper) :
|
|
@@ -75,8 +80,7 @@ function PluggableStorage(options) {
|
|
|
75
80
|
impressionCountsCache.start();
|
|
76
81
|
if (uniqueKeysCache && uniqueKeysCache.start)
|
|
77
82
|
uniqueKeysCache.start();
|
|
78
|
-
|
|
79
|
-
if (telemetry && telemetry.recordConfig && mode)
|
|
83
|
+
if (telemetry && telemetry.recordConfig && !isSyncronizer)
|
|
80
84
|
telemetry.recordConfig();
|
|
81
85
|
}).catch(function (e) {
|
|
82
86
|
e = e || new Error('Error connecting wrapper');
|
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
import { setToArray, _Set } from '../../utils/lang/sets';
|
|
2
2
|
import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
|
|
3
|
+
/**
|
|
4
|
+
* Converts `uniqueKeys` data from cache into request payload for SS.
|
|
5
|
+
*/
|
|
6
|
+
export function fromUniqueKeysCollector(uniqueKeys) {
|
|
7
|
+
var payload = [];
|
|
8
|
+
var featureNames = Object.keys(uniqueKeys);
|
|
9
|
+
for (var i = 0; i < featureNames.length; i++) {
|
|
10
|
+
var featureName = featureNames[i];
|
|
11
|
+
var userKeys = setToArray(uniqueKeys[featureName]);
|
|
12
|
+
var uniqueKeysPayload = {
|
|
13
|
+
f: featureName,
|
|
14
|
+
ks: userKeys
|
|
15
|
+
};
|
|
16
|
+
payload.push(uniqueKeysPayload);
|
|
17
|
+
}
|
|
18
|
+
return { keys: payload };
|
|
19
|
+
}
|
|
3
20
|
var UniqueKeysCacheInMemory = /** @class */ (function () {
|
|
4
21
|
function UniqueKeysCacheInMemory(uniqueKeysQueueSize) {
|
|
5
22
|
if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = DEFAULT_CACHE_SIZE; }
|
|
@@ -11,16 +28,14 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
|
|
|
11
28
|
this.onFullQueue = cb;
|
|
12
29
|
};
|
|
13
30
|
/**
|
|
14
|
-
* Store unique keys
|
|
15
|
-
* key: string = feature name.
|
|
16
|
-
* value: Set<string> = set of unique keys.
|
|
31
|
+
* Store unique keys per feature.
|
|
17
32
|
*/
|
|
18
|
-
UniqueKeysCacheInMemory.prototype.track = function (
|
|
33
|
+
UniqueKeysCacheInMemory.prototype.track = function (userKey, featureName) {
|
|
19
34
|
if (!this.uniqueKeysTracker[featureName])
|
|
20
35
|
this.uniqueKeysTracker[featureName] = new _Set();
|
|
21
36
|
var tracker = this.uniqueKeysTracker[featureName];
|
|
22
|
-
if (!tracker.has(
|
|
23
|
-
tracker.add(
|
|
37
|
+
if (!tracker.has(userKey)) {
|
|
38
|
+
tracker.add(userKey);
|
|
24
39
|
this.uniqueTrackerSize++;
|
|
25
40
|
}
|
|
26
41
|
if (this.uniqueTrackerSize >= this.maxStorage && this.onFullQueue) {
|
|
@@ -40,7 +55,7 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
|
|
|
40
55
|
UniqueKeysCacheInMemory.prototype.pop = function () {
|
|
41
56
|
var data = this.uniqueKeysTracker;
|
|
42
57
|
this.uniqueKeysTracker = {};
|
|
43
|
-
return
|
|
58
|
+
return fromUniqueKeysCollector(data);
|
|
44
59
|
};
|
|
45
60
|
/**
|
|
46
61
|
* Check if the cache is empty.
|
|
@@ -48,23 +63,6 @@ var UniqueKeysCacheInMemory = /** @class */ (function () {
|
|
|
48
63
|
UniqueKeysCacheInMemory.prototype.isEmpty = function () {
|
|
49
64
|
return Object.keys(this.uniqueKeysTracker).length === 0;
|
|
50
65
|
};
|
|
51
|
-
/**
|
|
52
|
-
* Converts `uniqueKeys` data from cache into request payload for SS.
|
|
53
|
-
*/
|
|
54
|
-
UniqueKeysCacheInMemory.prototype.fromUniqueKeysCollector = function (uniqueKeys) {
|
|
55
|
-
var payload = [];
|
|
56
|
-
var featureNames = Object.keys(uniqueKeys);
|
|
57
|
-
for (var i = 0; i < featureNames.length; i++) {
|
|
58
|
-
var featureName = featureNames[i];
|
|
59
|
-
var featureKeys = setToArray(uniqueKeys[featureName]);
|
|
60
|
-
var uniqueKeysPayload = {
|
|
61
|
-
f: featureName,
|
|
62
|
-
ks: featureKeys
|
|
63
|
-
};
|
|
64
|
-
payload.push(uniqueKeysPayload);
|
|
65
|
-
}
|
|
66
|
-
return { keys: payload };
|
|
67
|
-
};
|
|
68
66
|
return UniqueKeysCacheInMemory;
|
|
69
67
|
}());
|
|
70
68
|
export { UniqueKeysCacheInMemory };
|
|
@@ -16,14 +16,12 @@ var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
|
|
|
16
16
|
this.onFullQueue = cb;
|
|
17
17
|
};
|
|
18
18
|
/**
|
|
19
|
-
* Store unique keys
|
|
20
|
-
* key: string = key.
|
|
21
|
-
* value: HashSet<string> = set of split names.
|
|
19
|
+
* Store unique keys per feature.
|
|
22
20
|
*/
|
|
23
|
-
UniqueKeysCacheInMemoryCS.prototype.track = function (
|
|
24
|
-
if (!this.uniqueKeysTracker[
|
|
25
|
-
this.uniqueKeysTracker[
|
|
26
|
-
var tracker = this.uniqueKeysTracker[
|
|
21
|
+
UniqueKeysCacheInMemoryCS.prototype.track = function (userKey, featureName) {
|
|
22
|
+
if (!this.uniqueKeysTracker[userKey])
|
|
23
|
+
this.uniqueKeysTracker[userKey] = new _Set();
|
|
24
|
+
var tracker = this.uniqueKeysTracker[userKey];
|
|
27
25
|
if (!tracker.has(featureName)) {
|
|
28
26
|
tracker.add(featureName);
|
|
29
27
|
this.uniqueTrackerSize++;
|
|
@@ -58,12 +56,12 @@ var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
|
|
|
58
56
|
*/
|
|
59
57
|
UniqueKeysCacheInMemoryCS.prototype.fromUniqueKeysCollector = function (uniqueKeys) {
|
|
60
58
|
var payload = [];
|
|
61
|
-
var
|
|
62
|
-
for (var k = 0; k <
|
|
63
|
-
var
|
|
64
|
-
var featureNames = setToArray(uniqueKeys[
|
|
59
|
+
var userKeys = Object.keys(uniqueKeys);
|
|
60
|
+
for (var k = 0; k < userKeys.length; k++) {
|
|
61
|
+
var userKey = userKeys[k];
|
|
62
|
+
var featureNames = setToArray(uniqueKeys[userKey]);
|
|
65
63
|
var uniqueKeysPayload = {
|
|
66
|
-
k:
|
|
64
|
+
k: userKey,
|
|
67
65
|
fs: featureNames
|
|
68
66
|
};
|
|
69
67
|
payload.push(uniqueKeysPayload);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { __extends } from "tslib";
|
|
2
|
+
import { forOwn } from '../../utils/lang';
|
|
2
3
|
import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
|
|
3
4
|
import { LOG_PREFIX, REFRESH_RATE, TTL_REFRESH } from './constants';
|
|
4
5
|
var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
|
|
@@ -42,6 +43,40 @@ var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
|
|
|
42
43
|
clearInterval(this.intervalId);
|
|
43
44
|
return this.postImpressionCountsInRedis();
|
|
44
45
|
};
|
|
46
|
+
// Async consumer API, used by synchronizer
|
|
47
|
+
ImpressionCountsCacheInRedis.prototype.getImpressionsCount = function () {
|
|
48
|
+
var _this = this;
|
|
49
|
+
return this.redis.hgetall(this.key)
|
|
50
|
+
.then(function (counts) {
|
|
51
|
+
if (!Object.keys(counts).length)
|
|
52
|
+
return undefined;
|
|
53
|
+
_this.redis.del(_this.key).catch(function () { });
|
|
54
|
+
var impressionsCount = { pf: [] };
|
|
55
|
+
forOwn(counts, function (count, key) {
|
|
56
|
+
var nameAndTime = key.split('::');
|
|
57
|
+
if (nameAndTime.length !== 2) {
|
|
58
|
+
_this.log.error(LOG_PREFIX + "Error spliting key " + key);
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
var timeFrame = parseInt(nameAndTime[1]);
|
|
62
|
+
if (isNaN(timeFrame)) {
|
|
63
|
+
_this.log.error(LOG_PREFIX + "Error parsing time frame " + nameAndTime[1]);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
var rawCount = parseInt(count);
|
|
67
|
+
if (isNaN(rawCount)) {
|
|
68
|
+
_this.log.error(LOG_PREFIX + "Error parsing raw count " + count);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
impressionsCount.pf.push({
|
|
72
|
+
f: nameAndTime[0],
|
|
73
|
+
m: timeFrame,
|
|
74
|
+
rc: rawCount,
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
return impressionsCount;
|
|
78
|
+
});
|
|
79
|
+
};
|
|
45
80
|
return ImpressionCountsCacheInRedis;
|
|
46
81
|
}(ImpressionCountsCacheInMemory));
|
|
47
82
|
export { ImpressionCountsCacheInRedis };
|
|
@@ -51,6 +51,18 @@ var UniqueKeysCacheInRedis = /** @class */ (function (_super) {
|
|
|
51
51
|
clearInterval(this.intervalId);
|
|
52
52
|
return this.postUniqueKeysInRedis();
|
|
53
53
|
};
|
|
54
|
+
/**
|
|
55
|
+
* Async consumer API, used by synchronizer.
|
|
56
|
+
* @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
|
|
57
|
+
*/
|
|
58
|
+
UniqueKeysCacheInRedis.prototype.popNRaw = function (count) {
|
|
59
|
+
var _this = this;
|
|
60
|
+
if (count === void 0) { count = 0; }
|
|
61
|
+
return this.redis.lrange(this.key, 0, count - 1).then(function (uniqueKeyItems) {
|
|
62
|
+
return _this.redis.ltrim(_this.key, uniqueKeyItems.length, -1)
|
|
63
|
+
.then(function () { return uniqueKeyItems.map(function (uniqueKeyItem) { return JSON.parse(uniqueKeyItem); }); });
|
|
64
|
+
});
|
|
65
|
+
};
|
|
54
66
|
return UniqueKeysCacheInRedis;
|
|
55
67
|
}(UniqueKeysCacheInMemory));
|
|
56
68
|
export { UniqueKeysCacheInRedis };
|
|
@@ -35,6 +35,44 @@ var ImpressionCountsCachePluggable = /** @class */ (function (_super) {
|
|
|
35
35
|
clearInterval(this.intervalId);
|
|
36
36
|
return this.storeImpressionCounts();
|
|
37
37
|
};
|
|
38
|
+
// Async consumer API, used by synchronizer
|
|
39
|
+
ImpressionCountsCachePluggable.prototype.getImpressionsCount = function () {
|
|
40
|
+
var _this = this;
|
|
41
|
+
return this.wrapper.getKeysByPrefix(this.key)
|
|
42
|
+
.then(function (keys) {
|
|
43
|
+
return keys.length ? Promise.all(keys.map(function (key) { return _this.wrapper.get(key); }))
|
|
44
|
+
.then(function (counts) {
|
|
45
|
+
keys.forEach(function (key) { return _this.wrapper.del(key).catch(function () { }); });
|
|
46
|
+
var impressionsCount = { pf: [] };
|
|
47
|
+
for (var i = 0; i < keys.length; i++) {
|
|
48
|
+
var key = keys[i];
|
|
49
|
+
var count = counts[i];
|
|
50
|
+
var keyFeatureNameAndTime = key.split('::');
|
|
51
|
+
if (keyFeatureNameAndTime.length !== 3) {
|
|
52
|
+
_this.log.error(LOG_PREFIX + "Error spliting key " + key);
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
var timeFrame = parseInt(keyFeatureNameAndTime[2]);
|
|
56
|
+
if (isNaN(timeFrame)) {
|
|
57
|
+
_this.log.error(LOG_PREFIX + "Error parsing time frame " + keyFeatureNameAndTime[2]);
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
// @ts-ignore
|
|
61
|
+
var rawCount = parseInt(count);
|
|
62
|
+
if (isNaN(rawCount)) {
|
|
63
|
+
_this.log.error(LOG_PREFIX + "Error parsing raw count " + count);
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
impressionsCount.pf.push({
|
|
67
|
+
f: keyFeatureNameAndTime[1],
|
|
68
|
+
m: timeFrame,
|
|
69
|
+
rc: rawCount,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
return impressionsCount;
|
|
73
|
+
}) : undefined;
|
|
74
|
+
});
|
|
75
|
+
};
|
|
38
76
|
return ImpressionCountsCachePluggable;
|
|
39
77
|
}(ImpressionCountsCacheInMemory));
|
|
40
78
|
export { ImpressionCountsCachePluggable };
|
|
@@ -42,6 +42,17 @@ var UniqueKeysCachePluggable = /** @class */ (function (_super) {
|
|
|
42
42
|
clearInterval(this.intervalId);
|
|
43
43
|
return this.storeUniqueKeys();
|
|
44
44
|
};
|
|
45
|
+
/**
|
|
46
|
+
* Async consumer API, used by synchronizer.
|
|
47
|
+
* @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
|
|
48
|
+
*/
|
|
49
|
+
UniqueKeysCachePluggable.prototype.popNRaw = function (count) {
|
|
50
|
+
var _this = this;
|
|
51
|
+
if (count === void 0) { count = 0; }
|
|
52
|
+
return Promise.resolve(count || this.wrapper.getItemsCount(this.key))
|
|
53
|
+
.then(function (count) { return _this.wrapper.popItems(_this.key, count); })
|
|
54
|
+
.then(function (uniqueKeyItems) { return uniqueKeyItems.map(function (uniqueKeyItem) { return JSON.parse(uniqueKeyItem); }); });
|
|
55
|
+
};
|
|
45
56
|
return UniqueKeysCachePluggable;
|
|
46
57
|
}(UniqueKeysCacheInMemory));
|
|
47
58
|
export { UniqueKeysCachePluggable };
|
|
@@ -52,14 +52,19 @@ export function PluggableStorage(options) {
|
|
|
52
52
|
var log = params.log, metadata = params.metadata, onReadyCb = params.onReadyCb, mode = params.mode, eventsQueueSize = params.eventsQueueSize, impressionsQueueSize = params.impressionsQueueSize, impressionsMode = params.impressionsMode, matchingKey = params.matchingKey;
|
|
53
53
|
var keys = new KeyBuilderSS(prefix, metadata);
|
|
54
54
|
var wrapper = wrapperAdapter(log, options.wrapper);
|
|
55
|
+
var isSyncronizer = mode === undefined; // If mode is not defined, the synchronizer is running
|
|
55
56
|
var isPartialConsumer = mode === CONSUMER_PARTIAL_MODE;
|
|
56
|
-
var telemetry = shouldRecordTelemetry(params) ?
|
|
57
|
-
isPartialConsumer ?
|
|
57
|
+
var telemetry = shouldRecordTelemetry(params) || isSyncronizer ?
|
|
58
|
+
isPartialConsumer ?
|
|
59
|
+
new TelemetryCacheInMemory() :
|
|
60
|
+
new TelemetryCachePluggable(log, keys, wrapper) :
|
|
58
61
|
undefined;
|
|
59
|
-
var impressionCountsCache = impressionsMode !== DEBUG ?
|
|
60
|
-
isPartialConsumer ?
|
|
62
|
+
var impressionCountsCache = impressionsMode !== DEBUG || isSyncronizer ?
|
|
63
|
+
isPartialConsumer ?
|
|
64
|
+
new ImpressionCountsCacheInMemory() :
|
|
65
|
+
new ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
|
|
61
66
|
undefined;
|
|
62
|
-
var uniqueKeysCache = impressionsMode === NONE ?
|
|
67
|
+
var uniqueKeysCache = impressionsMode === NONE || isSyncronizer ?
|
|
63
68
|
isPartialConsumer ?
|
|
64
69
|
matchingKey === undefined ? new UniqueKeysCacheInMemory() : new UniqueKeysCacheInMemoryCS() :
|
|
65
70
|
new UniqueKeysCachePluggable(log, keys.buildUniqueKeysKey(), wrapper) :
|
|
@@ -72,8 +77,7 @@ export function PluggableStorage(options) {
|
|
|
72
77
|
impressionCountsCache.start();
|
|
73
78
|
if (uniqueKeysCache && uniqueKeysCache.start)
|
|
74
79
|
uniqueKeysCache.start();
|
|
75
|
-
|
|
76
|
-
if (telemetry && telemetry.recordConfig && mode)
|
|
80
|
+
if (telemetry && telemetry.recordConfig && !isSyncronizer)
|
|
77
81
|
telemetry.recordConfig();
|
|
78
82
|
}).catch(function (e) {
|
|
79
83
|
e = e || new Error('Error connecting wrapper');
|
package/package.json
CHANGED
|
@@ -3,12 +3,31 @@ import { ISet, setToArray, _Set } from '../../utils/lang/sets';
|
|
|
3
3
|
import { UniqueKeysPayloadSs } from '../../sync/submitters/types';
|
|
4
4
|
import { DEFAULT_CACHE_SIZE } from '../inRedis/constants';
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Converts `uniqueKeys` data from cache into request payload for SS.
|
|
8
|
+
*/
|
|
9
|
+
export function fromUniqueKeysCollector(uniqueKeys: { [featureName: string]: ISet<string> }): UniqueKeysPayloadSs {
|
|
10
|
+
const payload = [];
|
|
11
|
+
const featureNames = Object.keys(uniqueKeys);
|
|
12
|
+
for (let i = 0; i < featureNames.length; i++) {
|
|
13
|
+
const featureName = featureNames[i];
|
|
14
|
+
const userKeys = setToArray(uniqueKeys[featureName]);
|
|
15
|
+
const uniqueKeysPayload = {
|
|
16
|
+
f: featureName,
|
|
17
|
+
ks: userKeys
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
payload.push(uniqueKeysPayload);
|
|
21
|
+
}
|
|
22
|
+
return { keys: payload };
|
|
23
|
+
}
|
|
24
|
+
|
|
6
25
|
export class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
7
26
|
|
|
8
27
|
protected onFullQueue?: () => void;
|
|
9
28
|
private readonly maxStorage: number;
|
|
10
29
|
private uniqueTrackerSize = 0;
|
|
11
|
-
protected uniqueKeysTracker: { [
|
|
30
|
+
protected uniqueKeysTracker: { [featureName: string]: ISet<string> };
|
|
12
31
|
|
|
13
32
|
constructor(uniqueKeysQueueSize = DEFAULT_CACHE_SIZE) {
|
|
14
33
|
this.maxStorage = uniqueKeysQueueSize;
|
|
@@ -20,15 +39,13 @@ export class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
|
20
39
|
}
|
|
21
40
|
|
|
22
41
|
/**
|
|
23
|
-
* Store unique keys
|
|
24
|
-
* key: string = feature name.
|
|
25
|
-
* value: Set<string> = set of unique keys.
|
|
42
|
+
* Store unique keys per feature.
|
|
26
43
|
*/
|
|
27
|
-
track(
|
|
44
|
+
track(userKey: string, featureName: string) {
|
|
28
45
|
if (!this.uniqueKeysTracker[featureName]) this.uniqueKeysTracker[featureName] = new _Set();
|
|
29
46
|
const tracker = this.uniqueKeysTracker[featureName];
|
|
30
|
-
if (!tracker.has(
|
|
31
|
-
tracker.add(
|
|
47
|
+
if (!tracker.has(userKey)) {
|
|
48
|
+
tracker.add(userKey);
|
|
32
49
|
this.uniqueTrackerSize++;
|
|
33
50
|
}
|
|
34
51
|
if (this.uniqueTrackerSize >= this.maxStorage && this.onFullQueue) {
|
|
@@ -50,7 +67,7 @@ export class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
|
50
67
|
pop() {
|
|
51
68
|
const data = this.uniqueKeysTracker;
|
|
52
69
|
this.uniqueKeysTracker = {};
|
|
53
|
-
return
|
|
70
|
+
return fromUniqueKeysCollector(data);
|
|
54
71
|
}
|
|
55
72
|
|
|
56
73
|
/**
|
|
@@ -60,23 +77,4 @@ export class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
|
60
77
|
return Object.keys(this.uniqueKeysTracker).length === 0;
|
|
61
78
|
}
|
|
62
79
|
|
|
63
|
-
/**
|
|
64
|
-
* Converts `uniqueKeys` data from cache into request payload for SS.
|
|
65
|
-
*/
|
|
66
|
-
private fromUniqueKeysCollector(uniqueKeys: { [featureName: string]: ISet<string> }): UniqueKeysPayloadSs {
|
|
67
|
-
const payload = [];
|
|
68
|
-
const featureNames = Object.keys(uniqueKeys);
|
|
69
|
-
for (let i = 0; i < featureNames.length; i++) {
|
|
70
|
-
const featureName = featureNames[i];
|
|
71
|
-
const featureKeys = setToArray(uniqueKeys[featureName]);
|
|
72
|
-
const uniqueKeysPayload = {
|
|
73
|
-
f: featureName,
|
|
74
|
-
ks: featureKeys
|
|
75
|
-
};
|
|
76
|
-
|
|
77
|
-
payload.push(uniqueKeysPayload);
|
|
78
|
-
}
|
|
79
|
-
return { keys: payload };
|
|
80
|
-
}
|
|
81
|
-
|
|
82
80
|
}
|
|
@@ -8,7 +8,7 @@ export class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
|
|
|
8
8
|
private onFullQueue?: () => void;
|
|
9
9
|
private readonly maxStorage: number;
|
|
10
10
|
private uniqueTrackerSize = 0;
|
|
11
|
-
private uniqueKeysTracker: { [
|
|
11
|
+
private uniqueKeysTracker: { [userKey: string]: ISet<string> };
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
*
|
|
@@ -25,14 +25,12 @@ export class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
|
|
|
25
25
|
}
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
|
-
* Store unique keys
|
|
29
|
-
* key: string = key.
|
|
30
|
-
* value: HashSet<string> = set of split names.
|
|
28
|
+
* Store unique keys per feature.
|
|
31
29
|
*/
|
|
32
|
-
track(
|
|
30
|
+
track(userKey: string, featureName: string) {
|
|
33
31
|
|
|
34
|
-
if (!this.uniqueKeysTracker[
|
|
35
|
-
const tracker = this.uniqueKeysTracker[
|
|
32
|
+
if (!this.uniqueKeysTracker[userKey]) this.uniqueKeysTracker[userKey] = new _Set();
|
|
33
|
+
const tracker = this.uniqueKeysTracker[userKey];
|
|
36
34
|
if (!tracker.has(featureName)) {
|
|
37
35
|
tracker.add(featureName);
|
|
38
36
|
this.uniqueTrackerSize++;
|
|
@@ -69,14 +67,14 @@ export class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
|
|
|
69
67
|
/**
|
|
70
68
|
* Converts `uniqueKeys` data from cache into request payload.
|
|
71
69
|
*/
|
|
72
|
-
private fromUniqueKeysCollector(uniqueKeys: { [
|
|
70
|
+
private fromUniqueKeysCollector(uniqueKeys: { [userKey: string]: ISet<string> }): UniqueKeysPayloadCs {
|
|
73
71
|
const payload = [];
|
|
74
|
-
const
|
|
75
|
-
for (let k = 0; k <
|
|
76
|
-
const
|
|
77
|
-
const featureNames = setToArray(uniqueKeys[
|
|
72
|
+
const userKeys = Object.keys(uniqueKeys);
|
|
73
|
+
for (let k = 0; k < userKeys.length; k++) {
|
|
74
|
+
const userKey = userKeys[k];
|
|
75
|
+
const featureNames = setToArray(uniqueKeys[userKey]);
|
|
78
76
|
const uniqueKeysPayload = {
|
|
79
|
-
k:
|
|
77
|
+
k: userKey,
|
|
80
78
|
fs: featureNames
|
|
81
79
|
};
|
|
82
80
|
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { Redis } from 'ioredis';
|
|
2
2
|
import { ILogger } from '../../logger/types';
|
|
3
|
+
import { ImpressionCountsPayload } from '../../sync/submitters/types';
|
|
4
|
+
import { forOwn } from '../../utils/lang';
|
|
3
5
|
import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
|
|
4
6
|
import { LOG_PREFIX, REFRESH_RATE, TTL_REFRESH } from './constants';
|
|
5
7
|
|
|
@@ -20,7 +22,7 @@ export class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory
|
|
|
20
22
|
this.onFullQueue = () => { this.postImpressionCountsInRedis(); };
|
|
21
23
|
}
|
|
22
24
|
|
|
23
|
-
private postImpressionCountsInRedis(){
|
|
25
|
+
private postImpressionCountsInRedis() {
|
|
24
26
|
const counts = this.pop();
|
|
25
27
|
const keys = Object.keys(counts);
|
|
26
28
|
if (!keys.length) return Promise.resolve(false);
|
|
@@ -50,4 +52,44 @@ export class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory
|
|
|
50
52
|
clearInterval(this.intervalId);
|
|
51
53
|
return this.postImpressionCountsInRedis();
|
|
52
54
|
}
|
|
55
|
+
|
|
56
|
+
// Async consumer API, used by synchronizer
|
|
57
|
+
getImpressionsCount(): Promise<ImpressionCountsPayload | undefined> {
|
|
58
|
+
return this.redis.hgetall(this.key)
|
|
59
|
+
.then(counts => {
|
|
60
|
+
if (!Object.keys(counts).length) return undefined;
|
|
61
|
+
|
|
62
|
+
this.redis.del(this.key).catch(() => { /* no-op */ });
|
|
63
|
+
|
|
64
|
+
const impressionsCount: ImpressionCountsPayload = { pf: [] };
|
|
65
|
+
|
|
66
|
+
forOwn(counts, (count, key) => {
|
|
67
|
+
const nameAndTime = key.split('::');
|
|
68
|
+
if (nameAndTime.length !== 2) {
|
|
69
|
+
this.log.error(`${LOG_PREFIX}Error spliting key ${key}`);
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const timeFrame = parseInt(nameAndTime[1]);
|
|
74
|
+
if (isNaN(timeFrame)) {
|
|
75
|
+
this.log.error(`${LOG_PREFIX}Error parsing time frame ${nameAndTime[1]}`);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const rawCount = parseInt(count);
|
|
80
|
+
if (isNaN(rawCount)) {
|
|
81
|
+
this.log.error(`${LOG_PREFIX}Error parsing raw count ${count}`);
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
impressionsCount.pf.push({
|
|
86
|
+
f: nameAndTime[0],
|
|
87
|
+
m: timeFrame,
|
|
88
|
+
rc: rawCount,
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
return impressionsCount;
|
|
93
|
+
});
|
|
94
|
+
}
|
|
53
95
|
}
|
|
@@ -5,6 +5,7 @@ import { setToArray } from '../../utils/lang/sets';
|
|
|
5
5
|
import { DEFAULT_CACHE_SIZE, REFRESH_RATE, TTL_REFRESH } from './constants';
|
|
6
6
|
import { LOG_PREFIX } from './constants';
|
|
7
7
|
import { ILogger } from '../../logger/types';
|
|
8
|
+
import { UniqueKeysItemSs } from '../../sync/submitters/types';
|
|
8
9
|
|
|
9
10
|
export class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
10
11
|
|
|
@@ -20,7 +21,7 @@ export class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements I
|
|
|
20
21
|
this.key = key;
|
|
21
22
|
this.redis = redis;
|
|
22
23
|
this.refreshRate = refreshRate;
|
|
23
|
-
this.onFullQueue = () => {this.postUniqueKeysInRedis();};
|
|
24
|
+
this.onFullQueue = () => { this.postUniqueKeysInRedis(); };
|
|
24
25
|
}
|
|
25
26
|
|
|
26
27
|
private postUniqueKeysInRedis() {
|
|
@@ -62,4 +63,15 @@ export class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements I
|
|
|
62
63
|
return this.postUniqueKeysInRedis();
|
|
63
64
|
}
|
|
64
65
|
|
|
66
|
+
/**
|
|
67
|
+
* Async consumer API, used by synchronizer.
|
|
68
|
+
* @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
|
|
69
|
+
*/
|
|
70
|
+
popNRaw(count = 0): Promise<UniqueKeysItemSs[]> {
|
|
71
|
+
return this.redis.lrange(this.key, 0, count - 1).then(uniqueKeyItems => {
|
|
72
|
+
return this.redis.ltrim(this.key, uniqueKeyItems.length, -1)
|
|
73
|
+
.then(() => uniqueKeyItems.map(uniqueKeyItem => JSON.parse(uniqueKeyItem)));
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
|
|
65
77
|
}
|
|
@@ -52,7 +52,7 @@ export function InRedisStorage(options: InRedisStorageOptions = {}): IStorageAsy
|
|
|
52
52
|
|
|
53
53
|
// When using REDIS we should:
|
|
54
54
|
// 1- Disconnect from the storage
|
|
55
|
-
destroy(): Promise<void>{
|
|
55
|
+
destroy(): Promise<void> {
|
|
56
56
|
let promises = [];
|
|
57
57
|
if (impressionCountsCache) promises.push(impressionCountsCache.stop());
|
|
58
58
|
if (uniqueKeysCache) promises.push(uniqueKeysCache.stop());
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ILogger } from '../../logger/types';
|
|
2
|
+
import { ImpressionCountsPayload } from '../../sync/submitters/types';
|
|
2
3
|
import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
|
|
3
4
|
import { REFRESH_RATE } from '../inRedis/constants';
|
|
4
5
|
import { IPluggableStorageWrapper } from '../types';
|
|
@@ -44,4 +45,48 @@ export class ImpressionCountsCachePluggable extends ImpressionCountsCacheInMemor
|
|
|
44
45
|
clearInterval(this.intervalId);
|
|
45
46
|
return this.storeImpressionCounts();
|
|
46
47
|
}
|
|
48
|
+
|
|
49
|
+
// Async consumer API, used by synchronizer
|
|
50
|
+
getImpressionsCount(): Promise<ImpressionCountsPayload | undefined> {
|
|
51
|
+
return this.wrapper.getKeysByPrefix(this.key)
|
|
52
|
+
.then(keys => {
|
|
53
|
+
return keys.length ? Promise.all(keys.map(key => this.wrapper.get(key)))
|
|
54
|
+
.then(counts => {
|
|
55
|
+
keys.forEach(key => this.wrapper.del(key).catch(() => { /* noop */ }));
|
|
56
|
+
|
|
57
|
+
const impressionsCount: ImpressionCountsPayload = { pf: [] };
|
|
58
|
+
|
|
59
|
+
for (let i = 0; i < keys.length; i++) {
|
|
60
|
+
const key = keys[i];
|
|
61
|
+
const count = counts[i];
|
|
62
|
+
|
|
63
|
+
const keyFeatureNameAndTime = key.split('::');
|
|
64
|
+
if (keyFeatureNameAndTime.length !== 3) {
|
|
65
|
+
this.log.error(`${LOG_PREFIX}Error spliting key ${key}`);
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const timeFrame = parseInt(keyFeatureNameAndTime[2]);
|
|
70
|
+
if (isNaN(timeFrame)) {
|
|
71
|
+
this.log.error(`${LOG_PREFIX}Error parsing time frame ${keyFeatureNameAndTime[2]}`);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
// @ts-ignore
|
|
75
|
+
const rawCount = parseInt(count);
|
|
76
|
+
if (isNaN(rawCount)) {
|
|
77
|
+
this.log.error(`${LOG_PREFIX}Error parsing raw count ${count}`);
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
impressionsCount.pf.push({
|
|
82
|
+
f: keyFeatureNameAndTime[1],
|
|
83
|
+
m: timeFrame,
|
|
84
|
+
rc: rawCount,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return impressionsCount;
|
|
89
|
+
}) : undefined;
|
|
90
|
+
});
|
|
91
|
+
}
|
|
47
92
|
}
|
|
@@ -4,6 +4,7 @@ import { setToArray } from '../../utils/lang/sets';
|
|
|
4
4
|
import { DEFAULT_CACHE_SIZE, REFRESH_RATE } from '../inRedis/constants';
|
|
5
5
|
import { LOG_PREFIX } from './constants';
|
|
6
6
|
import { ILogger } from '../../logger/types';
|
|
7
|
+
import { UniqueKeysItemSs } from '../../sync/submitters/types';
|
|
7
8
|
|
|
8
9
|
export class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
9
10
|
|
|
@@ -53,4 +54,14 @@ export class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory implements
|
|
|
53
54
|
return this.storeUniqueKeys();
|
|
54
55
|
}
|
|
55
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Async consumer API, used by synchronizer.
|
|
59
|
+
* @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
|
|
60
|
+
*/
|
|
61
|
+
popNRaw(count = 0): Promise<UniqueKeysItemSs[]> {
|
|
62
|
+
return Promise.resolve(count || this.wrapper.getItemsCount(this.key))
|
|
63
|
+
.then(count => this.wrapper.popItems(this.key, count))
|
|
64
|
+
.then((uniqueKeyItems) => uniqueKeyItems.map(uniqueKeyItem => JSON.parse(uniqueKeyItem)));
|
|
65
|
+
}
|
|
66
|
+
|
|
56
67
|
}
|
|
@@ -64,14 +64,23 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
|
|
|
64
64
|
const { log, metadata, onReadyCb, mode, eventsQueueSize, impressionsQueueSize, impressionsMode, matchingKey } = params;
|
|
65
65
|
const keys = new KeyBuilderSS(prefix, metadata);
|
|
66
66
|
const wrapper = wrapperAdapter(log, options.wrapper);
|
|
67
|
+
|
|
68
|
+
const isSyncronizer = mode === undefined; // If mode is not defined, the synchronizer is running
|
|
67
69
|
const isPartialConsumer = mode === CONSUMER_PARTIAL_MODE;
|
|
68
|
-
|
|
69
|
-
|
|
70
|
+
|
|
71
|
+
const telemetry = shouldRecordTelemetry(params) || isSyncronizer ?
|
|
72
|
+
isPartialConsumer ?
|
|
73
|
+
new TelemetryCacheInMemory() :
|
|
74
|
+
new TelemetryCachePluggable(log, keys, wrapper) :
|
|
70
75
|
undefined;
|
|
71
|
-
|
|
72
|
-
|
|
76
|
+
|
|
77
|
+
const impressionCountsCache = impressionsMode !== DEBUG || isSyncronizer ?
|
|
78
|
+
isPartialConsumer ?
|
|
79
|
+
new ImpressionCountsCacheInMemory() :
|
|
80
|
+
new ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
|
|
73
81
|
undefined;
|
|
74
|
-
|
|
82
|
+
|
|
83
|
+
const uniqueKeysCache = impressionsMode === NONE || isSyncronizer ?
|
|
75
84
|
isPartialConsumer ?
|
|
76
85
|
matchingKey === undefined ? new UniqueKeysCacheInMemory() : new UniqueKeysCacheInMemoryCS() :
|
|
77
86
|
new UniqueKeysCachePluggable(log, keys.buildUniqueKeysKey(), wrapper) :
|
|
@@ -84,9 +93,7 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
|
|
|
84
93
|
// Start periodic flush of async storages
|
|
85
94
|
if (impressionCountsCache && (impressionCountsCache as ImpressionCountsCachePluggable).start) (impressionCountsCache as ImpressionCountsCachePluggable).start();
|
|
86
95
|
if (uniqueKeysCache && (uniqueKeysCache as UniqueKeysCachePluggable).start) (uniqueKeysCache as UniqueKeysCachePluggable).start();
|
|
87
|
-
|
|
88
|
-
// If mode is not defined, it means that the synchronizer is running and so we don't have to record telemetry
|
|
89
|
-
if (telemetry && (telemetry as ITelemetryCacheAsync).recordConfig && mode) (telemetry as ITelemetryCacheAsync).recordConfig();
|
|
96
|
+
if (telemetry && (telemetry as ITelemetryCacheAsync).recordConfig && !isSyncronizer) (telemetry as ITelemetryCacheAsync).recordConfig();
|
|
90
97
|
}).catch((e) => {
|
|
91
98
|
e = e || new Error('Error connecting wrapper');
|
|
92
99
|
onReadyCb(e);
|
package/src/storages/types.ts
CHANGED
|
@@ -349,12 +349,12 @@ export interface IEventsCacheAsync extends IEventsCacheBase, IRecorderCacheProdu
|
|
|
349
349
|
* Only in memory. Named `ImpressionsCounter` in spec.
|
|
350
350
|
*/
|
|
351
351
|
export interface IImpressionCountsCacheSync extends IRecorderCacheProducerSync<Record<string, number>> {
|
|
352
|
-
// Used by impressions tracker
|
|
352
|
+
// Used by impressions tracker
|
|
353
353
|
track(featureName: string, timeFrame: number, amount: number): void
|
|
354
354
|
|
|
355
355
|
// Used by impressions count submitter in standalone and producer mode
|
|
356
356
|
isEmpty(): boolean // check if cache is empty. Return true if the cache was just created or cleared.
|
|
357
|
-
pop(toMerge?: Record<string, number>
|
|
357
|
+
pop(toMerge?: Record<string, number>): Record<string, number> // pop cache data
|
|
358
358
|
}
|
|
359
359
|
|
|
360
360
|
export interface IUniqueKeysCacheBase {
|
|
@@ -37,13 +37,15 @@ export type ImpressionCountsPayload = {
|
|
|
37
37
|
}[]
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
+
export type UniqueKeysItemSs = {
|
|
41
|
+
/** Split name */
|
|
42
|
+
f: string
|
|
43
|
+
/** keyNames */
|
|
44
|
+
ks: string[]
|
|
45
|
+
}
|
|
46
|
+
|
|
40
47
|
export type UniqueKeysPayloadSs = {
|
|
41
|
-
keys:
|
|
42
|
-
/** Split name */
|
|
43
|
-
f: string
|
|
44
|
-
/** keyNames */
|
|
45
|
-
ks: string[]
|
|
46
|
-
}[]
|
|
48
|
+
keys: UniqueKeysItemSs[]
|
|
47
49
|
}
|
|
48
50
|
|
|
49
51
|
export type UniqueKeysPayloadCs = {
|
|
@@ -1,21 +1,25 @@
|
|
|
1
1
|
import { IUniqueKeysCacheBase } from '../types';
|
|
2
2
|
import { ISet } from '../../utils/lang/sets';
|
|
3
3
|
import { UniqueKeysPayloadSs } from '../../sync/submitters/types';
|
|
4
|
+
/**
|
|
5
|
+
* Converts `uniqueKeys` data from cache into request payload for SS.
|
|
6
|
+
*/
|
|
7
|
+
export declare function fromUniqueKeysCollector(uniqueKeys: {
|
|
8
|
+
[featureName: string]: ISet<string>;
|
|
9
|
+
}): UniqueKeysPayloadSs;
|
|
4
10
|
export declare class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
5
11
|
protected onFullQueue?: () => void;
|
|
6
12
|
private readonly maxStorage;
|
|
7
13
|
private uniqueTrackerSize;
|
|
8
14
|
protected uniqueKeysTracker: {
|
|
9
|
-
[
|
|
15
|
+
[featureName: string]: ISet<string>;
|
|
10
16
|
};
|
|
11
17
|
constructor(uniqueKeysQueueSize?: number);
|
|
12
18
|
setOnFullQueueCb(cb: () => void): void;
|
|
13
19
|
/**
|
|
14
|
-
* Store unique keys
|
|
15
|
-
* key: string = feature name.
|
|
16
|
-
* value: Set<string> = set of unique keys.
|
|
20
|
+
* Store unique keys per feature.
|
|
17
21
|
*/
|
|
18
|
-
track(
|
|
22
|
+
track(userKey: string, featureName: string): void;
|
|
19
23
|
/**
|
|
20
24
|
* Clear the data stored on the cache.
|
|
21
25
|
*/
|
|
@@ -28,8 +32,4 @@ export declare class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
|
28
32
|
* Check if the cache is empty.
|
|
29
33
|
*/
|
|
30
34
|
isEmpty(): boolean;
|
|
31
|
-
/**
|
|
32
|
-
* Converts `uniqueKeys` data from cache into request payload for SS.
|
|
33
|
-
*/
|
|
34
|
-
private fromUniqueKeysCollector;
|
|
35
35
|
}
|
|
@@ -13,11 +13,9 @@ export declare class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
|
|
|
13
13
|
constructor(uniqueKeysQueueSize?: number);
|
|
14
14
|
setOnFullQueueCb(cb: () => void): void;
|
|
15
15
|
/**
|
|
16
|
-
* Store unique keys
|
|
17
|
-
* key: string = key.
|
|
18
|
-
* value: HashSet<string> = set of split names.
|
|
16
|
+
* Store unique keys per feature.
|
|
19
17
|
*/
|
|
20
|
-
track(
|
|
18
|
+
track(userKey: string, featureName: string): void;
|
|
21
19
|
/**
|
|
22
20
|
* Clear the data stored on the cache.
|
|
23
21
|
*/
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Redis } from 'ioredis';
|
|
2
2
|
import { ILogger } from '../../logger/types';
|
|
3
|
+
import { ImpressionCountsPayload } from '../../sync/submitters/types';
|
|
3
4
|
import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
|
|
4
5
|
export declare class ImpressionCountsCacheInRedis extends ImpressionCountsCacheInMemory {
|
|
5
6
|
private readonly log;
|
|
@@ -11,4 +12,5 @@ export declare class ImpressionCountsCacheInRedis extends ImpressionCountsCacheI
|
|
|
11
12
|
private postImpressionCountsInRedis;
|
|
12
13
|
start(): void;
|
|
13
14
|
stop(): Promise<boolean | import("ioredis").BooleanResponse | undefined>;
|
|
15
|
+
getImpressionsCount(): Promise<ImpressionCountsPayload | undefined>;
|
|
14
16
|
}
|
|
@@ -2,6 +2,7 @@ import { IUniqueKeysCacheBase } from '../types';
|
|
|
2
2
|
import { Redis } from 'ioredis';
|
|
3
3
|
import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
|
|
4
4
|
import { ILogger } from '../../logger/types';
|
|
5
|
+
import { UniqueKeysItemSs } from '../../sync/submitters/types';
|
|
5
6
|
export declare class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
6
7
|
private readonly log;
|
|
7
8
|
private readonly key;
|
|
@@ -12,4 +13,9 @@ export declare class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory impl
|
|
|
12
13
|
private postUniqueKeysInRedis;
|
|
13
14
|
start(): void;
|
|
14
15
|
stop(): Promise<boolean | import("ioredis").BooleanResponse | undefined>;
|
|
16
|
+
/**
|
|
17
|
+
* Async consumer API, used by synchronizer.
|
|
18
|
+
* @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
|
|
19
|
+
*/
|
|
20
|
+
popNRaw(count?: number): Promise<UniqueKeysItemSs[]>;
|
|
15
21
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ILogger } from '../../logger/types';
|
|
2
|
+
import { ImpressionCountsPayload } from '../../sync/submitters/types';
|
|
2
3
|
import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
|
|
3
4
|
import { IPluggableStorageWrapper } from '../types';
|
|
4
5
|
export declare class ImpressionCountsCachePluggable extends ImpressionCountsCacheInMemory {
|
|
@@ -11,4 +12,5 @@ export declare class ImpressionCountsCachePluggable extends ImpressionCountsCach
|
|
|
11
12
|
private storeImpressionCounts;
|
|
12
13
|
start(): void;
|
|
13
14
|
stop(): Promise<any>;
|
|
15
|
+
getImpressionsCount(): Promise<ImpressionCountsPayload | undefined>;
|
|
14
16
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { IPluggableStorageWrapper, IUniqueKeysCacheBase } from '../types';
|
|
2
2
|
import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
|
|
3
3
|
import { ILogger } from '../../logger/types';
|
|
4
|
+
import { UniqueKeysItemSs } from '../../sync/submitters/types';
|
|
4
5
|
export declare class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
|
|
5
6
|
private readonly log;
|
|
6
7
|
private readonly key;
|
|
@@ -11,4 +12,9 @@ export declare class UniqueKeysCachePluggable extends UniqueKeysCacheInMemory im
|
|
|
11
12
|
storeUniqueKeys(): Promise<any>;
|
|
12
13
|
start(): void;
|
|
13
14
|
stop(): Promise<any>;
|
|
15
|
+
/**
|
|
16
|
+
* Async consumer API, used by synchronizer.
|
|
17
|
+
* @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
|
|
18
|
+
*/
|
|
19
|
+
popNRaw(count?: number): Promise<UniqueKeysItemSs[]>;
|
|
14
20
|
}
|
|
@@ -33,13 +33,14 @@ export declare type ImpressionCountsPayload = {
|
|
|
33
33
|
rc: number;
|
|
34
34
|
}[];
|
|
35
35
|
};
|
|
36
|
+
export declare type UniqueKeysItemSs = {
|
|
37
|
+
/** Split name */
|
|
38
|
+
f: string;
|
|
39
|
+
/** keyNames */
|
|
40
|
+
ks: string[];
|
|
41
|
+
};
|
|
36
42
|
export declare type UniqueKeysPayloadSs = {
|
|
37
|
-
keys:
|
|
38
|
-
/** Split name */
|
|
39
|
-
f: string;
|
|
40
|
-
/** keyNames */
|
|
41
|
-
ks: string[];
|
|
42
|
-
}[];
|
|
43
|
+
keys: UniqueKeysItemSs[];
|
|
43
44
|
};
|
|
44
45
|
export declare type UniqueKeysPayloadCs = {
|
|
45
46
|
keys: {
|