@splitsoftware/splitio-commons 1.6.2-rc.1 → 1.6.2-rc.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGES.txt +4 -0
- package/cjs/consent/sdkUserConsent.js +2 -2
- package/cjs/evaluator/index.js +15 -16
- package/cjs/integrations/ga/GaToSplit.js +8 -5
- package/cjs/sdkClient/client.js +19 -7
- package/cjs/sdkClient/sdkClient.js +3 -1
- package/cjs/sdkFactory/index.js +15 -6
- package/cjs/sdkManager/index.js +3 -11
- package/cjs/services/splitApi.js +6 -6
- package/cjs/storages/AbstractSplitsCacheAsync.js +8 -10
- package/cjs/storages/AbstractSplitsCacheSync.js +8 -10
- package/cjs/storages/KeyBuilderSS.js +54 -9
- package/cjs/storages/dataLoader.js +1 -1
- package/cjs/storages/inLocalStorage/SplitsCacheInLocal.js +5 -7
- package/cjs/storages/inLocalStorage/index.js +5 -1
- package/cjs/storages/inMemory/ImpressionCountsCacheInMemory.js +12 -1
- package/cjs/storages/inMemory/InMemoryStorage.js +6 -2
- package/cjs/storages/inMemory/InMemoryStorageCS.js +6 -2
- package/cjs/storages/inMemory/SplitsCacheInMemory.js +7 -10
- package/cjs/storages/inMemory/TelemetryCacheInMemory.js +10 -5
- package/cjs/storages/inMemory/UniqueKeysCacheInMemory.js +72 -0
- package/cjs/storages/inMemory/UniqueKeysCacheInMemoryCS.js +76 -0
- package/cjs/storages/inRedis/EventsCacheInRedis.js +1 -1
- package/cjs/storages/inRedis/ImpressionCountsCacheInRedis.js +85 -0
- package/cjs/storages/inRedis/SplitsCacheInRedis.js +15 -9
- package/cjs/storages/inRedis/TelemetryCacheInRedis.js +100 -0
- package/cjs/storages/inRedis/UniqueKeysCacheInRedis.js +71 -0
- package/cjs/storages/inRedis/constants.js +4 -1
- package/cjs/storages/inRedis/index.js +17 -2
- package/cjs/storages/pluggable/ImpressionCountsCachePluggable.js +81 -0
- package/cjs/storages/pluggable/SplitsCachePluggable.js +14 -9
- package/cjs/storages/pluggable/TelemetryCachePluggable.js +126 -0
- package/cjs/storages/pluggable/UniqueKeysCachePluggable.js +61 -0
- package/cjs/storages/pluggable/index.js +46 -17
- package/cjs/sync/offline/syncTasks/fromObjectSyncTask.js +2 -3
- package/cjs/sync/polling/updaters/splitChangesUpdater.js +1 -1
- package/cjs/sync/submitters/telemetrySubmitter.js +8 -4
- package/cjs/sync/submitters/uniqueKeysSubmitter.js +16 -59
- package/cjs/trackers/impressionsTracker.js +17 -15
- package/cjs/trackers/strategy/strategyNone.js +1 -1
- package/cjs/trackers/strategy/strategyOptimized.js +2 -1
- package/cjs/trackers/telemetryTracker.js +6 -0
- package/cjs/trackers/uniqueKeysTracker.js +11 -42
- package/cjs/utils/constants/index.js +3 -2
- package/cjs/utils/lang/maps.js +15 -7
- package/cjs/utils/redis/RedisMock.js +31 -0
- package/cjs/utils/settingsValidation/index.js +0 -3
- package/esm/consent/sdkUserConsent.js +2 -2
- package/esm/evaluator/index.js +15 -16
- package/esm/integrations/ga/GaToSplit.js +8 -5
- package/esm/sdkClient/client.js +19 -7
- package/esm/sdkClient/sdkClient.js +3 -1
- package/esm/sdkFactory/index.js +16 -7
- package/esm/sdkManager/index.js +3 -11
- package/esm/services/splitApi.js +6 -6
- package/esm/storages/AbstractSplitsCacheAsync.js +8 -10
- package/esm/storages/AbstractSplitsCacheSync.js +8 -10
- package/esm/storages/KeyBuilderSS.js +50 -8
- package/esm/storages/dataLoader.js +1 -1
- package/esm/storages/inLocalStorage/SplitsCacheInLocal.js +5 -7
- package/esm/storages/inLocalStorage/index.js +6 -2
- package/esm/storages/inMemory/ImpressionCountsCacheInMemory.js +12 -1
- package/esm/storages/inMemory/InMemoryStorage.js +8 -4
- package/esm/storages/inMemory/InMemoryStorageCS.js +7 -3
- package/esm/storages/inMemory/SplitsCacheInMemory.js +7 -10
- package/esm/storages/inMemory/TelemetryCacheInMemory.js +9 -5
- package/esm/storages/inMemory/UniqueKeysCacheInMemory.js +68 -0
- package/esm/storages/inMemory/UniqueKeysCacheInMemoryCS.js +73 -0
- package/esm/storages/inRedis/EventsCacheInRedis.js +1 -1
- package/esm/storages/inRedis/ImpressionCountsCacheInRedis.js +82 -0
- package/esm/storages/inRedis/SplitsCacheInRedis.js +15 -9
- package/esm/storages/inRedis/TelemetryCacheInRedis.js +100 -0
- package/esm/storages/inRedis/UniqueKeysCacheInRedis.js +68 -0
- package/esm/storages/inRedis/constants.js +3 -0
- package/esm/storages/inRedis/index.js +18 -3
- package/esm/storages/pluggable/ImpressionCountsCachePluggable.js +78 -0
- package/esm/storages/pluggable/SplitsCachePluggable.js +14 -9
- package/esm/storages/pluggable/TelemetryCachePluggable.js +126 -0
- package/esm/storages/pluggable/UniqueKeysCachePluggable.js +58 -0
- package/esm/storages/pluggable/index.js +47 -18
- package/esm/sync/offline/syncTasks/fromObjectSyncTask.js +2 -3
- package/esm/sync/polling/updaters/splitChangesUpdater.js +1 -1
- package/esm/sync/submitters/telemetrySubmitter.js +9 -5
- package/esm/sync/submitters/uniqueKeysSubmitter.js +15 -56
- package/esm/trackers/impressionsTracker.js +17 -15
- package/esm/trackers/strategy/strategyNone.js +1 -1
- package/esm/trackers/strategy/strategyOptimized.js +2 -1
- package/esm/trackers/telemetryTracker.js +6 -0
- package/esm/trackers/uniqueKeysTracker.js +11 -42
- package/esm/utils/constants/index.js +1 -0
- package/esm/utils/lang/maps.js +15 -7
- package/esm/utils/redis/RedisMock.js +28 -0
- package/esm/utils/settingsValidation/index.js +0 -3
- package/package.json +1 -2
- package/src/consent/sdkUserConsent.ts +2 -2
- package/src/evaluator/index.ts +14 -15
- package/src/integrations/ga/GaToSplit.ts +9 -5
- package/src/integrations/types.ts +2 -1
- package/src/logger/.DS_Store +0 -0
- package/src/sdkClient/client.ts +21 -8
- package/src/sdkClient/sdkClient.ts +3 -1
- package/src/sdkFactory/index.ts +17 -7
- package/src/sdkManager/index.ts +3 -12
- package/src/services/splitApi.ts +6 -6
- package/src/services/types.ts +2 -2
- package/src/storages/AbstractSplitsCacheAsync.ts +13 -15
- package/src/storages/AbstractSplitsCacheSync.ts +15 -17
- package/src/storages/KeyBuilderSS.ts +61 -9
- package/src/storages/dataLoader.ts +1 -1
- package/src/storages/inLocalStorage/SplitsCacheInLocal.ts +8 -11
- package/src/storages/inLocalStorage/index.ts +5 -2
- package/src/storages/inMemory/ImpressionCountsCacheInMemory.ts +16 -1
- package/src/storages/inMemory/InMemoryStorage.ts +7 -4
- package/src/storages/inMemory/InMemoryStorageCS.ts +6 -3
- package/src/storages/inMemory/SplitsCacheInMemory.ts +10 -14
- package/src/storages/inMemory/TelemetryCacheInMemory.ts +10 -6
- package/src/storages/inMemory/UniqueKeysCacheInMemory.ts +80 -0
- package/src/storages/inMemory/UniqueKeysCacheInMemoryCS.ts +86 -0
- package/src/storages/inRedis/EventsCacheInRedis.ts +1 -1
- package/src/storages/inRedis/ImpressionCountsCacheInRedis.ts +95 -0
- package/src/storages/inRedis/SplitsCacheInRedis.ts +21 -17
- package/src/storages/inRedis/TelemetryCacheInRedis.ts +122 -2
- package/src/storages/inRedis/UniqueKeysCacheInRedis.ts +77 -0
- package/src/storages/inRedis/constants.ts +3 -0
- package/src/storages/inRedis/index.ts +15 -5
- package/src/storages/pluggable/ImpressionCountsCachePluggable.ts +92 -0
- package/src/storages/pluggable/SplitsCachePluggable.ts +20 -17
- package/src/storages/pluggable/TelemetryCachePluggable.ts +147 -2
- package/src/storages/pluggable/UniqueKeysCachePluggable.ts +67 -0
- package/src/storages/pluggable/index.ts +51 -19
- package/src/storages/types.ts +38 -30
- package/src/sync/offline/syncTasks/fromObjectSyncTask.ts +5 -6
- package/src/sync/polling/updaters/splitChangesUpdater.ts +2 -2
- package/src/sync/submitters/telemetrySubmitter.ts +15 -8
- package/src/sync/submitters/types.ts +26 -12
- package/src/sync/submitters/uniqueKeysSubmitter.ts +18 -61
- package/src/trackers/impressionsTracker.ts +16 -15
- package/src/trackers/strategy/strategyNone.ts +1 -1
- package/src/trackers/strategy/strategyOptimized.ts +1 -1
- package/src/trackers/telemetryTracker.ts +7 -2
- package/src/trackers/types.ts +9 -7
- package/src/trackers/uniqueKeysTracker.ts +15 -47
- package/src/types.ts +0 -1
- package/src/utils/constants/index.ts +1 -0
- package/src/utils/lang/maps.ts +20 -8
- package/src/utils/redis/RedisMock.ts +33 -0
- package/src/utils/settingsValidation/index.ts +1 -4
- package/types/integrations/types.d.ts +2 -1
- package/types/services/types.d.ts +2 -2
- package/types/storages/AbstractSplitsCacheAsync.d.ts +7 -6
- package/types/storages/AbstractSplitsCacheSync.d.ts +6 -6
- package/types/storages/KeyBuilderSS.d.ts +9 -2
- package/types/storages/inLocalStorage/SplitsCacheInLocal.d.ts +3 -4
- package/types/storages/inMemory/ImpressionCountsCacheInMemory.d.ts +5 -1
- package/types/storages/inMemory/SplitsCacheInMemory.d.ts +3 -2
- package/types/storages/inMemory/TelemetryCacheInMemory.d.ts +6 -3
- package/types/storages/inMemory/uniqueKeysCacheInMemory.d.ts +35 -0
- package/types/storages/inMemory/uniqueKeysCacheInMemoryCS.d.ts +35 -0
- package/types/storages/inRedis/EventsCacheInRedis.d.ts +1 -1
- package/types/storages/inRedis/ImpressionCountsCacheInRedis.d.ts +16 -0
- package/types/storages/inRedis/SplitsCacheInRedis.d.ts +6 -5
- package/types/storages/inRedis/TelemetryCacheInRedis.d.ts +16 -1
- package/types/storages/inRedis/constants.d.ts +3 -0
- package/types/storages/inRedis/uniqueKeysCacheInRedis.d.ts +21 -0
- package/types/storages/pluggable/ImpressionCountsCachePluggable.d.ts +16 -0
- package/types/storages/pluggable/SplitsCachePluggable.d.ts +6 -5
- package/types/storages/pluggable/TelemetryCachePluggable.d.ts +17 -1
- package/types/storages/pluggable/UniqueKeysCachePluggable.d.ts +20 -0
- package/types/storages/types.d.ts +35 -35
- package/types/sync/polling/updaters/splitChangesUpdater.d.ts +1 -1
- package/types/sync/submitters/telemetrySubmitter.d.ts +1 -1
- package/types/sync/submitters/types.d.ts +19 -12
- package/types/sync/submitters/uniqueKeysSubmitter.d.ts +0 -14
- package/types/trackers/types.d.ts +9 -11
- package/types/trackers/uniqueKeysTracker.d.ts +3 -3
- package/types/types.d.ts +0 -1
- package/types/utils/constants/index.d.ts +1 -0
- package/types/utils/lang/maps.d.ts +6 -2
- package/types/utils/redis/RedisMock.d.ts +4 -0
- package/types/utils/settingsValidation/index.d.ts +0 -1
- package/types/sdkClient/types.d.ts +0 -18
- package/types/storages/inMemory/CountsCacheInMemory.d.ts +0 -20
- package/types/storages/inMemory/LatenciesCacheInMemory.d.ts +0 -20
- package/types/storages/inRedis/CountsCacheInRedis.d.ts +0 -9
- package/types/storages/inRedis/LatenciesCacheInRedis.d.ts +0 -9
- package/types/sync/offline/LocalhostFromFile.d.ts +0 -2
- package/types/sync/offline/splitsParser/splitsParserFromFile.d.ts +0 -2
- package/types/sync/submitters/eventsSyncTask.d.ts +0 -8
- package/types/sync/submitters/impressionCountsSyncTask.d.ts +0 -13
- package/types/sync/submitters/impressionsSyncTask.d.ts +0 -14
- package/types/sync/submitters/metricsSyncTask.d.ts +0 -12
- package/types/sync/submitters/submitterSyncTask.d.ts +0 -10
- package/types/sync/syncTaskComposite.d.ts +0 -5
- package/types/trackers/filter/bloomFilter.d.ts +0 -10
- package/types/trackers/filter/dictionaryFilter.d.ts +0 -8
- package/types/trackers/filter/types.d.ts +0 -5
- package/types/utils/timeTracker/index.d.ts +0 -70
|
@@ -8,6 +8,7 @@ var EventsCacheInMemory_1 = require("./EventsCacheInMemory");
|
|
|
8
8
|
var ImpressionCountsCacheInMemory_1 = require("./ImpressionCountsCacheInMemory");
|
|
9
9
|
var constants_1 = require("../../utils/constants");
|
|
10
10
|
var TelemetryCacheInMemory_1 = require("./TelemetryCacheInMemory");
|
|
11
|
+
var UniqueKeysCacheInMemoryCS_1 = require("./UniqueKeysCacheInMemoryCS");
|
|
11
12
|
/**
|
|
12
13
|
* InMemory storage factory for standalone client-side SplitFactory
|
|
13
14
|
*
|
|
@@ -18,16 +19,19 @@ function InMemoryStorageCSFactory(params) {
|
|
|
18
19
|
splits: new SplitsCacheInMemory_1.SplitsCacheInMemory(),
|
|
19
20
|
segments: new MySegmentsCacheInMemory_1.MySegmentsCacheInMemory(),
|
|
20
21
|
impressions: new ImpressionsCacheInMemory_1.ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
21
|
-
impressionCounts: params.
|
|
22
|
+
impressionCounts: params.impressionsMode !== constants_1.DEBUG ? new ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory() : undefined,
|
|
22
23
|
events: new EventsCacheInMemory_1.EventsCacheInMemory(params.eventsQueueSize),
|
|
23
|
-
telemetry:
|
|
24
|
+
telemetry: (0, TelemetryCacheInMemory_1.shouldRecordTelemetry)(params) ? new TelemetryCacheInMemory_1.TelemetryCacheInMemory() : undefined,
|
|
25
|
+
uniqueKeys: params.impressionsMode === constants_1.NONE ? new UniqueKeysCacheInMemoryCS_1.UniqueKeysCacheInMemoryCS() : undefined,
|
|
24
26
|
// When using MEMORY we should clean all the caches to leave them empty
|
|
25
27
|
destroy: function () {
|
|
28
|
+
var _a;
|
|
26
29
|
this.splits.clear();
|
|
27
30
|
this.segments.clear();
|
|
28
31
|
this.impressions.clear();
|
|
29
32
|
this.impressionCounts && this.impressionCounts.clear();
|
|
30
33
|
this.events.clear();
|
|
34
|
+
(_a = this.uniqueKeys) === null || _a === void 0 ? void 0 : _a.clear();
|
|
31
35
|
},
|
|
32
36
|
// When using shared instanciation with MEMORY we reuse everything but segments (they are unique per key)
|
|
33
37
|
shared: function () {
|
|
@@ -25,9 +25,8 @@ var SplitsCacheInMemory = /** @class */ (function (_super) {
|
|
|
25
25
|
this.splitsWithSegmentsCount = 0;
|
|
26
26
|
};
|
|
27
27
|
SplitsCacheInMemory.prototype.addSplit = function (name, split) {
|
|
28
|
-
var
|
|
29
|
-
if (
|
|
30
|
-
var previousSplit = JSON.parse(splitFromMemory);
|
|
28
|
+
var previousSplit = this.getSplit(name);
|
|
29
|
+
if (previousSplit) { // We had this Split already
|
|
31
30
|
if (previousSplit.trafficTypeName) {
|
|
32
31
|
var previousTtName = previousSplit.trafficTypeName;
|
|
33
32
|
this.ttCache[previousTtName]--;
|
|
@@ -38,19 +37,18 @@ var SplitsCacheInMemory = /** @class */ (function (_super) {
|
|
|
38
37
|
this.splitsWithSegmentsCount--;
|
|
39
38
|
}
|
|
40
39
|
}
|
|
41
|
-
|
|
42
|
-
if (parsedSplit) {
|
|
40
|
+
if (split) {
|
|
43
41
|
// Store the Split.
|
|
44
42
|
this.splitsCache[name] = split;
|
|
45
43
|
// Update TT cache
|
|
46
|
-
var ttName =
|
|
44
|
+
var ttName = split.trafficTypeName;
|
|
47
45
|
if (ttName) { // safeguard
|
|
48
46
|
if (!this.ttCache[ttName])
|
|
49
47
|
this.ttCache[ttName] = 0;
|
|
50
48
|
this.ttCache[ttName]++;
|
|
51
49
|
}
|
|
52
50
|
// Add to segments count for the new version of the Split
|
|
53
|
-
if ((0, AbstractSplitsCacheSync_1.usesSegments)(
|
|
51
|
+
if ((0, AbstractSplitsCacheSync_1.usesSegments)(split))
|
|
54
52
|
this.splitsWithSegmentsCount++;
|
|
55
53
|
return true;
|
|
56
54
|
}
|
|
@@ -63,15 +61,14 @@ var SplitsCacheInMemory = /** @class */ (function (_super) {
|
|
|
63
61
|
if (split) {
|
|
64
62
|
// Delete the Split
|
|
65
63
|
delete this.splitsCache[name];
|
|
66
|
-
var
|
|
67
|
-
var ttName = parsedSplit.trafficTypeName;
|
|
64
|
+
var ttName = split.trafficTypeName;
|
|
68
65
|
if (ttName) { // safeguard
|
|
69
66
|
this.ttCache[ttName]--; // Update tt cache
|
|
70
67
|
if (!this.ttCache[ttName])
|
|
71
68
|
delete this.ttCache[ttName];
|
|
72
69
|
}
|
|
73
70
|
// Update the segments count.
|
|
74
|
-
if ((0, AbstractSplitsCacheSync_1.usesSegments)(
|
|
71
|
+
if ((0, AbstractSplitsCacheSync_1.usesSegments)(split))
|
|
75
72
|
this.splitsWithSegmentsCount--;
|
|
76
73
|
return true;
|
|
77
74
|
}
|
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.TelemetryCacheInMemory = exports.shouldRecordTelemetry = void 0;
|
|
3
|
+
exports.TelemetryCacheInMemory = exports.shouldRecordTelemetry = exports.newBuckets = exports.MAX_LATENCY_BUCKET_COUNT = void 0;
|
|
4
|
+
var constants_1 = require("../../utils/constants");
|
|
4
5
|
var findLatencyIndex_1 = require("../findLatencyIndex");
|
|
5
6
|
var MAX_STREAMING_EVENTS = 20;
|
|
6
7
|
var MAX_TAGS = 10;
|
|
8
|
+
exports.MAX_LATENCY_BUCKET_COUNT = 23;
|
|
7
9
|
function newBuckets() {
|
|
8
|
-
// MAX_LATENCY_BUCKET_COUNT (length) is 23
|
|
10
|
+
// MAX_LATENCY_BUCKET_COUNT (length) is 23
|
|
11
|
+
// Not using Array.fill for old browsers compatibility
|
|
9
12
|
return [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
|
10
13
|
}
|
|
14
|
+
exports.newBuckets = newBuckets;
|
|
11
15
|
var ACCEPTANCE_RANGE = 0.001;
|
|
12
16
|
/**
|
|
13
|
-
*
|
|
17
|
+
* Record telemetry if mode is not localhost.
|
|
18
|
+
* All factory instances track telemetry on server-side, and 0.1% on client-side.
|
|
14
19
|
*/
|
|
15
|
-
function shouldRecordTelemetry() {
|
|
16
|
-
return Math.random() <= ACCEPTANCE_RANGE;
|
|
20
|
+
function shouldRecordTelemetry(params) {
|
|
21
|
+
return params.mode !== constants_1.LOCALHOST_MODE && (params.matchingKey === undefined || Math.random() <= ACCEPTANCE_RANGE);
|
|
17
22
|
}
|
|
18
23
|
exports.shouldRecordTelemetry = shouldRecordTelemetry;
|
|
19
24
|
var TelemetryCacheInMemory = /** @class */ (function () {
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UniqueKeysCacheInMemory = exports.fromUniqueKeysCollector = void 0;
|
|
4
|
+
var sets_1 = require("../../utils/lang/sets");
|
|
5
|
+
var constants_1 = require("../inRedis/constants");
|
|
6
|
+
/**
|
|
7
|
+
* Converts `uniqueKeys` data from cache into request payload for SS.
|
|
8
|
+
*/
|
|
9
|
+
function fromUniqueKeysCollector(uniqueKeys) {
|
|
10
|
+
var payload = [];
|
|
11
|
+
var featureNames = Object.keys(uniqueKeys);
|
|
12
|
+
for (var i = 0; i < featureNames.length; i++) {
|
|
13
|
+
var featureName = featureNames[i];
|
|
14
|
+
var userKeys = (0, sets_1.setToArray)(uniqueKeys[featureName]);
|
|
15
|
+
var uniqueKeysPayload = {
|
|
16
|
+
f: featureName,
|
|
17
|
+
ks: userKeys
|
|
18
|
+
};
|
|
19
|
+
payload.push(uniqueKeysPayload);
|
|
20
|
+
}
|
|
21
|
+
return { keys: payload };
|
|
22
|
+
}
|
|
23
|
+
exports.fromUniqueKeysCollector = fromUniqueKeysCollector;
|
|
24
|
+
var UniqueKeysCacheInMemory = /** @class */ (function () {
|
|
25
|
+
function UniqueKeysCacheInMemory(uniqueKeysQueueSize) {
|
|
26
|
+
if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = constants_1.DEFAULT_CACHE_SIZE; }
|
|
27
|
+
this.uniqueTrackerSize = 0;
|
|
28
|
+
this.maxStorage = uniqueKeysQueueSize;
|
|
29
|
+
this.uniqueKeysTracker = {};
|
|
30
|
+
}
|
|
31
|
+
UniqueKeysCacheInMemory.prototype.setOnFullQueueCb = function (cb) {
|
|
32
|
+
this.onFullQueue = cb;
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* Store unique keys per feature.
|
|
36
|
+
*/
|
|
37
|
+
UniqueKeysCacheInMemory.prototype.track = function (userKey, featureName) {
|
|
38
|
+
if (!this.uniqueKeysTracker[featureName])
|
|
39
|
+
this.uniqueKeysTracker[featureName] = new sets_1._Set();
|
|
40
|
+
var tracker = this.uniqueKeysTracker[featureName];
|
|
41
|
+
if (!tracker.has(userKey)) {
|
|
42
|
+
tracker.add(userKey);
|
|
43
|
+
this.uniqueTrackerSize++;
|
|
44
|
+
}
|
|
45
|
+
if (this.uniqueTrackerSize >= this.maxStorage && this.onFullQueue) {
|
|
46
|
+
this.uniqueTrackerSize = 0;
|
|
47
|
+
this.onFullQueue();
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Clear the data stored on the cache.
|
|
52
|
+
*/
|
|
53
|
+
UniqueKeysCacheInMemory.prototype.clear = function () {
|
|
54
|
+
this.uniqueKeysTracker = {};
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Pop the collected data, used as payload for posting.
|
|
58
|
+
*/
|
|
59
|
+
UniqueKeysCacheInMemory.prototype.pop = function () {
|
|
60
|
+
var data = this.uniqueKeysTracker;
|
|
61
|
+
this.uniqueKeysTracker = {};
|
|
62
|
+
return fromUniqueKeysCollector(data);
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* Check if the cache is empty.
|
|
66
|
+
*/
|
|
67
|
+
UniqueKeysCacheInMemory.prototype.isEmpty = function () {
|
|
68
|
+
return Object.keys(this.uniqueKeysTracker).length === 0;
|
|
69
|
+
};
|
|
70
|
+
return UniqueKeysCacheInMemory;
|
|
71
|
+
}());
|
|
72
|
+
exports.UniqueKeysCacheInMemory = UniqueKeysCacheInMemory;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UniqueKeysCacheInMemoryCS = void 0;
|
|
4
|
+
var sets_1 = require("../../utils/lang/sets");
|
|
5
|
+
var constants_1 = require("../inRedis/constants");
|
|
6
|
+
var UniqueKeysCacheInMemoryCS = /** @class */ (function () {
|
|
7
|
+
/**
|
|
8
|
+
*
|
|
9
|
+
* @param impressionsQueueSize number of queued impressions to call onFullQueueCb.
|
|
10
|
+
* Default value is 0, that means no maximum value, in case we want to avoid this being triggered.
|
|
11
|
+
*/
|
|
12
|
+
function UniqueKeysCacheInMemoryCS(uniqueKeysQueueSize) {
|
|
13
|
+
if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = constants_1.DEFAULT_CACHE_SIZE; }
|
|
14
|
+
this.uniqueTrackerSize = 0;
|
|
15
|
+
this.maxStorage = uniqueKeysQueueSize;
|
|
16
|
+
this.uniqueKeysTracker = {};
|
|
17
|
+
}
|
|
18
|
+
UniqueKeysCacheInMemoryCS.prototype.setOnFullQueueCb = function (cb) {
|
|
19
|
+
this.onFullQueue = cb;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Store unique keys per feature.
|
|
23
|
+
*/
|
|
24
|
+
UniqueKeysCacheInMemoryCS.prototype.track = function (userKey, featureName) {
|
|
25
|
+
if (!this.uniqueKeysTracker[userKey])
|
|
26
|
+
this.uniqueKeysTracker[userKey] = new sets_1._Set();
|
|
27
|
+
var tracker = this.uniqueKeysTracker[userKey];
|
|
28
|
+
if (!tracker.has(featureName)) {
|
|
29
|
+
tracker.add(featureName);
|
|
30
|
+
this.uniqueTrackerSize++;
|
|
31
|
+
}
|
|
32
|
+
if (this.uniqueTrackerSize >= this.maxStorage && this.onFullQueue) {
|
|
33
|
+
this.uniqueTrackerSize = 0;
|
|
34
|
+
this.onFullQueue();
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Clear the data stored on the cache.
|
|
39
|
+
*/
|
|
40
|
+
UniqueKeysCacheInMemoryCS.prototype.clear = function () {
|
|
41
|
+
this.uniqueKeysTracker = {};
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Pop the collected data, used as payload for posting.
|
|
45
|
+
*/
|
|
46
|
+
UniqueKeysCacheInMemoryCS.prototype.pop = function () {
|
|
47
|
+
var data = this.uniqueKeysTracker;
|
|
48
|
+
this.uniqueKeysTracker = {};
|
|
49
|
+
return this.fromUniqueKeysCollector(data);
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* Check if the cache is empty.
|
|
53
|
+
*/
|
|
54
|
+
UniqueKeysCacheInMemoryCS.prototype.isEmpty = function () {
|
|
55
|
+
return Object.keys(this.uniqueKeysTracker).length === 0;
|
|
56
|
+
};
|
|
57
|
+
/**
|
|
58
|
+
* Converts `uniqueKeys` data from cache into request payload.
|
|
59
|
+
*/
|
|
60
|
+
UniqueKeysCacheInMemoryCS.prototype.fromUniqueKeysCollector = function (uniqueKeys) {
|
|
61
|
+
var payload = [];
|
|
62
|
+
var userKeys = Object.keys(uniqueKeys);
|
|
63
|
+
for (var k = 0; k < userKeys.length; k++) {
|
|
64
|
+
var userKey = userKeys[k];
|
|
65
|
+
var featureNames = (0, sets_1.setToArray)(uniqueKeys[userKey]);
|
|
66
|
+
var uniqueKeysPayload = {
|
|
67
|
+
k: userKey,
|
|
68
|
+
fs: featureNames
|
|
69
|
+
};
|
|
70
|
+
payload.push(uniqueKeysPayload);
|
|
71
|
+
}
|
|
72
|
+
return { keys: payload };
|
|
73
|
+
};
|
|
74
|
+
return UniqueKeysCacheInMemoryCS;
|
|
75
|
+
}());
|
|
76
|
+
exports.UniqueKeysCacheInMemoryCS = UniqueKeysCacheInMemoryCS;
|
|
@@ -42,7 +42,7 @@ var EventsCacheInRedis = /** @class */ (function () {
|
|
|
42
42
|
};
|
|
43
43
|
/**
|
|
44
44
|
* Pop the given number of events from the storage.
|
|
45
|
-
* The returned promise rejects if the
|
|
45
|
+
* The returned promise rejects if the redis operation fails.
|
|
46
46
|
*
|
|
47
47
|
* NOTE: this method doesn't take into account MAX_EVENT_SIZE or MAX_QUEUE_BYTE_SIZE limits.
|
|
48
48
|
* It is the submitter responsability to handle that.
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ImpressionCountsCacheInRedis = void 0;
|
|
4
|
+
var tslib_1 = require("tslib");
|
|
5
|
+
var lang_1 = require("../../utils/lang");
|
|
6
|
+
var ImpressionCountsCacheInMemory_1 = require("../inMemory/ImpressionCountsCacheInMemory");
|
|
7
|
+
var constants_1 = require("./constants");
|
|
8
|
+
var ImpressionCountsCacheInRedis = /** @class */ (function (_super) {
|
|
9
|
+
(0, tslib_1.__extends)(ImpressionCountsCacheInRedis, _super);
|
|
10
|
+
function ImpressionCountsCacheInRedis(log, key, redis, impressionCountsCacheSize, refreshRate) {
|
|
11
|
+
if (refreshRate === void 0) { refreshRate = constants_1.REFRESH_RATE; }
|
|
12
|
+
var _this = _super.call(this, impressionCountsCacheSize) || this;
|
|
13
|
+
_this.log = log;
|
|
14
|
+
_this.key = key;
|
|
15
|
+
_this.redis = redis;
|
|
16
|
+
_this.refreshRate = refreshRate;
|
|
17
|
+
_this.onFullQueue = function () { _this.postImpressionCountsInRedis(); };
|
|
18
|
+
return _this;
|
|
19
|
+
}
|
|
20
|
+
ImpressionCountsCacheInRedis.prototype.postImpressionCountsInRedis = function () {
|
|
21
|
+
var _this = this;
|
|
22
|
+
var counts = this.pop();
|
|
23
|
+
var keys = Object.keys(counts);
|
|
24
|
+
if (!keys.length)
|
|
25
|
+
return Promise.resolve(false);
|
|
26
|
+
var pipeline = this.redis.pipeline();
|
|
27
|
+
keys.forEach(function (key) {
|
|
28
|
+
pipeline.hincrby(_this.key, key, counts[key]);
|
|
29
|
+
});
|
|
30
|
+
return pipeline.exec()
|
|
31
|
+
.then(function (data) {
|
|
32
|
+
// If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
|
|
33
|
+
if (data.length && data.length === keys.length) {
|
|
34
|
+
return _this.redis.expire(_this.key, constants_1.TTL_REFRESH);
|
|
35
|
+
}
|
|
36
|
+
})
|
|
37
|
+
.catch(function (err) {
|
|
38
|
+
_this.log.error(constants_1.LOG_PREFIX + "Error in impression counts pipeline: " + err + ".");
|
|
39
|
+
return false;
|
|
40
|
+
});
|
|
41
|
+
};
|
|
42
|
+
ImpressionCountsCacheInRedis.prototype.start = function () {
|
|
43
|
+
this.intervalId = setInterval(this.postImpressionCountsInRedis.bind(this), this.refreshRate);
|
|
44
|
+
};
|
|
45
|
+
ImpressionCountsCacheInRedis.prototype.stop = function () {
|
|
46
|
+
clearInterval(this.intervalId);
|
|
47
|
+
return this.postImpressionCountsInRedis();
|
|
48
|
+
};
|
|
49
|
+
// Async consumer API, used by synchronizer
|
|
50
|
+
ImpressionCountsCacheInRedis.prototype.getImpressionsCount = function () {
|
|
51
|
+
var _this = this;
|
|
52
|
+
return this.redis.hgetall(this.key)
|
|
53
|
+
.then(function (counts) {
|
|
54
|
+
if (!Object.keys(counts).length)
|
|
55
|
+
return undefined;
|
|
56
|
+
_this.redis.del(_this.key).catch(function () { });
|
|
57
|
+
var impressionsCount = { pf: [] };
|
|
58
|
+
(0, lang_1.forOwn)(counts, function (count, key) {
|
|
59
|
+
var nameAndTime = key.split('::');
|
|
60
|
+
if (nameAndTime.length !== 2) {
|
|
61
|
+
_this.log.error(constants_1.LOG_PREFIX + "Error spliting key " + key);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
var timeFrame = parseInt(nameAndTime[1]);
|
|
65
|
+
if (isNaN(timeFrame)) {
|
|
66
|
+
_this.log.error(constants_1.LOG_PREFIX + "Error parsing time frame " + nameAndTime[1]);
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
var rawCount = parseInt(count);
|
|
70
|
+
if (isNaN(rawCount)) {
|
|
71
|
+
_this.log.error(constants_1.LOG_PREFIX + "Error parsing raw count " + count);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
impressionsCount.pf.push({
|
|
75
|
+
f: nameAndTime[0],
|
|
76
|
+
m: timeFrame,
|
|
77
|
+
rc: rawCount,
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
return impressionsCount;
|
|
81
|
+
});
|
|
82
|
+
};
|
|
83
|
+
return ImpressionCountsCacheInRedis;
|
|
84
|
+
}(ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory));
|
|
85
|
+
exports.ImpressionCountsCacheInRedis = ImpressionCountsCacheInRedis;
|
|
@@ -62,17 +62,17 @@ var SplitsCacheInRedis = /** @class */ (function (_super) {
|
|
|
62
62
|
var splitKey = this.keys.buildSplitKey(name);
|
|
63
63
|
return this.redis.get(splitKey).then(function (splitFromStorage) {
|
|
64
64
|
// handling parsing errors
|
|
65
|
-
var parsedPreviousSplit,
|
|
65
|
+
var parsedPreviousSplit, newStringifiedSplit;
|
|
66
66
|
try {
|
|
67
67
|
parsedPreviousSplit = splitFromStorage ? JSON.parse(splitFromStorage) : undefined;
|
|
68
|
-
|
|
68
|
+
newStringifiedSplit = JSON.stringify(split);
|
|
69
69
|
}
|
|
70
70
|
catch (e) {
|
|
71
71
|
throw new Error('Error parsing split definition: ' + e);
|
|
72
72
|
}
|
|
73
73
|
return Promise.all([
|
|
74
|
-
_this.redis.set(splitKey,
|
|
75
|
-
_this._incrementCounts(
|
|
74
|
+
_this.redis.set(splitKey, newStringifiedSplit),
|
|
75
|
+
_this._incrementCounts(split),
|
|
76
76
|
// If it's an update, we decrement the traffic type of the existing split,
|
|
77
77
|
parsedPreviousSplit && _this._decrementCounts(parsedPreviousSplit)
|
|
78
78
|
]);
|
|
@@ -99,8 +99,7 @@ var SplitsCacheInRedis = /** @class */ (function (_super) {
|
|
|
99
99
|
var _this = this;
|
|
100
100
|
return this.getSplit(name).then(function (split) {
|
|
101
101
|
if (split) {
|
|
102
|
-
|
|
103
|
-
_this._decrementCounts(parsedSplit);
|
|
102
|
+
_this._decrementCounts(split);
|
|
104
103
|
}
|
|
105
104
|
return _this.redis.del(_this.keys.buildSplitKey(name));
|
|
106
105
|
});
|
|
@@ -123,7 +122,8 @@ var SplitsCacheInRedis = /** @class */ (function (_super) {
|
|
|
123
122
|
this.log.error(constants_1.LOG_PREFIX + this.redisError);
|
|
124
123
|
return Promise.reject(this.redisError);
|
|
125
124
|
}
|
|
126
|
-
return this.redis.get(this.keys.buildSplitKey(name))
|
|
125
|
+
return this.redis.get(this.keys.buildSplitKey(name))
|
|
126
|
+
.then(function (maybeSplit) { return maybeSplit && JSON.parse(maybeSplit); });
|
|
127
127
|
};
|
|
128
128
|
/**
|
|
129
129
|
* Set till number.
|
|
@@ -158,7 +158,12 @@ var SplitsCacheInRedis = /** @class */ (function (_super) {
|
|
|
158
158
|
*/
|
|
159
159
|
SplitsCacheInRedis.prototype.getAll = function () {
|
|
160
160
|
var _this = this;
|
|
161
|
-
return this.redis.keys(this.keys.searchPatternForSplitKeys())
|
|
161
|
+
return this.redis.keys(this.keys.searchPatternForSplitKeys())
|
|
162
|
+
.then(function (listOfKeys) { return _this.redis.pipeline(listOfKeys.map(function (k) { return ['get', k]; })).exec(); })
|
|
163
|
+
.then(processPipelineAnswer)
|
|
164
|
+
.then(function (splitDefinitions) { return splitDefinitions.map(function (splitDefinition) {
|
|
165
|
+
return JSON.parse(splitDefinition);
|
|
166
|
+
}); });
|
|
162
167
|
};
|
|
163
168
|
/**
|
|
164
169
|
* Get list of split names.
|
|
@@ -218,7 +223,8 @@ var SplitsCacheInRedis = /** @class */ (function (_super) {
|
|
|
218
223
|
var keys = names.map(function (name) { return _this.keys.buildSplitKey(name); });
|
|
219
224
|
return (_a = this.redis).mget.apply(_a, keys).then(function (splitDefinitions) {
|
|
220
225
|
names.forEach(function (name, idx) {
|
|
221
|
-
|
|
226
|
+
var split = splitDefinitions[idx];
|
|
227
|
+
splits[name] = split && JSON.parse(split);
|
|
222
228
|
});
|
|
223
229
|
return Promise.resolve(splits);
|
|
224
230
|
})
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.TelemetryCacheInRedis = void 0;
|
|
4
|
+
var KeyBuilderSS_1 = require("../KeyBuilderSS");
|
|
4
5
|
var findLatencyIndex_1 = require("../findLatencyIndex");
|
|
5
6
|
var telemetrySubmitter_1 = require("../../sync/submitters/telemetrySubmitter");
|
|
6
7
|
var constants_1 = require("../../utils/constants");
|
|
8
|
+
var lang_1 = require("../../utils/lang");
|
|
9
|
+
var maps_1 = require("../../utils/lang/maps");
|
|
10
|
+
var TelemetryCacheInMemory_1 = require("../inMemory/TelemetryCacheInMemory");
|
|
7
11
|
var TelemetryCacheInRedis = /** @class */ (function () {
|
|
8
12
|
/**
|
|
9
13
|
* Create a Telemetry cache that uses Redis as storage.
|
|
@@ -31,6 +35,102 @@ var TelemetryCacheInRedis = /** @class */ (function () {
|
|
|
31
35
|
var value = JSON.stringify((0, telemetrySubmitter_1.getTelemetryConfigStats)(constants_1.CONSUMER_MODE, constants_1.STORAGE_REDIS));
|
|
32
36
|
return this.redis.hset(key, field, value).catch(function () { });
|
|
33
37
|
};
|
|
38
|
+
/**
|
|
39
|
+
* Pop telemetry latencies.
|
|
40
|
+
* The returned promise rejects if redis operations fail.
|
|
41
|
+
*/
|
|
42
|
+
TelemetryCacheInRedis.prototype.popLatencies = function () {
|
|
43
|
+
var _this = this;
|
|
44
|
+
return this.redis.hgetall(this.keys.latencyPrefix).then(function (latencies) {
|
|
45
|
+
var result = new maps_1._Map();
|
|
46
|
+
Object.keys(latencies).forEach(function (field) {
|
|
47
|
+
var parsedField = (0, KeyBuilderSS_1.parseLatencyField)(field);
|
|
48
|
+
if ((0, lang_1.isString)(parsedField)) {
|
|
49
|
+
_this.log.error("Ignoring invalid latency field: " + field + ": " + parsedField);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
var count = parseInt(latencies[field]);
|
|
53
|
+
if ((0, lang_1.isNaNNumber)(count)) {
|
|
54
|
+
_this.log.error("Ignoring latency with invalid count: " + latencies[field]);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
var metadata = parsedField[0], method = parsedField[1], bucket = parsedField[2];
|
|
58
|
+
if (bucket >= TelemetryCacheInMemory_1.MAX_LATENCY_BUCKET_COUNT) {
|
|
59
|
+
_this.log.error("Ignoring latency with invalid bucket: " + bucket);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
if (!result.has(metadata))
|
|
63
|
+
result.set(metadata, {
|
|
64
|
+
t: (0, TelemetryCacheInMemory_1.newBuckets)(),
|
|
65
|
+
ts: (0, TelemetryCacheInMemory_1.newBuckets)(),
|
|
66
|
+
tc: (0, TelemetryCacheInMemory_1.newBuckets)(),
|
|
67
|
+
tcs: (0, TelemetryCacheInMemory_1.newBuckets)(),
|
|
68
|
+
tr: (0, TelemetryCacheInMemory_1.newBuckets)(),
|
|
69
|
+
});
|
|
70
|
+
result.get(metadata)[method][bucket] = count;
|
|
71
|
+
});
|
|
72
|
+
return _this.redis.del(_this.keys.latencyPrefix).then(function () { return result; });
|
|
73
|
+
});
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* Pop telemetry exceptions.
|
|
77
|
+
* The returned promise rejects if redis operations fail.
|
|
78
|
+
*/
|
|
79
|
+
TelemetryCacheInRedis.prototype.popExceptions = function () {
|
|
80
|
+
var _this = this;
|
|
81
|
+
return this.redis.hgetall(this.keys.exceptionPrefix).then(function (exceptions) {
|
|
82
|
+
var result = new maps_1._Map();
|
|
83
|
+
Object.keys(exceptions).forEach(function (field) {
|
|
84
|
+
var parsedField = (0, KeyBuilderSS_1.parseExceptionField)(field);
|
|
85
|
+
if ((0, lang_1.isString)(parsedField)) {
|
|
86
|
+
_this.log.error("Ignoring invalid exception field: " + field + ": " + parsedField);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
var count = parseInt(exceptions[field]);
|
|
90
|
+
if ((0, lang_1.isNaNNumber)(count)) {
|
|
91
|
+
_this.log.error("Ignoring exception with invalid count: " + exceptions[field]);
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
var metadata = parsedField[0], method = parsedField[1];
|
|
95
|
+
if (!result.has(metadata))
|
|
96
|
+
result.set(metadata, {
|
|
97
|
+
t: 0,
|
|
98
|
+
ts: 0,
|
|
99
|
+
tc: 0,
|
|
100
|
+
tcs: 0,
|
|
101
|
+
tr: 0,
|
|
102
|
+
});
|
|
103
|
+
result.get(metadata)[method] = count;
|
|
104
|
+
});
|
|
105
|
+
return _this.redis.del(_this.keys.exceptionPrefix).then(function () { return result; });
|
|
106
|
+
});
|
|
107
|
+
};
|
|
108
|
+
/**
|
|
109
|
+
* Pop telemetry configs.
|
|
110
|
+
* The returned promise rejects if redis operations fail.
|
|
111
|
+
*/
|
|
112
|
+
TelemetryCacheInRedis.prototype.popConfigs = function () {
|
|
113
|
+
var _this = this;
|
|
114
|
+
return this.redis.hgetall(this.keys.initPrefix).then(function (configs) {
|
|
115
|
+
var result = new maps_1._Map();
|
|
116
|
+
Object.keys(configs).forEach(function (field) {
|
|
117
|
+
var parsedField = (0, KeyBuilderSS_1.parseMetadata)(field);
|
|
118
|
+
if ((0, lang_1.isString)(parsedField)) {
|
|
119
|
+
_this.log.error("Ignoring invalid config field: " + field + ": " + parsedField);
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
var metadata = parsedField[0];
|
|
123
|
+
try {
|
|
124
|
+
var config = JSON.parse(configs[field]);
|
|
125
|
+
result.set(metadata, config);
|
|
126
|
+
}
|
|
127
|
+
catch (e) {
|
|
128
|
+
_this.log.error("Ignoring invalid config: " + configs[field]);
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
return _this.redis.del(_this.keys.initPrefix).then(function () { return result; });
|
|
132
|
+
});
|
|
133
|
+
};
|
|
34
134
|
return TelemetryCacheInRedis;
|
|
35
135
|
}());
|
|
36
136
|
exports.TelemetryCacheInRedis = TelemetryCacheInRedis;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UniqueKeysCacheInRedis = void 0;
|
|
4
|
+
var tslib_1 = require("tslib");
|
|
5
|
+
var UniqueKeysCacheInMemory_1 = require("../inMemory/UniqueKeysCacheInMemory");
|
|
6
|
+
var sets_1 = require("../../utils/lang/sets");
|
|
7
|
+
var constants_1 = require("./constants");
|
|
8
|
+
var constants_2 = require("./constants");
|
|
9
|
+
var UniqueKeysCacheInRedis = /** @class */ (function (_super) {
|
|
10
|
+
(0, tslib_1.__extends)(UniqueKeysCacheInRedis, _super);
|
|
11
|
+
function UniqueKeysCacheInRedis(log, key, redis, uniqueKeysQueueSize, refreshRate) {
|
|
12
|
+
if (uniqueKeysQueueSize === void 0) { uniqueKeysQueueSize = constants_1.DEFAULT_CACHE_SIZE; }
|
|
13
|
+
if (refreshRate === void 0) { refreshRate = constants_1.REFRESH_RATE; }
|
|
14
|
+
var _this = _super.call(this, uniqueKeysQueueSize) || this;
|
|
15
|
+
_this.log = log;
|
|
16
|
+
_this.key = key;
|
|
17
|
+
_this.redis = redis;
|
|
18
|
+
_this.refreshRate = refreshRate;
|
|
19
|
+
_this.onFullQueue = function () { _this.postUniqueKeysInRedis(); };
|
|
20
|
+
return _this;
|
|
21
|
+
}
|
|
22
|
+
UniqueKeysCacheInRedis.prototype.postUniqueKeysInRedis = function () {
|
|
23
|
+
var _this = this;
|
|
24
|
+
var featureNames = Object.keys(this.uniqueKeysTracker);
|
|
25
|
+
if (!featureNames.length)
|
|
26
|
+
return Promise.resolve(false);
|
|
27
|
+
var pipeline = this.redis.pipeline();
|
|
28
|
+
for (var i = 0; i < featureNames.length; i++) {
|
|
29
|
+
var featureName = featureNames[i];
|
|
30
|
+
var featureKeys = (0, sets_1.setToArray)(this.uniqueKeysTracker[featureName]);
|
|
31
|
+
var uniqueKeysPayload = {
|
|
32
|
+
f: featureName,
|
|
33
|
+
ks: featureKeys
|
|
34
|
+
};
|
|
35
|
+
pipeline.rpush(this.key, JSON.stringify(uniqueKeysPayload));
|
|
36
|
+
}
|
|
37
|
+
this.clear();
|
|
38
|
+
return pipeline.exec()
|
|
39
|
+
.then(function (data) {
|
|
40
|
+
// If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
|
|
41
|
+
if (data.length && data.length === featureNames.length) {
|
|
42
|
+
return _this.redis.expire(_this.key, constants_1.TTL_REFRESH);
|
|
43
|
+
}
|
|
44
|
+
})
|
|
45
|
+
.catch(function (err) {
|
|
46
|
+
_this.log.error(constants_2.LOG_PREFIX + "Error in uniqueKeys pipeline: " + err + ".");
|
|
47
|
+
return false;
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
UniqueKeysCacheInRedis.prototype.start = function () {
|
|
51
|
+
this.intervalId = setInterval(this.postUniqueKeysInRedis.bind(this), this.refreshRate);
|
|
52
|
+
};
|
|
53
|
+
UniqueKeysCacheInRedis.prototype.stop = function () {
|
|
54
|
+
clearInterval(this.intervalId);
|
|
55
|
+
return this.postUniqueKeysInRedis();
|
|
56
|
+
};
|
|
57
|
+
/**
|
|
58
|
+
* Async consumer API, used by synchronizer.
|
|
59
|
+
* @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
|
|
60
|
+
*/
|
|
61
|
+
UniqueKeysCacheInRedis.prototype.popNRaw = function (count) {
|
|
62
|
+
var _this = this;
|
|
63
|
+
if (count === void 0) { count = 0; }
|
|
64
|
+
return this.redis.lrange(this.key, 0, count - 1).then(function (uniqueKeyItems) {
|
|
65
|
+
return _this.redis.ltrim(_this.key, uniqueKeyItems.length, -1)
|
|
66
|
+
.then(function () { return uniqueKeyItems.map(function (uniqueKeyItem) { return JSON.parse(uniqueKeyItem); }); });
|
|
67
|
+
});
|
|
68
|
+
};
|
|
69
|
+
return UniqueKeysCacheInRedis;
|
|
70
|
+
}(UniqueKeysCacheInMemory_1.UniqueKeysCacheInMemory));
|
|
71
|
+
exports.UniqueKeysCacheInRedis = UniqueKeysCacheInRedis;
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.LOG_PREFIX = void 0;
|
|
3
|
+
exports.TTL_REFRESH = exports.REFRESH_RATE = exports.DEFAULT_CACHE_SIZE = exports.LOG_PREFIX = void 0;
|
|
4
4
|
exports.LOG_PREFIX = 'storage:redis: ';
|
|
5
|
+
exports.DEFAULT_CACHE_SIZE = 30000;
|
|
6
|
+
exports.REFRESH_RATE = 300000; // 300.000 ms = start after 5 mins
|
|
7
|
+
exports.TTL_REFRESH = 3600; // 1hr
|