@splitsoftware/splitio-commons 1.2.1-rc.3 → 1.2.1-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cjs/logger/constants.js +2 -2
- package/cjs/logger/messages/info.js +1 -1
- package/cjs/sdkFactory/index.js +1 -0
- package/cjs/storages/inLocalStorage/index.js +1 -1
- package/cjs/storages/inMemory/ImpressionsCacheInMemory.js +15 -1
- package/cjs/storages/inMemory/InMemoryStorage.js +1 -1
- package/cjs/storages/inMemory/InMemoryStorageCS.js +1 -1
- package/cjs/storages/pluggable/index.js +2 -2
- package/cjs/sync/submitters/eventsSyncTask.js +4 -3
- package/cjs/sync/submitters/impressionsSyncTask.js +9 -1
- package/cjs/utils/settingsValidation/index.js +2 -0
- package/cjs/utils/settingsValidation/{runtime/browser.js → runtime.js} +1 -0
- package/esm/logger/constants.js +1 -1
- package/esm/logger/messages/info.js +1 -1
- package/esm/sdkFactory/index.js +1 -0
- package/esm/storages/inLocalStorage/index.js +1 -1
- package/esm/storages/inMemory/ImpressionsCacheInMemory.js +15 -1
- package/esm/storages/inMemory/InMemoryStorage.js +1 -1
- package/esm/storages/inMemory/InMemoryStorageCS.js +1 -1
- package/esm/storages/pluggable/index.js +2 -2
- package/esm/sync/submitters/eventsSyncTask.js +5 -4
- package/esm/sync/submitters/impressionsSyncTask.js +9 -1
- package/esm/utils/settingsValidation/index.js +2 -0
- package/esm/utils/settingsValidation/{runtime/browser.js → runtime.js} +1 -0
- package/package.json +1 -2
- package/src/logger/constants.ts +1 -1
- package/src/logger/messages/info.ts +1 -1
- package/src/sdkFactory/index.ts +1 -0
- package/src/storages/inLocalStorage/index.ts +1 -1
- package/src/storages/inMemory/ImpressionsCacheInMemory.ts +22 -1
- package/src/storages/inMemory/InMemoryStorage.ts +1 -1
- package/src/storages/inMemory/InMemoryStorageCS.ts +1 -1
- package/src/storages/pluggable/index.ts +2 -2
- package/src/storages/types.ts +5 -1
- package/src/sync/submitters/eventsSyncTask.ts +6 -4
- package/src/sync/submitters/impressionsSyncTask.ts +12 -1
- package/src/types.ts +15 -0
- package/src/utils/settingsValidation/index.ts +2 -0
- package/src/utils/settingsValidation/runtime.ts +9 -0
- package/types/logger/constants.d.ts +1 -1
- package/types/storages/inMemory/ImpressionsCacheInMemory.d.ts +9 -0
- package/types/storages/types.d.ts +2 -0
- package/types/types.d.ts +15 -0
- package/types/utils/settingsValidation/runtime.d.ts +2 -0
- package/cjs/utils/settingsValidation/runtime/node.js +0 -22
- package/esm/utils/settingsValidation/runtime/node.js +0 -17
- package/src/utils/settingsValidation/runtime/browser.ts +0 -8
- package/src/utils/settingsValidation/runtime/node.ts +0 -22
package/cjs/logger/constants.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.SYNC_SPLITS_FETCH_RETRY = exports.POLLING_STOP = exports.POLLING_START = exports.POLLING_SMART_PAUSING = exports.NEW_FACTORY = exports.NEW_SHARED_CLIENT = exports.IMPRESSION_QUEUEING = exports.IMPRESSION = exports.CLIENT_READY = exports.CLIENT_READY_FROM_CACHE = exports.SETTINGS_SPLITS_FILTER = exports.SYNC_TASK_STOP = exports.SYNC_TASK_EXECUTE = exports.SYNC_TASK_START = exports.STREAMING_NEW_MESSAGE = exports.SYNC_SPLITS_SEGMENTS = exports.SYNC_SPLITS_REMOVED = exports.SYNC_SPLITS_NEW = exports.SYNC_SPLITS_FETCH = exports.SYNC_OFFLINE_DATA = exports.RETRIEVE_MANAGER = exports.RETRIEVE_CLIENT_EXISTING = exports.RETRIEVE_CLIENT_DEFAULT = exports.CLEANUP_DEREGISTERING = exports.CLEANUP_REGISTERING = exports.ENGINE_SANITIZE = exports.ENGINE_VALUE = exports.ENGINE_MATCHER_WHITELIST = exports.ENGINE_MATCHER_STARTS_WITH = exports.ENGINE_MATCHER_STRING_INVALID = exports.ENGINE_MATCHER_STRING = exports.ENGINE_MATCHER_SEGMENT = exports.ENGINE_MATCHER_PART_OF = exports.ENGINE_MATCHER_LESS = exports.ENGINE_MATCHER_GREATER = exports.ENGINE_MATCHER_ENDS_WITH = exports.ENGINE_MATCHER_EQUAL_TO_SET = exports.ENGINE_MATCHER_EQUAL = exports.ENGINE_MATCHER_DEPENDENCY_PRE = exports.ENGINE_MATCHER_DEPENDENCY = exports.ENGINE_MATCHER_CONTAINS_STRING = exports.ENGINE_MATCHER_CONTAINS_ANY = exports.ENGINE_MATCHER_CONTAINS_ALL = exports.ENGINE_MATCHER_BOOLEAN = exports.ENGINE_MATCHER_BETWEEN = exports.ENGINE_MATCHER_ALL = exports.ENGINE_BUCKET = exports.ENGINE_COMBINER_IFELSEIF_NO_TREATMENT = exports.ENGINE_COMBINER_IFELSEIF = exports.ENGINE_COMBINER_AND = void 0;
|
|
4
|
-
exports.ERROR_NOT_FINITE = exports.ERROR_SIZE_EXCEEDED = exports.ERROR_NOT_PLAIN_OBJECT = exports.ERROR_EVENT_TYPE_FORMAT = exports.ERROR_EVENTS_TRACKER = exports.ERROR_IMPRESSIONS_LISTENER = exports.ERROR_IMPRESSIONS_TRACKER = exports.ERROR_STREAMING_AUTH = exports.ERROR_STREAMING_SSE = exports.ERROR_SYNC_OFFLINE_LOADING = exports.ERROR_CLIENT_CANNOT_GET_READY = exports.ERROR_CLIENT_LISTENER = exports.ERROR_LOGLEVEL_INVALID = exports.ERROR_ENGINE_COMBINER_IFELSEIF = exports.STREAMING_PARSING_MY_SEGMENTS_UPDATE_V2 = exports.WARN_API_KEY = exports.WARN_SPLITS_FILTER_EMPTY = exports.WARN_SPLITS_FILTER_INVALID = exports.WARN_SPLITS_FILTER_IGNORED = exports.WARN_INTEGRATION_INVALID = exports.WARN_NOT_EXISTENT_TT = exports.WARN_LOWERCASE_TRAFFIC_TYPE = exports.WARN_NOT_EXISTENT_SPLIT = exports.WARN_TRIMMING = exports.WARN_CONVERTING = exports.WARN_TRIMMING_PROPERTIES = exports.WARN_SETTING_NULL = exports.SUBMITTERS_PUSH_RETRY = exports.SUBMITTERS_PUSH_FAILS = exports.STREAMING_FALLBACK = exports.STREAMING_PARSING_MESSAGE_FAILS = exports.STREAMING_PARSING_ERROR_FAILS = exports.SYNC_SPLITS_FETCH_FAILS = exports.SYNC_MYSEGMENTS_FETCH_RETRY = exports.CLIENT_NOT_READY = exports.CLIENT_NO_LISTENER = exports.ENGINE_VALUE_NO_ATTRIBUTES = exports.ENGINE_VALUE_INVALID = exports.IMPRESSIONS_TRACKER_SUCCESS = exports.EVENTS_TRACKER_SUCCESS = exports.SYNC_STOP_POLLING = exports.SYNC_CONTINUE_POLLING = exports.SYNC_START_POLLING = exports.SUBMITTERS_PUSH = exports.
|
|
4
|
+
exports.ERROR_NOT_FINITE = exports.ERROR_SIZE_EXCEEDED = exports.ERROR_NOT_PLAIN_OBJECT = exports.ERROR_EVENT_TYPE_FORMAT = exports.ERROR_EVENTS_TRACKER = exports.ERROR_IMPRESSIONS_LISTENER = exports.ERROR_IMPRESSIONS_TRACKER = exports.ERROR_STREAMING_AUTH = exports.ERROR_STREAMING_SSE = exports.ERROR_SYNC_OFFLINE_LOADING = exports.ERROR_CLIENT_CANNOT_GET_READY = exports.ERROR_CLIENT_LISTENER = exports.ERROR_LOGLEVEL_INVALID = exports.ERROR_ENGINE_COMBINER_IFELSEIF = exports.STREAMING_PARSING_MY_SEGMENTS_UPDATE_V2 = exports.WARN_API_KEY = exports.WARN_SPLITS_FILTER_EMPTY = exports.WARN_SPLITS_FILTER_INVALID = exports.WARN_SPLITS_FILTER_IGNORED = exports.WARN_INTEGRATION_INVALID = exports.WARN_NOT_EXISTENT_TT = exports.WARN_LOWERCASE_TRAFFIC_TYPE = exports.WARN_NOT_EXISTENT_SPLIT = exports.WARN_TRIMMING = exports.WARN_CONVERTING = exports.WARN_TRIMMING_PROPERTIES = exports.WARN_SETTING_NULL = exports.SUBMITTERS_PUSH_RETRY = exports.SUBMITTERS_PUSH_FAILS = exports.STREAMING_FALLBACK = exports.STREAMING_PARSING_MESSAGE_FAILS = exports.STREAMING_PARSING_ERROR_FAILS = exports.SYNC_SPLITS_FETCH_FAILS = exports.SYNC_MYSEGMENTS_FETCH_RETRY = exports.CLIENT_NOT_READY = exports.CLIENT_NO_LISTENER = exports.ENGINE_VALUE_NO_ATTRIBUTES = exports.ENGINE_VALUE_INVALID = exports.IMPRESSIONS_TRACKER_SUCCESS = exports.EVENTS_TRACKER_SUCCESS = exports.SYNC_STOP_POLLING = exports.SYNC_CONTINUE_POLLING = exports.SYNC_START_POLLING = exports.SUBMITTERS_PUSH = exports.SUBMITTERS_PUSH_FULL_QUEUE = exports.STREAMING_DISCONNECTING = exports.STREAMING_DISABLED = exports.STREAMING_CONNECTING = exports.STREAMING_RECONNECT = exports.STREAMING_REFRESH_TOKEN = void 0;
|
|
5
5
|
exports.LOG_PREFIX_CLEANUP = exports.LOG_PREFIX_EVENTS_TRACKER = exports.LOG_PREFIX_IMPRESSIONS_TRACKER = exports.LOG_PREFIX_SYNC_SUBMITTERS = exports.LOG_PREFIX_SYNC_POLLING = exports.LOG_PREFIX_SYNC_MYSEGMENTS = exports.LOG_PREFIX_SYNC_SEGMENTS = exports.LOG_PREFIX_SYNC_SPLITS = exports.LOG_PREFIX_SYNC_STREAMING = exports.LOG_PREFIX_SYNC_OFFLINE = exports.LOG_PREFIX_SYNC_MANAGER = exports.LOG_PREFIX_SYNC = exports.LOG_PREFIX_ENGINE_VALUE = exports.LOG_PREFIX_ENGINE_MATCHER = exports.LOG_PREFIX_ENGINE_COMBINER = exports.LOG_PREFIX_ENGINE = exports.LOG_PREFIX_INSTANTIATION = exports.LOG_PREFIX_SETTINGS = exports.ERROR_STORAGE_INVALID = exports.ERROR_LOCALHOST_MODULE_REQUIRED = exports.ERROR_HTTP = exports.ERROR_INVALID_IMPRESSIONS_MODE = exports.ERROR_EMPTY_ARRAY = exports.ERROR_EMPTY = exports.ERROR_INVALID = exports.ERROR_INVALID_KEY_OBJECT = exports.ERROR_TOO_LONG = exports.ERROR_NULL = exports.ERROR_CLIENT_DESTROYED = void 0;
|
|
6
6
|
/**
|
|
7
7
|
* Message codes used to trim string log messages from commons and client-side API modules,
|
|
@@ -65,7 +65,7 @@ exports.STREAMING_RECONNECT = 111;
|
|
|
65
65
|
exports.STREAMING_CONNECTING = 112;
|
|
66
66
|
exports.STREAMING_DISABLED = 113;
|
|
67
67
|
exports.STREAMING_DISCONNECTING = 114;
|
|
68
|
-
exports.
|
|
68
|
+
exports.SUBMITTERS_PUSH_FULL_QUEUE = 115;
|
|
69
69
|
exports.SUBMITTERS_PUSH = 116;
|
|
70
70
|
exports.SYNC_START_POLLING = 117;
|
|
71
71
|
exports.SYNC_CONTINUE_POLLING = 118;
|
|
@@ -21,7 +21,7 @@ exports.codesInfo = warn_1.codesWarn.concat([
|
|
|
21
21
|
[c.POLLING_START, c.LOG_PREFIX_SYNC_POLLING + 'Starting polling'],
|
|
22
22
|
[c.POLLING_STOP, c.LOG_PREFIX_SYNC_POLLING + 'Stopping polling'],
|
|
23
23
|
[c.SYNC_SPLITS_FETCH_RETRY, c.LOG_PREFIX_SYNC_SPLITS + 'Retrying download of splits #%s. Reason: %s'],
|
|
24
|
-
[c.
|
|
24
|
+
[c.SUBMITTERS_PUSH_FULL_QUEUE, c.LOG_PREFIX_SYNC_SUBMITTERS + 'Flushing full %s queue and reseting timer.'],
|
|
25
25
|
[c.SUBMITTERS_PUSH, c.LOG_PREFIX_SYNC_SUBMITTERS + 'Pushing %s %s.'],
|
|
26
26
|
[c.STREAMING_REFRESH_TOKEN, c.LOG_PREFIX_SYNC_STREAMING + 'Refreshing streaming token in %s seconds, and connecting streaming in %s seconds.'],
|
|
27
27
|
[c.STREAMING_RECONNECT, c.LOG_PREFIX_SYNC_STREAMING + 'Attempting to reconnect streaming in %s seconds.'],
|
package/cjs/sdkFactory/index.js
CHANGED
|
@@ -25,6 +25,7 @@ function sdkFactory(params) {
|
|
|
25
25
|
var readinessManager = sdkReadinessManager.readinessManager;
|
|
26
26
|
// @TODO consider passing the settings object, so that each storage access only what it needs
|
|
27
27
|
var storageFactoryParams = {
|
|
28
|
+
impressionsQueueSize: settings.scheduler.impressionsQueueSize,
|
|
28
29
|
eventsQueueSize: settings.scheduler.eventsQueueSize,
|
|
29
30
|
optimize: (0, utils_1.shouldBeOptimized)(settings),
|
|
30
31
|
// ATM, only used by InLocalStorage
|
|
@@ -33,7 +33,7 @@ function InLocalStorage(options) {
|
|
|
33
33
|
return {
|
|
34
34
|
splits: new SplitsCacheInLocal_1.SplitsCacheInLocal(log, keys, expirationTimestamp, params.splitFiltersValidation),
|
|
35
35
|
segments: new MySegmentsCacheInLocal_1.MySegmentsCacheInLocal(log, keys),
|
|
36
|
-
impressions: new ImpressionsCacheInMemory_1.ImpressionsCacheInMemory(),
|
|
36
|
+
impressions: new ImpressionsCacheInMemory_1.ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
37
37
|
impressionCounts: params.optimize ? new ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory() : undefined,
|
|
38
38
|
events: new EventsCacheInMemory_1.EventsCacheInMemory(params.eventsQueueSize),
|
|
39
39
|
destroy: function () {
|
|
@@ -2,15 +2,29 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ImpressionsCacheInMemory = void 0;
|
|
4
4
|
var ImpressionsCacheInMemory = /** @class */ (function () {
|
|
5
|
-
|
|
5
|
+
/**
|
|
6
|
+
*
|
|
7
|
+
* @param impressionsQueueSize number of queued impressions to call onFullQueueCb.
|
|
8
|
+
* Default value is 0, that means no maximum value, in case we want to avoid this being triggered.
|
|
9
|
+
*/
|
|
10
|
+
function ImpressionsCacheInMemory(impressionsQueueSize) {
|
|
11
|
+
if (impressionsQueueSize === void 0) { impressionsQueueSize = 0; }
|
|
12
|
+
this.maxQueue = impressionsQueueSize;
|
|
6
13
|
this.queue = [];
|
|
7
14
|
}
|
|
15
|
+
ImpressionsCacheInMemory.prototype.setOnFullQueueCb = function (cb) {
|
|
16
|
+
this.onFullQueue = cb;
|
|
17
|
+
};
|
|
8
18
|
/**
|
|
9
19
|
* Store impressions in sequential order
|
|
10
20
|
*/
|
|
11
21
|
ImpressionsCacheInMemory.prototype.track = function (data) {
|
|
12
22
|
var _a;
|
|
13
23
|
(_a = this.queue).push.apply(_a, data);
|
|
24
|
+
// Check if the cache queue is full and we need to flush it.
|
|
25
|
+
if (this.maxQueue > 0 && this.queue.length >= this.maxQueue && this.onFullQueue) {
|
|
26
|
+
this.onFullQueue();
|
|
27
|
+
}
|
|
14
28
|
};
|
|
15
29
|
/**
|
|
16
30
|
* Clear the data stored on the cache.
|
|
@@ -16,7 +16,7 @@ function InMemoryStorageFactory(params) {
|
|
|
16
16
|
return {
|
|
17
17
|
splits: new SplitsCacheInMemory_1.SplitsCacheInMemory(),
|
|
18
18
|
segments: new SegmentsCacheInMemory_1.SegmentsCacheInMemory(),
|
|
19
|
-
impressions: new ImpressionsCacheInMemory_1.ImpressionsCacheInMemory(),
|
|
19
|
+
impressions: new ImpressionsCacheInMemory_1.ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
20
20
|
impressionCounts: params.optimize ? new ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory() : undefined,
|
|
21
21
|
events: new EventsCacheInMemory_1.EventsCacheInMemory(params.eventsQueueSize),
|
|
22
22
|
// When using MEMORY we should clean all the caches to leave them empty
|
|
@@ -16,7 +16,7 @@ function InMemoryStorageCSFactory(params) {
|
|
|
16
16
|
return {
|
|
17
17
|
splits: new SplitsCacheInMemory_1.SplitsCacheInMemory(),
|
|
18
18
|
segments: new MySegmentsCacheInMemory_1.MySegmentsCacheInMemory(),
|
|
19
|
-
impressions: new ImpressionsCacheInMemory_1.ImpressionsCacheInMemory(),
|
|
19
|
+
impressions: new ImpressionsCacheInMemory_1.ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
20
20
|
impressionCounts: params.optimize ? new ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory() : undefined,
|
|
21
21
|
events: new EventsCacheInMemory_1.EventsCacheInMemory(params.eventsQueueSize),
|
|
22
22
|
// When using MEMORY we should clean all the caches to leave them empty
|
|
@@ -54,7 +54,7 @@ function PluggableStorage(options) {
|
|
|
54
54
|
validatePluggableStorageOptions(options);
|
|
55
55
|
var prefix = (0, KeyBuilder_1.validatePrefix)(options.prefix);
|
|
56
56
|
function PluggableStorageFactory(_a) {
|
|
57
|
-
var log = _a.log, metadata = _a.metadata, onReadyCb = _a.onReadyCb, mode = _a.mode, eventsQueueSize = _a.eventsQueueSize, optimize = _a.optimize;
|
|
57
|
+
var log = _a.log, metadata = _a.metadata, onReadyCb = _a.onReadyCb, mode = _a.mode, eventsQueueSize = _a.eventsQueueSize, impressionsQueueSize = _a.impressionsQueueSize, optimize = _a.optimize;
|
|
58
58
|
var keys = new KeyBuilderSS_1.KeyBuilderSS(prefix, metadata);
|
|
59
59
|
var wrapper = (0, wrapperAdapter_1.wrapperAdapter)(log, options.wrapper);
|
|
60
60
|
var isPartialConsumer = mode === constants_1.CONSUMER_PARTIAL_MODE;
|
|
@@ -63,7 +63,7 @@ function PluggableStorage(options) {
|
|
|
63
63
|
return {
|
|
64
64
|
splits: new SplitsCachePluggable_1.SplitsCachePluggable(log, keys, wrapper),
|
|
65
65
|
segments: new SegmentsCachePluggable_1.SegmentsCachePluggable(log, keys, wrapper),
|
|
66
|
-
impressions: isPartialConsumer ? new ImpressionsCacheInMemory_1.ImpressionsCacheInMemory() : new ImpressionsCachePluggable_1.ImpressionsCachePluggable(log, keys.buildImpressionsKey(), wrapper, metadata),
|
|
66
|
+
impressions: isPartialConsumer ? new ImpressionsCacheInMemory_1.ImpressionsCacheInMemory(impressionsQueueSize) : new ImpressionsCachePluggable_1.ImpressionsCachePluggable(log, keys.buildImpressionsKey(), wrapper, metadata),
|
|
67
67
|
impressionCounts: optimize ? new ImpressionCountsCacheInMemory_1.ImpressionCountsCacheInMemory() : undefined,
|
|
68
68
|
events: isPartialConsumer ? promisifyEventsTrack(new EventsCacheInMemory_1.EventsCacheInMemory(eventsQueueSize)) : new EventsCachePluggable_1.EventsCachePluggable(log, keys.buildEventsKey(), wrapper, metadata),
|
|
69
69
|
// @TODO add telemetry cache when required
|
|
@@ -3,12 +3,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.eventsSyncTaskFactory = void 0;
|
|
4
4
|
var submitterSyncTask_1 = require("./submitterSyncTask");
|
|
5
5
|
var constants_1 = require("../../logger/constants");
|
|
6
|
+
var DATA_NAME = 'events';
|
|
6
7
|
/**
|
|
7
8
|
* Sync task that periodically posts tracked events
|
|
8
9
|
*/
|
|
9
10
|
function eventsSyncTaskFactory(log, postEventsBulk, eventsCache, eventsPushRate, eventsFirstPushWindow, latencyTracker) {
|
|
10
11
|
// don't retry events.
|
|
11
|
-
var syncTask = (0, submitterSyncTask_1.submitterSyncTaskFactory)(log, postEventsBulk, eventsCache, eventsPushRate,
|
|
12
|
+
var syncTask = (0, submitterSyncTask_1.submitterSyncTaskFactory)(log, postEventsBulk, eventsCache, eventsPushRate, DATA_NAME, latencyTracker);
|
|
12
13
|
// Set a timer for the first push of events,
|
|
13
14
|
if (eventsFirstPushWindow > 0) {
|
|
14
15
|
var stopEventPublisherTimeout_1;
|
|
@@ -22,9 +23,9 @@ function eventsSyncTaskFactory(log, postEventsBulk, eventsCache, eventsPushRate,
|
|
|
22
23
|
originalStop_1();
|
|
23
24
|
};
|
|
24
25
|
}
|
|
25
|
-
// register
|
|
26
|
+
// register events submitter to be executed when events cache is full
|
|
26
27
|
eventsCache.setOnFullQueueCb(function () {
|
|
27
|
-
log.info(constants_1.
|
|
28
|
+
log.info(constants_1.SUBMITTERS_PUSH_FULL_QUEUE, [DATA_NAME]);
|
|
28
29
|
syncTask.execute();
|
|
29
30
|
});
|
|
30
31
|
return syncTask;
|
|
@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.impressionsSyncTaskFactory = exports.fromImpressionsCollector = void 0;
|
|
4
4
|
var lang_1 = require("../../utils/lang");
|
|
5
5
|
var submitterSyncTask_1 = require("./submitterSyncTask");
|
|
6
|
+
var constants_1 = require("../../logger/constants");
|
|
7
|
+
var DATA_NAME = 'impressions';
|
|
6
8
|
/**
|
|
7
9
|
* Converts `impressions` data from cache into request payload.
|
|
8
10
|
*/
|
|
@@ -36,6 +38,12 @@ exports.fromImpressionsCollector = fromImpressionsCollector;
|
|
|
36
38
|
function impressionsSyncTaskFactory(log, postTestImpressionsBulk, impressionsCache, impressionsRefreshRate, sendLabels, latencyTracker) {
|
|
37
39
|
if (sendLabels === void 0) { sendLabels = false; }
|
|
38
40
|
// retry impressions only once.
|
|
39
|
-
|
|
41
|
+
var syncTask = (0, submitterSyncTask_1.submitterSyncTaskFactory)(log, postTestImpressionsBulk, impressionsCache, impressionsRefreshRate, DATA_NAME, latencyTracker, fromImpressionsCollector.bind(undefined, sendLabels), 1);
|
|
42
|
+
// register impressions submitter to be executed when impressions cache is full
|
|
43
|
+
impressionsCache.setOnFullQueueCb(function () {
|
|
44
|
+
log.info(constants_1.SUBMITTERS_PUSH_FULL_QUEUE, [DATA_NAME]);
|
|
45
|
+
syncTask.execute();
|
|
46
|
+
});
|
|
47
|
+
return syncTask;
|
|
40
48
|
}
|
|
41
49
|
exports.impressionsSyncTaskFactory = impressionsSyncTaskFactory;
|
|
@@ -36,6 +36,8 @@ var base = {
|
|
|
36
36
|
eventsPushRate: 60,
|
|
37
37
|
// how many events will be queued before flushing
|
|
38
38
|
eventsQueueSize: 500,
|
|
39
|
+
// how many impressions will be queued before flushing
|
|
40
|
+
impressionsQueueSize: 30000,
|
|
39
41
|
// backoff base seconds to wait before re attempting to connect to push notifications
|
|
40
42
|
pushRetryBackoffBase: 1,
|
|
41
43
|
},
|
package/esm/logger/constants.js
CHANGED
|
@@ -60,7 +60,7 @@ export var STREAMING_RECONNECT = 111;
|
|
|
60
60
|
export var STREAMING_CONNECTING = 112;
|
|
61
61
|
export var STREAMING_DISABLED = 113;
|
|
62
62
|
export var STREAMING_DISCONNECTING = 114;
|
|
63
|
-
export var
|
|
63
|
+
export var SUBMITTERS_PUSH_FULL_QUEUE = 115;
|
|
64
64
|
export var SUBMITTERS_PUSH = 116;
|
|
65
65
|
export var SYNC_START_POLLING = 117;
|
|
66
66
|
export var SYNC_CONTINUE_POLLING = 118;
|
|
@@ -17,7 +17,7 @@ export var codesInfo = codesWarn.concat([
|
|
|
17
17
|
[c.POLLING_START, c.LOG_PREFIX_SYNC_POLLING + 'Starting polling'],
|
|
18
18
|
[c.POLLING_STOP, c.LOG_PREFIX_SYNC_POLLING + 'Stopping polling'],
|
|
19
19
|
[c.SYNC_SPLITS_FETCH_RETRY, c.LOG_PREFIX_SYNC_SPLITS + 'Retrying download of splits #%s. Reason: %s'],
|
|
20
|
-
[c.
|
|
20
|
+
[c.SUBMITTERS_PUSH_FULL_QUEUE, c.LOG_PREFIX_SYNC_SUBMITTERS + 'Flushing full %s queue and reseting timer.'],
|
|
21
21
|
[c.SUBMITTERS_PUSH, c.LOG_PREFIX_SYNC_SUBMITTERS + 'Pushing %s %s.'],
|
|
22
22
|
[c.STREAMING_REFRESH_TOKEN, c.LOG_PREFIX_SYNC_STREAMING + 'Refreshing streaming token in %s seconds, and connecting streaming in %s seconds.'],
|
|
23
23
|
[c.STREAMING_RECONNECT, c.LOG_PREFIX_SYNC_STREAMING + 'Attempting to reconnect streaming in %s seconds.'],
|
package/esm/sdkFactory/index.js
CHANGED
|
@@ -22,6 +22,7 @@ export function sdkFactory(params) {
|
|
|
22
22
|
var readinessManager = sdkReadinessManager.readinessManager;
|
|
23
23
|
// @TODO consider passing the settings object, so that each storage access only what it needs
|
|
24
24
|
var storageFactoryParams = {
|
|
25
|
+
impressionsQueueSize: settings.scheduler.impressionsQueueSize,
|
|
25
26
|
eventsQueueSize: settings.scheduler.eventsQueueSize,
|
|
26
27
|
optimize: shouldBeOptimized(settings),
|
|
27
28
|
// ATM, only used by InLocalStorage
|
|
@@ -30,7 +30,7 @@ export function InLocalStorage(options) {
|
|
|
30
30
|
return {
|
|
31
31
|
splits: new SplitsCacheInLocal(log, keys, expirationTimestamp, params.splitFiltersValidation),
|
|
32
32
|
segments: new MySegmentsCacheInLocal(log, keys),
|
|
33
|
-
impressions: new ImpressionsCacheInMemory(),
|
|
33
|
+
impressions: new ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
34
34
|
impressionCounts: params.optimize ? new ImpressionCountsCacheInMemory() : undefined,
|
|
35
35
|
events: new EventsCacheInMemory(params.eventsQueueSize),
|
|
36
36
|
destroy: function () {
|
|
@@ -1,13 +1,27 @@
|
|
|
1
1
|
var ImpressionsCacheInMemory = /** @class */ (function () {
|
|
2
|
-
|
|
2
|
+
/**
|
|
3
|
+
*
|
|
4
|
+
* @param impressionsQueueSize number of queued impressions to call onFullQueueCb.
|
|
5
|
+
* Default value is 0, that means no maximum value, in case we want to avoid this being triggered.
|
|
6
|
+
*/
|
|
7
|
+
function ImpressionsCacheInMemory(impressionsQueueSize) {
|
|
8
|
+
if (impressionsQueueSize === void 0) { impressionsQueueSize = 0; }
|
|
9
|
+
this.maxQueue = impressionsQueueSize;
|
|
3
10
|
this.queue = [];
|
|
4
11
|
}
|
|
12
|
+
ImpressionsCacheInMemory.prototype.setOnFullQueueCb = function (cb) {
|
|
13
|
+
this.onFullQueue = cb;
|
|
14
|
+
};
|
|
5
15
|
/**
|
|
6
16
|
* Store impressions in sequential order
|
|
7
17
|
*/
|
|
8
18
|
ImpressionsCacheInMemory.prototype.track = function (data) {
|
|
9
19
|
var _a;
|
|
10
20
|
(_a = this.queue).push.apply(_a, data);
|
|
21
|
+
// Check if the cache queue is full and we need to flush it.
|
|
22
|
+
if (this.maxQueue > 0 && this.queue.length >= this.maxQueue && this.onFullQueue) {
|
|
23
|
+
this.onFullQueue();
|
|
24
|
+
}
|
|
11
25
|
};
|
|
12
26
|
/**
|
|
13
27
|
* Clear the data stored on the cache.
|
|
@@ -13,7 +13,7 @@ export function InMemoryStorageFactory(params) {
|
|
|
13
13
|
return {
|
|
14
14
|
splits: new SplitsCacheInMemory(),
|
|
15
15
|
segments: new SegmentsCacheInMemory(),
|
|
16
|
-
impressions: new ImpressionsCacheInMemory(),
|
|
16
|
+
impressions: new ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
17
17
|
impressionCounts: params.optimize ? new ImpressionCountsCacheInMemory() : undefined,
|
|
18
18
|
events: new EventsCacheInMemory(params.eventsQueueSize),
|
|
19
19
|
// When using MEMORY we should clean all the caches to leave them empty
|
|
@@ -13,7 +13,7 @@ export function InMemoryStorageCSFactory(params) {
|
|
|
13
13
|
return {
|
|
14
14
|
splits: new SplitsCacheInMemory(),
|
|
15
15
|
segments: new MySegmentsCacheInMemory(),
|
|
16
|
-
impressions: new ImpressionsCacheInMemory(),
|
|
16
|
+
impressions: new ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
17
17
|
impressionCounts: params.optimize ? new ImpressionCountsCacheInMemory() : undefined,
|
|
18
18
|
events: new EventsCacheInMemory(params.eventsQueueSize),
|
|
19
19
|
// When using MEMORY we should clean all the caches to leave them empty
|
|
@@ -51,7 +51,7 @@ export function PluggableStorage(options) {
|
|
|
51
51
|
validatePluggableStorageOptions(options);
|
|
52
52
|
var prefix = validatePrefix(options.prefix);
|
|
53
53
|
function PluggableStorageFactory(_a) {
|
|
54
|
-
var log = _a.log, metadata = _a.metadata, onReadyCb = _a.onReadyCb, mode = _a.mode, eventsQueueSize = _a.eventsQueueSize, optimize = _a.optimize;
|
|
54
|
+
var log = _a.log, metadata = _a.metadata, onReadyCb = _a.onReadyCb, mode = _a.mode, eventsQueueSize = _a.eventsQueueSize, impressionsQueueSize = _a.impressionsQueueSize, optimize = _a.optimize;
|
|
55
55
|
var keys = new KeyBuilderSS(prefix, metadata);
|
|
56
56
|
var wrapper = wrapperAdapter(log, options.wrapper);
|
|
57
57
|
var isPartialConsumer = mode === CONSUMER_PARTIAL_MODE;
|
|
@@ -60,7 +60,7 @@ export function PluggableStorage(options) {
|
|
|
60
60
|
return {
|
|
61
61
|
splits: new SplitsCachePluggable(log, keys, wrapper),
|
|
62
62
|
segments: new SegmentsCachePluggable(log, keys, wrapper),
|
|
63
|
-
impressions: isPartialConsumer ? new ImpressionsCacheInMemory() : new ImpressionsCachePluggable(log, keys.buildImpressionsKey(), wrapper, metadata),
|
|
63
|
+
impressions: isPartialConsumer ? new ImpressionsCacheInMemory(impressionsQueueSize) : new ImpressionsCachePluggable(log, keys.buildImpressionsKey(), wrapper, metadata),
|
|
64
64
|
impressionCounts: optimize ? new ImpressionCountsCacheInMemory() : undefined,
|
|
65
65
|
events: isPartialConsumer ? promisifyEventsTrack(new EventsCacheInMemory(eventsQueueSize)) : new EventsCachePluggable(log, keys.buildEventsKey(), wrapper, metadata),
|
|
66
66
|
// @TODO add telemetry cache when required
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { submitterSyncTaskFactory } from './submitterSyncTask';
|
|
2
|
-
import {
|
|
2
|
+
import { SUBMITTERS_PUSH_FULL_QUEUE } from '../../logger/constants';
|
|
3
|
+
var DATA_NAME = 'events';
|
|
3
4
|
/**
|
|
4
5
|
* Sync task that periodically posts tracked events
|
|
5
6
|
*/
|
|
6
7
|
export function eventsSyncTaskFactory(log, postEventsBulk, eventsCache, eventsPushRate, eventsFirstPushWindow, latencyTracker) {
|
|
7
8
|
// don't retry events.
|
|
8
|
-
var syncTask = submitterSyncTaskFactory(log, postEventsBulk, eventsCache, eventsPushRate,
|
|
9
|
+
var syncTask = submitterSyncTaskFactory(log, postEventsBulk, eventsCache, eventsPushRate, DATA_NAME, latencyTracker);
|
|
9
10
|
// Set a timer for the first push of events,
|
|
10
11
|
if (eventsFirstPushWindow > 0) {
|
|
11
12
|
var stopEventPublisherTimeout_1;
|
|
@@ -19,9 +20,9 @@ export function eventsSyncTaskFactory(log, postEventsBulk, eventsCache, eventsPu
|
|
|
19
20
|
originalStop_1();
|
|
20
21
|
};
|
|
21
22
|
}
|
|
22
|
-
// register
|
|
23
|
+
// register events submitter to be executed when events cache is full
|
|
23
24
|
eventsCache.setOnFullQueueCb(function () {
|
|
24
|
-
log.info(
|
|
25
|
+
log.info(SUBMITTERS_PUSH_FULL_QUEUE, [DATA_NAME]);
|
|
25
26
|
syncTask.execute();
|
|
26
27
|
});
|
|
27
28
|
return syncTask;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { groupBy, forOwn } from '../../utils/lang';
|
|
2
2
|
import { submitterSyncTaskFactory } from './submitterSyncTask';
|
|
3
|
+
import { SUBMITTERS_PUSH_FULL_QUEUE } from '../../logger/constants';
|
|
4
|
+
var DATA_NAME = 'impressions';
|
|
3
5
|
/**
|
|
4
6
|
* Converts `impressions` data from cache into request payload.
|
|
5
7
|
*/
|
|
@@ -32,5 +34,11 @@ export function fromImpressionsCollector(sendLabels, data) {
|
|
|
32
34
|
export function impressionsSyncTaskFactory(log, postTestImpressionsBulk, impressionsCache, impressionsRefreshRate, sendLabels, latencyTracker) {
|
|
33
35
|
if (sendLabels === void 0) { sendLabels = false; }
|
|
34
36
|
// retry impressions only once.
|
|
35
|
-
|
|
37
|
+
var syncTask = submitterSyncTaskFactory(log, postTestImpressionsBulk, impressionsCache, impressionsRefreshRate, DATA_NAME, latencyTracker, fromImpressionsCollector.bind(undefined, sendLabels), 1);
|
|
38
|
+
// register impressions submitter to be executed when impressions cache is full
|
|
39
|
+
impressionsCache.setOnFullQueueCb(function () {
|
|
40
|
+
log.info(SUBMITTERS_PUSH_FULL_QUEUE, [DATA_NAME]);
|
|
41
|
+
syncTask.execute();
|
|
42
|
+
});
|
|
43
|
+
return syncTask;
|
|
36
44
|
}
|
|
@@ -33,6 +33,8 @@ var base = {
|
|
|
33
33
|
eventsPushRate: 60,
|
|
34
34
|
// how many events will be queued before flushing
|
|
35
35
|
eventsQueueSize: 500,
|
|
36
|
+
// how many impressions will be queued before flushing
|
|
37
|
+
impressionsQueueSize: 30000,
|
|
36
38
|
// backoff base seconds to wait before re attempting to connect to push notifications
|
|
37
39
|
pushRetryBackoffBase: 1,
|
|
38
40
|
},
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@splitsoftware/splitio-commons",
|
|
3
|
-
"version": "1.2.1-rc.
|
|
3
|
+
"version": "1.2.1-rc.4",
|
|
4
4
|
"description": "Split Javascript SDK common components",
|
|
5
5
|
"main": "cjs/index.js",
|
|
6
6
|
"module": "esm/index.js",
|
|
@@ -49,7 +49,6 @@
|
|
|
49
49
|
"devDependencies": {
|
|
50
50
|
"@types/google.analytics": "0.0.40",
|
|
51
51
|
"@types/ioredis": "^4.28.0",
|
|
52
|
-
"@types/ip": "^1.1.0",
|
|
53
52
|
"@types/jest": "^27.0.0",
|
|
54
53
|
"@types/lodash": "^4.14.162",
|
|
55
54
|
"@typescript-eslint/eslint-plugin": "^4.2.0",
|
package/src/logger/constants.ts
CHANGED
|
@@ -61,7 +61,7 @@ export const STREAMING_RECONNECT = 111;
|
|
|
61
61
|
export const STREAMING_CONNECTING = 112;
|
|
62
62
|
export const STREAMING_DISABLED = 113;
|
|
63
63
|
export const STREAMING_DISCONNECTING = 114;
|
|
64
|
-
export const
|
|
64
|
+
export const SUBMITTERS_PUSH_FULL_QUEUE = 115;
|
|
65
65
|
export const SUBMITTERS_PUSH = 116;
|
|
66
66
|
export const SYNC_START_POLLING = 117;
|
|
67
67
|
export const SYNC_CONTINUE_POLLING = 118;
|
|
@@ -20,7 +20,7 @@ export const codesInfo: [number, string][] = codesWarn.concat([
|
|
|
20
20
|
[c.POLLING_START, c.LOG_PREFIX_SYNC_POLLING + 'Starting polling'],
|
|
21
21
|
[c.POLLING_STOP, c.LOG_PREFIX_SYNC_POLLING + 'Stopping polling'],
|
|
22
22
|
[c.SYNC_SPLITS_FETCH_RETRY, c.LOG_PREFIX_SYNC_SPLITS + 'Retrying download of splits #%s. Reason: %s'],
|
|
23
|
-
[c.
|
|
23
|
+
[c.SUBMITTERS_PUSH_FULL_QUEUE, c.LOG_PREFIX_SYNC_SUBMITTERS + 'Flushing full %s queue and reseting timer.'],
|
|
24
24
|
[c.SUBMITTERS_PUSH, c.LOG_PREFIX_SYNC_SUBMITTERS + 'Pushing %s %s.'],
|
|
25
25
|
[c.STREAMING_REFRESH_TOKEN, c.LOG_PREFIX_SYNC_STREAMING + 'Refreshing streaming token in %s seconds, and connecting streaming in %s seconds.'],
|
|
26
26
|
[c.STREAMING_RECONNECT, c.LOG_PREFIX_SYNC_STREAMING + 'Attempting to reconnect streaming in %s seconds.'],
|
package/src/sdkFactory/index.ts
CHANGED
|
@@ -33,6 +33,7 @@ export function sdkFactory(params: ISdkFactoryParams): SplitIO.ICsSDK | SplitIO.
|
|
|
33
33
|
|
|
34
34
|
// @TODO consider passing the settings object, so that each storage access only what it needs
|
|
35
35
|
const storageFactoryParams: IStorageFactoryParams = {
|
|
36
|
+
impressionsQueueSize: settings.scheduler.impressionsQueueSize,
|
|
36
37
|
eventsQueueSize: settings.scheduler.eventsQueueSize,
|
|
37
38
|
optimize: shouldBeOptimized(settings),
|
|
38
39
|
|
|
@@ -40,7 +40,7 @@ export function InLocalStorage(options: InLocalStorageOptions = {}): IStorageSyn
|
|
|
40
40
|
return {
|
|
41
41
|
splits: new SplitsCacheInLocal(log, keys, expirationTimestamp, params.splitFiltersValidation),
|
|
42
42
|
segments: new MySegmentsCacheInLocal(log, keys),
|
|
43
|
-
impressions: new ImpressionsCacheInMemory(),
|
|
43
|
+
impressions: new ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
44
44
|
impressionCounts: params.optimize ? new ImpressionCountsCacheInMemory() : undefined,
|
|
45
45
|
events: new EventsCacheInMemory(params.eventsQueueSize),
|
|
46
46
|
|
|
@@ -3,13 +3,34 @@ import { ImpressionDTO } from '../../types';
|
|
|
3
3
|
|
|
4
4
|
export class ImpressionsCacheInMemory implements IImpressionsCacheSync {
|
|
5
5
|
|
|
6
|
-
private
|
|
6
|
+
private onFullQueue?: () => void;
|
|
7
|
+
private readonly maxQueue: number;
|
|
8
|
+
private queue: ImpressionDTO[];
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
*
|
|
12
|
+
* @param impressionsQueueSize number of queued impressions to call onFullQueueCb.
|
|
13
|
+
* Default value is 0, that means no maximum value, in case we want to avoid this being triggered.
|
|
14
|
+
*/
|
|
15
|
+
constructor(impressionsQueueSize: number = 0) {
|
|
16
|
+
this.maxQueue = impressionsQueueSize;
|
|
17
|
+
this.queue = [];
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
setOnFullQueueCb(cb: () => void) {
|
|
21
|
+
this.onFullQueue = cb;
|
|
22
|
+
}
|
|
7
23
|
|
|
8
24
|
/**
|
|
9
25
|
* Store impressions in sequential order
|
|
10
26
|
*/
|
|
11
27
|
track(data: ImpressionDTO[]) {
|
|
12
28
|
this.queue.push(...data);
|
|
29
|
+
|
|
30
|
+
// Check if the cache queue is full and we need to flush it.
|
|
31
|
+
if (this.maxQueue > 0 && this.queue.length >= this.maxQueue && this.onFullQueue) {
|
|
32
|
+
this.onFullQueue();
|
|
33
|
+
}
|
|
13
34
|
}
|
|
14
35
|
|
|
15
36
|
/**
|
|
@@ -16,7 +16,7 @@ export function InMemoryStorageFactory(params: IStorageFactoryParams): IStorageS
|
|
|
16
16
|
return {
|
|
17
17
|
splits: new SplitsCacheInMemory(),
|
|
18
18
|
segments: new SegmentsCacheInMemory(),
|
|
19
|
-
impressions: new ImpressionsCacheInMemory(),
|
|
19
|
+
impressions: new ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
20
20
|
impressionCounts: params.optimize ? new ImpressionCountsCacheInMemory() : undefined,
|
|
21
21
|
events: new EventsCacheInMemory(params.eventsQueueSize),
|
|
22
22
|
|
|
@@ -16,7 +16,7 @@ export function InMemoryStorageCSFactory(params: IStorageFactoryParams): IStorag
|
|
|
16
16
|
return {
|
|
17
17
|
splits: new SplitsCacheInMemory(),
|
|
18
18
|
segments: new MySegmentsCacheInMemory(),
|
|
19
|
-
impressions: new ImpressionsCacheInMemory(),
|
|
19
|
+
impressions: new ImpressionsCacheInMemory(params.impressionsQueueSize),
|
|
20
20
|
impressionCounts: params.optimize ? new ImpressionCountsCacheInMemory() : undefined,
|
|
21
21
|
events: new EventsCacheInMemory(params.eventsQueueSize),
|
|
22
22
|
|
|
@@ -63,7 +63,7 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
|
|
|
63
63
|
|
|
64
64
|
const prefix = validatePrefix(options.prefix);
|
|
65
65
|
|
|
66
|
-
function PluggableStorageFactory({ log, metadata, onReadyCb, mode, eventsQueueSize, optimize }: IStorageFactoryParams): IStorageAsync {
|
|
66
|
+
function PluggableStorageFactory({ log, metadata, onReadyCb, mode, eventsQueueSize, impressionsQueueSize, optimize }: IStorageFactoryParams): IStorageAsync {
|
|
67
67
|
const keys = new KeyBuilderSS(prefix, metadata);
|
|
68
68
|
const wrapper = wrapperAdapter(log, options.wrapper);
|
|
69
69
|
const isPartialConsumer = mode === CONSUMER_PARTIAL_MODE;
|
|
@@ -74,7 +74,7 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
|
|
|
74
74
|
return {
|
|
75
75
|
splits: new SplitsCachePluggable(log, keys, wrapper),
|
|
76
76
|
segments: new SegmentsCachePluggable(log, keys, wrapper),
|
|
77
|
-
impressions: isPartialConsumer ? new ImpressionsCacheInMemory() : new ImpressionsCachePluggable(log, keys.buildImpressionsKey(), wrapper, metadata),
|
|
77
|
+
impressions: isPartialConsumer ? new ImpressionsCacheInMemory(impressionsQueueSize) : new ImpressionsCachePluggable(log, keys.buildImpressionsKey(), wrapper, metadata),
|
|
78
78
|
impressionCounts: optimize ? new ImpressionCountsCacheInMemory() : undefined,
|
|
79
79
|
events: isPartialConsumer ? promisifyEventsTrack(new EventsCacheInMemory(eventsQueueSize)) : new EventsCachePluggable(log, keys.buildEventsKey(), wrapper, metadata),
|
|
80
80
|
// @TODO add telemetry cache when required
|
package/src/storages/types.ts
CHANGED
|
@@ -298,7 +298,7 @@ export interface IRecorderCacheProducerSync<T> {
|
|
|
298
298
|
// @TODO names are inconsistent with spec
|
|
299
299
|
/* Checks if cache is empty. Returns true if the cache was just created or cleared */
|
|
300
300
|
isEmpty(): boolean
|
|
301
|
-
/*
|
|
301
|
+
/* Clears cache data */
|
|
302
302
|
clear(): void
|
|
303
303
|
/* Gets cache data */
|
|
304
304
|
state(): T
|
|
@@ -307,10 +307,13 @@ export interface IRecorderCacheProducerSync<T> {
|
|
|
307
307
|
|
|
308
308
|
export interface IImpressionsCacheSync extends IImpressionsCacheBase, IRecorderCacheProducerSync<ImpressionDTO[]> {
|
|
309
309
|
track(data: ImpressionDTO[]): void
|
|
310
|
+
/* Registers callback for full queue */
|
|
311
|
+
setOnFullQueueCb(cb: () => void): void
|
|
310
312
|
}
|
|
311
313
|
|
|
312
314
|
export interface IEventsCacheSync extends IEventsCacheBase, IRecorderCacheProducerSync<SplitIO.EventData[]> {
|
|
313
315
|
track(data: SplitIO.EventData, size?: number): boolean
|
|
316
|
+
/* Registers callback for full queue */
|
|
314
317
|
setOnFullQueueCb(cb: () => void): void
|
|
315
318
|
}
|
|
316
319
|
|
|
@@ -423,6 +426,7 @@ export type DataLoader = (storage: IStorageSync, matchingKey: string) => void
|
|
|
423
426
|
|
|
424
427
|
export interface IStorageFactoryParams {
|
|
425
428
|
log: ILogger,
|
|
429
|
+
impressionsQueueSize?: number,
|
|
426
430
|
eventsQueueSize?: number,
|
|
427
431
|
optimize?: boolean /* whether create the `impressionCounts` cache (OPTIMIZED impression mode) or not (DEBUG impression mode) */,
|
|
428
432
|
|
|
@@ -3,7 +3,9 @@ import { IPostEventsBulk } from '../../services/types';
|
|
|
3
3
|
import { ISyncTask, ITimeTracker } from '../types';
|
|
4
4
|
import { submitterSyncTaskFactory } from './submitterSyncTask';
|
|
5
5
|
import { ILogger } from '../../logger/types';
|
|
6
|
-
import {
|
|
6
|
+
import { SUBMITTERS_PUSH_FULL_QUEUE } from '../../logger/constants';
|
|
7
|
+
|
|
8
|
+
const DATA_NAME = 'events';
|
|
7
9
|
|
|
8
10
|
/**
|
|
9
11
|
* Sync task that periodically posts tracked events
|
|
@@ -18,7 +20,7 @@ export function eventsSyncTaskFactory(
|
|
|
18
20
|
): ISyncTask {
|
|
19
21
|
|
|
20
22
|
// don't retry events.
|
|
21
|
-
const syncTask = submitterSyncTaskFactory(log, postEventsBulk, eventsCache, eventsPushRate,
|
|
23
|
+
const syncTask = submitterSyncTaskFactory(log, postEventsBulk, eventsCache, eventsPushRate, DATA_NAME, latencyTracker);
|
|
22
24
|
|
|
23
25
|
// Set a timer for the first push of events,
|
|
24
26
|
if (eventsFirstPushWindow > 0) {
|
|
@@ -34,9 +36,9 @@ export function eventsSyncTaskFactory(
|
|
|
34
36
|
};
|
|
35
37
|
}
|
|
36
38
|
|
|
37
|
-
// register
|
|
39
|
+
// register events submitter to be executed when events cache is full
|
|
38
40
|
eventsCache.setOnFullQueueCb(() => {
|
|
39
|
-
log.info(
|
|
41
|
+
log.info(SUBMITTERS_PUSH_FULL_QUEUE, [DATA_NAME]);
|
|
40
42
|
syncTask.execute();
|
|
41
43
|
});
|
|
42
44
|
|
|
@@ -6,6 +6,9 @@ import { ImpressionDTO } from '../../types';
|
|
|
6
6
|
import { submitterSyncTaskFactory } from './submitterSyncTask';
|
|
7
7
|
import { ImpressionsPayload } from './types';
|
|
8
8
|
import { ILogger } from '../../logger/types';
|
|
9
|
+
import { SUBMITTERS_PUSH_FULL_QUEUE } from '../../logger/constants';
|
|
10
|
+
|
|
11
|
+
const DATA_NAME = 'impressions';
|
|
9
12
|
|
|
10
13
|
/**
|
|
11
14
|
* Converts `impressions` data from cache into request payload.
|
|
@@ -50,5 +53,13 @@ export function impressionsSyncTaskFactory(
|
|
|
50
53
|
): ISyncTask {
|
|
51
54
|
|
|
52
55
|
// retry impressions only once.
|
|
53
|
-
|
|
56
|
+
const syncTask = submitterSyncTaskFactory(log, postTestImpressionsBulk, impressionsCache, impressionsRefreshRate, DATA_NAME, latencyTracker, fromImpressionsCollector.bind(undefined, sendLabels), 1);
|
|
57
|
+
|
|
58
|
+
// register impressions submitter to be executed when impressions cache is full
|
|
59
|
+
impressionsCache.setOnFullQueueCb(() => {
|
|
60
|
+
log.info(SUBMITTERS_PUSH_FULL_QUEUE, [DATA_NAME]);
|
|
61
|
+
syncTask.execute();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
return syncTask;
|
|
54
65
|
}
|
package/src/types.ts
CHANGED
|
@@ -73,6 +73,7 @@ export interface ISettings {
|
|
|
73
73
|
readonly scheduler: {
|
|
74
74
|
featuresRefreshRate: number,
|
|
75
75
|
impressionsRefreshRate: number,
|
|
76
|
+
impressionsQueueSize: number,
|
|
76
77
|
metricsRefreshRate: number,
|
|
77
78
|
segmentsRefreshRate: number,
|
|
78
79
|
offlineRefreshRate: number,
|
|
@@ -255,6 +256,13 @@ interface INodeBasicSettings extends ISharedSettings {
|
|
|
255
256
|
* @default 300
|
|
256
257
|
*/
|
|
257
258
|
impressionsRefreshRate?: number,
|
|
259
|
+
/**
|
|
260
|
+
* The maximum number of impression items we want to queue. If we queue more values, it will trigger a flush and reset the timer.
|
|
261
|
+
* If you use a 0 here, the queue will have no maximum size.
|
|
262
|
+
* @property {number} impressionsQueueSize
|
|
263
|
+
* @default 30000
|
|
264
|
+
*/
|
|
265
|
+
impressionsQueueSize?: number,
|
|
258
266
|
/**
|
|
259
267
|
* The SDK sends diagnostic metrics to Split servers. This parameters controls this metric flush period in seconds.
|
|
260
268
|
* @property {number} metricsRefreshRate
|
|
@@ -769,6 +777,13 @@ export namespace SplitIO {
|
|
|
769
777
|
* @default 60
|
|
770
778
|
*/
|
|
771
779
|
impressionsRefreshRate?: number,
|
|
780
|
+
/**
|
|
781
|
+
* The maximum number of impression items we want to queue. If we queue more values, it will trigger a flush and reset the timer.
|
|
782
|
+
* If you use a 0 here, the queue will have no maximum size.
|
|
783
|
+
* @property {number} impressionsQueueSize
|
|
784
|
+
* @default 30000
|
|
785
|
+
*/
|
|
786
|
+
impressionsQueueSize?: number,
|
|
772
787
|
/**
|
|
773
788
|
* The SDK sends diagnostic metrics to Split servers. This parameters controls this metric flush period in seconds.
|
|
774
789
|
* @property {number} metricsRefreshRate
|
|
@@ -38,6 +38,8 @@ const base = {
|
|
|
38
38
|
eventsPushRate: 60,
|
|
39
39
|
// how many events will be queued before flushing
|
|
40
40
|
eventsQueueSize: 500,
|
|
41
|
+
// how many impressions will be queued before flushing
|
|
42
|
+
impressionsQueueSize: 30000,
|
|
41
43
|
// backoff base seconds to wait before re attempting to connect to push notifications
|
|
42
44
|
pushRetryBackoffBase: 1,
|
|
43
45
|
},
|
|
@@ -60,7 +60,7 @@ export declare const STREAMING_RECONNECT = 111;
|
|
|
60
60
|
export declare const STREAMING_CONNECTING = 112;
|
|
61
61
|
export declare const STREAMING_DISABLED = 113;
|
|
62
62
|
export declare const STREAMING_DISCONNECTING = 114;
|
|
63
|
-
export declare const
|
|
63
|
+
export declare const SUBMITTERS_PUSH_FULL_QUEUE = 115;
|
|
64
64
|
export declare const SUBMITTERS_PUSH = 116;
|
|
65
65
|
export declare const SYNC_START_POLLING = 117;
|
|
66
66
|
export declare const SYNC_CONTINUE_POLLING = 118;
|
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
import { IImpressionsCacheSync } from '../types';
|
|
2
2
|
import { ImpressionDTO } from '../../types';
|
|
3
3
|
export declare class ImpressionsCacheInMemory implements IImpressionsCacheSync {
|
|
4
|
+
private onFullQueue?;
|
|
5
|
+
private readonly maxQueue;
|
|
4
6
|
private queue;
|
|
7
|
+
/**
|
|
8
|
+
*
|
|
9
|
+
* @param impressionsQueueSize number of queued impressions to call onFullQueueCb.
|
|
10
|
+
* Default value is 0, that means no maximum value, in case we want to avoid this being triggered.
|
|
11
|
+
*/
|
|
12
|
+
constructor(impressionsQueueSize?: number);
|
|
13
|
+
setOnFullQueueCb(cb: () => void): void;
|
|
5
14
|
/**
|
|
6
15
|
* Store impressions in sequential order
|
|
7
16
|
*/
|
|
@@ -269,6 +269,7 @@ export interface IRecorderCacheProducerSync<T> {
|
|
|
269
269
|
}
|
|
270
270
|
export interface IImpressionsCacheSync extends IImpressionsCacheBase, IRecorderCacheProducerSync<ImpressionDTO[]> {
|
|
271
271
|
track(data: ImpressionDTO[]): void;
|
|
272
|
+
setOnFullQueueCb(cb: () => void): void;
|
|
272
273
|
}
|
|
273
274
|
export interface IEventsCacheSync extends IEventsCacheBase, IRecorderCacheProducerSync<SplitIO.EventData[]> {
|
|
274
275
|
track(data: SplitIO.EventData, size?: number): boolean;
|
|
@@ -335,6 +336,7 @@ export declare type IStorageAsync = IStorageBase<ISplitsCacheAsync, ISegmentsCac
|
|
|
335
336
|
export declare type DataLoader = (storage: IStorageSync, matchingKey: string) => void;
|
|
336
337
|
export interface IStorageFactoryParams {
|
|
337
338
|
log: ILogger;
|
|
339
|
+
impressionsQueueSize?: number;
|
|
338
340
|
eventsQueueSize?: number;
|
|
339
341
|
optimize?: boolean;
|
|
340
342
|
matchingKey?: string;
|
package/types/types.d.ts
CHANGED
|
@@ -67,6 +67,7 @@ export interface ISettings {
|
|
|
67
67
|
readonly scheduler: {
|
|
68
68
|
featuresRefreshRate: number;
|
|
69
69
|
impressionsRefreshRate: number;
|
|
70
|
+
impressionsQueueSize: number;
|
|
70
71
|
metricsRefreshRate: number;
|
|
71
72
|
segmentsRefreshRate: number;
|
|
72
73
|
offlineRefreshRate: number;
|
|
@@ -249,6 +250,13 @@ interface INodeBasicSettings extends ISharedSettings {
|
|
|
249
250
|
* @default 300
|
|
250
251
|
*/
|
|
251
252
|
impressionsRefreshRate?: number;
|
|
253
|
+
/**
|
|
254
|
+
* The maximum number of impression items we want to queue. If we queue more values, it will trigger a flush and reset the timer.
|
|
255
|
+
* If you use a 0 here, the queue will have no maximum size.
|
|
256
|
+
* @property {number} impressionsQueueSize
|
|
257
|
+
* @default 30000
|
|
258
|
+
*/
|
|
259
|
+
impressionsQueueSize?: number;
|
|
252
260
|
/**
|
|
253
261
|
* The SDK sends diagnostic metrics to Split servers. This parameters controls this metric flush period in seconds.
|
|
254
262
|
* @property {number} metricsRefreshRate
|
|
@@ -766,6 +774,13 @@ export declare namespace SplitIO {
|
|
|
766
774
|
* @default 60
|
|
767
775
|
*/
|
|
768
776
|
impressionsRefreshRate?: number;
|
|
777
|
+
/**
|
|
778
|
+
* The maximum number of impression items we want to queue. If we queue more values, it will trigger a flush and reset the timer.
|
|
779
|
+
* If you use a 0 here, the queue will have no maximum size.
|
|
780
|
+
* @property {number} impressionsQueueSize
|
|
781
|
+
* @default 30000
|
|
782
|
+
*/
|
|
783
|
+
impressionsQueueSize?: number;
|
|
769
784
|
/**
|
|
770
785
|
* The SDK sends diagnostic metrics to Split servers. This parameters controls this metric flush period in seconds.
|
|
771
786
|
* @property {number} metricsRefreshRate
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.validateRuntime = void 0;
|
|
4
|
-
var tslib_1 = require("tslib");
|
|
5
|
-
var os_1 = (0, tslib_1.__importDefault)(require("os"));
|
|
6
|
-
var ip_1 = (0, tslib_1.__importDefault)(require("ip"));
|
|
7
|
-
var constants_1 = require("../../constants");
|
|
8
|
-
function validateRuntime(settings) {
|
|
9
|
-
var isIPAddressesEnabled = settings.core.IPAddressesEnabled === true;
|
|
10
|
-
var isConsumerMode = settings.mode === constants_1.CONSUMER_MODE;
|
|
11
|
-
// If the values are not available, default to false (for standalone) or "unknown" (for consumer mode, to be used on Redis keys)
|
|
12
|
-
var ip = ip_1.default.address() || (isConsumerMode ? constants_1.UNKNOWN : false);
|
|
13
|
-
var hostname = os_1.default.hostname() || (isConsumerMode ? constants_1.UNKNOWN : false);
|
|
14
|
-
if (!isIPAddressesEnabled) { // If IPAddresses setting is not enabled, set as false (for standalone) or "NA" (for consumer mode, to be used on Redis keys)
|
|
15
|
-
ip = hostname = isConsumerMode ? constants_1.NA : false;
|
|
16
|
-
}
|
|
17
|
-
return {
|
|
18
|
-
ip: ip,
|
|
19
|
-
hostname: hostname
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
|
-
exports.validateRuntime = validateRuntime;
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import osFunction from 'os';
|
|
2
|
-
import ipFunction from 'ip';
|
|
3
|
-
import { UNKNOWN, NA, CONSUMER_MODE } from '../../constants';
|
|
4
|
-
export function validateRuntime(settings) {
|
|
5
|
-
var isIPAddressesEnabled = settings.core.IPAddressesEnabled === true;
|
|
6
|
-
var isConsumerMode = settings.mode === CONSUMER_MODE;
|
|
7
|
-
// If the values are not available, default to false (for standalone) or "unknown" (for consumer mode, to be used on Redis keys)
|
|
8
|
-
var ip = ipFunction.address() || (isConsumerMode ? UNKNOWN : false);
|
|
9
|
-
var hostname = osFunction.hostname() || (isConsumerMode ? UNKNOWN : false);
|
|
10
|
-
if (!isIPAddressesEnabled) { // If IPAddresses setting is not enabled, set as false (for standalone) or "NA" (for consumer mode, to be used on Redis keys)
|
|
11
|
-
ip = hostname = isConsumerMode ? NA : false;
|
|
12
|
-
}
|
|
13
|
-
return {
|
|
14
|
-
ip: ip,
|
|
15
|
-
hostname: hostname
|
|
16
|
-
};
|
|
17
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import osFunction from 'os';
|
|
2
|
-
import ipFunction from 'ip';
|
|
3
|
-
|
|
4
|
-
import { UNKNOWN, NA, CONSUMER_MODE } from '../../constants';
|
|
5
|
-
import { ISettings } from '../../../types';
|
|
6
|
-
|
|
7
|
-
export function validateRuntime(settings: ISettings): ISettings['runtime'] {
|
|
8
|
-
const isIPAddressesEnabled = settings.core.IPAddressesEnabled === true;
|
|
9
|
-
const isConsumerMode = settings.mode === CONSUMER_MODE;
|
|
10
|
-
|
|
11
|
-
// If the values are not available, default to false (for standalone) or "unknown" (for consumer mode, to be used on Redis keys)
|
|
12
|
-
let ip = ipFunction.address() || (isConsumerMode ? UNKNOWN : false);
|
|
13
|
-
let hostname = osFunction.hostname() || (isConsumerMode ? UNKNOWN : false);
|
|
14
|
-
|
|
15
|
-
if (!isIPAddressesEnabled) { // If IPAddresses setting is not enabled, set as false (for standalone) or "NA" (for consumer mode, to be used on Redis keys)
|
|
16
|
-
ip = hostname = isConsumerMode ? NA : false;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
return {
|
|
20
|
-
ip, hostname
|
|
21
|
-
};
|
|
22
|
-
}
|