@splitsoftware/splitio-commons 1.6.2-rc.1 → 1.6.2-rc.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. package/CHANGES.txt +4 -0
  2. package/cjs/consent/sdkUserConsent.js +2 -2
  3. package/cjs/evaluator/index.js +15 -16
  4. package/cjs/integrations/ga/GaToSplit.js +8 -5
  5. package/cjs/sdkClient/client.js +19 -7
  6. package/cjs/sdkClient/sdkClient.js +3 -1
  7. package/cjs/sdkFactory/index.js +15 -6
  8. package/cjs/sdkManager/index.js +3 -11
  9. package/cjs/services/splitApi.js +6 -6
  10. package/cjs/storages/AbstractSplitsCacheAsync.js +8 -10
  11. package/cjs/storages/AbstractSplitsCacheSync.js +8 -10
  12. package/cjs/storages/KeyBuilderSS.js +54 -9
  13. package/cjs/storages/dataLoader.js +1 -1
  14. package/cjs/storages/inLocalStorage/SplitsCacheInLocal.js +5 -7
  15. package/cjs/storages/inLocalStorage/index.js +5 -1
  16. package/cjs/storages/inMemory/ImpressionCountsCacheInMemory.js +12 -1
  17. package/cjs/storages/inMemory/InMemoryStorage.js +6 -2
  18. package/cjs/storages/inMemory/InMemoryStorageCS.js +6 -2
  19. package/cjs/storages/inMemory/SplitsCacheInMemory.js +7 -10
  20. package/cjs/storages/inMemory/TelemetryCacheInMemory.js +10 -5
  21. package/cjs/storages/inMemory/UniqueKeysCacheInMemory.js +72 -0
  22. package/cjs/storages/inMemory/UniqueKeysCacheInMemoryCS.js +76 -0
  23. package/cjs/storages/inRedis/EventsCacheInRedis.js +1 -1
  24. package/cjs/storages/inRedis/ImpressionCountsCacheInRedis.js +85 -0
  25. package/cjs/storages/inRedis/SplitsCacheInRedis.js +15 -9
  26. package/cjs/storages/inRedis/TelemetryCacheInRedis.js +100 -0
  27. package/cjs/storages/inRedis/UniqueKeysCacheInRedis.js +71 -0
  28. package/cjs/storages/inRedis/constants.js +4 -1
  29. package/cjs/storages/inRedis/index.js +17 -2
  30. package/cjs/storages/pluggable/ImpressionCountsCachePluggable.js +81 -0
  31. package/cjs/storages/pluggable/SplitsCachePluggable.js +14 -9
  32. package/cjs/storages/pluggable/TelemetryCachePluggable.js +126 -0
  33. package/cjs/storages/pluggable/UniqueKeysCachePluggable.js +61 -0
  34. package/cjs/storages/pluggable/index.js +46 -17
  35. package/cjs/sync/offline/syncTasks/fromObjectSyncTask.js +2 -3
  36. package/cjs/sync/polling/updaters/splitChangesUpdater.js +1 -1
  37. package/cjs/sync/submitters/telemetrySubmitter.js +8 -4
  38. package/cjs/sync/submitters/uniqueKeysSubmitter.js +16 -59
  39. package/cjs/trackers/impressionsTracker.js +17 -15
  40. package/cjs/trackers/strategy/strategyNone.js +1 -1
  41. package/cjs/trackers/strategy/strategyOptimized.js +2 -1
  42. package/cjs/trackers/telemetryTracker.js +6 -0
  43. package/cjs/trackers/uniqueKeysTracker.js +11 -42
  44. package/cjs/utils/constants/index.js +3 -2
  45. package/cjs/utils/lang/maps.js +15 -7
  46. package/cjs/utils/redis/RedisMock.js +31 -0
  47. package/cjs/utils/settingsValidation/index.js +0 -3
  48. package/esm/consent/sdkUserConsent.js +2 -2
  49. package/esm/evaluator/index.js +15 -16
  50. package/esm/integrations/ga/GaToSplit.js +8 -5
  51. package/esm/sdkClient/client.js +19 -7
  52. package/esm/sdkClient/sdkClient.js +3 -1
  53. package/esm/sdkFactory/index.js +16 -7
  54. package/esm/sdkManager/index.js +3 -11
  55. package/esm/services/splitApi.js +6 -6
  56. package/esm/storages/AbstractSplitsCacheAsync.js +8 -10
  57. package/esm/storages/AbstractSplitsCacheSync.js +8 -10
  58. package/esm/storages/KeyBuilderSS.js +50 -8
  59. package/esm/storages/dataLoader.js +1 -1
  60. package/esm/storages/inLocalStorage/SplitsCacheInLocal.js +5 -7
  61. package/esm/storages/inLocalStorage/index.js +6 -2
  62. package/esm/storages/inMemory/ImpressionCountsCacheInMemory.js +12 -1
  63. package/esm/storages/inMemory/InMemoryStorage.js +8 -4
  64. package/esm/storages/inMemory/InMemoryStorageCS.js +7 -3
  65. package/esm/storages/inMemory/SplitsCacheInMemory.js +7 -10
  66. package/esm/storages/inMemory/TelemetryCacheInMemory.js +9 -5
  67. package/esm/storages/inMemory/UniqueKeysCacheInMemory.js +68 -0
  68. package/esm/storages/inMemory/UniqueKeysCacheInMemoryCS.js +73 -0
  69. package/esm/storages/inRedis/EventsCacheInRedis.js +1 -1
  70. package/esm/storages/inRedis/ImpressionCountsCacheInRedis.js +82 -0
  71. package/esm/storages/inRedis/SplitsCacheInRedis.js +15 -9
  72. package/esm/storages/inRedis/TelemetryCacheInRedis.js +100 -0
  73. package/esm/storages/inRedis/UniqueKeysCacheInRedis.js +68 -0
  74. package/esm/storages/inRedis/constants.js +3 -0
  75. package/esm/storages/inRedis/index.js +18 -3
  76. package/esm/storages/pluggable/ImpressionCountsCachePluggable.js +78 -0
  77. package/esm/storages/pluggable/SplitsCachePluggable.js +14 -9
  78. package/esm/storages/pluggable/TelemetryCachePluggable.js +126 -0
  79. package/esm/storages/pluggable/UniqueKeysCachePluggable.js +58 -0
  80. package/esm/storages/pluggable/index.js +47 -18
  81. package/esm/sync/offline/syncTasks/fromObjectSyncTask.js +2 -3
  82. package/esm/sync/polling/updaters/splitChangesUpdater.js +1 -1
  83. package/esm/sync/submitters/telemetrySubmitter.js +9 -5
  84. package/esm/sync/submitters/uniqueKeysSubmitter.js +15 -56
  85. package/esm/trackers/impressionsTracker.js +17 -15
  86. package/esm/trackers/strategy/strategyNone.js +1 -1
  87. package/esm/trackers/strategy/strategyOptimized.js +2 -1
  88. package/esm/trackers/telemetryTracker.js +6 -0
  89. package/esm/trackers/uniqueKeysTracker.js +11 -42
  90. package/esm/utils/constants/index.js +1 -0
  91. package/esm/utils/lang/maps.js +15 -7
  92. package/esm/utils/redis/RedisMock.js +28 -0
  93. package/esm/utils/settingsValidation/index.js +0 -3
  94. package/package.json +1 -2
  95. package/src/consent/sdkUserConsent.ts +2 -2
  96. package/src/evaluator/index.ts +14 -15
  97. package/src/integrations/ga/GaToSplit.ts +9 -5
  98. package/src/integrations/types.ts +2 -1
  99. package/src/logger/.DS_Store +0 -0
  100. package/src/sdkClient/client.ts +21 -8
  101. package/src/sdkClient/sdkClient.ts +3 -1
  102. package/src/sdkFactory/index.ts +17 -7
  103. package/src/sdkManager/index.ts +3 -12
  104. package/src/services/splitApi.ts +6 -6
  105. package/src/services/types.ts +2 -2
  106. package/src/storages/AbstractSplitsCacheAsync.ts +13 -15
  107. package/src/storages/AbstractSplitsCacheSync.ts +15 -17
  108. package/src/storages/KeyBuilderSS.ts +61 -9
  109. package/src/storages/dataLoader.ts +1 -1
  110. package/src/storages/inLocalStorage/SplitsCacheInLocal.ts +8 -11
  111. package/src/storages/inLocalStorage/index.ts +5 -2
  112. package/src/storages/inMemory/ImpressionCountsCacheInMemory.ts +16 -1
  113. package/src/storages/inMemory/InMemoryStorage.ts +7 -4
  114. package/src/storages/inMemory/InMemoryStorageCS.ts +6 -3
  115. package/src/storages/inMemory/SplitsCacheInMemory.ts +10 -14
  116. package/src/storages/inMemory/TelemetryCacheInMemory.ts +10 -6
  117. package/src/storages/inMemory/UniqueKeysCacheInMemory.ts +80 -0
  118. package/src/storages/inMemory/UniqueKeysCacheInMemoryCS.ts +86 -0
  119. package/src/storages/inRedis/EventsCacheInRedis.ts +1 -1
  120. package/src/storages/inRedis/ImpressionCountsCacheInRedis.ts +95 -0
  121. package/src/storages/inRedis/SplitsCacheInRedis.ts +21 -17
  122. package/src/storages/inRedis/TelemetryCacheInRedis.ts +122 -2
  123. package/src/storages/inRedis/UniqueKeysCacheInRedis.ts +77 -0
  124. package/src/storages/inRedis/constants.ts +3 -0
  125. package/src/storages/inRedis/index.ts +15 -5
  126. package/src/storages/pluggable/ImpressionCountsCachePluggable.ts +92 -0
  127. package/src/storages/pluggable/SplitsCachePluggable.ts +20 -17
  128. package/src/storages/pluggable/TelemetryCachePluggable.ts +147 -2
  129. package/src/storages/pluggable/UniqueKeysCachePluggable.ts +67 -0
  130. package/src/storages/pluggable/index.ts +51 -19
  131. package/src/storages/types.ts +38 -30
  132. package/src/sync/offline/syncTasks/fromObjectSyncTask.ts +5 -6
  133. package/src/sync/polling/updaters/splitChangesUpdater.ts +2 -2
  134. package/src/sync/submitters/telemetrySubmitter.ts +15 -8
  135. package/src/sync/submitters/types.ts +26 -12
  136. package/src/sync/submitters/uniqueKeysSubmitter.ts +18 -61
  137. package/src/trackers/impressionsTracker.ts +16 -15
  138. package/src/trackers/strategy/strategyNone.ts +1 -1
  139. package/src/trackers/strategy/strategyOptimized.ts +1 -1
  140. package/src/trackers/telemetryTracker.ts +7 -2
  141. package/src/trackers/types.ts +9 -7
  142. package/src/trackers/uniqueKeysTracker.ts +15 -47
  143. package/src/types.ts +0 -1
  144. package/src/utils/constants/index.ts +1 -0
  145. package/src/utils/lang/maps.ts +20 -8
  146. package/src/utils/redis/RedisMock.ts +33 -0
  147. package/src/utils/settingsValidation/index.ts +1 -4
  148. package/types/integrations/types.d.ts +2 -1
  149. package/types/services/types.d.ts +2 -2
  150. package/types/storages/AbstractSplitsCacheAsync.d.ts +7 -6
  151. package/types/storages/AbstractSplitsCacheSync.d.ts +6 -6
  152. package/types/storages/KeyBuilderSS.d.ts +9 -2
  153. package/types/storages/inLocalStorage/SplitsCacheInLocal.d.ts +3 -4
  154. package/types/storages/inMemory/ImpressionCountsCacheInMemory.d.ts +5 -1
  155. package/types/storages/inMemory/SplitsCacheInMemory.d.ts +3 -2
  156. package/types/storages/inMemory/TelemetryCacheInMemory.d.ts +6 -3
  157. package/types/storages/inMemory/uniqueKeysCacheInMemory.d.ts +35 -0
  158. package/types/storages/inMemory/uniqueKeysCacheInMemoryCS.d.ts +35 -0
  159. package/types/storages/inRedis/EventsCacheInRedis.d.ts +1 -1
  160. package/types/storages/inRedis/ImpressionCountsCacheInRedis.d.ts +16 -0
  161. package/types/storages/inRedis/SplitsCacheInRedis.d.ts +6 -5
  162. package/types/storages/inRedis/TelemetryCacheInRedis.d.ts +16 -1
  163. package/types/storages/inRedis/constants.d.ts +3 -0
  164. package/types/storages/inRedis/uniqueKeysCacheInRedis.d.ts +21 -0
  165. package/types/storages/pluggable/ImpressionCountsCachePluggable.d.ts +16 -0
  166. package/types/storages/pluggable/SplitsCachePluggable.d.ts +6 -5
  167. package/types/storages/pluggable/TelemetryCachePluggable.d.ts +17 -1
  168. package/types/storages/pluggable/UniqueKeysCachePluggable.d.ts +20 -0
  169. package/types/storages/types.d.ts +35 -35
  170. package/types/sync/polling/updaters/splitChangesUpdater.d.ts +1 -1
  171. package/types/sync/submitters/telemetrySubmitter.d.ts +1 -1
  172. package/types/sync/submitters/types.d.ts +19 -12
  173. package/types/sync/submitters/uniqueKeysSubmitter.d.ts +0 -14
  174. package/types/trackers/types.d.ts +9 -11
  175. package/types/trackers/uniqueKeysTracker.d.ts +3 -3
  176. package/types/types.d.ts +0 -1
  177. package/types/utils/constants/index.d.ts +1 -0
  178. package/types/utils/lang/maps.d.ts +6 -2
  179. package/types/utils/redis/RedisMock.d.ts +4 -0
  180. package/types/utils/settingsValidation/index.d.ts +0 -1
  181. package/types/sdkClient/types.d.ts +0 -18
  182. package/types/storages/inMemory/CountsCacheInMemory.d.ts +0 -20
  183. package/types/storages/inMemory/LatenciesCacheInMemory.d.ts +0 -20
  184. package/types/storages/inRedis/CountsCacheInRedis.d.ts +0 -9
  185. package/types/storages/inRedis/LatenciesCacheInRedis.d.ts +0 -9
  186. package/types/sync/offline/LocalhostFromFile.d.ts +0 -2
  187. package/types/sync/offline/splitsParser/splitsParserFromFile.d.ts +0 -2
  188. package/types/sync/submitters/eventsSyncTask.d.ts +0 -8
  189. package/types/sync/submitters/impressionCountsSyncTask.d.ts +0 -13
  190. package/types/sync/submitters/impressionsSyncTask.d.ts +0 -14
  191. package/types/sync/submitters/metricsSyncTask.d.ts +0 -12
  192. package/types/sync/submitters/submitterSyncTask.d.ts +0 -10
  193. package/types/sync/syncTaskComposite.d.ts +0 -5
  194. package/types/trackers/filter/bloomFilter.d.ts +0 -10
  195. package/types/trackers/filter/dictionaryFilter.d.ts +0 -8
  196. package/types/trackers/filter/types.d.ts +0 -5
  197. package/types/utils/timeTracker/index.d.ts +0 -70
@@ -1,11 +1,14 @@
1
1
  import { ILogger } from '../../logger/types';
2
- import { Method } from '../../sync/submitters/types';
3
- import { KeyBuilderSS } from '../KeyBuilderSS';
2
+ import { Method, MultiConfigs, MultiMethodExceptions, MultiMethodLatencies } from '../../sync/submitters/types';
3
+ import { KeyBuilderSS, parseExceptionField, parseLatencyField, parseMetadata } from '../KeyBuilderSS';
4
4
  import { ITelemetryCacheAsync } from '../types';
5
5
  import { findLatencyIndex } from '../findLatencyIndex';
6
6
  import { Redis } from 'ioredis';
7
7
  import { getTelemetryConfigStats } from '../../sync/submitters/telemetrySubmitter';
8
8
  import { CONSUMER_MODE, STORAGE_REDIS } from '../../utils/constants';
9
+ import { isNaNNumber, isString } from '../../utils/lang';
10
+ import { _Map } from '../../utils/lang/maps';
11
+ import { MAX_LATENCY_BUCKET_COUNT, newBuckets } from '../inMemory/TelemetryCacheInMemory';
9
12
 
10
13
  export class TelemetryCacheInRedis implements ITelemetryCacheAsync {
11
14
 
@@ -22,6 +25,7 @@ export class TelemetryCacheInRedis implements ITelemetryCacheAsync {
22
25
  return this.redis.hincrby(key, field, 1)
23
26
  .catch(() => { /* Handle rejections for telemetry */ });
24
27
  }
28
+
25
29
  recordException(method: Method) {
26
30
  const [key, field] = this.keys.buildExceptionKey(method).split('::');
27
31
  return this.redis.hincrby(key, field, 1)
@@ -33,4 +37,120 @@ export class TelemetryCacheInRedis implements ITelemetryCacheAsync {
33
37
  const value = JSON.stringify(getTelemetryConfigStats(CONSUMER_MODE, STORAGE_REDIS));
34
38
  return this.redis.hset(key, field, value).catch(() => { /* Handle rejections for telemetry */ });
35
39
  }
40
+
41
+ /**
42
+ * Pop telemetry latencies.
43
+ * The returned promise rejects if redis operations fail.
44
+ */
45
+ popLatencies(): Promise<MultiMethodLatencies> {
46
+ return this.redis.hgetall(this.keys.latencyPrefix).then(latencies => {
47
+
48
+ const result: MultiMethodLatencies = new _Map();
49
+
50
+ Object.keys(latencies).forEach(field => {
51
+
52
+ const parsedField = parseLatencyField(field);
53
+ if (isString(parsedField)) {
54
+ this.log.error(`Ignoring invalid latency field: ${field}: ${parsedField}`);
55
+ return;
56
+ }
57
+
58
+ const count = parseInt(latencies[field]);
59
+ if (isNaNNumber(count)) {
60
+ this.log.error(`Ignoring latency with invalid count: ${latencies[field]}`);
61
+ return;
62
+ }
63
+
64
+ const [metadata, method, bucket] = parsedField;
65
+
66
+ if (bucket >= MAX_LATENCY_BUCKET_COUNT) {
67
+ this.log.error(`Ignoring latency with invalid bucket: ${bucket}`);
68
+ return;
69
+ }
70
+
71
+ if (!result.has(metadata)) result.set(metadata, {
72
+ t: newBuckets(),
73
+ ts: newBuckets(),
74
+ tc: newBuckets(),
75
+ tcs: newBuckets(),
76
+ tr: newBuckets(),
77
+ });
78
+
79
+ result.get(metadata)![method][bucket] = count;
80
+ });
81
+
82
+ return this.redis.del(this.keys.latencyPrefix).then(() => result);
83
+ });
84
+ }
85
+
86
+ /**
87
+ * Pop telemetry exceptions.
88
+ * The returned promise rejects if redis operations fail.
89
+ */
90
+ popExceptions(): Promise<MultiMethodExceptions> {
91
+ return this.redis.hgetall(this.keys.exceptionPrefix).then(exceptions => {
92
+
93
+ const result: MultiMethodExceptions = new _Map();
94
+
95
+ Object.keys(exceptions).forEach(field => {
96
+
97
+ const parsedField = parseExceptionField(field);
98
+ if (isString(parsedField)) {
99
+ this.log.error(`Ignoring invalid exception field: ${field}: ${parsedField}`);
100
+ return;
101
+ }
102
+
103
+ const count = parseInt(exceptions[field]);
104
+ if (isNaNNumber(count)) {
105
+ this.log.error(`Ignoring exception with invalid count: ${exceptions[field]}`);
106
+ return;
107
+ }
108
+
109
+ const [metadata, method] = parsedField;
110
+
111
+ if (!result.has(metadata)) result.set(metadata, {
112
+ t: 0,
113
+ ts: 0,
114
+ tc: 0,
115
+ tcs: 0,
116
+ tr: 0,
117
+ });
118
+
119
+ result.get(metadata)![method] = count;
120
+ });
121
+
122
+ return this.redis.del(this.keys.exceptionPrefix).then(() => result);
123
+ });
124
+ }
125
+
126
+ /**
127
+ * Pop telemetry configs.
128
+ * The returned promise rejects if redis operations fail.
129
+ */
130
+ popConfigs(): Promise<MultiConfigs> {
131
+ return this.redis.hgetall(this.keys.initPrefix).then(configs => {
132
+
133
+ const result: MultiConfigs = new _Map();
134
+
135
+ Object.keys(configs).forEach(field => {
136
+
137
+ const parsedField = parseMetadata(field);
138
+ if (isString(parsedField)) {
139
+ this.log.error(`Ignoring invalid config field: ${field}: ${parsedField}`);
140
+ return;
141
+ }
142
+
143
+ const [metadata] = parsedField;
144
+
145
+ try {
146
+ const config = JSON.parse(configs[field]);
147
+ result.set(metadata, config);
148
+ } catch (e) {
149
+ this.log.error(`Ignoring invalid config: ${configs[field]}`);
150
+ }
151
+ });
152
+
153
+ return this.redis.del(this.keys.initPrefix).then(() => result);
154
+ });
155
+ }
36
156
  }
@@ -0,0 +1,77 @@
1
+ import { IUniqueKeysCacheBase } from '../types';
2
+ import { Redis } from 'ioredis';
3
+ import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
4
+ import { setToArray } from '../../utils/lang/sets';
5
+ import { DEFAULT_CACHE_SIZE, REFRESH_RATE, TTL_REFRESH } from './constants';
6
+ import { LOG_PREFIX } from './constants';
7
+ import { ILogger } from '../../logger/types';
8
+ import { UniqueKeysItemSs } from '../../sync/submitters/types';
9
+
10
+ export class UniqueKeysCacheInRedis extends UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
11
+
12
+ private readonly log: ILogger;
13
+ private readonly key: string;
14
+ private readonly redis: Redis;
15
+ private readonly refreshRate: number;
16
+ private intervalId: any;
17
+
18
+ constructor(log: ILogger, key: string, redis: Redis, uniqueKeysQueueSize = DEFAULT_CACHE_SIZE, refreshRate = REFRESH_RATE) {
19
+ super(uniqueKeysQueueSize);
20
+ this.log = log;
21
+ this.key = key;
22
+ this.redis = redis;
23
+ this.refreshRate = refreshRate;
24
+ this.onFullQueue = () => { this.postUniqueKeysInRedis(); };
25
+ }
26
+
27
+ private postUniqueKeysInRedis() {
28
+ const featureNames = Object.keys(this.uniqueKeysTracker);
29
+ if (!featureNames.length) return Promise.resolve(false);
30
+
31
+ const pipeline = this.redis.pipeline();
32
+ for (let i = 0; i < featureNames.length; i++) {
33
+ const featureName = featureNames[i];
34
+ const featureKeys = setToArray(this.uniqueKeysTracker[featureName]);
35
+ const uniqueKeysPayload = {
36
+ f: featureName,
37
+ ks: featureKeys
38
+ };
39
+
40
+ pipeline.rpush(this.key, JSON.stringify(uniqueKeysPayload));
41
+ }
42
+ this.clear();
43
+ return pipeline.exec()
44
+ .then(data => {
45
+ // If this is the creation of the key on Redis, set the expiration for it in 3600 seconds.
46
+ if (data.length && data.length === featureNames.length) {
47
+ return this.redis.expire(this.key, TTL_REFRESH);
48
+ }
49
+ })
50
+ .catch(err => {
51
+ this.log.error(`${LOG_PREFIX}Error in uniqueKeys pipeline: ${err}.`);
52
+ return false;
53
+ });
54
+ }
55
+
56
+
57
+ start() {
58
+ this.intervalId = setInterval(this.postUniqueKeysInRedis.bind(this), this.refreshRate);
59
+ }
60
+
61
+ stop() {
62
+ clearInterval(this.intervalId);
63
+ return this.postUniqueKeysInRedis();
64
+ }
65
+
66
+ /**
67
+ * Async consumer API, used by synchronizer.
68
+ * @param count number of items to pop from the queue. If not provided or equal 0, all items will be popped.
69
+ */
70
+ popNRaw(count = 0): Promise<UniqueKeysItemSs[]> {
71
+ return this.redis.lrange(this.key, 0, count - 1).then(uniqueKeyItems => {
72
+ return this.redis.ltrim(this.key, uniqueKeyItems.length, -1)
73
+ .then(() => uniqueKeyItems.map(uniqueKeyItem => JSON.parse(uniqueKeyItem)));
74
+ });
75
+ }
76
+
77
+ }
@@ -1 +1,4 @@
1
1
  export const LOG_PREFIX = 'storage:redis: ';
2
+ export const DEFAULT_CACHE_SIZE = 30000;
3
+ export const REFRESH_RATE = 300000; // 300.000 ms = start after 5 mins
4
+ export const TTL_REFRESH = 3600; // 1hr
@@ -6,8 +6,10 @@ import { SplitsCacheInRedis } from './SplitsCacheInRedis';
6
6
  import { SegmentsCacheInRedis } from './SegmentsCacheInRedis';
7
7
  import { ImpressionsCacheInRedis } from './ImpressionsCacheInRedis';
8
8
  import { EventsCacheInRedis } from './EventsCacheInRedis';
9
- import { STORAGE_REDIS } from '../../utils/constants';
9
+ import { DEBUG, NONE, STORAGE_REDIS } from '../../utils/constants';
10
10
  import { TelemetryCacheInRedis } from './TelemetryCacheInRedis';
11
+ import { UniqueKeysCacheInRedis } from './UniqueKeysCacheInRedis';
12
+ import { ImpressionCountsCacheInRedis } from './ImpressionCountsCacheInRedis';
11
13
 
12
14
  export interface InRedisStorageOptions {
13
15
  prefix?: string
@@ -22,15 +24,18 @@ export function InRedisStorage(options: InRedisStorageOptions = {}): IStorageAsy
22
24
 
23
25
  const prefix = validatePrefix(options.prefix);
24
26
 
25
- function InRedisStorageFactory({ log, metadata, onReadyCb }: IStorageFactoryParams): IStorageAsync {
26
-
27
+ function InRedisStorageFactory({ log, metadata, onReadyCb, impressionsMode }: IStorageFactoryParams): IStorageAsync {
27
28
  const keys = new KeyBuilderSS(prefix, metadata);
28
29
  const redisClient = new RedisAdapter(log, options.options || {});
29
30
  const telemetry = new TelemetryCacheInRedis(log, keys, redisClient);
31
+ const impressionCountsCache = impressionsMode !== DEBUG ? new ImpressionCountsCacheInRedis(log, keys.buildImpressionsCountKey(), redisClient) : undefined;
32
+ const uniqueKeysCache = impressionsMode === NONE ? new UniqueKeysCacheInRedis(log, keys.buildUniqueKeysKey(), redisClient) : undefined;
30
33
 
31
34
  // subscription to Redis connect event in order to emit SDK_READY event on consumer mode
32
35
  redisClient.on('connect', () => {
33
36
  onReadyCb();
37
+ if (impressionCountsCache) impressionCountsCache.start();
38
+ if (uniqueKeysCache) uniqueKeysCache.start();
34
39
 
35
40
  // Synchronize config
36
41
  telemetry.recordConfig();
@@ -40,13 +45,18 @@ export function InRedisStorage(options: InRedisStorageOptions = {}): IStorageAsy
40
45
  splits: new SplitsCacheInRedis(log, keys, redisClient),
41
46
  segments: new SegmentsCacheInRedis(log, keys, redisClient),
42
47
  impressions: new ImpressionsCacheInRedis(log, keys.buildImpressionsKey(), redisClient, metadata),
48
+ impressionCounts: impressionCountsCache,
43
49
  events: new EventsCacheInRedis(log, keys.buildEventsKey(), redisClient, metadata),
44
50
  telemetry,
51
+ uniqueKeys: uniqueKeysCache,
45
52
 
46
53
  // When using REDIS we should:
47
54
  // 1- Disconnect from the storage
48
- destroy() {
49
- redisClient.disconnect();
55
+ destroy(): Promise<void> {
56
+ let promises = [];
57
+ if (impressionCountsCache) promises.push(impressionCountsCache.stop());
58
+ if (uniqueKeysCache) promises.push(uniqueKeysCache.stop());
59
+ return Promise.all(promises).then(() => { redisClient.disconnect(); });
50
60
  // @TODO check that caches works as expected when redisClient is disconnected
51
61
  }
52
62
  };
@@ -0,0 +1,92 @@
1
+ import { ILogger } from '../../logger/types';
2
+ import { ImpressionCountsPayload } from '../../sync/submitters/types';
3
+ import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
4
+ import { REFRESH_RATE } from '../inRedis/constants';
5
+ import { IPluggableStorageWrapper } from '../types';
6
+ import { LOG_PREFIX } from './constants';
7
+
8
+ export class ImpressionCountsCachePluggable extends ImpressionCountsCacheInMemory {
9
+
10
+ private readonly log: ILogger;
11
+ private readonly key: string;
12
+ private readonly wrapper: IPluggableStorageWrapper;
13
+ private readonly refreshRate: number;
14
+ private intervalId: any;
15
+
16
+ constructor(log: ILogger, key: string, wrapper: IPluggableStorageWrapper, impressionCountsCacheSize?: number, refreshRate = REFRESH_RATE) {
17
+ super(impressionCountsCacheSize);
18
+ this.log = log;
19
+ this.key = key;
20
+ this.wrapper = wrapper;
21
+ this.refreshRate = refreshRate;
22
+ this.onFullQueue = () => { this.storeImpressionCounts(); };
23
+ }
24
+
25
+ private storeImpressionCounts() {
26
+ const counts = this.pop();
27
+ const keys = Object.keys(counts);
28
+ if (!keys.length) return Promise.resolve(false);
29
+
30
+ const pipeline = keys.reduce<Promise<any>>((pipeline, key) => {
31
+ return pipeline.then(() => this.wrapper.incr(`${this.key}::${key}`, counts[key]));
32
+ }, Promise.resolve());
33
+
34
+ return pipeline.catch(err => {
35
+ this.log.error(`${LOG_PREFIX}Error in impression counts pipeline: ${err}.`);
36
+ return false;
37
+ });
38
+ }
39
+
40
+ start() {
41
+ this.intervalId = setInterval(this.storeImpressionCounts.bind(this), this.refreshRate);
42
+ }
43
+
44
+ stop() {
45
+ clearInterval(this.intervalId);
46
+ return this.storeImpressionCounts();
47
+ }
48
+
49
+ // Async consumer API, used by synchronizer
50
+ getImpressionsCount(): Promise<ImpressionCountsPayload | undefined> {
51
+ return this.wrapper.getKeysByPrefix(this.key)
52
+ .then(keys => {
53
+ return keys.length ? Promise.all(keys.map(key => this.wrapper.get(key)))
54
+ .then(counts => {
55
+ keys.forEach(key => this.wrapper.del(key).catch(() => { /* noop */ }));
56
+
57
+ const impressionsCount: ImpressionCountsPayload = { pf: [] };
58
+
59
+ for (let i = 0; i < keys.length; i++) {
60
+ const key = keys[i];
61
+ const count = counts[i];
62
+
63
+ const keyFeatureNameAndTime = key.split('::');
64
+ if (keyFeatureNameAndTime.length !== 3) {
65
+ this.log.error(`${LOG_PREFIX}Error spliting key ${key}`);
66
+ continue;
67
+ }
68
+
69
+ const timeFrame = parseInt(keyFeatureNameAndTime[2]);
70
+ if (isNaN(timeFrame)) {
71
+ this.log.error(`${LOG_PREFIX}Error parsing time frame ${keyFeatureNameAndTime[2]}`);
72
+ continue;
73
+ }
74
+ // @ts-ignore
75
+ const rawCount = parseInt(count);
76
+ if (isNaN(rawCount)) {
77
+ this.log.error(`${LOG_PREFIX}Error parsing raw count ${count}`);
78
+ continue;
79
+ }
80
+
81
+ impressionsCount.pf.push({
82
+ f: keyFeatureNameAndTime[1],
83
+ m: timeFrame,
84
+ rc: rawCount,
85
+ });
86
+ }
87
+
88
+ return impressionsCount;
89
+ }) : undefined;
90
+ });
91
+ }
92
+ }
@@ -49,22 +49,22 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
49
49
  * The returned promise is resolved when the operation success
50
50
  * or rejected if it fails (e.g., wrapper operation fails)
51
51
  */
52
- addSplit(name: string, split: string): Promise<boolean> {
52
+ addSplit(name: string, split: ISplit): Promise<boolean> {
53
53
  const splitKey = this.keys.buildSplitKey(name);
54
54
  return this.wrapper.get(splitKey).then(splitFromStorage => {
55
55
 
56
56
  // handling parsing error
57
- let parsedPreviousSplit, parsedSplit;
57
+ let parsedPreviousSplit, stringifiedNewSplit;
58
58
  try {
59
59
  parsedPreviousSplit = splitFromStorage ? JSON.parse(splitFromStorage) : undefined;
60
- parsedSplit = JSON.parse(split);
60
+ stringifiedNewSplit = JSON.stringify(split);
61
61
  } catch (e) {
62
62
  throw new Error('Error parsing split definition: ' + e);
63
63
  }
64
64
 
65
65
  return Promise.all([
66
- this.wrapper.set(splitKey, split),
67
- this._incrementCounts(parsedSplit),
66
+ this.wrapper.set(splitKey, stringifiedNewSplit),
67
+ this._incrementCounts(split),
68
68
  // If it's an update, we decrement the traffic type and segment count of the existing split,
69
69
  parsedPreviousSplit && this._decrementCounts(parsedPreviousSplit)
70
70
  ]);
@@ -76,7 +76,7 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
76
76
  * The returned promise is resolved when the operation success
77
77
  * or rejected if it fails (e.g., wrapper operation fails)
78
78
  */
79
- addSplits(entries: [string, string][]): Promise<boolean[]> {
79
+ addSplits(entries: [string, ISplit][]): Promise<boolean[]> {
80
80
  return Promise.all(entries.map(keyValuePair => this.addSplit(keyValuePair[0], keyValuePair[1])));
81
81
  }
82
82
 
@@ -88,8 +88,7 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
88
88
  removeSplit(name: string) {
89
89
  return this.getSplit(name).then((split) => {
90
90
  if (split) {
91
- const parsedSplit = JSON.parse(split);
92
- this._decrementCounts(parsedSplit);
91
+ this._decrementCounts(split);
93
92
  }
94
93
  return this.wrapper.del(this.keys.buildSplitKey(name));
95
94
  });
@@ -109,8 +108,9 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
109
108
  * The returned promise is resolved with the split definition or null if it's not defined,
110
109
  * or rejected if wrapper operation fails.
111
110
  */
112
- getSplit(name: string): Promise<string | null> {
113
- return this.wrapper.get(this.keys.buildSplitKey(name));
111
+ getSplit(name: string): Promise<ISplit | null> {
112
+ return this.wrapper.get(this.keys.buildSplitKey(name))
113
+ .then(maybeSplit => maybeSplit && JSON.parse(maybeSplit));
114
114
  }
115
115
 
116
116
  /**
@@ -118,13 +118,14 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
118
118
  * The returned promise is resolved with a map of split names to their split definition or null if it's not defined,
119
119
  * or rejected if wrapper operation fails.
120
120
  */
121
- getSplits(names: string[]): Promise<Record<string, string | null>> {
121
+ getSplits(names: string[]): Promise<Record<string, ISplit | null>> {
122
122
  const keys = names.map(name => this.keys.buildSplitKey(name));
123
123
 
124
124
  return this.wrapper.getMany(keys).then(splitDefinitions => {
125
- const splits: Record<string, string | null> = {};
125
+ const splits: Record<string, ISplit | null> = {};
126
126
  names.forEach((name, idx) => {
127
- splits[name] = splitDefinitions[idx];
127
+ const split = splitDefinitions[idx];
128
+ splits[name] = split && JSON.parse(split);
128
129
  });
129
130
  return Promise.resolve(splits);
130
131
  });
@@ -135,10 +136,12 @@ export class SplitsCachePluggable extends AbstractSplitsCacheAsync {
135
136
  * The returned promise is resolved with the list of split definitions,
136
137
  * or rejected if wrapper operation fails.
137
138
  */
138
- getAll(): Promise<string[]> {
139
- return this.wrapper.getKeysByPrefix(this.keys.buildSplitKeyPrefix()).then(
140
- (listOfKeys) => Promise.all(listOfKeys.map(this.wrapper.get) as Promise<string>[])
141
- );
139
+ getAll(): Promise<ISplit[]> {
140
+ return this.wrapper.getKeysByPrefix(this.keys.buildSplitKeyPrefix())
141
+ .then((listOfKeys) => this.wrapper.getMany(listOfKeys))
142
+ .then((splitDefinitions) => splitDefinitions.map((splitDefinition) => {
143
+ return JSON.parse(splitDefinition as string);
144
+ }));
142
145
  }
143
146
 
144
147
  /**
@@ -1,8 +1,13 @@
1
1
  import { ILogger } from '../../logger/types';
2
- import { Method } from '../../sync/submitters/types';
3
- import { KeyBuilderSS } from '../KeyBuilderSS';
2
+ import { Method, MultiConfigs, MultiMethodExceptions, MultiMethodLatencies } from '../../sync/submitters/types';
3
+ import { KeyBuilderSS, parseExceptionField, parseLatencyField, parseMetadata } from '../KeyBuilderSS';
4
4
  import { IPluggableStorageWrapper, ITelemetryCacheAsync } from '../types';
5
5
  import { findLatencyIndex } from '../findLatencyIndex';
6
+ import { getTelemetryConfigStats } from '../../sync/submitters/telemetrySubmitter';
7
+ import { CONSUMER_MODE, STORAGE_PLUGGABLE } from '../../utils/constants';
8
+ import { isString, isNaNNumber } from '../../utils/lang';
9
+ import { _Map } from '../../utils/lang/maps';
10
+ import { MAX_LATENCY_BUCKET_COUNT, newBuckets } from '../inMemory/TelemetryCacheInMemory';
6
11
 
7
12
  export class TelemetryCachePluggable implements ITelemetryCacheAsync {
8
13
 
@@ -23,4 +28,144 @@ export class TelemetryCachePluggable implements ITelemetryCacheAsync {
23
28
  .catch(() => { /* Handle rejections for telemetry */ });
24
29
  }
25
30
 
31
+ recordConfig() {
32
+ const value = JSON.stringify(getTelemetryConfigStats(CONSUMER_MODE, STORAGE_PLUGGABLE));
33
+ return this.wrapper.set(this.keys.buildInitKey(), value).catch(() => { /* Handle rejections for telemetry */ });
34
+ }
35
+
36
+ /**
37
+ * Pop telemetry latencies.
38
+ * The returned promise rejects if wrapper operations fail.
39
+ */
40
+ popLatencies(): Promise<MultiMethodLatencies> {
41
+ return this.wrapper.getKeysByPrefix(this.keys.latencyPrefix).then(latencyKeys => {
42
+ return latencyKeys.length ?
43
+ this.wrapper.getMany(latencyKeys).then(latencies => {
44
+
45
+ const result: MultiMethodLatencies = new _Map();
46
+
47
+ for (let i = 0; i < latencyKeys.length; i++) {
48
+ const field = latencyKeys[i].split('::')[1];
49
+
50
+ const parsedField = parseLatencyField(field);
51
+ if (isString(parsedField)) {
52
+ this.log.error(`Ignoring invalid latency field: ${field}: ${parsedField}`);
53
+ continue;
54
+ }
55
+
56
+ // @ts-ignore
57
+ const count = parseInt(latencies[i]);
58
+ if (isNaNNumber(count)) {
59
+ this.log.error(`Ignoring latency with invalid count: ${latencies[i]}`);
60
+ continue;
61
+ }
62
+
63
+ const [metadata, method, bucket] = parsedField;
64
+
65
+ if (bucket >= MAX_LATENCY_BUCKET_COUNT) {
66
+ this.log.error(`Ignoring latency with invalid bucket: ${bucket}`);
67
+ continue;
68
+ }
69
+
70
+ if (!result.has(metadata)) result.set(metadata, {
71
+ t: newBuckets(),
72
+ ts: newBuckets(),
73
+ tc: newBuckets(),
74
+ tcs: newBuckets(),
75
+ tr: newBuckets(),
76
+ });
77
+
78
+ result.get(metadata)![method][bucket] = count;
79
+ }
80
+
81
+ return Promise.all(latencyKeys.map((latencyKey) => this.wrapper.del(latencyKey))).then(() => result);
82
+ }) :
83
+ // If latencyKeys is empty, return an empty map.
84
+ new _Map();
85
+ });
86
+ }
87
+
88
+ /**
89
+ * Pop telemetry exceptions.
90
+ * The returned promise rejects if wrapper operations fail.
91
+ */
92
+ popExceptions(): Promise<MultiMethodExceptions> {
93
+ return this.wrapper.getKeysByPrefix(this.keys.exceptionPrefix).then(exceptionKeys => {
94
+ return exceptionKeys.length ?
95
+ this.wrapper.getMany(exceptionKeys).then(exceptions => {
96
+
97
+ const result: MultiMethodExceptions = new _Map();
98
+
99
+ for (let i = 0; i < exceptionKeys.length; i++) {
100
+ const field = exceptionKeys[i].split('::')[1];
101
+
102
+ const parsedField = parseExceptionField(field);
103
+ if (isString(parsedField)) {
104
+ this.log.error(`Ignoring invalid exception field: ${field}: ${parsedField}`);
105
+ continue;
106
+ }
107
+
108
+ // @ts-ignore
109
+ const count = parseInt(exceptions[i]);
110
+ if (isNaNNumber(count)) {
111
+ this.log.error(`Ignoring exception with invalid count: ${exceptions[i]}`);
112
+ continue;
113
+ }
114
+
115
+ const [metadata, method] = parsedField;
116
+
117
+ if (!result.has(metadata)) result.set(metadata, {
118
+ t: 0,
119
+ ts: 0,
120
+ tc: 0,
121
+ tcs: 0,
122
+ tr: 0,
123
+ });
124
+
125
+ result.get(metadata)![method] = count;
126
+ }
127
+
128
+ return Promise.all(exceptionKeys.map((exceptionKey) => this.wrapper.del(exceptionKey))).then(() => result);
129
+ }) :
130
+ // If exceptionKeys is empty, return an empty map.
131
+ new _Map();
132
+ });
133
+ }
134
+
135
+ /**
136
+ * Pop telemetry configs.
137
+ * The returned promise rejects if wrapper operations fail.
138
+ */
139
+ popConfigs(): Promise<MultiConfigs> {
140
+ return this.wrapper.getKeysByPrefix(this.keys.initPrefix).then(configKeys => {
141
+ return configKeys.length ?
142
+ this.wrapper.getMany(configKeys).then(configs => {
143
+
144
+ const result: MultiConfigs = new _Map();
145
+
146
+ for (let i = 0; i < configKeys.length; i++) {
147
+ const field = configKeys[i].split('::')[1];
148
+
149
+ const parsedField = parseMetadata(field);
150
+ if (isString(parsedField)) {
151
+ this.log.error(`Ignoring invalid config field: ${field}: ${parsedField}`);
152
+ continue;
153
+ }
154
+
155
+ const [metadata] = parsedField;
156
+
157
+ try { // @ts-ignore
158
+ const config = JSON.parse(configs[i]);
159
+ result.set(metadata, config);
160
+ } catch (e) {
161
+ this.log.error(`Ignoring invalid config: ${configs[i]}`);
162
+ }
163
+ }
164
+
165
+ return Promise.all(configKeys.map((configKey) => this.wrapper.del(configKey))).then(() => result);
166
+ }) :
167
+ // If configKeys is empty, return an empty map.
168
+ new _Map();
169
+ });
170
+ }
26
171
  }