@splitsoftware/splitio-commons 1.6.2-rc.1 → 1.6.2-rc.2

package/src/storages/types.ts

@@ -1,8 +1,7 @@
 import { MaybeThenable, IMetadata, ISplitFiltersValidation } from '../dtos/types';
 import { ILogger } from '../logger/types';
-import { EventDataType, HttpErrors, HttpLatencies, ImpressionDataType, LastSync, Method, MethodExceptions, MethodLatencies, OperationType, StoredEventWithMetadata, StoredImpressionWithMetadata, StreamingEvent } from '../sync/submitters/types';
+import { EventDataType, HttpErrors, HttpLatencies, ImpressionDataType, LastSync, Method, MethodExceptions, MethodLatencies, OperationType, StoredEventWithMetadata, StoredImpressionWithMetadata, StreamingEvent, UniqueKeysPayloadCs, UniqueKeysPayloadSs } from '../sync/submitters/types';
 import { SplitIO, ImpressionDTO, SDKMode } from '../types';
-import { ISet } from '../utils/lang/sets';
 
 /**
  * Interface of a pluggable storage wrapper.
@@ -356,13 +355,16 @@ export interface IImpressionCountsCacheSync extends IRecorderCacheProducerSync<R
   pop(toMerge?: Record<string, number> ): Record<string, number> // pop cache data
 }
 
-export interface IUniqueKeysCacheBase extends IRecorderCacheProducerSync<{ [featureName: string]: ISet<string> }>{
+export interface IUniqueKeysCacheBase {
   // Used by unique Keys tracker
-  track(featureName: string, timeFrame: number, amount: number): void
+  track(key: string, value: string): void
 
   // Used by unique keys submitter in standalone and producer mode
   isEmpty(): boolean // check if cache is empty. Return true if the cache was just created or cleared.
-  pop(toMerge?: { [featureName: string]: ISet<string> } ): { [featureName: string]: ISet<string> } // pop cache data
+  pop(): UniqueKeysPayloadSs | UniqueKeysPayloadCs // pop cache data
+  /* Registers callback for full queue */
+  setOnFullQueueCb(cb: () => void): void,
+  clear(): void
 }
 
 /**
@@ -493,10 +495,11 @@ export type DataLoader = (storage: IStorageSync, matchingKey: string) => void
 export interface IStorageFactoryParams {
   log: ILogger,
   impressionsQueueSize?: number,
+  uniqueKeysCacheSize?: number;
   eventsQueueSize?: number,
   optimize?: boolean /* whether create the `impressionCounts` cache (OPTIMIZED impression mode) or not (DEBUG impression mode) */,
   mode: SDKMode,
-
+  impressionsMode?: string,
   // ATM, only used by InLocalStorage
   matchingKey?: string, /* undefined on server-side SDKs */
   splitFiltersValidation?: ISplitFiltersValidation,

package/src/sync/submitters/telemetrySubmitter.ts

@@ -1,7 +1,7 @@
 import { ISegmentsCacheSync, ISplitsCacheSync, ITelemetryCacheSync } from '../../storages/types';
 import { submitterFactory, firstPushWindowDecorator } from './submitter';
 import { TelemetryUsageStatsPayload, TelemetryConfigStatsPayload, TelemetryConfigStats } from './types';
-import { QUEUED, DEDUPED, DROPPED, CONSUMER_MODE, CONSUMER_ENUM, STANDALONE_MODE, CONSUMER_PARTIAL_MODE, STANDALONE_ENUM, CONSUMER_PARTIAL_ENUM, OPTIMIZED, DEBUG, DEBUG_ENUM, OPTIMIZED_ENUM, CONSENT_GRANTED, CONSENT_DECLINED, CONSENT_UNKNOWN } from '../../utils/constants';
+import { QUEUED, DEDUPED, DROPPED, CONSUMER_MODE, CONSUMER_ENUM, STANDALONE_MODE, CONSUMER_PARTIAL_MODE, STANDALONE_ENUM, CONSUMER_PARTIAL_ENUM, OPTIMIZED, DEBUG, NONE, DEBUG_ENUM, OPTIMIZED_ENUM, NONE_ENUM, CONSENT_GRANTED, CONSENT_DECLINED, CONSENT_UNKNOWN } from '../../utils/constants';
 import { SDK_READY, SDK_READY_FROM_CACHE } from '../../readiness/constants';
 import { ConsentStatus, ISettings, SDKMode } from '../../types';
 import { base } from '../../utils/settingsValidation';
@@ -52,8 +52,9 @@ const OPERATION_MODE_MAP = {
 
 const IMPRESSIONS_MODE_MAP = {
   [OPTIMIZED]: OPTIMIZED_ENUM,
-  [DEBUG]: DEBUG_ENUM
-} as Record<ISettings['sync']['impressionsMode'], (0 | 1)>;
+  [DEBUG]: DEBUG_ENUM,
+  [NONE]: NONE_ENUM
+} as Record<ISettings['sync']['impressionsMode'], (0 | 1 | 2)>;
 
 const USER_CONSENT_MAP = {
   [CONSENT_UNKNOWN]: 1,

package/src/sync/submitters/uniqueKeysSubmitter.ts

@@ -1,63 +1,8 @@
+import { SUBMITTERS_PUSH_FULL_QUEUE } from '../../logger/constants';
 import { ISdkFactoryContextSync } from '../../sdkFactory/types';
-import { ISet, setToArray } from '../../utils/lang/sets';
 import { submitterFactory } from './submitter';
-import { UniqueKeysPayloadCs, UniqueKeysPayloadSs } from './types';
 
-/** 
- * Invert keys for feature to features for key
- */
-function invertUniqueKeys(uniqueKeys: { [featureName: string]: ISet<string> }): { [key: string]: string[] } {
-  const featureNames = Object.keys(uniqueKeys);
-  const inverted: { [key: string]: string[] } = {};
-  for (let i = 0; i < featureNames.length; i++) {
-    const featureName = featureNames[i];
-    const featureKeys = setToArray(uniqueKeys[featureName]);
-    for (let j = 0; j< featureKeys.length; j++) {
-      const featureKey = featureKeys[j];
-      if (!inverted[featureKey]) inverted[featureKey] = []; 
-      inverted[featureKey].push(featureName);
-    }
-  }
-  return inverted;
-}
-
-/**
- * Converts `uniqueKeys` data from cache into request payload for CS.
- */
-export function fromUniqueKeysCollectorCs(uniqueKeys: { [featureName: string]: ISet<string> }): UniqueKeysPayloadCs {
-  const payload = [];
-  const featuresPerKey = invertUniqueKeys(uniqueKeys);
-  const keys = Object.keys(featuresPerKey);
-  for (let k = 0; k < keys.length; k++) {
-    const key = keys[k];
-    const uniqueKeysPayload = {
-      k: key,
-      fs: featuresPerKey[key]
-    };
-
-    payload.push(uniqueKeysPayload);
-  }
-  return { keys: payload };
-}
-
-/**
- * Converts `uniqueKeys` data from cache into request payload for SS.
- */
-export function fromUniqueKeysCollectorSs(uniqueKeys: { [featureName: string]: ISet<string> }): UniqueKeysPayloadSs {
-  const payload = [];
-  const featureNames = Object.keys(uniqueKeys);
-  for (let i = 0; i < featureNames.length; i++) {
-    const featureName = featureNames[i];
-    const featureKeys = setToArray(uniqueKeys[featureName]);
-    const uniqueKeysPayload = {
-      f: featureName,
-      ks: featureKeys
-    };
-
-    payload.push(uniqueKeysPayload);
-  }
-  return { keys: payload };
-}
+const DATA_NAME = 'uniqueKeys';
 
 /**
  * Submitter that periodically posts impression counts
@@ -72,8 +17,19 @@ export function uniqueKeysSubmitterFactory(params: ISdkFactoryContextSync) {
   
   const isClientSide = key !== undefined;
   const postUniqueKeysBulk = isClientSide ? postUniqueKeysBulkCs : postUniqueKeysBulkSs;
-  const fromUniqueKeysCollector = isClientSide ? fromUniqueKeysCollectorCs : fromUniqueKeysCollectorSs;
 
-  return submitterFactory(log, postUniqueKeysBulk, uniqueKeys!, uniqueKeysRefreshRate, 'unique keys', fromUniqueKeysCollector);
+  const syncTask = submitterFactory(log, postUniqueKeysBulk, uniqueKeys!, uniqueKeysRefreshRate, 'unique keys');
+
+  // register unique keys submitter to be executed when uniqueKeys cache is full
+  uniqueKeys!.setOnFullQueueCb(() => {
+    if (syncTask.isRunning()) {
+      log.info(SUBMITTERS_PUSH_FULL_QUEUE, [DATA_NAME]);
+      syncTask.execute();
+    }
+    // If submitter is stopped (e.g., user consent declined or unknown, or app state offline), we don't send the data.
+    // Data will be sent when submitter is resumed.
+  });
+  
+  return syncTask;
 }
 

package/src/trackers/impressionsTracker.ts

@@ -30,7 +30,6 @@ export function impressionsTrackerFactory(
       if (settings.userConsent === CONSENT_DECLINED) return;
 
       const impressionsCount = impressions.length;
-
       const { impressionsToStore, impressionsToListener, deduped } = strategy.process(impressions);
       
       const impressionsToListenerCount = impressionsToListener.length;

package/src/trackers/strategy/strategyNone.ts

@@ -21,7 +21,7 @@ export function strategyNoneFactory(
         // Increments impression counter per featureName
         impressionsCounter.track(impression.feature, now, 1);
         // Keep track by unique key
-        uniqueKeysTracker.track(impression.feature, impression.keyName);
+        uniqueKeysTracker.track(impression.keyName, impression.feature);
       });
       
       return {

package/src/trackers/types.ts

@@ -2,7 +2,6 @@ import { SplitIO, ImpressionDTO } from '../types';
 import { StreamingEventType, Method, OperationType } from '../sync/submitters/types';
 import { IEventsCacheBase } from '../storages/types';
 import { NetworkError } from '../services/types';
-import { ISet } from '../utils/lang/sets';
 
 /** Events tracker */
 
@@ -45,8 +44,8 @@ export interface ITelemetryTracker {
 }
 
 export interface IFilterAdapter {
-  add(featureName: string, key: string): boolean;
-  contains(featureName: string, key: string): boolean;
+  add(key: string, featureName: string): boolean;
+  contains(key: string, featureName: string): boolean;
   clear(): void;
 }
 
@@ -57,10 +56,7 @@ export interface IImpressionSenderAdapter {
 
 /** Unique keys tracker */
 export interface IUniqueKeysTracker {
-  track(featureName: string, key: string): void;
-  pop(toMerge?: { [featureName: string]: ISet<string> }): { [featureName: string]: ISet<string>; };
-  clear(): void;
-  isEmpty(): boolean;
+  track(key: string, featureName: string): void;
 }
 
 export interface IStrategyResult {

package/src/trackers/uniqueKeysTracker.ts

@@ -1,6 +1,6 @@
 import { LOG_PREFIX_UNIQUE_KEYS_TRACKER } from '../logger/constants';
 import { ILogger } from '../logger/types';
-import { ISet, _Set } from '../utils/lang/sets';
+import { IUniqueKeysCacheBase } from '../storages/types';
 import { IFilterAdapter, IUniqueKeysTracker } from './types';
 
 const noopFilterAdapter = {
@@ -9,7 +9,6 @@ const noopFilterAdapter = {
   clear() {}
 };
 
-const DEFAULT_CACHE_SIZE = 30000;
 /**
  * Trackes uniques keys
  * Unique Keys Tracker will be in charge of checking if the MTK was already sent to the BE in the last period
@@ -17,64 +16,22 @@ const DEFAULT_CACHE_SIZE = 30000;
  * 
  * @param log Logger instance
  * @param filterAdapter filter adapter
- * @param cacheSize optional internal cache size
- * @param maxBulkSize optional max MTKs bulk size
+ * @param uniqueKeysCache cache to save unique keys
  */
 export function uniqueKeysTrackerFactory(
   log: ILogger,
+  uniqueKeysCache: IUniqueKeysCacheBase,
   filterAdapter: IFilterAdapter = noopFilterAdapter,
-  cacheSize = DEFAULT_CACHE_SIZE,
-  // @TODO
-  // maxBulkSize: number = 5000,
 ): IUniqueKeysTracker {
   
-  let uniqueKeysTracker: { [featureName: string]: ISet<string> } = {};
-  let uniqueTrackerSize = 0;
-  
   return {
-    track(featureName: string, key: string): void {
-      if (!filterAdapter.add(featureName, key)) {
+    track(key: string, featureName: string): void {
+      if (!filterAdapter.add(key, featureName)) {
         log.debug(`${LOG_PREFIX_UNIQUE_KEYS_TRACKER}The feature ${featureName} and key ${key} exist in the filter`);
         return;
       }
-      if (!uniqueKeysTracker[featureName]) uniqueKeysTracker[featureName] = new _Set();
-      const tracker = uniqueKeysTracker[featureName];
-      if (!tracker.has(key)) {
-        tracker.add(key);
-        log.debug(`${LOG_PREFIX_UNIQUE_KEYS_TRACKER}Key ${key} added to feature ${featureName}`);
-        uniqueTrackerSize++;
-      }
-      
-      if (uniqueTrackerSize >= cacheSize) {
-        log.warn(`${LOG_PREFIX_UNIQUE_KEYS_TRACKER}The UniqueKeysTracker size reached the maximum limit`);
-        // @TODO trigger event to submitter to send mtk
-        uniqueTrackerSize = 0;
-      }
-    },
-    
-    /**
-     * Pop the collected data, used as payload for posting.
-     */
-    pop() {
-      const data = uniqueKeysTracker;
-      uniqueKeysTracker = {};
-      return data;
-    },
-
-    /**
-     * Clear the data stored on the cache.
-     */
-    clear() {
-      uniqueKeysTracker = {};
-    },
-
-    /**
-     * Check if the cache is empty.
-     */
-    isEmpty() {
-      return Object.keys(uniqueKeysTracker).length === 0;
+      uniqueKeysCache.track(key, featureName);
     }
-    
   };
 
 }

package/src/types.ts

@@ -81,6 +81,7 @@ export interface ISettings {
     impressionsRefreshRate: number,
     impressionsQueueSize: number,
     uniqueKeysRefreshRate: number,
+    uniqueKeysCacheSize: number,
     /**
      * @deprecated
      */

package/src/utils/constants/index.ts

@@ -50,6 +50,7 @@ export const CONSUMER_PARTIAL_ENUM = 2;
 
 export const OPTIMIZED_ENUM = 0;
 export const DEBUG_ENUM = 1;
+export const NONE_ENUM = 2;
 
 export const SPLITS = 'sp';
 export const IMPRESSIONS = 'im';

package/types/storages/inMemory/uniqueKeysCacheInMemory.d.ts

@@ -0,0 +1,32 @@
+import { IUniqueKeysCacheBase } from '../types';
+import { UniqueKeysPayloadSs } from '../../sync/submitters/types';
+export declare class UniqueKeysCacheInMemory implements IUniqueKeysCacheBase {
+    private onFullQueue?;
+    private readonly maxStorage;
+    private uniqueTrackerSize;
+    private uniqueKeysTracker;
+    constructor(uniqueKeysQueueSize?: number);
+    setOnFullQueueCb(cb: () => void): void;
+    /**
+     * Store unique keys in sequential order
+     * key: string = feature name.
+     * value: Set<string> = set of unique keys.
+     */
+    track(key: string, featureName: string): void;
+    /**
+     * Clear the data stored on the cache.
+     */
+    clear(): void;
+    /**
+     * Pop the collected data, used as payload for posting.
+     */
+    pop(): UniqueKeysPayloadSs;
+    /**
+     * Check if the cache is empty.
+     */
+    isEmpty(): boolean;
+    /**
+     * Converts `uniqueKeys` data from cache into request payload for SS.
+     */
+    private fromUniqueKeysCollector;
+}

package/types/storages/inMemory/uniqueKeysCacheInMemoryCS.d.ts

@@ -0,0 +1,37 @@
+import { IUniqueKeysCacheBase } from '../types';
+import { UniqueKeysPayloadCs } from '../../sync/submitters/types';
+export declare class UniqueKeysCacheInMemoryCS implements IUniqueKeysCacheBase {
+    private onFullQueue?;
+    private readonly maxStorage;
+    private uniqueTrackerSize;
+    private uniqueKeysTracker;
+    /**
+     *
+     * @param impressionsQueueSize number of queued impressions to call onFullQueueCb.
+     * Default value is 0, that means no maximum value, in case we want to avoid this being triggered.
+     */
+    constructor(uniqueKeysQueueSize?: number);
+    setOnFullQueueCb(cb: () => void): void;
+    /**
+     * Store unique keys in sequential order
+     * key: string = key.
+     * value: HashSet<string> = set of split names.
+     */
+    track(key: string, featureName: string): void;
+    /**
+     * Clear the data stored on the cache.
+     */
+    clear(): void;
+    /**
+     * Pop the collected data, used as payload for posting.
+     */
+    pop(): UniqueKeysPayloadCs;
+    /**
+     * Check if the cache is empty.
+     */
+    isEmpty(): boolean;
+    /**
+     * Converts `uniqueKeys` data from cache into request payload.
+     */
+    private fromUniqueKeysCollector;
+}

package/types/storages/types.d.ts

@@ -1,8 +1,7 @@
 import { MaybeThenable, IMetadata, ISplitFiltersValidation } from '../dtos/types';
 import { ILogger } from '../logger/types';
-import { EventDataType, HttpErrors, HttpLatencies, ImpressionDataType, LastSync, Method, MethodExceptions, MethodLatencies, OperationType, StoredEventWithMetadata, StoredImpressionWithMetadata, StreamingEvent } from '../sync/submitters/types';
+import { EventDataType, HttpErrors, HttpLatencies, ImpressionDataType, LastSync, Method, MethodExceptions, MethodLatencies, OperationType, StoredEventWithMetadata, StoredImpressionWithMetadata, StreamingEvent, UniqueKeysPayloadCs, UniqueKeysPayloadSs } from '../sync/submitters/types';
 import { SplitIO, ImpressionDTO, SDKMode } from '../types';
-import { ISet } from '../utils/lang/sets';
 /**
  * Interface of a pluggable storage wrapper.
  */
@@ -298,16 +297,12 @@ export interface IImpressionCountsCacheSync extends IRecorderCacheProducerSync<R
     isEmpty(): boolean;
     pop(toMerge?: Record<string, number>): Record<string, number>;
 }
-export interface IUniqueKeysCacheBase extends IRecorderCacheProducerSync<{
-    [featureName: string]: ISet<string>;
-}> {
-    track(featureName: string, timeFrame: number, amount: number): void;
+export interface IUniqueKeysCacheBase {
+    track(key: string, value: string): void;
     isEmpty(): boolean;
-    pop(toMerge?: {
-        [featureName: string]: ISet<string>;
-    }): {
-        [featureName: string]: ISet<string>;
-    };
+    pop(): UniqueKeysPayloadSs | UniqueKeysPayloadCs;
+    setOnFullQueueCb(cb: () => void): void;
+    clear(): void;
 }
 /**
  * Telemetry storage interface for standalone and partial consumer modes.
@@ -398,9 +393,11 @@ export declare type DataLoader = (storage: IStorageSync, matchingKey: string) =>
 export interface IStorageFactoryParams {
     log: ILogger;
     impressionsQueueSize?: number;
+    uniqueKeysCacheSize?: number;
     eventsQueueSize?: number;
     optimize?: boolean;
     mode: SDKMode;
+    impressionsMode?: string;
     matchingKey?: string;
     splitFiltersValidation?: ISplitFiltersValidation;
     onReadyCb: (error?: any) => void;

package/types/sync/submitters/uniqueKeysSubmitter.d.ts

@@ -1,18 +1,4 @@
 import { ISdkFactoryContextSync } from '../../sdkFactory/types';
-import { ISet } from '../../utils/lang/sets';
-import { UniqueKeysPayloadCs, UniqueKeysPayloadSs } from './types';
-/**
- * Converts `uniqueKeys` data from cache into request payload for CS.
- */
-export declare function fromUniqueKeysCollectorCs(uniqueKeys: {
-    [featureName: string]: ISet<string>;
-}): UniqueKeysPayloadCs;
-/**
- * Converts `uniqueKeys` data from cache into request payload for SS.
- */
-export declare function fromUniqueKeysCollectorSs(uniqueKeys: {
-    [featureName: string]: ISet<string>;
-}): UniqueKeysPayloadSs;
 /**
  * Submitter that periodically posts impression counts
  */

package/types/trackers/types.d.ts

@@ -2,7 +2,6 @@ import { SplitIO, ImpressionDTO } from '../types';
 import { StreamingEventType, Method, OperationType } from '../sync/submitters/types';
 import { IEventsCacheBase } from '../storages/types';
 import { NetworkError } from '../services/types';
-import { ISet } from '../utils/lang/sets';
 /** Events tracker */
 export interface IEventsHandler {
     handleEvent(eventData: SplitIO.EventData): any;
@@ -36,8 +35,8 @@ export interface ITelemetryTracker {
     streamingEvent(e: StreamingEventType | AUTH_REJECTION, d?: number): void;
 }
 export interface IFilterAdapter {
-    add(featureName: string, key: string): boolean;
-    contains(featureName: string, key: string): boolean;
+    add(key: string, featureName: string): boolean;
+    contains(key: string, featureName: string): boolean;
     clear(): void;
 }
 export interface IImpressionSenderAdapter {
@@ -46,14 +45,7 @@ export interface IImpressionSenderAdapter {
 }
 /** Unique keys tracker */
 export interface IUniqueKeysTracker {
-    track(featureName: string, key: string): void;
-    pop(toMerge?: {
-        [featureName: string]: ISet<string>;
-    }): {
-        [featureName: string]: ISet<string>;
-    };
-    clear(): void;
-    isEmpty(): boolean;
+    track(key: string, featureName: string): void;
 }
 export interface IStrategyResult {
     impressionsToStore: ImpressionDTO[];

package/types/trackers/uniqueKeysTracker.d.ts

@@ -1,4 +1,5 @@
 import { ILogger } from '../logger/types';
+import { IUniqueKeysCacheBase } from '../storages/types';
 import { IFilterAdapter, IUniqueKeysTracker } from './types';
 /**
  * Trackes uniques keys
@@ -7,7 +8,6 @@ import { IFilterAdapter, IUniqueKeysTracker } from './types';
  *
  * @param log Logger instance
  * @param filterAdapter filter adapter
- * @param cacheSize optional internal cache size
- * @param maxBulkSize optional max MTKs bulk size
+ * @param uniqueKeysCache cache to save unique keys
  */
-export declare function uniqueKeysTrackerFactory(log: ILogger, filterAdapter?: IFilterAdapter, cacheSize?: number): IUniqueKeysTracker;
+export declare function uniqueKeysTrackerFactory(log: ILogger, uniqueKeysCache: IUniqueKeysCacheBase, filterAdapter?: IFilterAdapter): IUniqueKeysTracker;

package/types/types.d.ts

@@ -75,6 +75,7 @@ export interface ISettings {
         impressionsRefreshRate: number;
         impressionsQueueSize: number;
         uniqueKeysRefreshRate: number;
+        uniqueKeysCacheSize: number;
         /**
          * @deprecated
          */

package/types/utils/constants/index.d.ts

@@ -32,6 +32,7 @@ export declare const CONSUMER_ENUM = 1;
 export declare const CONSUMER_PARTIAL_ENUM = 2;
 export declare const OPTIMIZED_ENUM = 0;
 export declare const DEBUG_ENUM = 1;
+export declare const NONE_ENUM = 2;
 export declare const SPLITS = "sp";
 export declare const IMPRESSIONS = "im";
 export declare const IMPRESSIONS_COUNT = "ic";