@splitsoftware/splitio-commons 1.6.2-rc.9 → 1.7.0

package/src/storages/pluggable/index.ts

@@ -8,12 +8,17 @@ import { EventsCachePluggable } from './EventsCachePluggable';
 import { wrapperAdapter, METHODS_TO_PROMISE_WRAP } from './wrapperAdapter';
 import { isObject } from '../../utils/lang';
 import { validatePrefix } from '../KeyBuilder';
-import { CONSUMER_PARTIAL_MODE, STORAGE_PLUGGABLE } from '../../utils/constants';
+import { CONSUMER_PARTIAL_MODE, DEBUG, NONE, STORAGE_PLUGGABLE } from '../../utils/constants';
 import { ImpressionsCacheInMemory } from '../inMemory/ImpressionsCacheInMemory';
 import { EventsCacheInMemory } from '../inMemory/EventsCacheInMemory';
 import { ImpressionCountsCacheInMemory } from '../inMemory/ImpressionCountsCacheInMemory';
 import { shouldRecordTelemetry, TelemetryCacheInMemory } from '../inMemory/TelemetryCacheInMemory';
 import { TelemetryCachePluggable } from './TelemetryCachePluggable';
+import { ImpressionCountsCachePluggable } from './ImpressionCountsCachePluggable';
+import { UniqueKeysCachePluggable } from './UniqueKeysCachePluggable';
+import { UniqueKeysCacheInMemory } from '../inMemory/UniqueKeysCacheInMemory';
+import { UniqueKeysCacheInMemoryCS } from '../inMemory/UniqueKeysCacheInMemoryCS';
+import { metadataBuilder } from '../utils';
 
 const NO_VALID_WRAPPER = 'Expecting pluggable storage `wrapper` in options, but no valid wrapper instance was provided.';
 const NO_VALID_WRAPPER_INTERFACE = 'The provided wrapper instance doesn’t follow the expected interface. Check our docs.';
@@ -57,19 +62,42 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
   const prefix = validatePrefix(options.prefix);
 
   function PluggableStorageFactory(params: IStorageFactoryParams): IStorageAsync {
-    const { log, metadata, onReadyCb, mode, eventsQueueSize, impressionsQueueSize, optimize } = params;
+    const { onReadyCb, settings, settings: { log, mode, sync: { impressionsMode }, scheduler: { impressionsQueueSize, eventsQueueSize } } } = params;
+    const metadata = metadataBuilder(settings);
     const keys = new KeyBuilderSS(prefix, metadata);
     const wrapper = wrapperAdapter(log, options.wrapper);
+
+    const isSyncronizer = mode === undefined; // If mode is not defined, the synchronizer is running
     const isPartialConsumer = mode === CONSUMER_PARTIAL_MODE;
-    const telemetry = shouldRecordTelemetry(params) ?
-      isPartialConsumer ? new TelemetryCacheInMemory() : new TelemetryCachePluggable(log, keys, wrapper) :
+
+    const telemetry = shouldRecordTelemetry(params) || isSyncronizer ?
+      isPartialConsumer ?
+        new TelemetryCacheInMemory() :
+        new TelemetryCachePluggable(log, keys, wrapper) :
+      undefined;
+
+    const impressionCountsCache = impressionsMode !== DEBUG || isSyncronizer ?
+      isPartialConsumer ?
+        new ImpressionCountsCacheInMemory() :
+        new ImpressionCountsCachePluggable(log, keys.buildImpressionsCountKey(), wrapper) :
+      undefined;
+
+    const uniqueKeysCache = impressionsMode === NONE || isSyncronizer ?
+      isPartialConsumer ?
+        settings.core.key === undefined ? new UniqueKeysCacheInMemory() : new UniqueKeysCacheInMemoryCS() :
+        new UniqueKeysCachePluggable(log, keys.buildUniqueKeysKey(), wrapper) :
       undefined;
 
     // Connects to wrapper and emits SDK_READY event on main client
     const connectPromise = wrapper.connect().then(() => {
       onReadyCb();
-      // If mode is not defined, it means that the synchronizer is running and so we don't have to record telemetry
-      if (telemetry && (telemetry as ITelemetryCacheAsync).recordConfig && mode) (telemetry as ITelemetryCacheAsync).recordConfig();
+
+      // Start periodic flush of async storages if not running synchronizer (producer mode)
+      if (!isSyncronizer) {
+        if (impressionCountsCache && (impressionCountsCache as ImpressionCountsCachePluggable).start) (impressionCountsCache as ImpressionCountsCachePluggable).start();
+        if (uniqueKeysCache && (uniqueKeysCache as UniqueKeysCachePluggable).start) (uniqueKeysCache as UniqueKeysCachePluggable).start();
+        if (telemetry && (telemetry as ITelemetryCacheAsync).recordConfig) (telemetry as ITelemetryCacheAsync).recordConfig();
+      }
     }).catch((e) => {
       e = e || new Error('Error connecting wrapper');
       onReadyCb(e);
@@ -80,13 +108,17 @@ export function PluggableStorage(options: PluggableStorageOptions): IStorageAsyn
       splits: new SplitsCachePluggable(log, keys, wrapper),
       segments: new SegmentsCachePluggable(log, keys, wrapper),
       impressions: isPartialConsumer ? new ImpressionsCacheInMemory(impressionsQueueSize) : new ImpressionsCachePluggable(log, keys.buildImpressionsKey(), wrapper, metadata),
-      impressionCounts: optimize ? new ImpressionCountsCacheInMemory() : undefined,
+      impressionCounts: impressionCountsCache,
       events: isPartialConsumer ? promisifyEventsTrack(new EventsCacheInMemory(eventsQueueSize)) : new EventsCachePluggable(log, keys.buildEventsKey(), wrapper, metadata),
       telemetry,
+      uniqueKeys: uniqueKeysCache,
 
-      // Disconnect the underlying storage
+      // Stop periodic flush and disconnect the underlying storage
       destroy() {
-        return wrapper.disconnect();
+        return Promise.all(isSyncronizer ? [] : [
+          impressionCountsCache && (impressionCountsCache as ImpressionCountsCachePluggable).stop && (impressionCountsCache as ImpressionCountsCachePluggable).stop(),
+          uniqueKeysCache && (uniqueKeysCache as UniqueKeysCachePluggable).stop && (uniqueKeysCache as UniqueKeysCachePluggable).stop(),
+        ]).then(() => wrapper.disconnect());
       },
 
       // emits SDK_READY event on shared clients and returns a reference to the storage

package/src/storages/types.ts

@@ -1,7 +1,6 @@
-import { MaybeThenable, IMetadata, ISplitFiltersValidation, ISplit } from '../dtos/types';
-import { ILogger } from '../logger/types';
-import { EventDataType, HttpErrors, HttpLatencies, ImpressionDataType, LastSync, Method, MethodExceptions, MethodLatencies, MultiMethodExceptions, MultiMethodLatencies, MultiConfigs, OperationType, StoredEventWithMetadata, StoredImpressionWithMetadata, StreamingEvent } from '../sync/submitters/types';
-import { SplitIO, ImpressionDTO, SDKMode } from '../types';
+import { MaybeThenable, ISplit } from '../dtos/types';
+import { EventDataType, HttpErrors, HttpLatencies, ImpressionDataType, LastSync, Method, MethodExceptions, MethodLatencies, MultiMethodExceptions, MultiMethodLatencies, MultiConfigs, OperationType, StoredEventWithMetadata, StoredImpressionWithMetadata, StreamingEvent, UniqueKeysPayloadCs, UniqueKeysPayloadSs, TelemetryUsageStatsPayload } from '../sync/submitters/types';
+import { SplitIO, ImpressionDTO, ISettings } from '../types';
 
 /**
  * Interface of a pluggable storage wrapper.
@@ -70,23 +69,25 @@ export interface IPluggableStorageWrapper {
   /** Integer operations */
 
   /**
-   * Increments in 1 the given `key` value or set it to 1 if the value doesn't exist.
+   * Increments the number stored at `key` by `increment`, or set it to `increment` if the value doesn't exist.
    *
    * @function incr
    * @param {string} key Key to increment
+   * @param {number} increment Value to increment by. Defaults to 1.
    * @returns {Promise<number>} A promise that resolves with the value of key after the increment. The promise rejects if the operation fails,
    * for example, if there is a connection error or the key contains a string that can not be represented as integer.
    */
-  incr: (key: string) => Promise<number>
+  incr: (key: string, increment?: number) => Promise<number>
   /**
-   * Decrements in 1 the given `key` value or set it to -1 if the value doesn't exist.
+   * Decrements the number stored at `key` by `decrement`, or set it to minus `decrement` if the value doesn't exist.
    *
    * @function decr
    * @param {string} key Key to decrement
+   * @param {number} decrement Value to decrement by. Defaults to 1.
    * @returns {Promise<number>} A promise that resolves with the value of key after the decrement. The promise rejects if the operation fails,
    * for example, if there is a connection error or the key contains a string that can not be represented as integer.
    */
-  decr: (key: string) => Promise<number>
+  decr: (key: string, decrement?: number) => Promise<number>
 
   /** Queue operations */
 
@@ -283,19 +284,29 @@ export interface ISegmentsCacheAsync extends ISegmentsCacheBase {
 /** Recorder storages (impressions, events and telemetry) */
 
 export interface IImpressionsCacheBase {
-  // Consumer API method, used by impressions tracker, in standalone and consumer modes, to push impressions into the storage.
+  // Used by impressions tracker, in DEBUG and OPTIMIZED impression modes, to push impressions into the storage.
   track(data: ImpressionDTO[]): MaybeThenable<void>
 }
 
 export interface IEventsCacheBase {
-  // Consumer API method, used by events tracker, in standalone and consumer modes, to push events into the storage.
+  // Used by events tracker to push events into the storage.
   track(data: SplitIO.EventData, size?: number): MaybeThenable<boolean>
 }
 
-/** Impressions and events cache for standalone mode (sync) */
+export interface IImpressionCountsCacheBase {
+  // Used by impressions tracker, in OPTIMIZED and NONE impression modes, to count impressions.
+  track(featureName: string, timeFrame: number, amount: number): void
+}
+
+export interface IUniqueKeysCacheBase {
+  // Used by impressions tracker, in NONE impression mode, to track unique keys.
+  track(key: string, value: string): void
+}
+
+/** Impressions and events cache for standalone and partial consumer modes (sync methods) */
 
-// Producer API methods for sync recorder storages, used by submitters in standalone mode to pop data and post it to Split BE.
-export interface IRecorderCacheProducerSync<T> {
+// API methods for sync recorder storages, used by submitters in standalone mode to pop data and post it to Split BE.
+export interface IRecorderCacheSync<T> {
   // @TODO names are inconsistent with spec
   /* Checks if cache is empty. Returns true if the cache was just created or cleared */
   isEmpty(): boolean
@@ -305,23 +316,29 @@ export interface IRecorderCacheProducerSync<T> {
   pop(toMerge?: T): T
 }
 
-
-export interface IImpressionsCacheSync extends IImpressionsCacheBase, IRecorderCacheProducerSync<ImpressionDTO[]> {
+export interface IImpressionsCacheSync extends IImpressionsCacheBase, IRecorderCacheSync<ImpressionDTO[]> {
   track(data: ImpressionDTO[]): void
   /* Registers callback for full queue */
   setOnFullQueueCb(cb: () => void): void
 }
 
-export interface IEventsCacheSync extends IEventsCacheBase, IRecorderCacheProducerSync<SplitIO.EventData[]> {
+export interface IEventsCacheSync extends IEventsCacheBase, IRecorderCacheSync<SplitIO.EventData[]> {
   track(data: SplitIO.EventData, size?: number): boolean
   /* Registers callback for full queue */
   setOnFullQueueCb(cb: () => void): void
 }
 
-/** Impressions and events cache for consumer and producer mode (async) */
+/* Named `ImpressionsCounter` in spec */
+export interface IImpressionCountsCacheSync extends IImpressionCountsCacheBase, IRecorderCacheSync<Record<string, number>> { }
+
+export interface IUniqueKeysCacheSync extends IUniqueKeysCacheBase, IRecorderCacheSync<UniqueKeysPayloadSs | UniqueKeysPayloadCs> {
+  setOnFullQueueCb(cb: () => void): void,
+}
+
+/** Impressions and events cache for consumer and producer modes (async methods) */
 
-// Producer API methods for async recorder storages, used by submitters in producer mode to pop data and post it to Split BE.
-export interface IRecorderCacheProducerAsync<T> {
+// API methods for async recorder storages, used by submitters in producer mode (synchronizer) to pop data and post it to Split BE.
+export interface IRecorderCacheAsync<T> {
   /* returns the number of stored items */
   count(): Promise<number>
   /* removes the given number of items from the store. If not provided, it deletes all items */
@@ -330,32 +347,18 @@ export interface IRecorderCacheProducerAsync<T> {
   popNWithMetadata(count: number): Promise<T>
 }
 
-export interface IImpressionsCacheAsync extends IImpressionsCacheBase, IRecorderCacheProducerAsync<StoredImpressionWithMetadata[]> {
+export interface IImpressionsCacheAsync extends IImpressionsCacheBase, IRecorderCacheAsync<StoredImpressionWithMetadata[]> {
   // Consumer API method, used by impressions tracker (in standalone and consumer modes) to push data into.
   // The result promise can reject.
   track(data: ImpressionDTO[]): Promise<void>
 }
 
-export interface IEventsCacheAsync extends IEventsCacheBase, IRecorderCacheProducerAsync<StoredEventWithMetadata[]> {
+export interface IEventsCacheAsync extends IEventsCacheBase, IRecorderCacheAsync<StoredEventWithMetadata[]> {
   // Consumer API method, used by events tracker (in standalone and consumer modes) to push data into.
   // The result promise cannot reject.
   track(data: SplitIO.EventData, size?: number): Promise<boolean>
 }
 
-/**
- * Impression counts cache for impressions dedup in standalone and producer mode.
- * Only in memory. Named `ImpressionsCounter` in spec.
- */
-export interface IImpressionCountsCacheSync extends IRecorderCacheProducerSync<Record<string, number>> {
-  // Used by impressions tracker
-  track(featureName: string, timeFrame: number, amount: number): void
-
-  // Used by impressions count submitter in standalone and producer mode
-  isEmpty(): boolean // check if cache is empty. Return true if the cache was just created or cleared.
-  pop(toMerge?: Record<string, number> ): Record<string, number> // pop cache data
-}
-
-
 /**
  * Telemetry storage interface for standalone and partial consumer modes.
  * Methods are sync because data is stored in memory.
@@ -415,7 +418,7 @@ export interface ITelemetryEvaluationProducerSync {
 
 export interface ITelemetryStorageProducerSync extends ITelemetryInitProducerSync, ITelemetryRuntimeProducerSync, ITelemetryEvaluationProducerSync { }
 
-export interface ITelemetryCacheSync extends ITelemetryStorageConsumerSync, ITelemetryStorageProducerSync { }
+export interface ITelemetryCacheSync extends ITelemetryStorageConsumerSync, ITelemetryStorageProducerSync, IRecorderCacheSync<TelemetryUsageStatsPayload> { }
 
 /**
  * Telemetry storage interface for consumer mode.
@@ -445,15 +448,18 @@ export interface IStorageBase<
   TSplitsCache extends ISplitsCacheBase,
   TSegmentsCache extends ISegmentsCacheBase,
   TImpressionsCache extends IImpressionsCacheBase,
+  TImpressionsCountCache extends IImpressionCountsCacheBase,
   TEventsCache extends IEventsCacheBase,
-  TTelemetryCache extends ITelemetryCacheSync | ITelemetryCacheAsync
+  TTelemetryCache extends ITelemetryCacheSync | ITelemetryCacheAsync,
+  TUniqueKeysCache extends IUniqueKeysCacheBase
   > {
   splits: TSplitsCache,
   segments: TSegmentsCache,
   impressions: TImpressionsCache,
-  impressionCounts?: IImpressionCountsCacheSync,
+  impressionCounts?: TImpressionsCountCache,
   events: TEventsCache,
-  telemetry?: TTelemetryCache
+  telemetry?: TTelemetryCache,
+  uniqueKeys?: TUniqueKeysCache,
   destroy(): void | Promise<void>,
   shared?: (matchingKey: string, onReadyCb: (error?: any) => void) => this
 }
@@ -462,16 +468,20 @@ export interface IStorageSync extends IStorageBase<
   ISplitsCacheSync,
   ISegmentsCacheSync,
   IImpressionsCacheSync,
+  IImpressionCountsCacheSync,
   IEventsCacheSync,
-  ITelemetryCacheSync
+  ITelemetryCacheSync,
+  IUniqueKeysCacheSync
   > { }
 
 export interface IStorageAsync extends IStorageBase<
   ISplitsCacheAsync,
   ISegmentsCacheAsync,
   IImpressionsCacheAsync | IImpressionsCacheSync,
+  IImpressionCountsCacheBase,
   IEventsCacheAsync | IEventsCacheSync,
-  ITelemetryCacheAsync | ITelemetryCacheSync
+  ITelemetryCacheAsync | ITelemetryCacheSync,
+  IUniqueKeysCacheBase
   > { }
 
 /** StorageFactory */
@@ -479,21 +489,12 @@ export interface IStorageAsync extends IStorageBase<
 export type DataLoader = (storage: IStorageSync, matchingKey: string) => void
 
 export interface IStorageFactoryParams {
-  log: ILogger,
-  impressionsQueueSize?: number,
-  eventsQueueSize?: number,
-  optimize?: boolean /* whether create the `impressionCounts` cache (OPTIMIZED impression mode) or not (DEBUG impression mode) */,
-  mode: SDKMode,
-
-  // ATM, only used by InLocalStorage
-  matchingKey?: string, /* undefined on server-side SDKs */
-  splitFiltersValidation?: ISplitFiltersValidation,
-
-  // This callback is invoked when the storage is ready to be used. Error-first callback style: if an error is passed,
-  // it means that the storge fail to connect and shouldn't be used.
-  // It is meant for emitting SDK_READY event in consumer mode, and for synchronizer to wait before using the storage.
+  settings: ISettings,
+  /**
+   * Error-first callback invoked when the storage is ready to be used. An error means that the storage failed to connect and shouldn't be used.
+   * It is meant for emitting SDK_READY event in consumer mode, and waiting before using the storage in the synchronizer.
+   */
   onReadyCb: (error?: any) => void,
-  metadata: IMetadata,
 }
 
 export type StorageType = 'MEMORY' | 'LOCALSTORAGE' | 'REDIS' | 'PLUGGABLE';

package/src/storages/utils.ts

@@ -0,0 +1,78 @@
+// Shared utils for Redis and Pluggable storage
+
+import { IMetadata } from '../dtos/types';
+import { Method, StoredImpressionWithMetadata } from '../sync/submitters/types';
+import { ImpressionDTO, ISettings } from '../types';
+import { UNKNOWN } from '../utils/constants';
+import { MAX_LATENCY_BUCKET_COUNT } from './inMemory/TelemetryCacheInMemory';
+import { METHOD_NAMES } from './KeyBuilderSS';
+
+export function metadataBuilder(settings: Pick<ISettings, 'version' | 'runtime'>): IMetadata {
+  return {
+    s: settings.version,
+    i: settings.runtime.ip || UNKNOWN,
+    n: settings.runtime.hostname || UNKNOWN,
+  };
+}
+
+// Converts impressions to be stored in Redis or pluggable storage.
+export function impressionsToJSON(impressions: ImpressionDTO[], metadata: IMetadata): string[] {
+  return impressions.map(impression => {
+    const impressionWithMetadata: StoredImpressionWithMetadata = {
+      m: metadata,
+      i: {
+        k: impression.keyName,
+        b: impression.bucketingKey,
+        f: impression.feature,
+        t: impression.treatment,
+        r: impression.label,
+        c: impression.changeNumber,
+        m: impression.time,
+        pt: impression.pt,
+      }
+    };
+
+    return JSON.stringify(impressionWithMetadata);
+  });
+}
+
+// Utilities used by TelemetryCacheInRedis and TelemetryCachePluggable
+
+const REVERSE_METHOD_NAMES = Object.keys(METHOD_NAMES).reduce((acc, key) => {
+  acc[METHOD_NAMES[key as Method]] = key as Method;
+  return acc;
+}, {} as Record<string, Method>);
+
+
+export function parseMetadata(field: string): [metadata: string] | string {
+  const parts = field.split('/');
+  if (parts.length !== 3) return `invalid subsection count. Expected 3, got: ${parts.length}`;
+
+  const [s /* metadata.s */, n /* metadata.n */, i /* metadata.i */] = parts;
+  return [JSON.stringify({ s, n, i })];
+}
+
+export function parseExceptionField(field: string): [metadata: string, method: Method] | string {
+  const parts = field.split('/');
+  if (parts.length !== 4) return `invalid subsection count. Expected 4, got: ${parts.length}`;
+
+  const [s /* metadata.s */, n /* metadata.n */, i /* metadata.i */, m] = parts;
+  const method = REVERSE_METHOD_NAMES[m];
+  if (!method) return `unknown method '${m}'`;
+
+  return [JSON.stringify({ s, n, i }), method];
+}
+
+export function parseLatencyField(field: string): [metadata: string, method: Method, bucket: number] | string {
+  const parts = field.split('/');
+  if (parts.length !== 5) return `invalid subsection count. Expected 5, got: ${parts.length}`;
+
+  const [s /* metadata.s */, n /* metadata.n */, i /* metadata.i */, m, b] = parts;
+  const method = REVERSE_METHOD_NAMES[m];
+  if (!method) return `unknown method '${m}'`;
+
+  const bucket = parseInt(b);
+  if (isNaN(bucket) || bucket >= MAX_LATENCY_BUCKET_COUNT) return `invalid bucket. Expected a number between 0 and ${MAX_LATENCY_BUCKET_COUNT - 1}, got: ${b}`;
+
+  return [JSON.stringify({ s, n, i }), method, bucket];
+}

package/src/sync/submitters/submitter.ts

@@ -1,6 +1,6 @@
 import { syncTaskFactory } from '../syncTask';
 import { ISyncTask } from '../types';
-import { IRecorderCacheProducerSync } from '../../storages/types';
+import { IRecorderCacheSync } from '../../storages/types';
 import { ILogger } from '../../logger/types';
 import { SUBMITTERS_PUSH, SUBMITTERS_PUSH_FAILS, SUBMITTERS_PUSH_RETRY } from '../../logger/constants';
 import { IResponse } from '../../services/types';
@@ -11,7 +11,7 @@ import { IResponse } from '../../services/types';
 export function submitterFactory<T>(
   log: ILogger,
   postClient: (body: string) => Promise<IResponse>,
-  sourceCache: IRecorderCacheProducerSync<T>,
+  sourceCache: IRecorderCacheSync<T>,
   postRate: number,
   dataName: string,
   fromCacheToPayload?: (cacheData: T) => any,

package/src/sync/submitters/submitterManager.ts

@@ -4,6 +4,7 @@ import { impressionCountsSubmitterFactory } from './impressionCountsSubmitter';
 import { telemetrySubmitterFactory } from './telemetrySubmitter';
 import { ISdkFactoryContextSync } from '../../sdkFactory/types';
 import { ISubmitterManager } from './types';
+import { uniqueKeysSubmitterFactory } from './uniqueKeysSubmitter';
 
 export function submitterManagerFactory(params: ISdkFactoryContextSync): ISubmitterManager {
 
@@ -15,6 +16,7 @@ export function submitterManagerFactory(params: ISdkFactoryContextSync): ISubmit
   const impressionCountsSubmitter = impressionCountsSubmitterFactory(params);
   if (impressionCountsSubmitter) submitters.push(impressionCountsSubmitter);
   const telemetrySubmitter = telemetrySubmitterFactory(params);
+  if (params.storage.uniqueKeys) submitters.push(uniqueKeysSubmitterFactory(params));
 
   return {
     // `onlyTelemetry` true if SDK is created with userConsent not GRANTED

package/src/sync/submitters/telemetrySubmitter.ts

@@ -1,7 +1,7 @@
-import { ISegmentsCacheSync, ISplitsCacheSync, ITelemetryCacheSync } from '../../storages/types';
+import { ITelemetryCacheSync } from '../../storages/types';
 import { submitterFactory, firstPushWindowDecorator } from './submitter';
-import { TelemetryUsageStatsPayload, TelemetryConfigStatsPayload, TelemetryConfigStats } from './types';
-import { QUEUED, DEDUPED, DROPPED, CONSUMER_MODE, CONSUMER_ENUM, STANDALONE_MODE, CONSUMER_PARTIAL_MODE, STANDALONE_ENUM, CONSUMER_PARTIAL_ENUM, OPTIMIZED, DEBUG, DEBUG_ENUM, OPTIMIZED_ENUM, CONSENT_GRANTED, CONSENT_DECLINED, CONSENT_UNKNOWN } from '../../utils/constants';
+import { TelemetryConfigStatsPayload, TelemetryConfigStats } from './types';
+import { CONSUMER_MODE, CONSUMER_ENUM, STANDALONE_MODE, CONSUMER_PARTIAL_MODE, STANDALONE_ENUM, CONSUMER_PARTIAL_ENUM, OPTIMIZED, DEBUG, NONE, DEBUG_ENUM, OPTIMIZED_ENUM, NONE_ENUM, CONSENT_GRANTED, CONSENT_DECLINED, CONSENT_UNKNOWN } from '../../utils/constants';
 import { SDK_READY, SDK_READY_FROM_CACHE } from '../../readiness/constants';
 import { ConsentStatus, ISettings, SDKMode } from '../../types';
 import { base } from '../../utils/settingsValidation';
@@ -9,41 +9,6 @@ import { usedKeysMap } from '../../utils/inputValidation/apiKey';
 import { timer } from '../../utils/timeTracker/timer';
 import { ISdkFactoryContextSync } from '../../sdkFactory/types';
 import { objectAssign } from '../../utils/lang/objectAssign';
-import { isStorageSync } from '../../trackers/impressionObserver/utils';
-
-/**
- * Converts data from telemetry cache into /metrics/usage request payload.
- */
-export function telemetryCacheStatsAdapter(telemetry: ITelemetryCacheSync, splits?: ISplitsCacheSync, segments?: ISegmentsCacheSync) {
-  return {
-    isEmpty() { return false; }, // There is always data in telemetry cache
-    clear() { }, //  No-op
-
-    // @TODO consider moving inside telemetry cache for code size reduction
-    pop(): TelemetryUsageStatsPayload {
-      return {
-        lS: telemetry.getLastSynchronization(),
-        mL: telemetry.popLatencies(),
-        mE: telemetry.popExceptions(),
-        hE: telemetry.popHttpErrors(),
-        hL: telemetry.popHttpLatencies(),
-        tR: telemetry.popTokenRefreshes(),
-        aR: telemetry.popAuthRejections(),
-        iQ: telemetry.getImpressionStats(QUEUED),
-        iDe: telemetry.getImpressionStats(DEDUPED),
-        iDr: telemetry.getImpressionStats(DROPPED),
-        spC: splits && splits.getSplitNames().length,
-        seC: segments && segments.getRegisteredSegments().length,
-        skC: segments && segments.getKeysCount(),
-        sL: telemetry.getSessionLength(),
-        eQ: telemetry.getEventStats(QUEUED),
-        eD: telemetry.getEventStats(DROPPED),
-        sE: telemetry.popStreamingEvents(),
-        t: telemetry.popTags(),
-      };
-    }
-  };
-}
 
 const OPERATION_MODE_MAP = {
   [STANDALONE_MODE]: STANDALONE_ENUM,
@@ -53,8 +18,9 @@ const OPERATION_MODE_MAP = {
 
 const IMPRESSIONS_MODE_MAP = {
   [OPTIMIZED]: OPTIMIZED_ENUM,
-  [DEBUG]: DEBUG_ENUM
-} as Record<ISettings['sync']['impressionsMode'], (0 | 1)>;
+  [DEBUG]: DEBUG_ENUM,
+  [NONE]: NONE_ENUM
+} as Record<ISettings['sync']['impressionsMode'], (0 | 1 | 2)>;
 
 const USER_CONSENT_MAP = {
   [CONSENT_UNKNOWN]: 1,
@@ -130,7 +96,7 @@ export function telemetryCacheConfigAdapter(telemetry: ITelemetryCacheSync, sett
  * Submitter that periodically posts telemetry data
  */
 export function telemetrySubmitterFactory(params: ISdkFactoryContextSync) {
-  const { storage: { splits, segments, telemetry }, platform: { now } } = params;
+  const { storage: { telemetry }, platform: { now } } = params;
   if (!telemetry || !now) return; // No submitter created if telemetry cache is not defined
 
   const { settings, settings: { log, scheduler: { telemetryRefreshRate } }, splitApi, readiness, sdkReadinessManager } = params;
@@ -139,8 +105,7 @@ export function telemetrySubmitterFactory(params: ISdkFactoryContextSync) {
   const submitter = firstPushWindowDecorator(
     submitterFactory(
       log, splitApi.postMetricsUsage,
-      // @TODO cannot provide splits and segments cache if they are async, because `submitterFactory` expects a sync storage source
-      isStorageSync(params.settings) ? telemetryCacheStatsAdapter(telemetry, splits, segments) : telemetryCacheStatsAdapter(telemetry),
+      telemetry,
       telemetryRefreshRate, 'telemetry stats', undefined, 0, true
     ),
     telemetryRefreshRate

package/src/sync/submitters/types.ts

@@ -37,6 +37,26 @@ export type ImpressionCountsPayload = {
   }[]
 }
 
+export type UniqueKeysItemSs = {
+  /** Split name */
+  f: string
+  /** keyNames */
+  ks: string[]
+}
+
+export type UniqueKeysPayloadSs = {
+  keys: UniqueKeysItemSs[]
+}
+
+export type UniqueKeysPayloadCs = {
+  keys: {
+    /** keyNames */
+    k: string
+    /** Split name */
+    fs: string[]
+  }[]
+}
+
 export type StoredImpressionWithMetadata = {
   /** Metadata */
   m: IMetadata,
@@ -56,6 +76,8 @@ export type StoredImpressionWithMetadata = {
     c: number,
     /** time */
     m: number
+    /** previous time */
+    pt?: number
   }
 }
 
@@ -72,27 +94,6 @@ export type MultiMethodExceptions = IMap<string, MethodExceptions>
 
 export type MultiConfigs = IMap<string, TelemetryConfigStats>
 
-// export type StoredLatenciesWithMetadata = {
-//   /** Metadata */
-//   m: IMetadata,
-//   /** Stored latencies */
-//   l: MethodLatencies
-// }
-
-// export type StoredExceptionsWithMetadata = {
-//   /** Metadata */
-//   m: IMetadata,
-//   /** Stored exceptions */
-//   e: MethodExceptions
-// }
-
-// export type StoredConfigsWithMetadata = {
-//   /** Metadata */
-//   m: IMetadata,
-//   /** Stored configs */
-//   c: TelemetryConfigStats
-// }
-
 /**
  * Telemetry usage stats
  */
@@ -113,9 +114,9 @@ export type SEGMENT = 'se';
 export type MY_SEGMENT = 'ms';
 export type OperationType = SPLITS | IMPRESSIONS | IMPRESSIONS_COUNT | EVENTS | TELEMETRY | TOKEN | SEGMENT | MY_SEGMENT;
 
-export type LastSync = Record<OperationType, number | undefined>
-export type HttpErrors = Record<OperationType, { [statusCode: string]: number }>
-export type HttpLatencies = Record<OperationType, Array<number>>
+export type LastSync = Partial<Record<OperationType, number | undefined>>
+export type HttpErrors = Partial<Record<OperationType, { [statusCode: string]: number }>>
+export type HttpLatencies = Partial<Record<OperationType, Array<number>>>
 
 export type TREATMENT = 't';
 export type TREATMENTS = 'ts';
@@ -124,9 +125,9 @@ export type TREATMENTS_WITH_CONFIG = 'tcs';
 export type TRACK = 'tr';
 export type Method = TREATMENT | TREATMENTS | TREATMENT_WITH_CONFIG | TREATMENTS_WITH_CONFIG | TRACK;
 
-export type MethodLatencies = Record<Method, Array<number>>;
+export type MethodLatencies = Partial<Record<Method, Array<number>>>;
 
-export type MethodExceptions = Record<Method, number>;
+export type MethodExceptions = Partial<Record<Method, number>>;
 
 export type CONNECTION_ESTABLISHED = 0;
 export type OCCUPANCY_PRI = 10;
@@ -181,7 +182,8 @@ export type OperationMode = STANDALONE_ENUM | CONSUMER_ENUM | CONSUMER_PARTIAL_E
 
 export type OPTIMIZED_ENUM = 0;
 export type DEBUG_ENUM = 1;
-export type ImpressionsMode = OPTIMIZED_ENUM | DEBUG_ENUM;
+export type NONE_ENUM = 2;
+export type ImpressionsMode = OPTIMIZED_ENUM | DEBUG_ENUM | NONE_ENUM;
 
 export type RefreshRates = {
   sp: number, // splits

package/src/sync/submitters/uniqueKeysSubmitter.ts

@@ -0,0 +1,36 @@
+import { SUBMITTERS_PUSH_FULL_QUEUE } from '../../logger/constants';
+import { ISdkFactoryContextSync } from '../../sdkFactory/types';
+import { submitterFactory } from './submitter';
+
+const DATA_NAME = 'unique keys';
+const UNIQUE_KEYS_RATE = 900000; // 15 minutes
+
+/**
+ * Submitter that periodically posts impression counts
+ */
+export function uniqueKeysSubmitterFactory(params: ISdkFactoryContextSync) {
+
+  const {
+    settings: { log, core: { key } },
+    splitApi: { postUniqueKeysBulkCs, postUniqueKeysBulkSs },
+    storage: { uniqueKeys }
+  } = params;
+
+  const isClientSide = key !== undefined;
+  const postUniqueKeysBulk = isClientSide ? postUniqueKeysBulkCs : postUniqueKeysBulkSs;
+
+  const syncTask = submitterFactory(log, postUniqueKeysBulk, uniqueKeys!, UNIQUE_KEYS_RATE, DATA_NAME);
+
+  // register unique keys submitter to be executed when uniqueKeys cache is full
+  uniqueKeys!.setOnFullQueueCb(() => {
+    if (syncTask.isRunning()) {
+      log.info(SUBMITTERS_PUSH_FULL_QUEUE, [DATA_NAME]);
+      syncTask.execute();
+    }
+    // If submitter is stopped (e.g., user consent declined or unknown, or app state offline), we don't send the data.
+    // Data will be sent when submitter is resumed.
+  });
+
+  return syncTask;
+}
+