@splitsoftware/splitio-commons 1.16.0 → 1.16.1-rc.0

package/src/sync/streaming/SSEHandler/types.ts

@@ -1,5 +1,5 @@
 import { ControlType } from '../constants';
-import { MY_SEGMENTS_UPDATE, MY_SEGMENTS_UPDATE_V2, SEGMENT_UPDATE, SPLIT_UPDATE, SPLIT_KILL, CONTROL, OCCUPANCY } from '../types';
+import { MY_SEGMENTS_UPDATE, MY_SEGMENTS_UPDATE_V2, SEGMENT_UPDATE, SPLIT_UPDATE, SPLIT_KILL, CONTROL, OCCUPANCY, MY_LARGE_SEGMENTS_UPDATE } from '../types';
 
 export interface IMySegmentsUpdateData {
   type: MY_SEGMENTS_UPDATE,
@@ -35,6 +35,18 @@ export interface IMySegmentsUpdateV2Data {
   u: UpdateStrategy,
 }
 
+export interface IMyLargeSegmentsUpdateData {
+  type: MY_LARGE_SEGMENTS_UPDATE,
+  changeNumber: number,
+  largeSegments: string[],
+  c: Compression,
+  d: string,
+  u: UpdateStrategy,
+  i?: number, // time interval in millis
+  h?: number, // hash function. 0 for murmur3_32, 1 for murmur3_64
+  s?: number, // seed for hash function
+}
+
 export interface ISegmentUpdateData {
   type: SEGMENT_UPDATE,
   changeNumber: number,
@@ -68,6 +80,6 @@ export interface IOccupancyData {
   }
 }
 
-export type INotificationData = IMySegmentsUpdateData | IMySegmentsUpdateV2Data | ISegmentUpdateData | ISplitUpdateData | ISplitKillData | IControlData | IOccupancyData
+export type INotificationData = IMySegmentsUpdateData | IMySegmentsUpdateV2Data | IMyLargeSegmentsUpdateData | ISegmentUpdateData | ISplitUpdateData | ISplitKillData | IControlData | IOccupancyData
 export type INotificationMessage = { parsedData: INotificationData, channel: string, timestamp: number, data: string }
 export type INotificationError = Event & { parsedData?: any, message?: string }

package/src/sync/streaming/UpdateWorkers/MySegmentsUpdateWorker.ts

@@ -1,19 +1,20 @@
 import { IMySegmentsSyncTask, MySegmentsData } from '../../polling/types';
 import { Backoff } from '../../../utils/Backoff';
 import { IUpdateWorker } from './types';
-import { MY_SEGMENT } from '../../../utils/constants';
 import { ITelemetryTracker } from '../../../trackers/types';
+import { UpdatesFromSSEEnum } from '../../submitters/types';
 
 /**
  * MySegmentsUpdateWorker factory
  */
-export function MySegmentsUpdateWorker(mySegmentsSyncTask: IMySegmentsSyncTask, telemetryTracker: ITelemetryTracker): IUpdateWorker {
+export function MySegmentsUpdateWorker(mySegmentsSyncTask: IMySegmentsSyncTask, telemetryTracker: ITelemetryTracker, updateType: UpdatesFromSSEEnum): IUpdateWorker {
 
   let maxChangeNumber = 0; // keeps the maximum changeNumber among queued events
   let currentChangeNumber = -1;
   let handleNewEvent = false;
   let isHandlingEvent: boolean;
   let _segmentsData: MySegmentsData | undefined; // keeps the segmentsData (if included in notification payload) from the queued event with maximum changeNumber
+  let _delay: undefined | number;
   const backoff = new Backoff(__handleMySegmentsUpdateCall);
 
   function __handleMySegmentsUpdateCall() {
@@ -23,10 +24,10 @@ export function MySegmentsUpdateWorker(mySegmentsSyncTask: IMySegmentsSyncTask,
       const currentMaxChangeNumber = maxChangeNumber;
 
       // fetch mySegments revalidating data if cached
-      mySegmentsSyncTask.execute(_segmentsData, true).then((result) => {
+      mySegmentsSyncTask.execute(_segmentsData, true, _delay).then((result) => {
         if (!isHandlingEvent) return; // halt if `stop` has been called
         if (result !== false) {// Unlike `Splits|SegmentsUpdateWorker`, we cannot use `mySegmentsCache.getChangeNumber` since `/mySegments` endpoint doesn't provide this value.
-          if (_segmentsData) telemetryTracker.trackUpdatesFromSSE(MY_SEGMENT);
+          if (_segmentsData) telemetryTracker.trackUpdatesFromSSE(updateType);
           currentChangeNumber = Math.max(currentChangeNumber, currentMaxChangeNumber); // use `currentMaxChangeNumber`, in case that `maxChangeNumber` was updated during fetch.
         }
         if (handleNewEvent) {
@@ -47,12 +48,13 @@ export function MySegmentsUpdateWorker(mySegmentsSyncTask: IMySegmentsSyncTask,
      * @param {number} changeNumber change number of the MY_SEGMENTS_UPDATE notification
      * @param {SegmentsData | undefined} segmentsData might be undefined
      */
-    put(changeNumber: number, segmentsData?: MySegmentsData) {
+    put(changeNumber: number, segmentsData?: MySegmentsData, delay?: number) {
       if (changeNumber <= currentChangeNumber || changeNumber <= maxChangeNumber) return;
 
       maxChangeNumber = changeNumber;
       handleNewEvent = true;
       _segmentsData = segmentsData;
+      _delay = delay;
 
       if (backoff.timeoutID || !isHandlingEvent) __handleMySegmentsUpdateCall();
       backoff.reset();

package/src/sync/streaming/constants.ts

@@ -30,6 +30,7 @@ export const MY_SEGMENTS_UPDATE_V2 = 'MY_SEGMENTS_UPDATE_V2';
 export const SEGMENT_UPDATE = 'SEGMENT_UPDATE';
 export const SPLIT_KILL = 'SPLIT_KILL';
 export const SPLIT_UPDATE = 'SPLIT_UPDATE';
+export const MY_LARGE_SEGMENTS_UPDATE = 'MY_LARGE_SEGMENTS_UPDATE';
 
 // Control-type push notifications, handled by NotificationKeeper
 export const CONTROL = 'CONTROL';

package/src/sync/streaming/pushManager.ts

@@ -11,17 +11,26 @@ import { authenticateFactory, hashUserKey } from './AuthClient';
 import { forOwn } from '../../utils/lang';
 import { SSEClient } from './SSEClient';
 import { getMatching } from '../../utils/key';
-import { MY_SEGMENTS_UPDATE, MY_SEGMENTS_UPDATE_V2, PUSH_NONRETRYABLE_ERROR, PUSH_SUBSYSTEM_DOWN, SECONDS_BEFORE_EXPIRATION, SEGMENT_UPDATE, SPLIT_KILL, SPLIT_UPDATE, PUSH_RETRYABLE_ERROR, PUSH_SUBSYSTEM_UP, ControlType } from './constants';
+import { MY_SEGMENTS_UPDATE, MY_SEGMENTS_UPDATE_V2, PUSH_NONRETRYABLE_ERROR, PUSH_SUBSYSTEM_DOWN, SECONDS_BEFORE_EXPIRATION, SEGMENT_UPDATE, SPLIT_KILL, SPLIT_UPDATE, PUSH_RETRYABLE_ERROR, PUSH_SUBSYSTEM_UP, ControlType, MY_LARGE_SEGMENTS_UPDATE } from './constants';
 import { STREAMING_FALLBACK, STREAMING_REFRESH_TOKEN, STREAMING_CONNECTING, STREAMING_DISABLED, ERROR_STREAMING_AUTH, STREAMING_DISCONNECTING, STREAMING_RECONNECT, STREAMING_PARSING_MY_SEGMENTS_UPDATE_V2, STREAMING_PARSING_SPLIT_UPDATE } from '../../logger/constants';
-import { KeyList, UpdateStrategy } from './SSEHandler/types';
+import { IMyLargeSegmentsUpdateData, IMySegmentsUpdateV2Data, KeyList, UpdateStrategy } from './SSEHandler/types';
 import { isInBitmap, parseBitmap, parseFFUpdatePayload, parseKeyList } from './parseUtils';
 import { ISet, _Set } from '../../utils/lang/sets';
+import { hash } from '../../utils/murmur3/murmur3';
 import { Hash64, hash64 } from '../../utils/murmur3/murmur3_64';
 import { IAuthTokenPushEnabled } from './AuthClient/types';
-import { TOKEN_REFRESH, AUTH_REJECTION } from '../../utils/constants';
+import { TOKEN_REFRESH, AUTH_REJECTION, MY_LARGE_SEGMENT, MY_SEGMENT } from '../../utils/constants';
 import { ISdkFactoryContextSync } from '../../sdkFactory/types';
 import { IUpdateWorker } from './UpdateWorkers/types';
 
+export function getDelay(parsedData: IMyLargeSegmentsUpdateData, matchingKey: string) {
+  const interval = parsedData.i || 60000;
+  // const hashType = parsedData.h || 0;
+  const seed = parsedData.s || 0;
+
+  return hash(matchingKey, seed) % interval;
+}
+
 /**
  * PushManager factory:
  * - for server-side if key is not provided in settings.
@@ -64,7 +73,7 @@ export function pushManagerFactory(
   const userKeyHashes: Record<string, string> = {};
   // [Only for client-side] map of user keys to their corresponding hash64 and MySegmentsUpdateWorkers.
   // Hash64 is used to process MY_SEGMENTS_UPDATE_V2 events and dispatch actions to the corresponding MySegmentsUpdateWorker.
-  const clients: Record<string, { hash64: Hash64, worker: IUpdateWorker }> = {};
+  const clients: Record<string, { hash64: Hash64, worker: IUpdateWorker, workerLarge?: IUpdateWorker }> = {};
 
   // [Only for client-side] variable to flag that a new client was added. It is needed to reconnect streaming.
   let connectForNewClient = false;
@@ -171,7 +180,10 @@ export function pushManagerFactory(
   // cancel scheduled fetch retries of Splits, Segments, and MySegments Update Workers
   function stopWorkers() {
     splitsUpdateWorker.stop();
-    if (userKey) forOwn(clients, ({ worker }) => worker.stop());
+    if (userKey) forOwn(clients, ({ worker, workerLarge }) => {
+      worker.stop();
+      workerLarge && workerLarge.stop();
+    });
     else segmentsUpdateWorker!.stop();
   }
 
@@ -236,76 +248,98 @@ export function pushManagerFactory(
     splitsUpdateWorker.put(parsedData);
   });
 
-  if (userKey) {
-    pushEmitter.on(MY_SEGMENTS_UPDATE, function handleMySegmentsUpdate(parsedData, channel) {
-      const userKeyHash = channel.split('_')[2];
-      const userKey = userKeyHashes[userKeyHash];
-      if (userKey && clients[userKey]) { // check existence since it can be undefined if client has been destroyed
-        clients[userKey].worker.put(
-          parsedData.changeNumber,
-          parsedData.includesPayload ? parsedData.segmentList ? parsedData.segmentList : [] : undefined);
-      }
-    });
-    pushEmitter.on(MY_SEGMENTS_UPDATE_V2, function handleMySegmentsUpdate(parsedData) {
-      switch (parsedData.u) {
-        case UpdateStrategy.BoundedFetchRequest: {
-          let bitmap: Uint8Array;
-          try {
-            bitmap = parseBitmap(parsedData.d, parsedData.c);
-          } catch (e) {
-            log.warn(STREAMING_PARSING_MY_SEGMENTS_UPDATE_V2, ['BoundedFetchRequest', e]);
-            break;
-          }
-
-          forOwn(clients, ({ hash64, worker }) => {
-            if (isInBitmap(bitmap, hash64.hex)) {
-              worker.put(parsedData.changeNumber); // fetch mySegments
-            }
-          });
-          return;
+  function handleMySegmentsUpdate(parsedData: IMySegmentsUpdateV2Data | IMyLargeSegmentsUpdateData) {
+    const isLS = parsedData.type === MY_LARGE_SEGMENTS_UPDATE;
+
+    switch (parsedData.u) {
+      case UpdateStrategy.BoundedFetchRequest: {
+        let bitmap: Uint8Array;
+        try {
+          bitmap = parseBitmap(parsedData.d, parsedData.c);
+        } catch (e) {
+          log.warn(STREAMING_PARSING_MY_SEGMENTS_UPDATE_V2, ['BoundedFetchRequest', e]);
+          break;
         }
-        case UpdateStrategy.KeyList: {
-          let keyList: KeyList, added: ISet<string>, removed: ISet<string>;
-          try {
-            keyList = parseKeyList(parsedData.d, parsedData.c);
-            added = new _Set(keyList.a);
-            removed = new _Set(keyList.r);
-          } catch (e) {
-            log.warn(STREAMING_PARSING_MY_SEGMENTS_UPDATE_V2, ['KeyList', e]);
-            break;
+
+        forOwn(clients, ({ hash64, worker, workerLarge }, matchingKey) => {
+          if (isInBitmap(bitmap, hash64.hex)) {
+            isLS ?
+              workerLarge && workerLarge.put(parsedData.changeNumber, undefined, getDelay(parsedData, matchingKey)) :
+              worker.put(parsedData.changeNumber);
           }
+        });
+        return;
+      }
+      case UpdateStrategy.KeyList: {
+        let keyList: KeyList, added: ISet<string>, removed: ISet<string>;
+        try {
+          keyList = parseKeyList(parsedData.d, parsedData.c);
+          added = new _Set(keyList.a);
+          removed = new _Set(keyList.r);
+        } catch (e) {
+          log.warn(STREAMING_PARSING_MY_SEGMENTS_UPDATE_V2, ['KeyList', e]);
+          break;
+        }
 
-          forOwn(clients, ({ hash64, worker }) => {
-            const add = added.has(hash64.dec) ? true : removed.has(hash64.dec) ? false : undefined;
-            if (add !== undefined) {
+        forOwn(clients, ({ hash64, worker, workerLarge }) => {
+          const add = added.has(hash64.dec) ? true : removed.has(hash64.dec) ? false : undefined;
+          if (add !== undefined) {
+            isLS ?
+              workerLarge && workerLarge.put(parsedData.changeNumber, {
+                name: parsedData.largeSegments[0],
+                add
+              }) :
               worker.put(parsedData.changeNumber, {
                 name: parsedData.segmentName,
                 add
               });
-            }
-          });
-          return;
-        }
-        case UpdateStrategy.SegmentRemoval:
-          if (!parsedData.segmentName) {
-            log.warn(STREAMING_PARSING_MY_SEGMENTS_UPDATE_V2, ['SegmentRemoval', 'No segment name was provided']);
-            break;
           }
+        });
+        return;
+      }
+      case UpdateStrategy.SegmentRemoval:
+        if ((isLS && parsedData.largeSegments.length === 0) || (!isLS && !parsedData.segmentName)) {
+          log.warn(STREAMING_PARSING_MY_SEGMENTS_UPDATE_V2, ['SegmentRemoval', 'No segment name was provided']);
+          break;
+        }
 
-          forOwn(clients, ({ worker }) =>
+        forOwn(clients, ({ worker, workerLarge }) => {
+          isLS ?
+            workerLarge && parsedData.largeSegments.forEach(largeSegment => {
+              workerLarge.put(parsedData.changeNumber, {
+                name: largeSegment,
+                add: false
+              });
+            }) :
             worker.put(parsedData.changeNumber, {
               name: parsedData.segmentName,
               add: false
-            })
-          );
-          return;
-      }
+            });
+        });
+        return;
+    }
 
-      // `UpdateStrategy.UnboundedFetchRequest` and fallbacks of other cases
-      forOwn(clients, ({ worker }) => {
+    // `UpdateStrategy.UnboundedFetchRequest` and fallbacks of other cases
+    forOwn(clients, ({ worker, workerLarge }, matchingKey) => {
+      isLS ?
+        workerLarge && workerLarge.put(parsedData.changeNumber, undefined, getDelay(parsedData, matchingKey)) :
         worker.put(parsedData.changeNumber);
-      });
     });
+  }
+
+  if (userKey) {
+    pushEmitter.on(MY_SEGMENTS_UPDATE, function handleMySegmentsUpdate(parsedData, channel) {
+      const userKeyHash = channel.split('_')[2];
+      const userKey = userKeyHashes[userKeyHash];
+      if (userKey && clients[userKey]) { // check existence since it can be undefined if client has been destroyed
+        clients[userKey].worker.put(
+          parsedData.changeNumber,
+          parsedData.includesPayload ? parsedData.segmentList ? parsedData.segmentList : [] : undefined);
+      }
+    });
+
+    pushEmitter.on(MY_SEGMENTS_UPDATE_V2, handleMySegmentsUpdate);
+    pushEmitter.on(MY_LARGE_SEGMENTS_UPDATE, handleMySegmentsUpdate);
   } else {
     pushEmitter.on(SEGMENT_UPDATE, segmentsUpdateWorker!.put);
   }
@@ -328,7 +362,7 @@ export function pushManagerFactory(
         if (disabled || disconnected === false) return;
         disconnected = false;
 
-        if (userKey) this.add(userKey, pollingManager.segmentsSyncTask as IMySegmentsSyncTask); // client-side
+        if (userKey) this.add(userKey, pollingManager.segmentsSyncTask, pollingManager.largeSegmentsSyncTask!); // client-side
         else setTimeout(connectPush); // server-side runs in next cycle as in client-side, for consistency with client-side
       },
 
@@ -338,12 +372,16 @@ export function pushManagerFactory(
       },
 
       // [Only for client-side]
-      add(userKey: string, mySegmentsSyncTask: IMySegmentsSyncTask) {
+      add(userKey: string, mySegmentsSyncTask: IMySegmentsSyncTask, myLargeSegmentsSyncTask?: IMySegmentsSyncTask) {
         const hash = hashUserKey(userKey);
 
         if (!userKeyHashes[hash]) {
           userKeyHashes[hash] = userKey;
-          clients[userKey] = { hash64: hash64(userKey), worker: MySegmentsUpdateWorker(mySegmentsSyncTask, telemetryTracker) };
+          clients[userKey] = {
+            hash64: hash64(userKey),
+            worker: MySegmentsUpdateWorker(mySegmentsSyncTask, telemetryTracker, MY_SEGMENT),
+            workerLarge: myLargeSegmentsSyncTask ? MySegmentsUpdateWorker(myLargeSegmentsSyncTask, telemetryTracker, MY_LARGE_SEGMENT) : undefined
+          };
           connectForNewClient = true; // we must reconnect on start, to listen the channel for the new user key
 
           // Reconnects in case of a new client.

package/src/sync/streaming/types.ts

@@ -1,4 +1,4 @@
-import { IMySegmentsUpdateData, IMySegmentsUpdateV2Data, ISegmentUpdateData, ISplitUpdateData, ISplitKillData } from './SSEHandler/types';
+import { IMySegmentsUpdateData, IMySegmentsUpdateV2Data, ISegmentUpdateData, ISplitUpdateData, ISplitKillData, IMyLargeSegmentsUpdateData } from './SSEHandler/types';
 import { ITask } from '../types';
 import { IMySegmentsSyncTask } from '../polling/types';
 import { IEventEmitter } from '../../types';
@@ -16,16 +16,18 @@ export type MY_SEGMENTS_UPDATE_V2 = 'MY_SEGMENTS_UPDATE_V2';
 export type SEGMENT_UPDATE = 'SEGMENT_UPDATE';
 export type SPLIT_KILL = 'SPLIT_KILL';
 export type SPLIT_UPDATE = 'SPLIT_UPDATE';
+export type MY_LARGE_SEGMENTS_UPDATE = 'MY_LARGE_SEGMENTS_UPDATE';
 
 // Control-type push notifications, handled by NotificationKeeper
 export type CONTROL = 'CONTROL';
 export type OCCUPANCY = 'OCCUPANCY';
 
-export type IPushEvent = PUSH_SUBSYSTEM_UP | PUSH_SUBSYSTEM_DOWN | PUSH_NONRETRYABLE_ERROR | PUSH_RETRYABLE_ERROR | MY_SEGMENTS_UPDATE | MY_SEGMENTS_UPDATE_V2 | SEGMENT_UPDATE | SPLIT_UPDATE | SPLIT_KILL | ControlType.STREAMING_RESET
+export type IPushEvent = PUSH_SUBSYSTEM_UP | PUSH_SUBSYSTEM_DOWN | PUSH_NONRETRYABLE_ERROR | PUSH_RETRYABLE_ERROR | MY_SEGMENTS_UPDATE | MY_SEGMENTS_UPDATE_V2 | SEGMENT_UPDATE | SPLIT_UPDATE | SPLIT_KILL | MY_LARGE_SEGMENTS_UPDATE | ControlType.STREAMING_RESET
 
 type IParsedData<T extends IPushEvent> =
   T extends MY_SEGMENTS_UPDATE ? IMySegmentsUpdateData :
   T extends MY_SEGMENTS_UPDATE_V2 ? IMySegmentsUpdateV2Data :
+  T extends MY_LARGE_SEGMENTS_UPDATE ? IMyLargeSegmentsUpdateData :
   T extends SEGMENT_UPDATE ? ISegmentUpdateData :
   T extends SPLIT_UPDATE ? ISplitUpdateData :
   T extends SPLIT_KILL ? ISplitKillData : undefined;
@@ -45,6 +47,6 @@ export interface IPushEventEmitter extends IEventEmitter {
  */
 export interface IPushManager extends ITask, IPushEventEmitter {
   // Methods used in client-side, to support multiple clients
-  add(userKey: string, mySegmentsSyncTask: IMySegmentsSyncTask): void,
+  add(userKey: string, mySegmentsSyncTask: IMySegmentsSyncTask, myLargeSegmentsSyncTask?: IMySegmentsSyncTask): void,
   remove(userKey: string): void
 }

package/src/sync/submitters/telemetrySubmitter.ts

@@ -76,10 +76,12 @@ export function telemetryCacheConfigAdapter(telemetry: ITelemetryCacheSync, sett
 
       return objectAssign(getTelemetryConfigStats(settings.mode, settings.storage.type), {
         sE: settings.streamingEnabled,
+        lE: isClientSide ? settings.sync.largeSegmentsEnabled : undefined,
         rR: {
           sp: scheduler.featuresRefreshRate / 1000,
           se: isClientSide ? undefined : scheduler.segmentsRefreshRate / 1000,
           ms: isClientSide ? scheduler.segmentsRefreshRate / 1000 : undefined,
+          mls: isClientSide && settings.sync.largeSegmentsEnabled ? scheduler.largeSegmentsRefreshRate / 1000 : undefined,
           im: scheduler.impressionsRefreshRate / 1000,
           ev: scheduler.eventsPushRate / 1000,
           te: scheduler.telemetryRefreshRate / 1000,

package/src/sync/submitters/types.ts

@@ -103,7 +103,7 @@ export type DROPPED = 1;
 export type DEDUPED = 2;
 export type ImpressionDataType = QUEUED | DROPPED | DEDUPED
 export type EventDataType = QUEUED | DROPPED;
-export type UpdatesFromSSEEnum = SPLITS | MY_SEGMENT;
+export type UpdatesFromSSEEnum = SPLITS | MY_SEGMENT | MY_LARGE_SEGMENT;
 
 export type SPLITS = 'sp';
 export type IMPRESSIONS = 'im';
@@ -113,7 +113,8 @@ export type TELEMETRY = 'te';
 export type TOKEN = 'to';
 export type SEGMENT = 'se';
 export type MY_SEGMENT = 'ms';
-export type OperationType = SPLITS | IMPRESSIONS | IMPRESSIONS_COUNT | EVENTS | TELEMETRY | TOKEN | SEGMENT | MY_SEGMENT;
+export type MY_LARGE_SEGMENT = 'mls';
+export type OperationType = SPLITS | IMPRESSIONS | IMPRESSIONS_COUNT | EVENTS | TELEMETRY | TOKEN | SEGMENT | MY_SEGMENT | MY_LARGE_SEGMENT;
 
 export type LastSync = Partial<Record<OperationType, number | undefined>>
 export type HttpErrors = Partial<Record<OperationType, { [statusCode: string]: number }>>
@@ -158,8 +159,9 @@ export type TelemetryUsageStats = {
 
 // amount of instant updates that we are doing by avoiding fetching to Split servers
 export type UpdatesFromSSE = {
-  sp: number, // splits
+  sp?: number, // splits
   ms?: number, // my segments
+  mls?: number // my large segments
 }
 
 // 'metrics/usage' JSON request body
@@ -175,12 +177,14 @@ export type TelemetryUsageStatsPayload = TelemetryUsageStats & {
   spC?: number, // splitCount
   seC?: number, // segmentCount
   skC?: number, // segmentKeyCount
+  lseC?: number, // largeSegmentCount
+  lskC?: number, // largeSegmentKeyCount
   sL?: number, // sessionLengthMs
   eQ: number, // eventsQueued
   eD: number, // eventsDropped
   sE: Array<StreamingEvent>, // streamingEvents
   t?: Array<string>, // tags
-  ufs?: UpdatesFromSSE, //UpdatesFromSSE
+  ufs?: UpdatesFromSSE, // instant updates
 }
 
 /**
@@ -201,6 +205,7 @@ export type RefreshRates = {
   sp: number, // splits
   se?: number, // segments
   ms?: number, // mySegments
+  mls?: number, // myLargeSegments
   im: number, // impressions
   ev: number, // events
   te: number, // telemetry
@@ -226,6 +231,7 @@ export type TelemetryConfigStats = {
 // 'metrics/config' JSON request body
 export type TelemetryConfigStatsPayload = TelemetryConfigStats & {
   sE: boolean, // streamingEnabled
+  lE?: boolean, // largeSegmentsEnabled
   rR: RefreshRates, // refreshRates
   uO: UrlOverrides, // urlOverrides
   iQ: number, // impressionsQueueSize

package/src/sync/syncManagerOnline.ts

@@ -7,7 +7,7 @@ import { IPollingManager, IPollingManagerCS } from './polling/types';
 import { PUSH_SUBSYSTEM_UP, PUSH_SUBSYSTEM_DOWN } from './streaming/constants';
 import { SYNC_START_POLLING, SYNC_CONTINUE_POLLING, SYNC_STOP_POLLING } from '../logger/constants';
 import { isConsentGranted } from '../consent';
-import { POLLING, STREAMING, SYNC_MODE_UPDATE } from '../utils/constants';
+import { IN_LARGE_SEGMENT, IN_SEGMENT, POLLING, STREAMING, SYNC_MODE_UPDATE } from '../utils/constants';
 import { ISdkFactoryContextSync } from '../sdkFactory/types';
 
 /**
@@ -141,36 +141,44 @@ export function syncManagerOnlineFactory(
       shared(matchingKey: string, readinessManager: IReadinessManager, storage: IStorageSync) {
         if (!pollingManager) return;
 
-        const mySegmentsSyncTask = (pollingManager as IPollingManagerCS).add(matchingKey, readinessManager, storage);
+        const { msSyncTask, mlsSyncTask } = (pollingManager as IPollingManagerCS).add(matchingKey, readinessManager, storage);
 
         return {
-          isRunning: mySegmentsSyncTask.isRunning,
+          isRunning: msSyncTask.isRunning,
           start() {
             if (syncEnabled) {
               if (pushManager) {
                 if (pollingManager!.isRunning()) {
                   // if doing polling, we must start the periodic fetch of data
-                  if (storage.splits.usesSegments()) mySegmentsSyncTask.start();
+                  if (storage.splits.usesMatcher(IN_SEGMENT)) msSyncTask.start();
+                  if (mlsSyncTask && storage.splits.usesMatcher(IN_LARGE_SEGMENT)) mlsSyncTask.start();
                 } else {
                   // if not polling, we must execute the sync task for the initial fetch
                   // of segments since `syncAll` was already executed when starting the main client
-                  mySegmentsSyncTask.execute();
+                  msSyncTask.execute();
+                  mlsSyncTask && mlsSyncTask.execute();
                 }
-                pushManager.add(matchingKey, mySegmentsSyncTask);
+                pushManager.add(matchingKey, msSyncTask, mlsSyncTask);
               } else {
-                if (storage.splits.usesSegments()) mySegmentsSyncTask.start();
+                if (storage.splits.usesMatcher(IN_SEGMENT)) msSyncTask.start();
+                if (mlsSyncTask && storage.splits.usesMatcher(IN_LARGE_SEGMENT)) mlsSyncTask.start();
               }
             } else {
-              if (!readinessManager.isReady()) mySegmentsSyncTask.execute();
+              if (!readinessManager.isReady()) {
+                msSyncTask.execute();
+                mlsSyncTask && mlsSyncTask.execute();
+              }
             }
           },
           stop() {
             // check in case `client.destroy()` has been invoked more than once for the same client
-            const mySegmentsSyncTask = (pollingManager as IPollingManagerCS).get(matchingKey);
-            if (mySegmentsSyncTask) {
+            const syncTasks = (pollingManager as IPollingManagerCS).get(matchingKey);
+            if (syncTasks) {
+              const { msSyncTask, mlsSyncTask } = syncTasks;
               // stop syncing
               if (pushManager) pushManager.remove(matchingKey);
-              if (mySegmentsSyncTask.isRunning()) mySegmentsSyncTask.stop();
+              if (msSyncTask.isRunning()) msSyncTask.stop();
+              if (mlsSyncTask && mlsSyncTask.isRunning()) mlsSyncTask.stop();
 
               (pollingManager as IPollingManagerCS).remove(matchingKey);
             }

package/src/types.ts

@@ -86,6 +86,7 @@ export interface ISettings {
     metricsRefreshRate?: number,
     telemetryRefreshRate: number,
     segmentsRefreshRate: number,
+    largeSegmentsRefreshRate: number,
     offlineRefreshRate: number,
     eventsPushRate: number,
     eventsQueueSize: number,
@@ -95,7 +96,8 @@ export interface ISettings {
     readyTimeout: number,
     requestTimeoutBeforeReady: number,
     retriesOnFailureBeforeReady: number,
-    eventsFirstPushWindow: number
+    eventsFirstPushWindow: number,
+    waitForLargeSegments: boolean
   },
   readonly storage: IStorageSyncFactory | IStorageAsyncFactory,
   readonly integrations: Array<{
@@ -119,6 +121,7 @@ export interface ISettings {
     __splitFiltersValidation: ISplitFiltersValidation,
     localhostMode?: SplitIO.LocalhostFactory,
     enabled: boolean,
+    largeSegmentsEnabled: boolean,
     flagSpecVersion: string
   },
   readonly runtime: {
@@ -813,6 +816,13 @@ export namespace SplitIO {
        * @default 10
        */
       eventsFirstPushWindow?: number,
+      /**
+       * Whether the SDK should wait for large segments to be ready before emitting SDK_READY event.
+       * It only applies if largeSegmentsEnabled is true.
+       * @property {number} waitForLargeSegments
+       * @default true
+       */
+      waitForLargeSegments?: boolean
     },
     /**
      * SDK scheduler settings.
@@ -857,6 +867,13 @@ export namespace SplitIO {
        * @default 60
        */
       segmentsRefreshRate?: number,
+      /**
+       * The SDK polls Split servers for changes to large segment definitions. This parameter controls this polling period in seconds.
+       * It only applies if largeSegmentsEnabled is true.
+       * @property {number} largeSegmentsRefreshRate
+       * @default 60
+       */
+      largeSegmentsRefreshRate?: number,
       /**
        * The SDK posts the queued events data in bulks. This parameter controls the posting rate in seconds.
        * @property {number} eventsPushRate
@@ -929,6 +946,14 @@ export namespace SplitIO {
      * @property {Object} urls
      */
     urls?: UrlSettings,
+    sync?: ISharedSettings['sync'] & {
+      /**
+       * Enables synchronization of large segments.
+       * @property {boolean} largeSegmentsEnabled
+       * @default false
+       */
+      largeSegmentsEnabled?: boolean
+    }
   }
   /**
    * Settings interface for SDK instances created on NodeJS.

package/src/utils/constants/index.ts

@@ -76,6 +76,7 @@ export const TELEMETRY = 'te';
 export const TOKEN = 'to';
 export const SEGMENT = 'se';
 export const MY_SEGMENT = 'ms';
+export const MY_LARGE_SEGMENT = 'mls';
 
 export const TREATMENT = 't';
 export const TREATMENTS = 'ts';
@@ -106,3 +107,7 @@ export const ENABLED = 1;
 export const PAUSED = 2;
 
 export const FLAG_SPEC_VERSION = '1.1';
+
+// Matcher types
+export const IN_SEGMENT = 'IN_SEGMENT';
+export const IN_LARGE_SEGMENT = 'IN_LARGE_SEGMENT';

package/src/utils/settingsValidation/index.ts

@@ -7,7 +7,7 @@ import { ISettingsValidationParams } from './types';
 import { ISettings } from '../../types';
 import { validateKey } from '../inputValidation/key';
 import { validateTrafficType } from '../inputValidation/trafficType';
-import { ERROR_MIN_CONFIG_PARAM } from '../../logger/constants';
+import { ERROR_MIN_CONFIG_PARAM, LOG_PREFIX_CLIENT_INSTANTIATION } from '../../logger/constants';
 
 // Exported for telemetry
 export const base = {
@@ -32,6 +32,8 @@ export const base = {
     featuresRefreshRate: 60,
     // fetch segments updates each 60 sec
     segmentsRefreshRate: 60,
+    // fetch large segments updates each 60 sec
+    largeSegmentsRefreshRate: 60,
     // publish telemetry stats each 3600 secs (1 hour)
     telemetryRefreshRate: 3600,
     // publish evaluations each 300 sec (default value for OPTIMIZED impressions mode)
@@ -85,7 +87,8 @@ export const base = {
     impressionsMode: OPTIMIZED,
     localhostMode: undefined,
     enabled: true,
-    flagSpecVersion: FLAG_SPEC_VERSION
+    flagSpecVersion: FLAG_SPEC_VERSION,
+    largeSegmentsEnabled: false
   },
 
   // Logger
@@ -132,6 +135,7 @@ export function settingsValidation(config: unknown, validationParams: ISettingsV
   const { scheduler, startup } = withDefaults;
   scheduler.featuresRefreshRate = fromSecondsToMillis(scheduler.featuresRefreshRate);
   scheduler.segmentsRefreshRate = fromSecondsToMillis(scheduler.segmentsRefreshRate);
+  scheduler.largeSegmentsRefreshRate = fromSecondsToMillis(scheduler.largeSegmentsRefreshRate);
   scheduler.offlineRefreshRate = fromSecondsToMillis(scheduler.offlineRefreshRate);
   scheduler.eventsPushRate = fromSecondsToMillis(scheduler.eventsPushRate);
   scheduler.telemetryRefreshRate = fromSecondsToMillis(validateMinValue('telemetryRefreshRate', scheduler.telemetryRefreshRate, 60));
@@ -166,13 +170,13 @@ export function settingsValidation(config: unknown, validationParams: ISettingsV
       // Keeping same behaviour than JS SDK: if settings key or TT are invalid,
       // `false` value is used as bound key/TT of the default client, which leads to some issues.
       // @ts-ignore, @TODO handle invalid keys as a non-recoverable error?
-      withDefaults.core.key = validateKey(log, maybeKey, 'Client instantiation');
+      withDefaults.core.key = validateKey(log, maybeKey, LOG_PREFIX_CLIENT_INSTANTIATION);
     }
 
     if (validationParams.acceptTT) {
       const maybeTT = withDefaults.core.trafficType;
       if (maybeTT !== undefined) { // @ts-ignore
-        withDefaults.core.trafficType = validateTrafficType(log, maybeTT, 'Client instantiation');
+        withDefaults.core.trafficType = validateTrafficType(log, maybeTT, LOG_PREFIX_CLIENT_INSTANTIATION);
       }
     }
   } else {
@@ -209,11 +213,12 @@ export function settingsValidation(config: unknown, validationParams: ISettingsV
   const splitFiltersValidation = validateSplitFilters(log, sync.splitFilters, withDefaults.mode);
   sync.splitFilters = splitFiltersValidation.validFilters;
   sync.__splitFiltersValidation = splitFiltersValidation;
-  sync.flagSpecVersion = flagSpec ? flagSpec(withDefaults) : FLAG_SPEC_VERSION;
 
+  // ensure a valid flag spec version
+  sync.flagSpecVersion = flagSpec ? flagSpec(withDefaults) : FLAG_SPEC_VERSION;
   // ensure a valid user consent value
   // @ts-ignore, modify readonly prop
-  withDefaults.userConsent = consent(withDefaults);
+  withDefaults.userConsent = consent ? consent(withDefaults) : undefined;
 
   return withDefaults;
 }