@splitsoftware/splitio-commons 0.1.1-canary.6 → 0.1.1-rc.18

package/src/sync/streaming/SSEClient/index.ts

@@ -1,3 +1,4 @@
+import { ISettings } from '../../../types';
 import { IAuthTokenPushEnabled } from '../AuthClient/types';
 import { ISSEClient, ISseEventHandler } from './types';
 
@@ -5,6 +6,26 @@ const VERSION = '1.1';
 
 const CONTROL_CHANNEL_REGEX = /^control_/;
 
+/**
+ * Build metadata headers for SSE connection.
+ *
+ * @param {ISettings} settings Validated settings.
+ * @returns {Record<string, string>} Headers object
+ */
+function buildSSEHeaders(settings: ISettings) {
+  const headers: Record<string, string> = {
+    SplitSDKClientKey: settings.core.authorizationKey.slice(-4),
+    SplitSDKVersion: settings.version,
+  };
+
+  // ip and hostname are false if IPAddressesEnabled is false
+  const { ip, hostname } = settings.runtime;
+  if (ip) headers['SplitSDKMachineIP'] = ip;
+  if (hostname) headers['SplitSDKMachineName'] = hostname;
+
+  return headers;
+}
+
 /**
  * Handles streaming connections with EventSource API
  */
@@ -14,23 +35,27 @@ export default class SSEClient implements ISSEClient {
   streamingUrl: string;
   connection?: InstanceType<typeof EventSource>;
   handler?: ISseEventHandler;
-  authToken?: IAuthTokenPushEnabled;
+  useHeaders?: boolean;
+  headers: Record<string, string>;
 
   /**
    * SSEClient constructor.
    *
-   * @param streamingUrl URL
-   * @param eventSource optional EventSource constructor to use instead of the global one
-   * @throws 'EventSource API is not available. ' if EventSource is not passed and global one is not available
+   * @param settings Validated settings.
+   * @param useHeaders True to send metadata as headers or false to send as query params. If `true`, the provided EventSource must support headers.
+   * @param getEventSource Function to get the EventSource constructor.
+   * @throws 'EventSource API is not available. ' if EventSource is not available.
    */
-  constructor(streamingUrl: string, getEventSource?: () => typeof EventSource | undefined) {
+  constructor(settings: ISettings, useHeaders?: boolean, getEventSource?: () => (typeof EventSource | undefined)) {
     // @ts-expect-error
     this.eventSource = getEventSource && getEventSource();
     // if eventSource is not available, throw an exception
     if (!this.eventSource) throw new Error('EventSource API is not available. ');
 
-    this.streamingUrl = streamingUrl + '/sse';
-    this.reopen = this.reopen.bind(this);
+    this.streamingUrl = settings.urls.streaming + '/sse';
+    // @TODO get `useHeaders` flag from `getEventSource`, to use EventSource headers on client-side SDKs when possible.
+    this.useHeaders = useHeaders;
+    this.headers = buildSSEHeaders(settings);
   }
 
   setEventHandler(handler: ISseEventHandler) {
@@ -41,13 +66,11 @@ export default class SSEClient implements ISSEClient {
    * Open the connection with a given authToken
    *
    * @param {IAuthTokenPushEnabled} authToken
-   * @throws {TypeError} if `authToken` is undefined
+   * @throws {TypeError} Will throw an error if `authToken` is undefined
    */
   open(authToken: IAuthTokenPushEnabled) {
     this.close(); // it closes connection if previously opened
 
-    this.authToken = authToken;
-
     const channelsQueryParam = Object.keys(authToken.channels).map(
       function (channel) {
         const params = CONTROL_CHANNEL_REGEX.test(channel) ? '[?occupancy=metrics.publishers]' : '';
@@ -56,12 +79,18 @@ export default class SSEClient implements ISSEClient {
     ).join(',');
     const url = `${this.streamingUrl}?channels=${channelsQueryParam}&accessToken=${authToken.token}&v=${VERSION}&heartbeats=true`; // same results using `&heartbeats=false`
 
-    this.connection = new this.eventSource(url);
+    this.connection = new this.eventSource(
+      // For client-side SDKs, SplitSDKClientKey and SplitSDKClientKey metadata is passed as query params,
+      // because native EventSource implementations for browser doesn't support headers.
+      this.useHeaders ? url : url + `&SplitSDKVersion=${this.headers.SplitSDKVersion}&SplitSDKClientKey=${this.headers.SplitSDKClientKey}`,
+      // @ts-ignore. For server-side SDKs, metadata is passed via headers. EventSource must support headers, like 'eventsource' package for Node.
+      this.useHeaders ? { headers: this.headers } : undefined
+    );
 
     if (this.handler) { // no need to check if SSEClient is used only by PushManager
-      this.connection.onopen = this.handler.handleOpen;
-      this.connection.onmessage = this.handler.handleMessage;
-      this.connection.onerror = this.handler.handleError;
+      this.connection.addEventListener('open', this.handler.handleOpen);
+      this.connection.addEventListener('message', this.handler.handleMessage);
+      this.connection.addEventListener('error', this.handler.handleError);
     }
   }
 
@@ -69,13 +98,4 @@ export default class SSEClient implements ISSEClient {
   close() {
     if (this.connection) this.connection.close();
   }
-
-  /**
-   * Re-open the connection with the last given authToken.
-   *
-   * @throws {TypeError} if `open` has not been previously called with an authToken
-   */
-  reopen() {
-    this.open(this.authToken as IAuthTokenPushEnabled);
-  }
 }

package/src/sync/streaming/SSEClient/types.ts

@@ -8,7 +8,6 @@ export interface ISseEventHandler {
 
 export interface ISSEClient {
   open(authToken: IAuthTokenPushEnabled): void,
-  reopen(): void,
   close(): void,
   setEventHandler(handler: ISseEventHandler): void
 }

package/src/sync/streaming/SSEHandler/NotificationKeeper.ts

@@ -62,6 +62,14 @@ export default function notificationKeeperFactory(pushEmitter: IPushEventEmitter
     },
 
     handleControlEvent(controlType: ControlType, channel: string, timestamp: number) {
+      /* STREAMING_RESET control event is handled by PushManager directly since it doesn't require
+       * tracking timestamp and state like OCCUPANCY or CONTROL. It also ignores previous
+       * OCCUPANCY and CONTROL notifications, and whether PUSH_SUBSYSTEM_DOWN has been emitted or not */
+      if (controlType === ControlType.STREAMING_RESET) {
+        pushEmitter.emit(controlType);
+        return;
+      }
+
       for (let i = 0; i < channels.length; i++) {
         const c = channels[i];
         if (c.regex.test(channel)) {

package/src/sync/streaming/SSEHandler/NotificationParser.ts

@@ -22,10 +22,12 @@ export function errorParser(error: Event): INotificationError {
  * Also assigns the type OCCUPANCY, if it corresponds, so that all supported messages (e.g., SPLIT_UPDATE, CONTROL) have a type.
  *
  * @param message
- * @returns parsed notification message
+ * @returns parsed notification message or undefined if the given event data is falsy (e.g, '' or undefined).
+ * For example, the EventSource implementation of React-Native for iOS emits a message event with empty data for Ably keepalive comments.
  * @throws {SyntaxError} if `message.data` or `JSON.parse(message.data).data` are invalid JSON strings
  */
-export function messageParser(message: MessageEvent): INotificationMessage {
+export function messageParser(message: MessageEvent): INotificationMessage | undefined {
+  if (!message.data) return;
   const messageData = JSON.parse(message.data);
   messageData.parsedData = JSON.parse(messageData.data);
 

package/src/sync/streaming/SSEHandler/index.ts

@@ -1,9 +1,9 @@
 import { errorParser, messageParser } from './NotificationParser';
 import notificationKeeperFactory from './NotificationKeeper';
-import { PUSH_RETRYABLE_ERROR, PUSH_NONRETRYABLE_ERROR, OCCUPANCY, CONTROL, MY_SEGMENTS_UPDATE, SEGMENT_UPDATE, SPLIT_KILL, SPLIT_UPDATE } from '../constants';
+import { PUSH_RETRYABLE_ERROR, PUSH_NONRETRYABLE_ERROR, OCCUPANCY, CONTROL, MY_SEGMENTS_UPDATE, MY_SEGMENTS_UPDATE_V2, SEGMENT_UPDATE, SPLIT_KILL, SPLIT_UPDATE } from '../constants';
 import { IPushEventEmitter } from '../types';
 import { ISseEventHandler } from '../SSEClient/types';
-import { INotificationError } from './types';
+import { INotificationError, INotificationMessage } from './types';
 import { ILogger } from '../../../logger/types';
 import { STREAMING_PARSING_ERROR_FAILS, ERROR_STREAMING_SSE, STREAMING_PARSING_MESSAGE_FAILS, STREAMING_NEW_MESSAGE } from '../../../logger/constants';
 
@@ -43,7 +43,7 @@ export default function SSEHandlerFactory(log: ILogger, pushEmitter: IPushEventE
         log.warn(STREAMING_PARSING_ERROR_FAILS, [err]);
       }
 
-      let errorMessage = errorWithParsedData.parsedData && errorWithParsedData.parsedData.message;
+      let errorMessage = (errorWithParsedData.parsedData && errorWithParsedData.parsedData.message) || errorWithParsedData.message;
       log.error(ERROR_STREAMING_SSE, [errorMessage]);
 
       if (isRetryableError(errorWithParsedData)) {
@@ -55,9 +55,10 @@ export default function SSEHandlerFactory(log: ILogger, pushEmitter: IPushEventE
 
     /* NotificationProcessor */
     handleMessage(message) {
-      let messageWithParsedData;
+      let messageWithParsedData: INotificationMessage | undefined;
       try {
         messageWithParsedData = messageParser(message);
+        if (!messageWithParsedData) return; // Messages with empty data are ignored
       } catch (err) {
         log.warn(STREAMING_PARSING_MESSAGE_FAILS, [err]);
         return;
@@ -67,30 +68,19 @@ export default function SSEHandlerFactory(log: ILogger, pushEmitter: IPushEventE
       log.debug(STREAMING_NEW_MESSAGE, [data]);
 
       // we only handle update events if streaming is up.
-      if (!notificationKeeper.isStreamingUp() && parsedData.type !== OCCUPANCY && parsedData.type !== CONTROL)
+      if (!notificationKeeper.isStreamingUp() && [OCCUPANCY, CONTROL].indexOf(parsedData.type) === -1)
         return;
 
       switch (parsedData.type) {
         /* update events */
         case SPLIT_UPDATE:
-          pushEmitter.emit(SPLIT_UPDATE,
-            parsedData.changeNumber);
-          break;
         case SEGMENT_UPDATE:
-          pushEmitter.emit(SEGMENT_UPDATE,
-            parsedData.changeNumber,
-            parsedData.segmentName);
+        case MY_SEGMENTS_UPDATE_V2:
+        case SPLIT_KILL:
+          pushEmitter.emit(parsedData.type, parsedData);
           break;
         case MY_SEGMENTS_UPDATE:
-          pushEmitter.emit(MY_SEGMENTS_UPDATE,
-            parsedData,
-            channel);
-          break;
-        case SPLIT_KILL:
-          pushEmitter.emit(SPLIT_KILL,
-            parsedData.changeNumber,
-            parsedData.splitName,
-            parsedData.defaultTreatment);
+          pushEmitter.emit(parsedData.type, parsedData, channel);
           break;
 
         /* occupancy & control events, handled by NotificationManagerKeeper */
@@ -100,6 +90,7 @@ export default function SSEHandlerFactory(log: ILogger, pushEmitter: IPushEventE
         case CONTROL:
           notificationKeeper.handleControlEvent(parsedData.controlType, channel, timestamp);
           break;
+
         default:
           break;
       }

package/src/sync/streaming/SSEHandler/types.ts

@@ -1,5 +1,5 @@
 import { ControlType } from '../constants';
-import { MY_SEGMENTS_UPDATE, SEGMENT_UPDATE, SPLIT_UPDATE, SPLIT_KILL, CONTROL, OCCUPANCY } from '../types';
+import { MY_SEGMENTS_UPDATE, MY_SEGMENTS_UPDATE_V2, SEGMENT_UPDATE, SPLIT_UPDATE, SPLIT_KILL, CONTROL, OCCUPANCY } from '../types';
 
 export interface IMySegmentsUpdateData {
   type: MY_SEGMENTS_UPDATE,
@@ -8,6 +8,33 @@ export interface IMySegmentsUpdateData {
   segmentList?: string[]
 }
 
+export enum Compression {
+  None = 0,
+  Gzip = 1,
+  Zlib = 2
+}
+
+export enum UpdateStrategy {
+  UnboundedFetchRequest = 0,
+  BoundedFetchRequest = 1,
+  KeyList = 2,
+  SegmentRemoval = 3
+}
+
+export interface KeyList {
+  a?: string[], // decimal hash64 of user keys
+  r?: string[], // decimal hash64 of user keys
+}
+
+export interface IMySegmentsUpdateV2Data {
+  type: MY_SEGMENTS_UPDATE_V2,
+  changeNumber: number,
+  segmentName: string,
+  c: Compression,
+  d: string,
+  u: UpdateStrategy,
+}
+
 export interface ISegmentUpdateData {
   type: SEGMENT_UPDATE,
   changeNumber: number,
@@ -38,6 +65,13 @@ export interface IOccupancyData {
   }
 }
 
-export type INotificationData = IMySegmentsUpdateData | ISegmentUpdateData | ISplitUpdateData | ISplitKillData | IControlData | IOccupancyData
+export type INotificationData = IMySegmentsUpdateData | IMySegmentsUpdateV2Data | ISegmentUpdateData | ISplitUpdateData | ISplitKillData | IControlData | IOccupancyData
 export type INotificationMessage = { parsedData: INotificationData, channel: string, timestamp: number, data: string }
-export type INotificationError = Event & { parsedData?: any }
+export type INotificationError = Event & { parsedData?: any, message?: string }
+
+export type SegmentsData = string[] | {
+  /* segment name */
+  name: string,
+  /* action: `true` for add, and `false` for delete */
+  add: boolean
+}

package/src/sync/streaming/UpdateWorkers/MySegmentsUpdateWorker.ts

@@ -1,6 +1,7 @@
 import { ISegmentsSyncTask } from '../../polling/types';
 import Backoff from '../../../utils/Backoff';
 import { IUpdateWorker } from './types';
+import { SegmentsData } from '../SSEHandler/types';
 
 /**
  * MySegmentsUpdateWorker class
@@ -10,7 +11,7 @@ export default class MySegmentsUpdateWorker implements IUpdateWorker {
   private readonly mySegmentsSyncTask: ISegmentsSyncTask;
   private maxChangeNumber: number;
   private handleNewEvent: boolean;
-  private segmentList?: string[];
+  private segmentsData?: SegmentsData;
   private currentChangeNumber: number;
   readonly backoff: Backoff;
 
@@ -21,7 +22,7 @@ export default class MySegmentsUpdateWorker implements IUpdateWorker {
     this.mySegmentsSyncTask = mySegmentsSyncTask;
     this.maxChangeNumber = 0; // keeps the maximum changeNumber among queued events
     this.handleNewEvent = false;
-    this.segmentList = undefined; // keeps the segmentList (if included in payload) from the queued event with maximum changeNumber
+    this.segmentsData = undefined; // keeps the segmentsData (if included in notification payload) from the queued event with maximum changeNumber
     this.currentChangeNumber = -1; // @TODO: remove once `/mySegments` endpoint provides the changeNumber
     this.put = this.put.bind(this);
     this.__handleMySegmentsUpdateCall = this.__handleMySegmentsUpdateCall.bind(this);
@@ -36,7 +37,7 @@ export default class MySegmentsUpdateWorker implements IUpdateWorker {
       const currentMaxChangeNumber = this.maxChangeNumber;
 
       // fetch mySegments revalidating data if cached
-      this.mySegmentsSyncTask.execute(this.segmentList, true).then((result) => {
+      this.mySegmentsSyncTask.execute(this.segmentsData, true).then((result) => {
         if (result !== false) // Unlike `Splits|SegmentsUpdateWorker`, we cannot use `mySegmentsCache.getChangeNumber` since `/mySegments` endpoint doesn't provide this value.
           this.currentChangeNumber = Math.max(this.currentChangeNumber, currentMaxChangeNumber); // use `currentMaxChangeNumber`, in case that `this.maxChangeNumber` was updated during fetch.
         if (this.handleNewEvent) {
@@ -52,15 +53,15 @@ export default class MySegmentsUpdateWorker implements IUpdateWorker {
    * Invoked by NotificationProcessor on MY_SEGMENTS_UPDATE event
    *
    * @param {number} changeNumber change number of the MY_SEGMENTS_UPDATE notification
-   * @param {string[] | undefined} segmentList might be undefined
+   * @param {SegmentsData | undefined} segmentsData might be undefined
    */
-  put(changeNumber: number, segmentList?: string[]) {
+  put(changeNumber: number, segmentsData?: SegmentsData) {
     if (changeNumber <= this.currentChangeNumber || changeNumber <= this.maxChangeNumber) return;
 
     this.maxChangeNumber = changeNumber;
     this.handleNewEvent = true;
     this.backoff.reset();
-    this.segmentList = segmentList;
+    this.segmentsData = segmentsData;
 
     if (this.mySegmentsSyncTask.isExecuting()) return;
 

package/src/sync/streaming/UpdateWorkers/SegmentsUpdateWorker.ts

@@ -1,6 +1,7 @@
 import { ISegmentsCacheSync } from '../../../storages/types';
 import Backoff from '../../../utils/Backoff';
 import { ISegmentsSyncTask } from '../../polling/types';
+import { ISegmentUpdateData } from '../SSEHandler/types';
 import { IUpdateWorker } from './types';
 
 /**
@@ -66,7 +67,7 @@ export default class SegmentsUpdateWorker implements IUpdateWorker {
    * @param {number} changeNumber change number of the SEGMENT_UPDATE notification
    * @param {string} segmentName segment name of the SEGMENT_UPDATE notification
    */
-  put(changeNumber: number, segmentName: string) {
+  put({ changeNumber, segmentName }: ISegmentUpdateData) {
     const currentChangeNumber = this.segmentsCache.getChangeNumber(segmentName);
 
     if (changeNumber <= currentChangeNumber || changeNumber <= this.maxChangeNumbers[segmentName]) return;

package/src/sync/streaming/UpdateWorkers/SplitsUpdateWorker.ts

@@ -3,6 +3,7 @@ import { ISplitsEventEmitter } from '../../../readiness/types';
 import { ISplitsCacheSync } from '../../../storages/types';
 import Backoff from '../../../utils/Backoff';
 import { ISegmentsSyncTask, ISplitsSyncTask } from '../../polling/types';
+import { ISplitKillData, ISplitUpdateData } from '../SSEHandler/types';
 import { IUpdateWorker } from './types';
 
 /**
@@ -60,7 +61,7 @@ export default class SplitsUpdateWorker implements IUpdateWorker {
    *
    * @param {number} changeNumber change number of the SPLIT_UPDATE notification
    */
-  put(changeNumber: number) {
+  put({ changeNumber }: Pick<ISplitUpdateData, 'changeNumber'>) {
     const currentChangeNumber = this.splitsCache.getChangeNumber();
 
     if (changeNumber <= currentChangeNumber || changeNumber <= this.maxChangeNumber) return;
@@ -81,14 +82,14 @@ export default class SplitsUpdateWorker implements IUpdateWorker {
    * @param {string} splitName name of split to kill
    * @param {string} defaultTreatment default treatment value
    */
-  killSplit(changeNumber: number, splitName: string, defaultTreatment: string) {
+  killSplit({ changeNumber, splitName, defaultTreatment }: ISplitKillData) {
     // @TODO handle retry due to errors in storage, once we allow the definition of custom async storages
     if (this.splitsCache.killLocally(splitName, defaultTreatment, changeNumber)) {
       // trigger an SDK_UPDATE if Split was killed locally
       this.splitsEventEmitter.emit(SDK_SPLITS_ARRIVED, true);
     }
     // queues the SplitChanges fetch (only if changeNumber is newer)
-    this.put(changeNumber);
+    this.put({ changeNumber });
   }
 
 }

package/src/sync/streaming/UpdateWorkers/types.ts

@@ -2,5 +2,5 @@ import Backoff from '../../../utils/Backoff';
 
 export interface IUpdateWorker {
   readonly backoff: Backoff,
-  put(changeNumber: number, ...args: any[]): void
+  put(...args: any[]): void
 }

package/src/sync/streaming/constants.ts

@@ -26,6 +26,7 @@ export const PUSH_SUBSYSTEM_DOWN = 'PUSH_SUBSYSTEM_DOWN';
 
 // Update-type push notifications, handled by NotificationProcessor
 export const MY_SEGMENTS_UPDATE = 'MY_SEGMENTS_UPDATE';
+export const MY_SEGMENTS_UPDATE_V2 = 'MY_SEGMENTS_UPDATE_V2';
 export const SEGMENT_UPDATE = 'SEGMENT_UPDATE';
 export const SPLIT_KILL = 'SPLIT_KILL';
 export const SPLIT_UPDATE = 'SPLIT_UPDATE';
@@ -38,4 +39,5 @@ export enum ControlType {
   STREAMING_DISABLED = 'STREAMING_DISABLED',
   STREAMING_PAUSED = 'STREAMING_PAUSED',
   STREAMING_RESUMED = 'STREAMING_RESUMED',
+  STREAMING_RESET = 'STREAMING_RESET',
 }

package/src/sync/streaming/mySegmentsV2utils.ts

@@ -0,0 +1,77 @@
+import { algorithms } from '../../utils/decompress';
+import { decodeFromBase64 } from '../../utils/base64';
+import { Compression, KeyList } from './SSEHandler/types';
+
+const GZIP = 1;
+const ZLIB = 2;
+
+function Uint8ArrayToString(myUint8Arr: Uint8Array) { // @ts-ignore
+  return String.fromCharCode.apply(null, myUint8Arr);
+}
+
+function StringToUint8Array(myString: string) {
+  const charCodes = myString.split('').map((e) => e.charCodeAt(0));
+  return new Uint8Array(charCodes);
+}
+
+/**
+ * Decode and decompress 'data' with 'compression' algorithm
+ *
+ * @param {string} data
+ * @param {number} compression 1 GZIP, 2 ZLIB
+ * @returns {Uint8Array}
+ * @throws if data string cannot be decoded, decompressed or the provided compression value is invalid (not 1 or 2)
+ */
+function decompress(data: string, compression: Compression) {
+  let compressData = decodeFromBase64(data);
+  const binData = StringToUint8Array(compressData);
+
+  if (typeof algorithms === 'string') throw new Error(algorithms);
+  if (compression === GZIP) return algorithms.gunzipSync(binData);
+  if (compression === ZLIB) return algorithms.unzlibSync(binData);
+  throw new Error(`Invalid compression algorithm #${compression}`);
+}
+
+/**
+ * Decode, decompress and parse the provided 'data' into a KeyList object
+ *
+ * @param {string} data
+ * @param {number} compression
+ * @returns {{a?: string[], r?: string[] }}
+ * @throws if data string cannot be decoded, decompressed or parsed
+ */
+export function parseKeyList(data: string, compression: Compression): KeyList {
+  const binKeyList = decompress(data, compression);
+  const strKeyList = Uint8ArrayToString(binKeyList);
+
+  // replace numbers to strings, to avoid losing precision
+  return JSON.parse(strKeyList.replace(/\d+/g, '"$&"'));
+}
+
+/**
+ * Decode, decompress and parse the provided 'data' into a Bitmap object
+ *
+ * @param {string} data
+ * @param {number} compression
+ * @returns {Uint8Array}
+ * @throws if data string cannot be decoded or decompressed
+ */
+export function parseBitmap(data: string, compression: Compression) {
+  return decompress(data, compression);
+}
+
+/**
+ * Check if the 'bitmap' bit at 'hash64hex' position is 1
+ *
+ * @param {Uint8Array} bitmap
+ * @param {string} hash64hex 16-chars string, representing a number in hexa
+ * @returns {boolean}
+ */
+export function isInBitmap(bitmap: Uint8Array, hash64hex: string) {
+  // using the lowest 32 bits as index, to avoid losing precision when converting to number
+  const index = parseInt(hash64hex.slice(8), 16) % (bitmap.length * 8);
+
+  const internal = Math.floor(index / 8);
+  const offset = index % 8;
+  return (bitmap[internal] & 1 << offset) > 0;
+}