@mparticle/web-sdk 2.20.2 → 2.21.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@mparticle/web-sdk",
-    "version": "2.20.2",
+    "version": "2.21.0",
     "description": "mParticle core SDK for web applications",
     "license": "Apache-2.0",
     "keywords": [
@@ -78,7 +78,7 @@
         "@mparticle/event-models": "^1.1.6",
         "@rollup/plugin-json": "^4.1.0",
         "@rollup/plugin-typescript": "^8.3.3",
-        "@rollup/plugin-babel": "5.3.1",
+        "@rollup/plugin-babel": "6.0.3",
         "@rollup/plugin-commonjs": "22.0.2",
         "@rollup/plugin-node-resolve": "13.3.0",
         "@semantic-release/changelog": "^5.0.1",
@@ -93,8 +93,8 @@
         "browserify": "^17.0.0",
         "chai": "^4.2.0",
         "cross-env": "^7.0.3",
-        "eslint": "^7.32.0",
-        "eslint-config-prettier": "8.3.0",
+        "eslint": "^8.36.0",
+        "eslint-config-prettier": "8.8.0",
         "eslint-plugin-prettier": "3.4.1",
         "fetch-mock": "^7.5.1",
         "gts": "^3.1.0",
@@ -116,7 +116,7 @@
         "should": "^7.1.0",
         "sinon": "^11.0.0",
         "tslib": "^2.1.0",
-        "typescript": "^4.2.4",
+        "typescript": "^5.0.2",
         "uglify-js": "^3.4.9",
         "webpack": "^5.36.2",
         "webpack-cli": "^4.6.0"

package/src/apiClient.ts

@@ -8,7 +8,7 @@ import {
     SDKEvent,
 } from './sdkRuntimeModels';
 import KitBlocker from './kitBlocking';
-import { Dictionary } from './utils';
+import { Dictionary, getRampNumber } from './utils';
 import { IUploadObject } from './serverModel';
 
 const Messages = Constants.Messages;
@@ -67,9 +67,7 @@ export default function APIClient(
             return false;
         }
 
-        const rampNumber = mpInstance._Helpers.getRampNumber(
-            mpInstance._Store.deviceId
-        );
+        const rampNumber = getRampNumber(mpInstance._Store.deviceId);
         return eventsV3Percentage >= rampNumber;
     };
 

package/src/batchUploader.ts

@@ -1,4 +1,5 @@
 import { Batch } from '@mparticle/event-models';
+import Constants from './constants';
 import {
     SDKEvent,
     MParticleUser,
@@ -7,12 +8,17 @@ import {
 } from './sdkRuntimeModels';
 import { convertEvents } from './sdkToEventsApiConverter';
 import Types from './types';
-import { isEmpty } from './utils';
+import { getRampNumber, isEmpty } from './utils';
+import { SessionStorageVault, LocalStorageVault } from './vault';
 
 /**
- * BatchUploader contains all the logic to upload batches to mParticle.
- * It queues events as they come in and at set intervals turns them into batches.
- * It then attempts to upload them to mParticle.
+ * BatchUploader contains all the logic to store/retrieve events and batches
+ * to/from persistence, and upload batches to mParticle.
+ * It queues events as they come in, storing them in persistence, then at set
+ * intervals turns them into batches and transfers between event and batch
+ * persistence.
+ * It then attempts to upload them to mParticle, purging batch persistence if
+ * the upload is successful
  *
  * These uploads happen on an interval basis using window.fetch or XHR
  * requests, depending on what is available in the browser.
@@ -31,6 +37,9 @@ export class BatchUploader {
     mpInstance: MParticleWebSDK;
     uploadUrl: string;
     batchingEnabled: boolean;
+    private eventVault: SessionStorageVault<SDKEvent[]>;
+    private batchVault: LocalStorageVault<Batch[]>;
+    private offlineStorageEnabled: boolean = false;
     private uploader: AsyncUploader;
 
     /**
@@ -46,9 +55,38 @@ export class BatchUploader {
         if (this.uploadIntervalMillis < BatchUploader.MINIMUM_INTERVAL_MILLIS) {
             this.uploadIntervalMillis = BatchUploader.MINIMUM_INTERVAL_MILLIS;
         }
+
+        // Events will be queued during `queueEvents` method
         this.eventsQueuedForProcessing = [];
+
+        // Batch queue should start empty and will be populated during
+        // `prepareAndUpload` method, either via Local Storage or after
+        // new batches are created.
         this.batchesQueuedForProcessing = [];
 
+        // Cache Offline Storage Availability boolean
+        // so that we don't have to check it every time
+        this.offlineStorageEnabled = this.isOfflineStorageAvailable();
+
+        if (this.offlineStorageEnabled) {
+            this.eventVault = new SessionStorageVault<SDKEvent[]>(
+                `${mpInstance._Store.storageName}-events`,
+                {
+                    logger: mpInstance.Logger,
+                }
+            );
+
+            this.batchVault = new LocalStorageVault<Batch[]>(
+                `${mpInstance._Store.storageName}-batches`,
+                {
+                    logger: mpInstance.Logger,
+                }
+            );
+
+            // Load Events from Session Storage in case we have any in storage
+            this.eventsQueuedForProcessing.push(...this.eventVault.retrieve());
+        }
+
         const { SDKConfig, devToken } = this.mpInstance._Store;
         const baseUrl = this.mpInstance._Helpers.createServiceUrl(
             SDKConfig.v3SecureServiceUrl,
@@ -64,6 +102,29 @@ export class BatchUploader {
         this.addEventListeners();
     }
 
+    private isOfflineStorageAvailable(): boolean {
+        const {
+            _Helpers: { getFeatureFlag },
+            _Store: { deviceId },
+        } = this.mpInstance;
+
+        const offlineStorageFeatureFlagValue = getFeatureFlag(
+            Constants.FeatureFlags.OfflineStorage
+        );
+
+        const offlineStoragePercentage = parseInt(
+            offlineStorageFeatureFlagValue,
+            10
+        );
+
+        const rampNumber = getRampNumber(deviceId);
+
+        // TODO: Handle cases where Local Storage is unavailable
+        //       Potentially shared between Vault and Persistence as well
+        //       https://go.mparticle.com/work/SQDSDKS-5022
+        return offlineStoragePercentage >= rampNumber;
+    }
+
     // Adds listeners to be used trigger Navigator.sendBeacon if the browser
     // loses focus for any reason, such as closing browser tab or minimizing window
     private addEventListeners() {
@@ -102,9 +163,12 @@ export class BatchUploader {
      * This method will queue a single Event which will eventually be processed into a Batch
      * @param event event that should be queued
      */
-    queueEvent(event: SDKEvent): void {
+    public queueEvent(event: SDKEvent): void {
         if (!isEmpty(event)) {
             this.eventsQueuedForProcessing.push(event);
+            if (this.offlineStorageEnabled && this.eventVault) {
+                this.eventVault.store(this.eventsQueuedForProcessing);
+            }
             this.mpInstance.Logger.verbose(
                 `Queuing event: ${JSON.stringify(event)}`
             );
@@ -214,14 +278,32 @@ export class BatchUploader {
         // Fetch current user so that events can be grouped by MPID
         const currentUser = this.mpInstance.Identity.getCurrentUser();
 
-        const currentEvents = this.eventsQueuedForProcessing;
+        const currentEvents: SDKEvent[] = this.eventsQueuedForProcessing;
+
         this.eventsQueuedForProcessing = [];
+        if (this.offlineStorageEnabled && this.eventVault) {
+            this.eventVault.store([]);
+        }
 
-        const newBatches = BatchUploader.createNewBatches(
-            currentEvents,
-            currentUser,
-            this.mpInstance
-        );
+        let newBatches: Batch[] = [];
+        if (!isEmpty(currentEvents)) {
+            newBatches = BatchUploader.createNewBatches(
+                currentEvents,
+                currentUser,
+                this.mpInstance
+            );
+        }
+
+        // Top Load any older Batches from Offline Storage so they go out first
+        if (this.offlineStorageEnabled && this.batchVault) {
+            this.batchesQueuedForProcessing.unshift(
+                ...this.batchVault.retrieve()
+            );
+
+            // Remove batches from local storage before transmit to
+            // prevent duplication
+            this.batchVault.purge();
+        }
 
         if (!isEmpty(newBatches)) {
             this.batchesQueuedForProcessing.push(...newBatches);
@@ -231,6 +313,13 @@ export class BatchUploader {
         const batchesToUpload = this.batchesQueuedForProcessing;
         this.batchesQueuedForProcessing = [];
 
+        // If `useBeacon` is true, the browser has been closed suddently
+        // so we should save `batchesToUpload` to Offline Storage before
+        // an upload is attempted.
+        if (useBeacon && this.offlineStorageEnabled && this.batchVault) {
+            this.batchVault.store(batchesToUpload);
+        }
+
         const batchesThatDidNotUpload = await this.uploadBatches(
             this.mpInstance.Logger,
             batchesToUpload,
@@ -245,6 +334,19 @@ export class BatchUploader {
             this.batchesQueuedForProcessing.unshift(...batchesThatDidNotUpload);
         }
 
+        // Update Offline Storage with current state of batch queue
+        if (!useBeacon && this.offlineStorageEnabled && this.batchVault) {
+            // Note: since beacon is "Fire and forget" it will empty `batchesThatDidNotUplod`
+            // regardless of whether the batches were successfully uploaded or not. We should
+            // therefore NOT overwrite Offline Storage when beacon returns, so that we can retry
+            // uploading saved batches at a later time. Batches should only be removed from
+            // Local Storage once we can confirm they are successfully uploaded.
+            this.batchVault.store(this.batchesQueuedForProcessing);
+
+            // Clear batch queue since everything should be in Offline Storage
+            this.batchesQueuedForProcessing = [];
+        }
+
         if (triggerFuture) {
             this.triggerUploadInterval(triggerFuture, false);
         }

package/src/configAPIClient.ts

@@ -29,8 +29,9 @@ export interface IKitConfigs {
     eventNameFilters: number[];
     eventTypeFilters: number[];
     attributeFilters: number[];
-    filteringEventAttributeValue: Dictionary;
-    filteringConsentRuleValues: Dictionary;
+    filteringEventAttributeValue: IFilteringAttributeValue;
+    filteringUserAttributeValue: IFilteringAttributeValue;
+    filteringConsentRuleValues: IFilteringConsentRulesValue;
     consentRegulationFilters: number[];
     consentRegulationPurposeFilters: number[];
     messageTypeFilters: number[];
@@ -39,6 +40,22 @@ export interface IKitConfigs {
     excludeAnonymousUser: boolean;
 }
 
+export interface IFilteringAttributeValue {
+    eventAttributeName: string;
+    eventAttributeValue: string;
+    includeOnMatch: boolean
+}
+export interface IFilteringConsentRulesValue {
+    includeOnMatch: boolean;
+    values: IConsentRuleValue[];
+}
+
+export interface IConsentRuleValue {
+    consentPurpose: string;
+    hasConsented: boolean
+}
+
+
 export interface IPixelConfig {
     name: string;
     moduleId: number;

package/src/constants.ts

@@ -161,6 +161,7 @@ const Constants = {
         ReportBatching: 'reportBatching',
         EventsV3: 'eventsV3',
         EventBatchingIntervalMillis: 'eventBatchingIntervalMillis',
+        OfflineStorage: 'offlineStorage',
     },
     DefaultInstance: 'default_instance',
 } as const;

package/src/forwarders.interfaces.ts

@@ -0,0 +1,61 @@
+import { MParticleUser, SDKEvent, SDKEventCustomFlags, SDKUserIdentity } from './sdkRuntimeModels';
+import { Dictionary } from './utils';
+import { IKitConfigs } from './configAPIClient';
+import { UserAttributes } from './persistence.interfaces';
+import { IdentityApiData } from '@mparticle/web-sdk';
+
+// The state of the kit when accessed via window.KitName via CDN
+// or imported as an NPM package
+export interface UnregisteredKit {
+    register(config): void;
+}
+
+// The state of the kit after being added to forwarderConstructors in the CDN 
+// or after registered to SDKConfig.kits via NPM
+export interface RegisteredKit {
+    constructor(): void;
+    name: string;
+    getId(): number;
+}
+
+// The state of the kit after being configured. This is what the kit looks like when acted on.
+export interface ConfiguredKit
+    extends Omit<IKitConfigs, 'isDebugString' | 'hasDebugString'> {
+    common: Dictionary<unknown>;
+    init(
+        settings: Dictionary<unknown>,
+        service: forwardingStatsCallback,
+        testMode: boolean,
+        trackerId: string | null,
+        userAttributes: UserAttributes,
+        userIdentities: SDKUserIdentity,
+        appVersion: string,
+        appName: string,
+        customFlags: SDKEventCustomFlags,
+        clientId: string): string;
+    onIdentifyComplete(user: MParticleUser, filteredIdentityRequest: IdentityApiData): string | KitMappedMethodFailure;
+    onLoginComplete(user: MParticleUser, filteredIdentityRequest: IdentityApiData): string | KitMappedMethodFailure;
+    onLogoutComplete(user: MParticleUser, filteredIdentityRequest: IdentityApiData): string | KitMappedMethodFailure;
+    onModifyComplete(user: MParticleUser, filteredIdentityRequest: IdentityApiData): string | KitMappedMethodFailure;
+    onUserIdentified(user: MParticleUser): string | KitMappedMethodFailure;
+    process(event: SDKEvent): string;
+    setOptOut(isOptingOut: boolean): string | KitMappedMethodFailure;
+    removeUserAttribute(key: string): string;
+    setUserAttribute(key: string, value:string): string;
+
+    // TODO: Convert type to enum during Identity migration
+    // https://go.mparticle.com/work/SQDSDKS-5218
+    setUserIdentity(id: UserIdentityId, type: UserIdentityType): void;
+
+    // TODO: https://go.mparticle.com/work/SQDSDKS-5156
+    isSandbox: boolean;  
+    hasSandbox: boolean;
+}
+export interface KitMappedMethodFailure {
+    error: string
+}
+
+export type UserIdentityId = string;
+export type UserIdentityType = number;
+
+export type forwardingStatsCallback = (forwarder: ConfiguredKit, event: SDKEvent) => void;

package/src/forwarders.js

@@ -650,7 +650,7 @@ export default function Forwarders(mpInstance, kitBlocker) {
         newForwarder.id = config.moduleId;
 
         // TODO: isSandbox, hasSandbox is never used in any kit or in core SDK.
-        // isVisibleInvestigate is only used in 1 place. It is always true if
+        // isVisible.  Investigate is only used in 1 place. It is always true if
         // it is sent to JS. Investigate further to determine if these can be removed.
         // https://go.mparticle.com/work/SQDSDKS-5156
         newForwarder.isSandbox = config.isDebug || config.isSandbox;

package/src/helpers.js

@@ -26,17 +26,6 @@ export default function Helpers(mpInstance) {
         return null;
     };
 
-    /**
-     * Returns a value between 1-100 inclusive.
-     */
-    this.getRampNumber = function(deviceId) {
-        if (!deviceId) {
-            return 100;
-        }
-        var hash = self.generateHash(deviceId);
-        return Math.abs(hash % 100) + 1;
-    };
-
     this.invokeCallback = function(
         callback,
         code,
@@ -284,39 +273,6 @@ export default function Helpers(mpInstance) {
         return xhr;
     };
 
-    function generateRandomValue(a) {
-        var randomValue;
-        if (window.crypto && window.crypto.getRandomValues) {
-            randomValue = window.crypto.getRandomValues(new Uint8Array(1)); // eslint-disable-line no-undef
-        }
-        if (randomValue) {
-            return (a ^ (randomValue[0] % 16 >> (a / 4))).toString(16);
-        }
-
-        return (a ^ ((Math.random() * 16) >> (a / 4))).toString(16);
-    }
-
-    this.generateUniqueId = function(a) {
-        // https://gist.github.com/jed/982883
-        // Added support for crypto for better random
-
-        return a // if the placeholder was passed, return
-            ? generateRandomValue(a) // a random number
-            : // or otherwise a concatenated string:
-              (
-                  [1e7] + // 10000000 +
-                  -1e3 + // -1000 +
-                  -4e3 + // -4000 +
-                  -8e3 + // -80000000 +
-                  -1e11
-              ) // -100000000000,
-                  .replace(
-                      // replacing
-                      /[018]/g, // zeroes, ones, and eights with
-                      self.generateUniqueId // random hex digits
-                  );
-    };
-
     this.filterUserIdentities = function(userIdentitiesObject, filterList) {
         var filteredUserIdentities = [];
 
@@ -403,37 +359,6 @@ export default function Helpers(mpInstance) {
         return false;
     };
 
-    this.generateHash = function(name) {
-        var hash = 0,
-            i = 0,
-            character;
-
-        if (name === undefined || name === null) {
-            return 0;
-        }
-
-        name = name.toString().toLowerCase();
-
-        if (Array.prototype.reduce) {
-            return name.split('').reduce(function(a, b) {
-                a = (a << 5) - a + b.charCodeAt(0);
-                return a & a;
-            }, 0);
-        }
-
-        if (name.length === 0) {
-            return hash;
-        }
-
-        for (i = 0; i < name.length; i++) {
-            character = name.charCodeAt(i);
-            hash = (hash << 5) - hash + character;
-            hash = hash & hash;
-        }
-
-        return hash;
-    };
-
     this.sanitizeAttributes = function(attrs, name) {
         if (!attrs || !self.isObject(attrs)) {
             return null;
@@ -501,6 +426,8 @@ export default function Helpers(mpInstance) {
         }
     };
 
+    // TODO: Refactor SDK to directly use these methods
+    // https://go.mparticle.com/work/SQDSDKS-5239
     // Utility Functions
     this.converted = utils.converted;
     this.findKeyInObject = utils.findKeyInObject;
@@ -510,6 +437,8 @@ export default function Helpers(mpInstance) {
     this.decoded = utils.decoded;
     this.returnConvertedBoolean = utils.returnConvertedBoolean;
     this.parseStringOrNumber = utils.parseStringOrNumber;
+    this.generateHash = utils.generateHash;
+    this.generateUniqueId = utils.generateUniqueId;
 
     // Imported Validators
     this.Validators = Validators;

package/src/mockBatchCreator.ts

@@ -4,6 +4,7 @@ import ServerModel from './serverModel';
 import { SDKEvent, BaseEvent, MParticleWebSDK } from './sdkRuntimeModels';
 import { convertEvents } from './sdkToEventsApiConverter';
 import * as EventsApi from '@mparticle/event-models';
+import { Batch } from '@mparticle/event-models';
 
 const mockFunction = function () {
     return null;
@@ -119,16 +120,19 @@ export default class _BatchValidator {
         } as MParticleWebSDK;
     }
 
-    returnBatch(event: BaseEvent) {
+    private createSDKEventFunction(event): SDKEvent {
+        return new ServerModel(this.getMPInstance()).createEventObject(event);
+    }
+
+    public returnBatch(events: BaseEvent | BaseEvent[]): Batch | null {
         const mpInstance = this.getMPInstance();
-        const sdkEvent: SDKEvent = new ServerModel(
-            mpInstance
-        ).createEventObject(event);
-        const batch: EventsApi.Batch | null = convertEvents(
-            '0',
-            [sdkEvent],
-            mpInstance as any
-        );
+
+        const sdkEvents: SDKEvent[] = Array.isArray(events)
+            ? events.map((event) => this.createSDKEventFunction(event))
+            : [this.createSDKEventFunction(events)];
+
+        const batch: Batch = convertEvents('0', sdkEvents, mpInstance as any);
+
         return batch;
     }
 }

package/src/sdkRuntimeModels.ts

@@ -232,7 +232,6 @@ export interface SDKHelpersApi {
     generateUniqueId();
     generateHash?(value: string): string;
     getFeatureFlag?(feature: string); // TODO: Feature Constants should be converted to enum
-    getRampNumber?(deviceId: string): number;
     isDelayedByIntegration?(
         delayedIntegrations: Dictionary<boolean>,
         timeoutStart: number,

package/src/store.ts

@@ -444,5 +444,12 @@ export default function Store(
         ) {
             this.SDKConfig.flags[Constants.FeatureFlags.ReportBatching] = false;
         }
+        if (
+            !this.SDKConfig.flags.hasOwnProperty(
+                Constants.FeatureFlags.OfflineStorage
+            )
+        ) {
+            this.SDKConfig.flags[Constants.FeatureFlags.OfflineStorage] = 0;
+        }
     }
 }

package/src/utils.ts

@@ -36,6 +36,81 @@ const findKeyInObject = (obj: any, key: string): string => {
     return null;
 };
 
+function generateHash(name: string): number {
+    let hash: number = 0;
+    let character: number;
+
+    if (name === undefined || name === null) {
+        return 0;
+    }
+
+    name = name.toString().toLowerCase();
+
+    if (Array.prototype.reduce) {
+        return name.split('').reduce(function (a: number, b: string) {
+            a = (a << 5) - a + b.charCodeAt(0);
+            return a & a;
+        }, 0);
+    }
+
+    if (name.length === 0) {
+        return hash;
+    }
+
+    for (let i = 0; i < name.length; i++) {
+        character = name.charCodeAt(i);
+        hash = (hash << 5) - hash + character;
+        hash = hash & hash;
+    }
+
+    return hash;
+}
+
+const generateRandomValue = (value?: string): string => {
+    let randomValue: string;
+    let a: number;
+
+    if (window.crypto && window.crypto.getRandomValues) {
+        // @ts-ignore
+        randomValue = window.crypto.getRandomValues(new Uint8Array(1)); // eslint-disable-line no-undef
+    }
+    if (randomValue) {
+        // @ts-ignore
+        return (a ^ (randomValue[0] % 16 >> (a / 4))).toString(16);
+    }
+
+    return (a ^ ((Math.random() * 16) >> (a / 4))).toString(16);
+};
+
+const generateUniqueId = (a: string = ''): string =>
+    // https://gist.github.com/jed/982883
+    // Added support for crypto for better random
+
+    a // if the placeholder was passed, return
+        ? generateRandomValue(a) // if the placeholder was passed, return
+        : // [1e7] -> // 10000000 +
+          // -1e3  -> // -1000 +
+          // -4e3  -> // -4000 +
+          // -8e3  -> // -80000000 +
+          // -1e11 -> //-100000000000,
+          `${1e7}-${1e3}-${4e3}-${8e3}-${1e11}`.replace(
+              /[018]/g, // zeroes, ones, and eights with
+              generateUniqueId // random hex digits
+          );
+
+/**
+ * Returns a value between 1-100 inclusive.
+ */
+const getRampNumber = (value?: string): number => {
+    if (!value) {
+        return 100;
+    }
+
+    const hash = generateHash(value);
+
+    return Math.abs(hash % 100) + 1;
+};
+
 const isObject = (value: any): boolean => {
     var objType = Object.prototype.toString.call(value);
     return objType === '[object Object]' || objType === '[object Error]';
@@ -76,10 +151,7 @@ const decoded = (s: string): string =>
 
 const converted = (s: string): string => {
     if (s.indexOf('"') === 0) {
-        s = s
-            .slice(1, -1)
-            .replace(/\\"/g, '"')
-            .replace(/\\\\/g, '\\');
+        s = s.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, '\\');
     }
 
     return s;
@@ -104,6 +176,9 @@ export {
     converted,
     decoded,
     findKeyInObject,
+    generateHash,
+    generateUniqueId,
+    getRampNumber,
     inArray,
     isObject,
     isStringOrNumber,