clarity-js 0.6.31 → 0.6.34

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-js",
-  "version": "0.6.31",
+  "version": "0.6.34",
   "description": "An analytics library that uses web page interactions to generate aggregated insights",
   "author": "Microsoft Corp.",
   "license": "MIT",

package/src/core/api.ts

@@ -0,0 +1,8 @@
+import { Constant } from "@clarity-types/core";
+
+export default function api(method: string): string {
+    // Zone.js, a popular package for Angular, overrides native browser APIs which can lead to inconsistent state for single page applications.
+    // Example issue: https://github.com/angular/angular/issues/31712
+    // As a work around, we ensuring Clarity access APIs outside of Zone (and use native implementation instead)
+    return window[Constant.Zone] && Constant.Symbol in window[Constant.Zone] ? window[Constant.Zone][Constant.Symbol](method) : method;
+}

package/src/core/config.ts

@@ -9,8 +9,7 @@ let config: Config = {
     mask: [],
     unmask: [],
     regions: [],
-    metrics: [],
-    dimensions: [],
+    extract: [],
     cookies: [],
     report: null,
     upload: null,

package/src/core/event.ts

@@ -1,4 +1,5 @@
-import { BrowserEvent } from "@clarity-types/core";
+import { BrowserEvent, Constant } from "@clarity-types/core";
+import api from "./api";
 import measure from "./measure";
 
 let bindings: BrowserEvent[] = [];
@@ -8,7 +9,7 @@ export function bind(target: EventTarget, event: string, listener: EventListener
     // Wrapping following lines inside try / catch to cover edge cases where we might try to access an inaccessible element.
     // E.g. Iframe may start off as same-origin but later turn into cross-origin, and the following lines will throw an exception.
     try {
-      target.addEventListener(event, listener, capture);
+      target[api(Constant.AddEventListener)](event, listener, capture);
       bindings.push({ event, target, listener, capture });
     } catch { /* do nothing */ }
 }
@@ -18,7 +19,7 @@ export function reset(): void {
   for (let binding of bindings) {
     // Wrapping inside try / catch to avoid situations where the element may be destroyed before we get a chance to unbind
     try {
-      binding.target.removeEventListener(binding.event, binding.listener, binding.capture);
+      binding.target[api(Constant.RemoveEventListener)](binding.event, binding.listener, binding.capture);
     } catch { /* do nothing */ }
   }
   bindings = [];

package/src/core/history.ts

@@ -2,7 +2,7 @@ import { Code, Constant, Setting, Severity } from "@clarity-types/data";
 import * as clarity from "@src/clarity";
 import { bind } from "@src/core/event";
 import * as internal from "@src/diagnostic/internal";
-
+import * as core from "@src/core"
 let pushState = null;
 let replaceState = null;
 let url = null;
@@ -12,24 +12,29 @@ export function start(): void {
     url = getCurrentUrl();
     count = 0;
     bind(window, "popstate", compute);
-    
+
     // Add a proxy to history.pushState function
-    if (pushState === null) { pushState = history.pushState; }
-    history.pushState = function(): void {
-        if (check()) {
+    if (pushState === null) { 
+        pushState = history.pushState; 
+        history.pushState = function(): void {
             pushState.apply(this, arguments);
-            compute();
-        }
-    };
+            if (core.active() && check()) {
+                compute();
+            }
+        };
+    }
 
     // Add a proxy to history.replaceState function
-    if (replaceState === null) { replaceState = history.replaceState; }
-    history.replaceState = function(): void {
-        if (check()) {
+    if (replaceState === null) 
+    { 
+        replaceState = history.replaceState; 
+        history.replaceState = function(): void {
             replaceState.apply(this, arguments);
-            compute();
-        }
-    };
+            if (core.active() && check()) {
+                compute();
+            }
+        };
+    }
 }
 
 function check(): boolean {
@@ -54,18 +59,6 @@ function getCurrentUrl(): string {
 }
 
 export function stop(): void {
-    // Restore original function definition of history.pushState
-    if (pushState !== null) {
-        history.pushState = pushState;
-        pushState = null;
-    }
-
-    // Restore original function definition of history.replaceState
-    if (replaceState !== null) {
-        history.replaceState = replaceState;
-        replaceState = null;
-    }
-    
     url = null;
     count = 0;
 }

package/src/core/version.ts

@@ -1,2 +1,2 @@
-let version = "0.6.31";
+let version = "0.6.34";
 export default version;

package/src/data/encode.ts

@@ -9,6 +9,7 @@ import * as ping from "@src/data/ping";
 import * as summary from "@src/data/summary";
 import * as upgrade from "@src/data/upgrade";
 import * as variable from "@src/data/variable";
+import * as extract from "@src/data/extract";
 import { queue, track } from "./upload";
 
 export default function(event: Event): void {
@@ -105,5 +106,13 @@ export default function(event: Event): void {
                 queue(tokens, false);
             }
             break;
+        case Event.Extract:
+            let extractKeys = extract.keys;
+            for (let e of extractKeys) {
+                tokens.push(e);
+                tokens.push(extract.data[e]);
+            }
+            extract.reset();
+            queue(tokens, false);
     }
 }

package/src/data/extract.ts

@@ -0,0 +1,143 @@
+import { ExtractSource, Syntax, Type } from "@clarity-types/core";
+import { Event, Setting, ExtractData } from "@clarity-types/data";
+import config from "@src/core/config";
+import encode from "./encode";
+import * as internal from "@src/diagnostic/internal";
+import { Code, Constant, Severity } from "@clarity-types/data";
+
+export let data: ExtractData = {};
+export let keys: (number | string)[] = [];
+
+let variables : { [key: number]: Syntax[] } = {};
+let selectors : { [key: number]: string } = {};
+export let fragments: string[] = [];
+
+export function start(): void {
+    try {
+        let e = config.extract;
+        if (!e) { return; }
+        for (let i = 0; i < e.length; i+=3) {
+            let source = e[i] as ExtractSource;
+            let key = e[i+1] as number;
+            switch (source) {
+                case ExtractSource.Javascript:
+                    let variable = e[i+2] as string;
+                    variables[key] = parse(variable);
+                    break;
+                case ExtractSource.Cookie:
+                    /*Todo: Add cookie extract logic*/
+                    break;
+                case ExtractSource.Text:
+                    let match = e[i+2] as string;
+                    selectors[key] = match;
+                    break;
+                case ExtractSource.Fragment:
+                    fragments =  e[i+2] as string[];
+                    break;
+            }
+        }
+    }
+    catch(e) {
+        internal.log(Code.Config, Severity.Warning, e ? e.name : null);
+    }
+}
+
+export function clone(v: Syntax[]): Syntax[] {
+    return JSON.parse(JSON.stringify(v));
+}
+
+export function compute(): void {
+    try {
+        for (let v in variables) {
+            let value = str(evaluate(clone(variables[v])));
+            if (value) { update(v, value); }
+        }
+
+        for (let s in selectors) {
+            let node = document.querySelector(selectors[s] as string) as HTMLElement;
+            if (node) { update(s, node.innerText); }
+        }
+    }
+    catch (e) { internal.log(Code.Selector, Severity.Warning, e ? e.name : null); }
+
+    encode(Event.Extract);
+}
+
+export function reset(): void {
+    keys = [];
+}
+
+export function update(key: string, value: string | number, force: boolean = false): void {
+    if (!(key in data) || (key in data && data[key] !== value) || force ) {
+        data[key] = value;
+        keys.push(key);
+    }
+}
+
+export function stop(): void {
+    data = {};
+    keys = [];
+    variables = {};
+    selectors = {};
+}
+
+function parse(variable: string): Syntax[] {
+    let syntax: Syntax[] = [];
+    let parts = variable.split(Constant.Dot);
+    while (parts.length > 0) {
+        let part = parts.shift();
+        let arrayStart = part.indexOf(Constant.ArrayStart);
+        let conditionStart = part.indexOf(Constant.ConditionStart);
+        let conditionEnd = part.indexOf(Constant.ConditionEnd);
+        syntax.push({
+            name : arrayStart > 0 ? part.substring(0, arrayStart) : (conditionStart > 0 ? part.substring(0, conditionStart) : part),
+            type : arrayStart > 0 ? Type.Array : (conditionStart > 0 ? Type.Object : Type.Simple),
+            condition : conditionStart > 0 ? part.substring(conditionStart + 1, conditionEnd) : null
+        });
+    }
+
+    return syntax;
+}
+
+// The function below takes in a variable name in following format: "a.b.c" and safely evaluates its value in javascript context
+// For instance, for a.b.c, it will first check window["a"]. If it exists, it will recursively look at: window["a"]["b"] and finally,
+// return the value for window["a"]["b"]["c"].
+function evaluate(variable: Syntax[], base: Object = window): any {
+    if (variable.length == 0) { return base; }
+    let part = variable.shift();
+    let output;
+    if (base && base[part.name]) {
+        let obj = base[part.name];
+        if (part.type !== Type.Array && match(obj, part.condition)) {
+            output = evaluate(variable, obj);
+        }
+        else if (Array.isArray(obj)) {
+            let filtered = [];
+            for (var value of obj) {
+                if (match(value, part.condition)) {
+                    let op = evaluate(variable, value)
+                    if (op) { filtered.push(op); }
+                }
+            }
+            output = filtered;
+        }
+        
+        return output;
+    }
+
+    return null;
+}
+
+function str(input: string): string {
+    // Automatically trim string to max of Setting.ExtractLimit to avoid fetching long strings
+    return input ? JSON.stringify(input).substring(0, Setting.ExtractLimit) : input;
+}
+
+function match(base: Object, condition: string): boolean {
+    if (condition) {
+        let prop = condition.split(":");
+        return prop.length > 1 ? base[prop[0]] == prop[1] : base[prop[0]]
+    }
+
+    return true;
+}

package/src/data/index.ts

@@ -11,12 +11,13 @@ import * as summary from "@src/data/summary";
 import * as upgrade from "@src/data/upgrade";
 import * as upload from "@src/data/upload";
 import * as variable from "@src/data/variable";
+import * as extract from "@src/data/extract";
 export { event } from "@src/data/custom";
 export { consent, metadata } from "@src/data/metadata";
 export { upgrade } from "@src/data/upgrade";
 export { set, identify } from "@src/data/variable";
 
-const modules: Module[] = [baseline, dimension, variable, limit, summary, metadata, envelope, upload, ping, upgrade];
+const modules: Module[] = [baseline, dimension, variable, limit, summary, metadata, envelope, upload, ping, upgrade, extract];
 
 export function start(): void {
     // Metric needs to be initialized before we can start measuring. so metric is not wrapped in measure
@@ -40,4 +41,5 @@ export function compute(): void {
     metric.compute();
     summary.compute();
     limit.compute();
+    extract.compute();
 }

package/src/data/metadata.ts

@@ -1,5 +1,5 @@
 import { Time } from "@clarity-types/core";
-import { BooleanFlag, Constant, Dimension, Metadata, MetadataCallback, Metric, Session, User, Setting } from "@clarity-types/data";
+import { BooleanFlag, Constant, Dimension, Metadata, MetadataCallback, MetadataCallbackOptions, Metric, Session, User, Setting } from "@clarity-types/data";
 import * as core from "@src/core";
 import config from "@src/core/config";
 import hash from "@src/core/hash";
@@ -8,11 +8,10 @@ import * as metric from "@src/data/metric";
 import { set } from "@src/data/variable";
 
 export let data: Metadata = null;
-export let callback: MetadataCallback = null;
+export let callbacks: MetadataCallbackOptions[] = [];
 let rootDomain = null;
 
 export function start(): void {
-  callback = null;
   rootDomain = null;
   const ua = navigator && "userAgent" in navigator ? navigator.userAgent : Constant.Empty;
   const title = document && document.title ? document.title : Constant.Empty;
@@ -84,7 +83,6 @@ export function userAgentData(): void {
 }
 
 export function stop(): void {
-  callback = null;
   rootDomain = null;
   data = null;
 }
@@ -93,10 +91,9 @@ export function metadata(cb: MetadataCallback, wait: boolean = true): void {
   if (data && wait === false) {
     // Immediately invoke the callback if the caller explicitly doesn't want to wait for the upgrade confirmation
     cb(data, !config.lean);
-  } else {
-    // Save the callback for future reference; so we can inform the caller when page gets upgraded and we have a valid playback flag
-    callback = cb;
   }
+  
+  callbacks.push({callback: cb, wait: wait });
 }
 
 export function id(): string {
@@ -127,12 +124,20 @@ function tab(): string {
 
 export function save(): void {
   let ts = Math.round(Date.now());
-  let upgrade = config.lean ? BooleanFlag.False : BooleanFlag.True;
   let upload = config.upload && typeof config.upload === Constant.String ? (config.upload as string).replace(Constant.HTTPS, Constant.Empty) : Constant.Empty;
-  if (upgrade && callback) { callback(data, !config.lean); }
+  let upgrade = config.lean ? BooleanFlag.False : BooleanFlag.True;
+  processCallback(upgrade);
   setCookie(Constant.SessionKey, [data.sessionId, ts, data.pageNum, upgrade, upload].join(Constant.Pipe), Setting.SessionExpire);
 }
 
+function processCallback(upgrade: BooleanFlag) {
+  if (callbacks.length > 0) {
+    callbacks.forEach(x => {
+      if (x.callback && (!x.wait || upgrade)) { x.callback(data, !config.lean); }
+    })
+  }
+}
+
 function supported(target: Window | Document, api: string): boolean {
   try { return !!target[api]; } catch { return false; }
 }

package/src/data/upload.ts

@@ -42,7 +42,7 @@ export function queue(tokens: Token[], transmit: boolean = true): void {
         let now = time();
         let type = tokens.length > 1 ? tokens[1] : null;
         let event = JSON.stringify(tokens);
-        
+
         switch (type) {
             case Event.Discover:
                 discoverBytes += event.length;
@@ -118,12 +118,12 @@ async function upload(final: boolean = false): Promise<void> {
 
     let p = sendPlaybackBytes ? `[${playback.join()}]` : Constant.Empty;
     let encoded: EncodedPayload = {e, a, p};
-    
+
     // Get the payload ready for sending over the wire
     // We also attempt to compress the payload if it is not the last payload and the browser supports it
     // In all other cases, we continue to send back string value
     let payload = stringify(encoded);
-    let zipped = last ? null : await compress(payload) 
+    let zipped = last ? null : await compress(payload)
     metric.sum(Metric.TotalBytes, zipped ? zipped.length : payload.length);
     send(payload, zipped, envelope.data.sequence, last);
 
@@ -151,8 +151,11 @@ function send(payload: string, zipped: Uint8Array, sequence: number, beacon: boo
         // However, we don't want to rely on it for every payload, since we have no ability to retry if the upload failed.
         // Also, in case of sendBeacon, we do not have a way to alter HTTP headers and therefore can't send compressed payload
         if (beacon && "sendBeacon" in navigator) {
-            dispatched = navigator.sendBeacon(url, payload);
-            if (dispatched) { done(sequence); }
+            try {
+                // Navigator needs to be bound to sendBeacon before it is used to avoid errors in some browsers
+                dispatched = navigator.sendBeacon.bind(navigator)(url, payload);
+                if (dispatched) { done(sequence); }
+            } catch { /* do nothing - and we will automatically fallback to XHR below */ }
         }
 
         // Before initiating XHR upload, we check if the data has already been uploaded using sendBeacon
@@ -189,7 +192,7 @@ function check(xhr: XMLHttpRequest, sequence: number): void {
     if (xhr && xhr.readyState === XMLReadyState.Done && transitData) {
         // Attempt send payload again (as configured in settings) if we do not receive a success (2XX) response code back from the server
         if ((xhr.status < 200 || xhr.status > 208) && transitData.attempts <= Setting.RetryLimit) {
-            // We re-attempt in all cases except when server explicitly rejects our request with 4XX error 
+            // We re-attempt in all cases except when server explicitly rejects our request with 4XX error
             if (xhr.status >= 400 && xhr.status < 500) {
                 // In case of a 4XX response from the server, we bail out instead of trying again
                 limit.trigger(Check.Server);
@@ -212,7 +215,7 @@ function check(xhr: XMLHttpRequest, sequence: number): void {
             // Handle response if it was a 200 response with a valid body
             if (xhr.status === 200 && xhr.responseText) { response(xhr.responseText); }
             // If we exhausted our retries then trigger Clarity's shutdown for this page since the data will be incomplete
-            if (xhr.status === 0) { 
+            if (xhr.status === 0) {
                 // And, right before we terminate the session, we will attempt one last time to see if we can use
                 // different transport option (sendBeacon vs. XHR) to get this data to the server for analysis purposes
                 send(transitData.data, null, sequence, true);
@@ -233,7 +236,7 @@ function done(sequence: number): void {
 
 function delay(): number {
     // Progressively increase delay as we continue to send more payloads from the client to the server
-    // If we are not uploading data to a server, and instead invoking UploadCallback, in that case keep returning configured value 
+    // If we are not uploading data to a server, and instead invoking UploadCallback, in that case keep returning configured value
     let gap = config.lean === false && discoverBytes > 0 ? Setting.MinUploadDelay : envelope.data.sequence * config.delay;
     return typeof config.upload === Constant.String ? Math.max(Math.min(gap, Setting.MaxUploadDelay), Setting.MinUploadDelay) : config.delay;
 }

package/src/diagnostic/internal.ts

@@ -1,7 +1,5 @@
-import { Code, Constant, Event, Severity } from "@clarity-types/data";
+import { Code, Event, Severity } from "@clarity-types/data";
 import { LogData } from "@clarity-types/diagnostic";
-import config from "@src/core/config";
-import { bind } from "@src/core/event";
 import encode from "./encode";
 
 let history: { [key: number]: string[] } = {};
@@ -9,7 +7,6 @@ export let data: LogData;
 
 export function start(): void {
     history = {};
-    bind(document, "securitypolicyviolation", csp);
 }
 
 export function log(code: Code, severity: Severity, name: string = null, message: string = null, stack: string = null): void {
@@ -26,17 +23,6 @@ export function log(code: Code, severity: Severity, name: string = null, message
     encode(Event.Log);
 }
 
-function csp(e: SecurityPolicyViolationEvent): void {
-    let upload = config.upload as string;
-    // Look for first "/" starting after initial "https://" string
-    let parts = upload && typeof upload === Constant.String ? upload.substr(0, upload.indexOf("/", Constant.HTTPS.length)).split(Constant.Dot) : []; 
-    let domain = parts.length >= 2 ? parts.splice(-2).join(Constant.Dot) : null;
-    // Capture content security policy violation only if disposition value is not explicitly set to "report"
-    if (domain && e.blockedURI && e.blockedURI.indexOf(domain) >= 0 && e["disposition"] !== Constant.Report) {
-        log(Code.ContentSecurityPolicy, Severity.Warning, e.blockedURI);
-    }
-}
-
 export function stop(): void {
     history = {};
 }

package/src/layout/dom.ts

@@ -4,10 +4,10 @@ import { Constant, NodeInfo, NodeValue, Selector, SelectorInput, Source } from "
 import config from "@src/core/config";
 import hash from "@src/core/hash";
 import * as internal from "@src/diagnostic/internal";
-import * as extract from "@src/layout/extract";
 import * as region from "@src/layout/region";
 import selector from "@src/layout/selector";
-
+import * as mutation from "@src/layout/mutation";
+import * as extract from "@src/data/extract";
 let index: number = 1;
 
 // Reference: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/Input#%3Cinput%3E_types
@@ -21,6 +21,7 @@ let updateMap: number[] = [];
 let hashMap: { [hash: string]: number } = {};
 let override = [];
 let unmask = [];
+let updatedFragments: { [fragment: number]: string } = {};
 
 // The WeakMap object is a collection of key/value pairs in which the keys are weakly referenced
 let idMap: WeakMap<Node, number> = null; // Maps node => id.
@@ -61,9 +62,7 @@ export function parse(root: ParentNode, init: boolean = false): void {
         // Since mutations may happen on leaf nodes too, e.g. text nodes, which may not support all selector APIs.
         // We ensure that the root note supports querySelectorAll API before executing the code below to identify new regions.
         if ("querySelectorAll" in root) {
-            extract.regions(root, config.regions);
-            extract.metrics(root, config.metrics);
-            extract.dimensions(root, config.dimensions);
+            config.regions.forEach(x => root.querySelectorAll(x[1]).forEach(e => region.observe(e, `${x[0]}`))); // Regions
             config.mask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.TextImage))); // Masked Elements
             unmask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.None))); // Unmasked Elements
         }
@@ -88,11 +87,13 @@ export function add(node: Node, parent: Node, data: NodeInfo, source: Source): v
     let privacy = config.content ? Privacy.Sensitive : Privacy.Text;
     let parentValue = null;
     let regionId = region.exists(node) ? id : null;
+    let fragmentId = null;
 
     if (parentId >= 0 && values[parentId]) {
         parentValue = values[parentId];
         parentValue.children.push(id);
         regionId = regionId === null ? parentValue.region : regionId;
+        fragmentId = parentValue.fragment;
         privacy = parentValue.metadata.privacy;
     }
 
@@ -115,12 +116,13 @@ export function add(node: Node, parent: Node, data: NodeInfo, source: Source): v
         selector: null,
         hash: null,
         region: regionId,
-        metadata: { active: true, suspend: false, privacy, position: null, size: null }
+        metadata: { active: true, suspend: false, privacy, position: null, size: null },
+        fragment: fragmentId,
     };
 
     updateSelector(values[id]);
     size(values[id], parentValue);
-    track(id, source);
+    track(id, source, values[id].fragment);
 }
 
 export function update(node: Node, parent: Node, data: NodeInfo, source: Source): void {
@@ -174,9 +176,14 @@ export function update(node: Node, parent: Node, data: NodeInfo, source: Source)
             }
         }
 
+        // track node if it is a part of scheduled fragment mutation
+        if(value.fragment && updatedFragments[value.fragment]) {
+            changed = true;
+        }
+
         // Update selector
         updateSelector(value);
-        track(id, source, changed, parentChanged);
+        track(id, source, values[id].fragment, changed, parentChanged);
     }
 }
 
@@ -296,6 +303,9 @@ function updateSelector(value: NodeValue): void {
     value.selector = [selector(s), selector(s, true)];
     value.hash = value.selector.map(x => x ? hash(x) : null) as [string, string];
     value.hash.forEach(h => hashMap[h] = value.id);
+    if (value.hash.some(h => extract.fragments.indexOf(h) !== -1)) {
+        value.fragment = value.id;
+    }
 }
 
 export function getNode(id: number): Node {
@@ -331,6 +341,11 @@ export function updates(): NodeValue[] {
         if (id in values) { output.push(values[id]); }
     }
     updateMap = [];
+    for (let id in updatedFragments) {
+        extract.update(updatedFragments[id], id, true)
+    }
+
+    updatedFragments = {}
     return output;
 }
 
@@ -368,7 +383,19 @@ function getPreviousId(node: Node): number {
     return id;
 }
 
-function track(id: number, source: Source, changed: boolean = true, parentChanged: boolean = false): void {
+function track(id: number, source: Source, fragment: number = null, changed: boolean = true, parentChanged: boolean = false): void {
+    // if updated node is a part of fragment and the fragment is not being tracked currently, schedule a mutation on the fragment node
+    if (fragment && !updatedFragments[fragment]) {
+        let node = getNode(fragment)
+        let value = getValue(fragment);
+        if (node && value) {
+            mutation.schedule(node, true);
+            value.hash.forEach(h => {
+                if(extract.fragments.indexOf(h) !== -1) { updatedFragments[fragment] = h;}
+            });
+        }
+    }
+
     // Keep track of the order in which mutations happened, they may not be sequential
     // Edge case: If an element is added later on, and pre-discovered element is moved as a child.
     // In that case, we need to reorder the pre-discovered element in the update list to keep visualization consistent.