clarity-js 0.6.32 → 0.6.35

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-js",
-  "version": "0.6.32",
+  "version": "0.6.35",
   "description": "An analytics library that uses web page interactions to generate aggregated insights",
   "author": "Microsoft Corp.",
   "license": "MIT",

package/src/core/config.ts

@@ -9,9 +9,9 @@ let config: Config = {
     mask: [],
     unmask: [],
     regions: [],
-    metrics: [],
-    dimensions: [],
+    extract: [],
     cookies: [],
+    fraud: [],
     report: null,
     upload: null,
     fallback: null,

package/src/core/history.ts

@@ -1,7 +1,9 @@
-import { Code, Constant, Setting, Severity } from "@clarity-types/data";
+import { BooleanFlag, Code, Constant, Metric, Setting, Severity } from "@clarity-types/data";
 import * as clarity from "@src/clarity";
+import * as core from "@src/core"
 import { bind } from "@src/core/event";
 import * as internal from "@src/diagnostic/internal";
+import * as metric from "@src/data/metric";
 
 let pushState = null;
 let replaceState = null;
@@ -12,24 +14,29 @@ export function start(): void {
     url = getCurrentUrl();
     count = 0;
     bind(window, "popstate", compute);
-    
+
     // Add a proxy to history.pushState function
-    if (pushState === null) { pushState = history.pushState; }
-    history.pushState = function(): void {
-        if (check()) {
+    if (pushState === null) { 
+        pushState = history.pushState; 
+        history.pushState = function(): void {
             pushState.apply(this, arguments);
-            compute();
-        }
-    };
+            if (core.active() && check()) {
+                compute();
+            }
+        };
+    }
 
     // Add a proxy to history.replaceState function
-    if (replaceState === null) { replaceState = history.replaceState; }
-    history.replaceState = function(): void {
-        if (check()) {
+    if (replaceState === null) 
+    { 
+        replaceState = history.replaceState; 
+        history.replaceState = function(): void {
             replaceState.apply(this, arguments);
-            compute();
-        }
-    };
+            if (core.active() && check()) {
+                compute();
+            }
+        };
+    }
 }
 
 function check(): boolean {
@@ -45,27 +52,20 @@ function compute(): void {
     if (url !== getCurrentUrl()) {
         // If the url changed, start tracking it as a new page
         clarity.stop();
-        window.setTimeout(clarity.start, Setting.RestartDelay);
+        window.setTimeout(restart, Setting.RestartDelay);
     }
 }
 
+function restart(): void {
+    clarity.start();
+    metric.max(Metric.SinglePage, BooleanFlag.True);
+}
+
 function getCurrentUrl(): string {
     return location.href ? location.href.replace(location.hash, Constant.Empty) : location.href;
 }
 
 export function stop(): void {
-    // Restore original function definition of history.pushState
-    if (pushState !== null) {
-        history.pushState = pushState;
-        pushState = null;
-    }
-
-    // Restore original function definition of history.replaceState
-    if (replaceState !== null) {
-        history.replaceState = replaceState;
-        replaceState = null;
-    }
-    
     url = null;
     count = 0;
 }

package/src/core/version.ts

@@ -1,2 +1,2 @@
-let version = "0.6.32";
+let version = "0.6.35";
 export default version;

package/src/data/encode.ts

@@ -9,6 +9,7 @@ import * as ping from "@src/data/ping";
 import * as summary from "@src/data/summary";
 import * as upgrade from "@src/data/upgrade";
 import * as variable from "@src/data/variable";
+import * as extract from "@src/data/extract";
 import { queue, track } from "./upload";
 
 export default function(event: Event): void {
@@ -105,5 +106,13 @@ export default function(event: Event): void {
                 queue(tokens, false);
             }
             break;
+        case Event.Extract:
+            let extractKeys = extract.keys;
+            for (let e of extractKeys) {
+                tokens.push(e);
+                tokens.push(extract.data[e]);
+            }
+            extract.reset();
+            queue(tokens, false);
     }
 }

package/src/data/extract.ts

@@ -0,0 +1,143 @@
+import { ExtractSource, Syntax, Type } from "@clarity-types/core";
+import { Event, Setting, ExtractData } from "@clarity-types/data";
+import config from "@src/core/config";
+import encode from "./encode";
+import * as internal from "@src/diagnostic/internal";
+import { Code, Constant, Severity } from "@clarity-types/data";
+
+export let data: ExtractData = {};
+export let keys: (number | string)[] = [];
+
+let variables : { [key: number]: Syntax[] } = {};
+let selectors : { [key: number]: string } = {};
+export let fragments: string[] = [];
+
+export function start(): void {
+    try {
+        let e = config.extract;
+        if (!e) { return; }
+        for (let i = 0; i < e.length; i+=3) {
+            let source = e[i] as ExtractSource;
+            let key = e[i+1] as number;
+            switch (source) {
+                case ExtractSource.Javascript:
+                    let variable = e[i+2] as string;
+                    variables[key] = parse(variable);
+                    break;
+                case ExtractSource.Cookie:
+                    /*Todo: Add cookie extract logic*/
+                    break;
+                case ExtractSource.Text:
+                    let match = e[i+2] as string;
+                    selectors[key] = match;
+                    break;
+                case ExtractSource.Fragment:
+                    fragments =  e[i+2] as string[];
+                    break;
+            }
+        }
+    }
+    catch(e) {
+        internal.log(Code.Config, Severity.Warning, e ? e.name : null);
+    }
+}
+
+export function clone(v: Syntax[]): Syntax[] {
+    return JSON.parse(JSON.stringify(v));
+}
+
+export function compute(): void {
+    try {
+        for (let v in variables) {
+            let value = str(evaluate(clone(variables[v])));
+            if (value) { update(v, value); }
+        }
+
+        for (let s in selectors) {
+            let node = document.querySelector(selectors[s] as string) as HTMLElement;
+            if (node) { update(s, node.innerText); }
+        }
+    }
+    catch (e) { internal.log(Code.Selector, Severity.Warning, e ? e.name : null); }
+
+    encode(Event.Extract);
+}
+
+export function reset(): void {
+    keys = [];
+}
+
+export function update(key: string, value: string | number, force: boolean = false): void {
+    if (!(key in data) || (key in data && data[key] !== value) || force ) {
+        data[key] = value;
+        keys.push(key);
+    }
+}
+
+export function stop(): void {
+    data = {};
+    keys = [];
+    variables = {};
+    selectors = {};
+}
+
+function parse(variable: string): Syntax[] {
+    let syntax: Syntax[] = [];
+    let parts = variable.split(Constant.Dot);
+    while (parts.length > 0) {
+        let part = parts.shift();
+        let arrayStart = part.indexOf(Constant.ArrayStart);
+        let conditionStart = part.indexOf(Constant.ConditionStart);
+        let conditionEnd = part.indexOf(Constant.ConditionEnd);
+        syntax.push({
+            name : arrayStart > 0 ? part.substring(0, arrayStart) : (conditionStart > 0 ? part.substring(0, conditionStart) : part),
+            type : arrayStart > 0 ? Type.Array : (conditionStart > 0 ? Type.Object : Type.Simple),
+            condition : conditionStart > 0 ? part.substring(conditionStart + 1, conditionEnd) : null
+        });
+    }
+
+    return syntax;
+}
+
+// The function below takes in a variable name in following format: "a.b.c" and safely evaluates its value in javascript context
+// For instance, for a.b.c, it will first check window["a"]. If it exists, it will recursively look at: window["a"]["b"] and finally,
+// return the value for window["a"]["b"]["c"].
+function evaluate(variable: Syntax[], base: Object = window): any {
+    if (variable.length == 0) { return base; }
+    let part = variable.shift();
+    let output;
+    if (base && base[part.name]) {
+        let obj = base[part.name];
+        if (part.type !== Type.Array && match(obj, part.condition)) {
+            output = evaluate(variable, obj);
+        }
+        else if (Array.isArray(obj)) {
+            let filtered = [];
+            for (var value of obj) {
+                if (match(value, part.condition)) {
+                    let op = evaluate(variable, value)
+                    if (op) { filtered.push(op); }
+                }
+            }
+            output = filtered;
+        }
+        
+        return output;
+    }
+
+    return null;
+}
+
+function str(input: string): string {
+    // Automatically trim string to max of Setting.ExtractLimit to avoid fetching long strings
+    return input ? JSON.stringify(input).substring(0, Setting.ExtractLimit) : input;
+}
+
+function match(base: Object, condition: string): boolean {
+    if (condition) {
+        let prop = condition.split(":");
+        return prop.length > 1 ? base[prop[0]] == prop[1] : base[prop[0]]
+    }
+
+    return true;
+}

package/src/data/index.ts

@@ -11,12 +11,13 @@ import * as summary from "@src/data/summary";
 import * as upgrade from "@src/data/upgrade";
 import * as upload from "@src/data/upload";
 import * as variable from "@src/data/variable";
+import * as extract from "@src/data/extract";
 export { event } from "@src/data/custom";
 export { consent, metadata } from "@src/data/metadata";
 export { upgrade } from "@src/data/upgrade";
 export { set, identify } from "@src/data/variable";
 
-const modules: Module[] = [baseline, dimension, variable, limit, summary, metadata, envelope, upload, ping, upgrade];
+const modules: Module[] = [baseline, dimension, variable, limit, summary, metadata, envelope, upload, ping, upgrade, extract];
 
 export function start(): void {
     // Metric needs to be initialized before we can start measuring. so metric is not wrapped in measure
@@ -40,4 +41,5 @@ export function compute(): void {
     metric.compute();
     summary.compute();
     limit.compute();
+    extract.compute();
 }

package/src/data/metadata.ts

@@ -1,5 +1,5 @@
 import { Time } from "@clarity-types/core";
-import { BooleanFlag, Constant, Dimension, Metadata, MetadataCallback, Metric, Session, User, Setting } from "@clarity-types/data";
+import { BooleanFlag, Constant, Dimension, Metadata, MetadataCallback, MetadataCallbackOptions, Metric, Session, User, Setting } from "@clarity-types/data";
 import * as core from "@src/core";
 import config from "@src/core/config";
 import hash from "@src/core/hash";
@@ -8,11 +8,10 @@ import * as metric from "@src/data/metric";
 import { set } from "@src/data/variable";
 
 export let data: Metadata = null;
-export let callback: MetadataCallback = null;
+export let callbacks: MetadataCallbackOptions[] = [];
 let rootDomain = null;
 
 export function start(): void {
-  callback = null;
   rootDomain = null;
   const ua = navigator && "userAgent" in navigator ? navigator.userAgent : Constant.Empty;
   const title = document && document.title ? document.title : Constant.Empty;
@@ -40,7 +39,6 @@ export function start(): void {
   dimension.log(Dimension.DocumentDirection, document.dir);
   if (navigator) {
     dimension.log(Dimension.Language, (<any>navigator).userLanguage || navigator.language);
-    metric.max(Metric.Automation, navigator.webdriver ? BooleanFlag.True : BooleanFlag.False);
     userAgentData();
   }
 
@@ -64,7 +62,7 @@ export function start(): void {
   track(u);
 }
 
-export function userAgentData(): void {
+function userAgentData(): void {
   if (navigator["userAgentData"] && navigator["userAgentData"].getHighEntropyValues) {
     navigator["userAgentData"].getHighEntropyValues(
       ["model",
@@ -84,7 +82,6 @@ export function userAgentData(): void {
 }
 
 export function stop(): void {
-  callback = null;
   rootDomain = null;
   data = null;
 }
@@ -93,10 +90,9 @@ export function metadata(cb: MetadataCallback, wait: boolean = true): void {
   if (data && wait === false) {
     // Immediately invoke the callback if the caller explicitly doesn't want to wait for the upgrade confirmation
     cb(data, !config.lean);
-  } else {
-    // Save the callback for future reference; so we can inform the caller when page gets upgraded and we have a valid playback flag
-    callback = cb;
   }
+  
+  callbacks.push({callback: cb, wait: wait });
 }
 
 export function id(): string {
@@ -127,12 +123,20 @@ function tab(): string {
 
 export function save(): void {
   let ts = Math.round(Date.now());
-  let upgrade = config.lean ? BooleanFlag.False : BooleanFlag.True;
   let upload = config.upload && typeof config.upload === Constant.String ? (config.upload as string).replace(Constant.HTTPS, Constant.Empty) : Constant.Empty;
-  if (upgrade && callback) { callback(data, !config.lean); }
+  let upgrade = config.lean ? BooleanFlag.False : BooleanFlag.True;
+  processCallback(upgrade);
   setCookie(Constant.SessionKey, [data.sessionId, ts, data.pageNum, upgrade, upload].join(Constant.Pipe), Setting.SessionExpire);
 }
 
+function processCallback(upgrade: BooleanFlag) {
+  if (callbacks.length > 0) {
+    callbacks.forEach(x => {
+      if (x.callback && (!x.wait || upgrade)) { x.callback(data, !config.lean); }
+    })
+  }
+}
+
 function supported(target: Window | Document, api: string): boolean {
   try { return !!target[api]; } catch { return false; }
 }

package/src/data/metric.ts

@@ -15,11 +15,11 @@ export function stop(): void {
     updates = {};
 }
 
-export function count(metric: Metric, increment: number = 1): void {
+export function count(metric: Metric): void {
     if (!(metric in data)) { data[metric] = 0; }
     if (!(metric in updates)) { updates[metric] = 0; }
-    data[metric] += increment;
-    updates[metric] += increment;
+    data[metric]++;
+    updates[metric]++;
 }
 
 export function sum(metric: Metric, value: number): void {

package/src/diagnostic/encode.ts

@@ -1,6 +1,7 @@
 import { Event, Token } from "@clarity-types/data";
 import { time } from "@src/core/time";
 import { queue } from "@src/data/upload";
+import * as fraud from "@src/diagnostic/fraud";
 import * as internal from "@src/diagnostic/internal";
 import * as script from "@src/diagnostic/script";
 
@@ -26,5 +27,13 @@ export default async function (type: Event): Promise<void> {
                 queue(tokens, false);
             }
             break;
+        case Event.Fraud:
+            if (fraud.data) {
+                tokens.push(fraud.data.id);
+                tokens.push(fraud.data.target);
+                tokens.push(fraud.data.hash);
+                queue(tokens, false);
+            }
+            break;
     }
 }

package/src/diagnostic/fraud.ts

@@ -0,0 +1,29 @@
+import { BooleanFlag, Event, Metric, Setting } from "@clarity-types/data";
+import { FraudData } from "@clarity-types/diagnostic";
+import hash from "@src/core/hash";
+import * as metric from "@src/data/metric";
+import encode from "./encode";
+
+let history = [];
+export let data: FraudData;
+
+export function start(): void {
+    history = [];
+    metric.max(Metric.Automation, navigator.webdriver ? BooleanFlag.True : BooleanFlag.False);
+}
+
+export function check(id: number, target: number, input: string): void {
+    // Compute hash for fraud detection. Hash is computed only if input meets the minimum length criteria
+    if (id !== null && input && input.length >= Setting.WordLength) {
+        data = { id, target, hash: hash(input) };
+        // Only encode this event if we haven't already reported this hash
+        if (history.indexOf(data.hash) < 0) {
+            history.push(data.hash);
+            encode(Event.Fraud);
+        }
+    }
+}
+
+export function stop(): void {
+    history = [];
+}

package/src/diagnostic/index.ts

@@ -1,7 +1,9 @@
+import * as fraud from "./fraud";
 import * as internal from "./internal";
 import * as script from "./script";
 
 export function start(): void {
+    fraud.start();
     script.start();
     internal.start();
 }

package/src/diagnostic/script.ts

@@ -1,7 +1,6 @@
-import { Constant, Event, Setting } from "@clarity-types/data";
+import { Event, Setting } from "@clarity-types/data";
 import { ScriptErrorData } from "@clarity-types/diagnostic";
 import { bind } from "@src/core/event";
-import * as box from "@src/layout/box";
 import encode from "./encode";
 
 let history: { [key: string]: number } = {};
@@ -29,15 +28,6 @@ function handler(error: ErrorEvent): boolean {
             source: error["filename"]
         };
 
-        // In certain cases, ResizeObserver could lead to flood of benign errors - especially when video element is involved.
-        // Reference Chromium issue: https://bugs.chromium.org/p/chromium/issues/detail?id=809574
-        // Even though it doesn't impact user experience, or show up in console, it can still flood error reporting through on error
-        // To mitigate that, we turn off Clarity's ResizeObserver on getting the first instance of this error
-        if (e.message.indexOf(Constant.ResizeObserver) >= 0) {
-            box.stop();
-            return false;
-        }
-
         encode(Event.ScriptError);
     }
 

package/src/interaction/click.ts

@@ -65,7 +65,8 @@ function handler(event: Event, root: Node, evt: MouseEvent): void {
                 context: context(a),
                 text: text(t),
                 link: a ? a.href : null,
-                hash: null
+                hash: null,
+                trust: evt.isTrusted ? BooleanFlag.True : BooleanFlag.False
             }
         });
         schedule(encode.bind(this, event));

package/src/interaction/encode.ts

@@ -44,7 +44,7 @@ export default async function (type: Event): Promise<void> {
             break;
         case Event.Click:
             for (let entry of click.state) {
-                let cTarget = metadata(entry.data.target as Node, entry.event);
+                let cTarget = metadata(entry.data.target as Node, entry.event, entry.data.text);
                 tokens = [entry.time, entry.event];
                 let cHash = cTarget.hash.join(Constant.Dot);
                 tokens.push(cTarget.id);
@@ -58,6 +58,7 @@ export default async function (type: Event): Promise<void> {
                 tokens.push(scrub(entry.data.text, "click", cTarget.privacy));
                 tokens.push(entry.data.link);
                 tokens.push(cHash);
+                tokens.push(entry.data.trust);
                 queue(tokens);
                 timeline.track(entry.time, entry.event, cHash, entry.data.x, entry.data.y, entry.data.reaction, entry.data.context);
             }
@@ -91,10 +92,10 @@ export default async function (type: Event): Promise<void> {
             break;
         case Event.Input:
             for (let entry of input.state) {
-                let iTarget = metadata(entry.data.target as Node, entry.event);
+                let iTarget = metadata(entry.data.target as Node, entry.event, entry.data.value);
                 tokens = [entry.time, entry.event];
                 tokens.push(iTarget.id);
-                tokens.push(entry.data.value);
+                tokens.push(scrub(entry.data.value, "input", iTarget.privacy));
                 queue(tokens);
             }
             input.reset();

package/src/interaction/input.ts

@@ -1,7 +1,6 @@
 import { Event } from "@clarity-types/data";
 import { InputData, InputState, Setting } from "@clarity-types/interaction";
 import { bind } from "@src/core/event";
-import scrub from "@src/core/scrub";
 import { schedule } from "@src/core/task";
 import { time } from "@src/core/time";
 import { clearTimeout, setTimeout } from "@src/core/timeout";
@@ -24,18 +23,12 @@ function recompute(evt: UIEvent): void {
     let input = target(evt) as HTMLInputElement;
     let value = get(input);
     if (input && input.type && value) {
-        let v;
+        let v = input.value;
         switch (input.type) {
             case "radio":
             case "checkbox":
                 v = input.checked ? "true" : "false";
                 break;
-            case "range":
-                v = input.value;
-                break;
-            default:
-                v = scrub(input.value, "input", value.metadata.privacy);
-                break;
         }
 
         let data: InputData = { target: input, value: v };