clarity-js 0.7.3 → 0.7.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-js",
-  "version": "0.7.3",
+  "version": "0.7.5",
   "description": "An analytics library that uses web page interactions to generate aggregated insights",
   "author": "Microsoft Corp.",
   "license": "MIT",

package/src/core/config.ts

@@ -10,7 +10,6 @@ let config: Config = {
     mask: [],
     unmask: [],
     regions: [],
-    extract: [],
     cookies: [],
     fraud: true,
     checksum: [],

package/src/core/report.ts

@@ -19,7 +19,7 @@ export function report(e: Error): Error {
             // Using POST request instead of a GET request (img-src) to not violate existing CSP rules
             // Since, Clarity already uses XHR to upload data, we stick with similar POST mechanism for reporting too
             let xhr = new XMLHttpRequest();
-            xhr.open("POST", url);
+            xhr.open("POST", url, true);
             xhr.send(JSON.stringify(payload));
             history.push(e.message);
         }

package/src/core/version.ts

@@ -1,2 +1,2 @@
-let version = "0.7.3";
+let version = "0.7.5";
 export default version;

package/src/data/encode.ts

@@ -110,7 +110,7 @@ export default function(event: Event): void {
             let extractKeys = extract.keys;
             for (let e of extractKeys) {
                 tokens.push(e);
-                tokens.push(extract.data[e]);
+                tokens.push([].concat(...extract.data[e]));
             }
             extract.reset();
             queue(tokens, false);

package/src/data/extract.ts

@@ -1,38 +1,36 @@
 import { ExtractSource, Syntax, Type } from "@clarity-types/core";
 import { Event, Setting, ExtractData } from "@clarity-types/data";
-import config from "@src/core/config";
 import encode from "./encode";
 import * as internal from "@src/diagnostic/internal";
 import { Code, Constant, Severity } from "@clarity-types/data";
 
 export let data: ExtractData = {};
-export let keys: (number | string)[] = [];
-
-let variables : { [key: number]: Syntax[] } = {};
-let selectors : { [key: number]: string } = {};
-export let fragments: string[] = [];
+export let keys: number[] = [];
 
+let variables : { [key: number]: { [key: number]: Syntax[] }} = {};
+let selectors : { [key: number]: { [key: number]: string }} = {};
 export function start(): void {
+    reset();
+}
+
+export function trigger(input: string): void { 
     try {
-        let e = config.extract;
-        if (!e) { return; }
-        for (let i = 0; i < e.length; i+=3) {
-            let source = e[i] as ExtractSource;
-            let key = e[i+1] as number;
+        var parts = input && input.length > 0 ? input.split(/ (.*)/) : [Constant.Empty];
+        var key = parseInt(parts[0]);
+        var values = parts.length > 1 ? JSON.parse(parts[1]) : {};
+        variables[key] = {};
+        selectors[key] = {};
+        for (var v in values) {
+            let id = parseInt(v);
+            let value = values[v] as string;
+            let source = value.startsWith(Constant.Tilde) ? ExtractSource.Javascript : ExtractSource.Text;
             switch (source) {
                 case ExtractSource.Javascript:
-                    let variable = e[i+2] as string;
-                    variables[key] = parse(variable);
-                    break;
-                case ExtractSource.Cookie:
-                    /*Todo: Add cookie extract logic*/
+                    let variable = value.substring(1, value.length);
+                    variables[key][id] = parse(variable);
                     break;
                 case ExtractSource.Text:
-                    let match = e[i+2] as string;
-                    selectors[key] = match;
-                    break;
-                case ExtractSource.Fragment:
-                    fragments =  e[i+2] as string[];
+                    selectors[key][id] = value;
                     break;
             }
         }
@@ -49,13 +47,25 @@ export function clone(v: Syntax[]): Syntax[] {
 export function compute(): void {
     try {
         for (let v in variables) {
-            let value = str(evaluate(clone(variables[v])));
-            if (value) { update(v, value); }
-        }
+            let key = parseInt(v);
+            if (!(key in keys)) {
+                let variableData = variables[key];
+                for (let v in variableData) {
+                    let variableKey = parseInt(v);
+                    let value = str(evaluate(clone(variableData[variableKey])));
+                    if (value) { update(key, variableKey, value); }
+                }
 
-        for (let s in selectors) {
-            let node = document.querySelector(selectors[s] as string) as HTMLElement;
-            if (node) { update(s, node.innerText); }
+                let selectorData = selectors[key];
+                for (let s in selectorData) {
+                    let selectorKey = parseInt(s);
+                    let nodes = document.querySelectorAll(selectorData[selectorKey]) as NodeListOf<HTMLElement>;
+                    if (nodes) {
+                        let text = Array.from(nodes).map(e => e.innerText)
+                        update(key, selectorKey, text.join(Constant.Seperator).substring(0, Setting.ExtractLimit));
+                    }
+                }
+            }
         }
     }
     catch (e) { internal.log(Code.Selector, Severity.Warning, e ? e.name : null); }
@@ -64,21 +74,22 @@ export function compute(): void {
 }
 
 export function reset(): void {
+    data = {};
     keys = [];
+    variables = {};
+    selectors = {};
 }
 
-export function update(key: string, value: string | number, force: boolean = false): void {
-    if (!(key in data) || (key in data && data[key] !== value) || force ) {
-        data[key] = value;
+export function update(key: number, subkey: number,  value: string): void {
+    if (!(key in data)) {
+        data[key] = []
         keys.push(key);
     }
+    data[key].push([subkey, value]);
 }
 
 export function stop(): void {
-    data = {};
-    keys = [];
-    variables = {};
-    selectors = {};
+   reset();
 }
 
 function parse(variable: string): Syntax[] {

package/src/data/upload.ts

@@ -15,6 +15,8 @@ import * as metric from "@src/data/metric";
 import * as ping from "@src/data/ping";
 import * as timeline from "@src/interaction/timeline";
 import * as region from "@src/layout/region";
+import * as extract from "@src/data/extract";
+import { report } from "@src/core/report";
 
 let discoverBytes: number = 0;
 let playbackBytes: number = 0;
@@ -168,7 +170,9 @@ function send(payload: string, zipped: Uint8Array, sequence: number, beacon: boo
             // Not all browsers support compression API and the support for it in supported browsers is still experimental
             if (sequence in transit) { transit[sequence].attempts++; } else { transit[sequence] = { data: payload, attempts: 1 }; }
             let xhr = new XMLHttpRequest();
-            xhr.open("POST", url);
+            xhr.open("POST", url, true);
+            xhr.timeout = Setting.UploadTimeout;
+            xhr.ontimeout = () => { report(new Error(`${Constant.Timeout} : ${url}`)) };
             if (sequence !== null) { xhr.onreadystatechange = (): void => { measure(check)(xhr, sequence); }; }
             xhr.withCredentials = true;
             if (zipped) {
@@ -242,19 +246,26 @@ function delay(): number {
 }
 
 function response(payload: string): void {
-    let parts = payload && payload.length > 0 ? payload.split(" ") : [Constant.Empty];
-    switch (parts[0]) {
-        case Constant.End:
-            // Clear out session storage and end the session so we can start fresh the next time
-            limit.trigger(Check.Server);
-            break;
-        case Constant.Upgrade:
-            // Upgrade current session to send back playback information
-            clarity.upgrade(Constant.Auto);
-            break;
-        case Constant.Action:
-            // Invoke action callback, if configured and has a valid value
-            if (config.action && parts.length > 1) { config.action(parts[1]); }
-            break;
+    let lines = payload && payload.length > 0 ? payload.split("\n") : [];
+    for (var line of lines)
+    {
+        let parts = line && line.length > 0 ? line.split(/ (.*)/) : [Constant.Empty];
+        switch (parts[0]) {
+            case Constant.End:
+                // Clear out session storage and end the session so we can start fresh the next time
+                limit.trigger(Check.Server);
+                break;
+            case Constant.Upgrade:
+                // Upgrade current session to send back playback information
+                clarity.upgrade(Constant.Auto);
+                break;
+            case Constant.Action:
+                // Invoke action callback, if configured and has a valid value
+                if (config.action && parts.length > 1) { config.action(parts[1]); }
+                break;
+            case Constant.Extract:
+                if (parts.length > 1) { extract.trigger(parts[1]); }
+                break;
+        }
     }
 }

package/src/layout/dom.ts

@@ -6,8 +6,6 @@ import hash from "@src/core/hash";
 import * as internal from "@src/diagnostic/internal";
 import * as region from "@src/layout/region";
 import * as selector from "@src/layout/selector";
-import * as mutation from "@src/layout/mutation";
-import * as extract from "@src/data/extract";
 let index: number = 1;
 let nodes: Node[] = [];
 let values: NodeValue[] = [];
@@ -15,7 +13,6 @@ let updateMap: number[] = [];
 let hashMap: { [hash: string]: number } = {};
 let override = [];
 let unmask = [];
-let updatedFragments: { [fragment: number]: string } = {};
 let maskText = [];
 let maskExclude = [];
 let maskDisable = [];
@@ -92,14 +89,12 @@ export function add(node: Node, parent: Node, data: NodeInfo, source: Source): v
     let previousId = getPreviousId(node);
     let parentValue: NodeValue = null;
     let regionId = region.exists(node) ? id : null;
-    let fragmentId = null;
     let fraudId = fraudMap.has(node) ? fraudMap.get(node) : null;
     let privacyId = config.content ? Privacy.Sensitive : Privacy.TextImage
     if (parentId >= 0 && values[parentId]) {
         parentValue = values[parentId];
         parentValue.children.push(id);
         regionId = regionId === null ? parentValue.region : regionId;
-        fragmentId = parentValue.fragment;
         fraudId = fraudId === null ? parentValue.metadata.fraud : fraudId;
         privacyId = parentValue.metadata.privacy;
     }
@@ -121,13 +116,12 @@ export function add(node: Node, parent: Node, data: NodeInfo, source: Source): v
         hash: null,
         region: regionId,
         metadata: { active: true, suspend: false, privacy: privacyId, position: null, fraud: fraudId, size: null },
-        fragment: fragmentId,
     };
 
     privacy(node, values[id], parentValue);
     updateSelector(values[id]);
     size(values[id]);
-    track(id, source, values[id].fragment);
+    track(id, source);
 }
 
 export function update(node: Node, parent: Node, data: NodeInfo, source: Source): void {
@@ -181,14 +175,9 @@ export function update(node: Node, parent: Node, data: NodeInfo, source: Source)
             }
         }
 
-        // track node if it is a part of scheduled fragment mutation
-        if(value.fragment && updatedFragments[value.fragment]) {
-            changed = true;
-        }
-
         // Update selector
         updateSelector(value);
-        track(id, source, values[id].fragment, changed, parentChanged);
+        track(id, source, changed, parentChanged);
     }
 }
 
@@ -299,10 +288,6 @@ function updateSelector(value: NodeValue): void {
     value.selector = [selector.get(s, Selector.Alpha), selector.get(s, Selector.Beta)];
     value.hash = value.selector.map(x => x ? hash(x) : null) as [string, string];
     value.hash.forEach(h => hashMap[h] = value.id);
-    // Match fragment configuration against both alpha and beta hash
-    if (value.hash.some(h => extract.fragments.indexOf(h) !== -1)) {
-        value.fragment = value.id;
-    }
 }
 
 export function hashText(hash: string): string {
@@ -344,11 +329,7 @@ export function updates(): NodeValue[] {
         if (id in values) { output.push(values[id]); }
     }
     updateMap = [];
-    for (let id in updatedFragments) {
-        extract.update(updatedFragments[id], id, true)
-    }
-
-    updatedFragments = {}
+   
     return output;
 }
 
@@ -377,19 +358,7 @@ function getPreviousId(node: Node): number {
     return id;
 }
 
-function track(id: number, source: Source, fragment: number = null, changed: boolean = true, parentChanged: boolean = false): void {
-    // if updated node is a part of fragment and the fragment is not being tracked currently, schedule a mutation on the fragment node
-    if (fragment && !updatedFragments[fragment]) {
-        let node = getNode(fragment)
-        let value = getValue(fragment);
-        if (node && value) {
-            mutation.schedule(node, true);
-            value.hash.forEach(h => {
-                if(extract.fragments.indexOf(h) !== -1) { updatedFragments[fragment] = h;}
-            });
-        }
-    }
-
+function track(id: number, source: Source, changed: boolean = true, parentChanged: boolean = false): void {
     // Keep track of the order in which mutations happened, they may not be sequential
     // Edge case: If an element is added later on, and pre-discovered element is moved as a child.
     // In that case, we need to reorder the pre-discovered element in the update list to keep visualization consistent.

package/src/layout/mutation.ts

@@ -210,7 +210,7 @@ async function processNodeList(list: NodeList, source: Source, timer: Timer): Pr
   }
 }
 
-export function schedule(node: Node, fragment: boolean = false): Node {
+export function schedule(node: Node): Node {
   // Only schedule manual trigger for this node if it's not already in the queue
   if (queue.indexOf(node) < 0) { queue.push(node); }
 
@@ -218,19 +218,19 @@ export function schedule(node: Node, fragment: boolean = false): Node {
   // It's common for a webpage to call multiple synchronous "insertRule" / "deleteRule" calls.
   // And in those cases we do not wish to monitor changes multiple times for the same node.
   if (timeout) { clearTimeout(timeout); }
-  timeout = setTimeout(() => { trigger(fragment) }, Setting.LookAhead);
+  timeout = setTimeout(() => { trigger() }, Setting.LookAhead);
 
   return node;
 }
 
-function trigger(fragment: boolean): void {
+function trigger(): void {
   for (let node of queue) {
     // Generate a mutation for this node only if it still exists
     if (node) {
       let shadowRoot = node.nodeType === Node.DOCUMENT_FRAGMENT_NODE;
       // Skip re-processing shadowRoot if it was already discovered
       if (shadowRoot && dom.has(node)) { continue; }
-      generate(node, shadowRoot || fragment ? Constant.ChildList : Constant.CharacterData);
+      generate(node, shadowRoot ? Constant.ChildList : Constant.CharacterData);
     }
   }
   queue = [];

package/src/layout/node.ts

@@ -128,7 +128,7 @@ export default function (node: Node, source: Source): Node {
                 case "HEAD":
                     let head = { tag, attributes };
                     let l = insideFrame && node.ownerDocument?.location ? node.ownerDocument.location : location;
-                    head.attributes[Constant.Base] = l.protocol + "//" + l.hostname + l.pathname;
+                    head.attributes[Constant.Base] = l.protocol + "//" + l.host + l.pathname;
                     dom[call](node, parent, head, source);
                     break;
                 case "BASE":
@@ -138,7 +138,7 @@ export default function (node: Node, source: Source): Node {
                         // We create "a" element so we can generate protocol and hostname for relative paths like "/path/"
                         let a = document.createElement("a");
                         a.href = attributes["href"];
-                        baseHead.data.attributes[Constant.Base] = a.protocol + "//" + a.hostname + a.pathname;
+                        baseHead.data.attributes[Constant.Base] = a.protocol + "//" + a.host + a.pathname;
                     }
                     break;
                 case "STYLE":

package/src/performance/observer.ts

@@ -91,5 +91,5 @@ export function stop(): void {
 function host(url: string): string {
     let a = document.createElement("a");
     a.href = url;
-    return a.hostname;
+    return a.host;
 }

package/types/core.d.ts

@@ -5,7 +5,6 @@ type TaskResolve = () => void;
 type UploadCallback = (data: string) => void;
 type Region = [number /* RegionId */, string /* Query Selector */];
 type Checksum = [number /* FraudId */, string /* Query Selector */];
-export type Extract = ExtractSource /* Extraction Source */ | number /* Extract Id */ | string  | string[] /* Hash or Query Selector or String Token */;
 
 /* Enum */
 
@@ -127,7 +126,6 @@ export interface Config {
     mask?: string[];
     unmask?: string[];
     regions?: Region[];
-    extract?: Extract[];
     cookies?: string[];
     fraud?: boolean;
     checksum?: Checksum[];

package/types/data.d.ts

@@ -1,6 +1,6 @@
 import { Time } from "@clarity-types/core";
 export type Target = (number | Node);
-export type Token = (string | number | number[] | string[]);
+export type Token = (string | number | number[] | string[] | (string | number)[]);
 export type DecodedToken = (any | any[]);
 
 export type MetadataCallback = (data: Metadata, playback: boolean) => void;
@@ -217,7 +217,8 @@ export const enum Setting {
     MinUploadDelay = 100, // Minimum time before we are ready to flush events to the server
     MaxUploadDelay = 30 * Time.Second, // Do flush out payload once every 30s,
     ExtractLimit = 10000, // Do not extract more than 10000 characters
-    ChecksumPrecision = 24 // n-bit integer to represent token hash 
+    ChecksumPrecision = 24, // n-bit integer to represent token hash 
+    UploadTimeout = 15000 // Timeout in ms for XHR requests
 }
 
 export const enum Character {
@@ -263,6 +264,7 @@ export const enum Constant {
     End = "END",
     Upgrade = "UPGRADE",
     Action = "ACTION",
+    Extract = "EXTRACT",
     UserId = "userId",
     SessionId = "sessionId",
     PageId = "pageId",
@@ -284,7 +286,9 @@ export const enum Constant {
     Tilde = "~",
     ArrayStart = "[",
     ConditionStart = "{",
-    ConditionEnd = "}"
+    ConditionEnd = "}",
+    Seperator = "<SEP>",
+    Timeout = "Timeout"
 }
 
 export const enum XMLReadyState {
@@ -404,7 +408,7 @@ export interface UpgradeData {
 }
 
 export interface ExtractData {
-    [key: string]: string | number;
+    [key: number]: [number, string][]; // Array of [id, value] for every extracted data
 }
 
 export interface UploadData {

package/types/layout.d.ts

@@ -168,7 +168,6 @@ export interface NodeValue {
     hash: [string, string];
     region: number;
     metadata: NodeMeta;
-    fragment: number;
 }
 
 export interface NodeMeta {