clarity-js 0.7.5 → 0.7.7

package/src/core/scrub.ts

@@ -30,6 +30,7 @@ export function text(value: string, hint: string, privacy: Privacy, mangle: bool
             case Privacy.TextImage:
                 switch (hint) {
                     case Layout.Constant.TextTag:
+                    case Layout.Constant.DataAttribute:
                         return mangle ? mangleText(value) : mask(value);
                     case "src":
                     case "srcset":
@@ -48,6 +49,7 @@ export function text(value: string, hint: string, privacy: Privacy, mangle: bool
             case Privacy.Exclude:
                 switch (hint) {
                     case Layout.Constant.TextTag:
+                    case Layout.Constant.DataAttribute:
                         return mangle ? mangleText(value) : mask(value);
                     case "value":
                     case "input":
@@ -57,6 +59,25 @@ export function text(value: string, hint: string, privacy: Privacy, mangle: bool
                     case "checksum":
                         return Data.Constant.Empty;
                 }
+                break;
+            case Privacy.Snapshot:
+                switch (hint) {
+                    case Layout.Constant.TextTag:
+                    case Layout.Constant.DataAttribute:
+                        return scrub(value);
+                    case "value":
+                    case "input":
+                    case "click":
+                    case "change":
+                        return Array(Data.Setting.WordLength).join(Data.Constant.Mask);
+                    case "checksum":
+                    case "src":
+                    case "srcset":
+                    case "alt":
+                    case "title":
+                        return Data.Constant.Empty;
+                }
+                break;
         }
     }
     return value;
@@ -83,11 +104,16 @@ function mangleText(value: string): string {
     }
     return value;
 }
-
+ 
 function mask(value: string): string {
     return value.replace(catchallRegex, Data.Constant.Mask);
 }
 
+function scrub(value: string): string {
+    regex(); // Initialize regular expressions
+    return value.replace(letterRegex, Data.Constant.Letter).replace(digitRegex, Data.Constant.Digit);
+}
+
 function mangleToken(value: string): string {
     let length = ((Math.floor(value.length / Data.Setting.WordLength) + 1) * Data.Setting.WordLength);
     let output: string = Layout.Constant.Empty;
@@ -97,14 +123,7 @@ function mangleToken(value: string): string {
     return output;
 }
 
-function redact(value: string): string {
-    let spaceIndex = -1;
-    let gap = 0;
-    let hasDigit = false;
-    let hasEmail = false;
-    let hasWhitespace = false;
-    let array = null;
-
+function regex(): void {
     // Initialize unicode regex, if supported by the browser
     // Reference: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Unicode_Property_Escapes
     if (unicodeRegex && digitRegex === null) {
@@ -114,6 +133,17 @@ function redact(value: string): string {
             currencyRegex = new RegExp("\\p{Sc}", "gu");
         } catch { unicodeRegex = false; }
     }
+}
+
+function redact(value: string): string {
+    let spaceIndex = -1;
+    let gap = 0;
+    let hasDigit = false;
+    let hasEmail = false;
+    let hasWhitespace = false;
+    let array = null;
+    
+    regex(); // Initialize regular expressions
 
     for (let i = 0; i < value.length; i++) {
         let c = value.charCodeAt(i);

package/src/core/version.ts

@@ -1,2 +1,2 @@
-let version = "0.7.5";
+let version = "0.7.7";
 export default version;

package/src/data/encode.ts

@@ -108,10 +108,17 @@ export default function(event: Event): void {
             break;
         case Event.Extract:
             let extractKeys = extract.keys;
-            for (let e of extractKeys) {
+            extractKeys.forEach((e => {
                 tokens.push(e);
-                tokens.push([].concat(...extract.data[e]));
-            }
+                let token = []
+                for (let d in extract.data[e]) {
+                    let key = parseInt(d, 10);
+                    token.push(key);
+                    token.push(extract.data[e][d]);
+                }
+                tokens.push(token);
+            }));
+            
             extract.reset();
             queue(tokens, false);
     }

package/src/data/extract.ts

@@ -3,16 +3,23 @@ import { Event, Setting, ExtractData } from "@clarity-types/data";
 import encode from "./encode";
 import * as internal from "@src/diagnostic/internal";
 import { Code, Constant, Severity } from "@clarity-types/data";
+import { hashText } from "@src/clarity";
 
 export let data: ExtractData = {};
-export let keys: number[] = [];
+export let keys: Set<number> = new Set();
 
 let variables : { [key: number]: { [key: number]: Syntax[] }} = {};
 let selectors : { [key: number]: { [key: number]: string }} = {};
+let hashes : { [key: number]: { [key: number]: string }} = {};
 export function start(): void {
     reset();
 }
 
+// Input string is of the following form:
+// EXTRACT 101 { "1": ".class1", "2": "~window.a.b", "3": "!abc"}
+// Which will set up event 101 to grab the contents of the class1 selector into component 1,
+// the javascript evaluated contents of window.a.b into component 2,
+// and the contents of Clarity's hash abc into component 3
 export function trigger(input: string): void { 
     try {
         var parts = input && input.length > 0 ? input.split(/ (.*)/) : [Constant.Empty];
@@ -20,10 +27,18 @@ export function trigger(input: string): void {
         var values = parts.length > 1 ? JSON.parse(parts[1]) : {};
         variables[key] = {};
         selectors[key] = {};
+        hashes[key] = {};
         for (var v in values) {
+            // values is a set of strings for proper JSON parsing, but it's more efficient 
+            // to interact with them as numbers
             let id = parseInt(v);
             let value = values[v] as string;
-            let source = value.startsWith(Constant.Tilde) ? ExtractSource.Javascript : ExtractSource.Text;
+            let source = ExtractSource.Text;
+            if (value.startsWith(Constant.Tilde)) {
+                source = ExtractSource.Javascript
+            } else if (value.startsWith(Constant.Bang)) {
+                source = ExtractSource.Hash
+            }
             switch (source) {
                 case ExtractSource.Javascript:
                     let variable = value.substring(1, value.length);
@@ -32,6 +47,10 @@ export function trigger(input: string): void {
                 case ExtractSource.Text:
                     selectors[key][id] = value;
                     break;
+                case ExtractSource.Hash:
+                    let hash = value.substring(1, value.length);
+                    hashes[key][id] = hash;
+                    break;
             }
         }
     }
@@ -48,44 +67,63 @@ export function compute(): void {
     try {
         for (let v in variables) {
             let key = parseInt(v);
-            if (!(key in keys)) {
-                let variableData = variables[key];
-                for (let v in variableData) {
-                    let variableKey = parseInt(v);
-                    let value = str(evaluate(clone(variableData[variableKey])));
-                    if (value) { update(key, variableKey, value); }
+            let variableData = variables[key];
+            for (let v in variableData) {
+                let variableKey = parseInt(v);
+                let value = str(evaluate(clone(variableData[variableKey])));
+                if (value) { 
+                    update(key, variableKey, value);
                 }
+            }
 
-                let selectorData = selectors[key];
-                for (let s in selectorData) {
-                    let selectorKey = parseInt(s);
-                    let nodes = document.querySelectorAll(selectorData[selectorKey]) as NodeListOf<HTMLElement>;
-                    if (nodes) {
-                        let text = Array.from(nodes).map(e => e.innerText)
-                        update(key, selectorKey, text.join(Constant.Seperator).substring(0, Setting.ExtractLimit));
-                    }
+            let selectorData = selectors[key];
+            for (let s in selectorData) {
+                let selectorKey = parseInt(s);
+                let nodes = document.querySelectorAll(selectorData[selectorKey]) as NodeListOf<HTMLElement>;
+                if (nodes) {
+                    let text = Array.from(nodes).map(e => e.innerText)
+                    update(key, selectorKey, text.join(Constant.Seperator).substring(0, Setting.ExtractLimit));
                 }
             }
+
+            let hashData = hashes[key];
+            for (let h in hashData) {
+                let hashKey = parseInt(h);
+                let content = hashText(hashData[hashKey]).trim().substring(0, Setting.ExtractLimit);
+                update(key, hashKey, content);
+            }            
+        }
+
+        if (keys.size > 0) {
+            encode(Event.Extract);
         }
     }
     catch (e) { internal.log(Code.Selector, Severity.Warning, e ? e.name : null); }
-
-    encode(Event.Extract);
 }
 
 export function reset(): void {
-    data = {};
-    keys = [];
-    variables = {};
-    selectors = {};
+    keys.clear();
 }
 
-export function update(key: number, subkey: number,  value: string): void {
+export function update(key: number, subkey: number, value: string): void {
+    var update = false;
     if (!(key in data)) {
-        data[key] = []
-        keys.push(key);
+        data[key] = {};
+        update = true;
     }
-    data[key].push([subkey, value]);
+    
+    if (!isEmpty(hashes[key]) 
+        && (!(subkey in data[key]) || data[key][subkey] != value))
+    {
+        update = true;
+    }
+
+    data[key][subkey] = value;
+    if (update) {
+        keys.add(key);
+    }
+
+    return;
 }
 
 export function stop(): void {
@@ -151,4 +189,8 @@ function match(base: Object, condition: string): boolean {
     }
 
     return true;
-}
+}
+
+function isEmpty(obj: Object): boolean {
+    return Object.keys(obj).length == 0;
+}

package/src/data/metadata.ts

@@ -85,11 +85,16 @@ export function stop(): void {
 }
 
 export function metadata(cb: MetadataCallback, wait: boolean = true): void {
-  if (data && wait === false) {
+  let upgraded = config.lean ? BooleanFlag.False : BooleanFlag.True;
+  // if caller hasn't specified that they want to skip waiting for upgrade but we've already upgraded, we need to
+  // directly execute the callback rather than adding to our list as we only process callbacks at the moment
+  // we go through the upgrading flow.
+  if (data && (upgraded || wait === false)) {
     // Immediately invoke the callback if the caller explicitly doesn't want to wait for the upgrade confirmation
     cb(data, !config.lean);
+  } else {
+    callbacks.push({callback: cb, wait: wait });
   }
-  callbacks.push({callback: cb, wait: wait });
 }
 
 export function id(): string {

package/src/data/upload.ts

@@ -49,7 +49,8 @@ export function queue(tokens: Token[], transmit: boolean = true): void {
             case Event.Discover:
                 discoverBytes += event.length;
             case Event.Box:
-            case Event.Mutation:
+            case Event.Mutation:                
+            case Event.Snapshot:
                 playbackBytes += event.length;
                 playback.push(event);
                 break;

package/src/{core → insight}/blank.ts

@@ -4,6 +4,8 @@ export let data = null;
 /* Intentionally blank module with empty code */
 
 export function start(): void {}
-export function observe(): void {}
 export function reset(): void {}
 export function stop(): void {}
+export function log(): void {}
+export function observe(): void {}
+export function compute(): void {}

package/src/insight/encode.ts

@@ -0,0 +1,61 @@
+import { Privacy } from "@clarity-types/core";
+import { Event, Token } from "@clarity-types/data";
+import { Constant, NodeInfo } from "@clarity-types/layout";
+import * as scrub from "@src/core/scrub";
+import { time } from "@src/core/time";
+import * as baseline from "@src/data/baseline";
+import tokenize from "@src/data/token";
+import { queue } from "@src/data/upload";
+import * as snapshot from "@src/insight/snapshot";
+import * as doc from "@src/layout/document";
+
+export default async function (type: Event): Promise<void> {
+    let eventTime = time()
+    let tokens: Token[] = [eventTime, type];
+    switch (type) {
+        case Event.Document:
+            let d = doc.data;
+            tokens.push(d.width);
+            tokens.push(d.height);
+            baseline.track(type, d.width, d.height);
+            queue(tokens);
+            break;
+        case Event.Snapshot:
+            let values = snapshot.values;
+            // Only encode and queue DOM updates if we have valid updates to report back
+            if (values.length > 0) {
+                for (let value of values) {
+                    let privacy = value.metadata.privacy;
+                    let data: NodeInfo = value.data;
+                    for (let key of ["tag", "attributes", "value"]) {
+                        if (data[key]) {
+                            switch (key) {
+                                case "tag":
+                                    tokens.push(value.id);
+                                    if (value.parent) { tokens.push(value.parent); }
+                                    if (value.previous) { tokens.push(value.previous); }
+                                    tokens.push(data[key]);
+                                    break;
+                                case "attributes":
+                                    for (let attr in data[key]) {
+                                        if (data[key][attr] !== undefined) {
+                                            tokens.push(attribute(attr, data[key][attr], privacy));
+                                        }
+                                    }
+                                    break;
+                                case "value":
+                                    tokens.push(scrub.text(data[key], data.tag, privacy));
+                                    break;
+                            }
+                        }
+                    }
+                }
+                queue(tokenize(tokens), true);
+            }
+            break;
+    }
+}
+
+function attribute(key: string, value: string, privacy: Privacy): string {
+    return `${key}=${scrub.text(value, key.indexOf(Constant.DataAttribute) === 0 ? Constant.DataAttribute : key, privacy)}`;
+}

package/src/insight/snapshot.ts

@@ -0,0 +1,112 @@
+import { OffsetDistance, Privacy } from "@clarity-types/core";
+import { Event } from "@clarity-types/data";
+import { Constant, NodeInfo, NodeValue, TargetMetadata } from "@clarity-types/layout";
+import * as doc from "@src/layout/document";
+import encode from "@src/insight/encode";
+import * as interaction from "@src/interaction";
+export let values: NodeValue[] = [];
+let index: number = 1;
+let idMap: WeakMap<Node, number> = null; // Maps node => id.
+
+export function start(): void {
+    reset();
+    doc.start();
+    getId(document.documentElement); // Pre-discover ID for page root
+    interaction.observe(document);
+}
+
+export function stop(): void {
+    reset();
+    doc.stop();
+}
+
+export function compute(): void { /* Intentionally Blank */ }
+export function iframe(): boolean { return false; }
+export function offset(): OffsetDistance { return { x: 0, y: 0 }; }
+export function hashText(): void { /* Intentionally Blank */ }
+
+export function target(evt: UIEvent): Node {
+    let path = evt.composed && evt.composedPath ? evt.composedPath() : null;
+    let node = (path && path.length > 0 ? path[0] : evt.target) as Node;
+    return node.nodeType === Node.DOCUMENT_NODE ? (node as Document).documentElement : node;
+}
+
+export function metadata(node: Node): TargetMetadata {
+    let output: TargetMetadata = { id: 0, hash: null, privacy: Privacy.Snapshot, node };
+    if (node) { output.id = idMap.has(node) ? idMap.get(node) : getId(node); }
+    return output;
+}
+
+export function snapshot(): void {    
+    values = [];
+    traverse(document);
+    encode(Event.Snapshot);
+}
+
+function reset(): void {
+    idMap = new WeakMap();
+}
+
+function traverse(root: Node): void {
+    let queue = [root];
+    while (queue.length > 0) {
+        let attributes = null, tag = null, value = null;
+        let node = queue.shift();
+        let next = node.firstChild;
+        let parent = node.parentElement ? node.parentElement : (node.parentNode ?  node.parentNode : null);
+
+        while (next) {
+            queue.push(next);
+            next = next.nextSibling;
+        }
+      
+        // Process the node
+        let type = node.nodeType;
+        switch (type) {
+            case Node.DOCUMENT_TYPE_NODE:
+                let doctype = node as DocumentType;
+                tag = Constant.DocumentTag;
+                attributes = { name: doctype.name, publicId: doctype.publicId, systemId: doctype.systemId }
+                break;
+            case Node.TEXT_NODE: 
+                value = node.nodeValue;
+                tag = idMap.get(parent) ? Constant.TextTag : tag;
+                break;
+            case Node.ELEMENT_NODE:
+                let element = node as HTMLElement;
+                attributes = getAttributes(element);
+                tag = ["NOSCRIPT", "SCRIPT", "STYLE"].indexOf(element.tagName) < 0 ? element.tagName : tag;
+                break;
+        }
+        add(node, parent, { tag, attributes, value });     
+    }
+}
+
+/* Helper Functions - Snapshot Traversal */
+function getAttributes(element: HTMLElement): { [key: string]: string } {
+    let output = {};
+    let attributes = element.attributes;
+    if (attributes && attributes.length > 0) {
+        for (let i = 0; i < attributes.length; i++) {            
+            output[attributes[i].name] = attributes[i].value;
+        }
+    }
+    return output;
+}
+
+function getId(node: Node): number {
+    if (node === null) { return null; }
+    if (idMap.has(node)) { return idMap.get(node); }
+    idMap.set(node, index);
+    return index++;
+}
+
+function add(node: Node, parent: Node, data: NodeInfo): void {
+    if (node && data && data.tag) {
+        let id = getId(node);
+        let parentId = parent ? idMap.get(parent) : null;
+        let previous = node.previousSibling ? idMap.get(node.previousSibling) : null;
+        let metadata = { active: true, suspend: false, privacy: Privacy.Snapshot, position: null, fraud: null, size: null };
+        values.push({ id, parent: parentId, previous, children: [], data, selector: null, hash: null, region: null, metadata });
+    }
+}

package/src/interaction/click.ts

@@ -5,8 +5,8 @@ import { bind } from "@src/core/event";
 import { schedule } from "@src/core/task";
 import { time } from "@src/core/time";
 import { iframe } from "@src/layout/dom";
-import offset from "@src/layout/offset";
-import { link, target } from "@src/layout/target";
+import { offset } from "@src/layout/offset";
+import { target } from "@src/layout/target";
 import encode from "./encode";
 
 const UserInputTags = ["input", "textarea", "radio", "button", "canvas"];
@@ -73,6 +73,19 @@ function handler(event: Event, root: Node, evt: MouseEvent): void {
     }
 }
 
+function link(node: Node): HTMLAnchorElement {
+    while (node && node !== document) {
+        if (node.nodeType === Node.ELEMENT_NODE) {
+            let element = node as HTMLElement;
+            if (element.tagName === "A") {
+                return element as HTMLAnchorElement;
+            }
+        }
+        node = node.parentNode;
+    }
+    return null;
+}
+
 function text(element: Node): string {
     let output = null;
     if (element) {

package/src/interaction/encode.ts

@@ -47,7 +47,7 @@ export default async function (type: Event, ts: number = null): Promise<void> {
             for (let entry of click.state) {
                 let cTarget = metadata(entry.data.target as Node, entry.event, entry.data.text);
                 tokens = [entry.time, entry.event];
-                let cHash = cTarget.hash.join(Constant.Dot);
+                let cHash = cTarget.hash ? cTarget.hash.join(Constant.Dot) : Constant.Empty;
                 tokens.push(cTarget.id);
                 tokens.push(entry.data.x);
                 tokens.push(entry.data.y);

package/src/interaction/pointer.ts

@@ -5,7 +5,7 @@ import { schedule } from "@src/core/task";
 import { time } from "@src/core/time";
 import { clearTimeout, setTimeout } from "@src/core/timeout";
 import { iframe } from "@src/layout/dom";
-import offset from "@src/layout/offset";
+import { offset } from "@src/layout/offset";
 import { target } from "@src/layout/target";
 import encode from "./encode";
 

package/src/layout/document.ts

@@ -1,6 +1,6 @@
 import { Event } from "@clarity-types/data";
 import { DocumentData } from "@clarity-types/layout";
-import encode from "./encode";
+import encode from "@src/layout/encode";
 
 export let data: DocumentData;
 
@@ -41,6 +41,6 @@ export function compute(): void {
     }
 }
 
-export function end(): void {
+export function stop(): void {
     reset();
 }

package/src/layout/encode.ts

@@ -106,5 +106,5 @@ function str(input: number): string {
 }
 
 function attribute(key: string, value: string, privacy: Privacy): string {
-    return `${key}=${scrub.text(value, key, privacy)}`;
+    return `${key}=${scrub.text(value, key.indexOf(Constant.DataAttribute) === 0 ? Constant.DataAttribute : key, privacy)}`;
 }

package/src/layout/index.ts

@@ -20,5 +20,5 @@ export function stop(): void {
     region.stop();
     dom.stop();
     mutation.stop();
-    doc.end();
+    doc.stop();
 }

package/src/layout/offset.ts

@@ -1,7 +1,7 @@
 import { OffsetDistance } from "@clarity-types/core";
 import { iframe } from "@src/layout/dom";
 
-export default function(element: HTMLElement): OffsetDistance {
+export function offset(element: HTMLElement): OffsetDistance {
     let output: OffsetDistance = { x: 0, y: 0 };
 
     // Walk up the chain to ensure we compute offset distance correctly

package/src/layout/target.ts

@@ -13,19 +13,6 @@ export function target(evt: UIEvent): Node {
     return node.nodeType === Node.DOCUMENT_NODE ? (node as Document).documentElement : node;
 }
 
-export function link(node: Node): HTMLAnchorElement {
-    while (node && node !== document) {
-        if (node.nodeType === Node.ELEMENT_NODE) {
-            let element = node as HTMLElement;
-            if (element.tagName === "A") {
-                return element as HTMLAnchorElement;
-            }
-        }
-        node = node.parentNode;
-    }
-    return null;
-}
-
 export function metadata(node: Node, event: Event, text: string = null): TargetMetadata {
     // If the node is null, we return a reserved value for id: 0. Valid assignment of id begins from 1+.
     let output: TargetMetadata = { id: 0, hash: null, privacy: Privacy.Text, node };

package/src/layout/traverse.ts

@@ -1,13 +1,13 @@
 import { Task, Timer } from "@clarity-types/core";
 import { Source } from "@clarity-types/layout";
 import * as task from "@src/core/task";
-import processNode from "./node";
+import node from "@src/layout/node";
 
 export default async function(root: Node, timer: Timer, source: Source): Promise<void> {
     let queue = [root];
     while (queue.length > 0) {
-        let node = queue.shift();
-        let next = node.firstChild;
+        let entry = queue.shift();
+        let next = entry.firstChild;
 
         while (next) {
             queue.push(next);
@@ -22,7 +22,7 @@ export default async function(root: Node, timer: Timer, source: Source): Promise
         // Check if processing a node gives us a pointer to one of its sub nodes for traversal
         // E.g. an element node may give us a pointer to traverse shadowDom if shadowRoot property is set
         // Or, an iframe from the same origin could give a pointer to it's document for traversing contents of iframe.
-        let subnode = processNode(node, source);
+        let subnode = node(entry, source);
         if (subnode) { queue.push(subnode); }
     }
 }

package/src/queue.ts

@@ -9,6 +9,8 @@ export function setup() {
     if (typeof w !== "undefined") {
         w[c] = function() {
             (w[c].q = w[c].q || []).push(arguments);
+            // if the start function was called, don't queue it and instead process the queue
+            arguments[0] === "start" && w[c].q.unshift(w[c].q.pop()) && process();
         };
     }
 }

package/types/core.d.ts

@@ -34,7 +34,8 @@ export const enum ExtractSource {
     Javascript = 0,
     Cookie = 1,
     Text = 2,
-    Fragment = 3
+    Fragment = 3,
+    Hash = 4
 }
 
 export const enum Type {
@@ -54,7 +55,8 @@ export const enum Privacy {
     Sensitive = 1,
     Text = 2,
     TextImage = 3,
-    Exclude = 4
+    Exclude = 4,
+    Snapshot = 5
 }
 
 /* Helper Interfaces */