clarity-js 0.6.32 → 0.6.33

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-js",
-  "version": "0.6.32",
+  "version": "0.6.33",
   "description": "An analytics library that uses web page interactions to generate aggregated insights",
   "author": "Microsoft Corp.",
   "license": "MIT",

package/src/core/config.ts

@@ -9,8 +9,7 @@ let config: Config = {
     mask: [],
     unmask: [],
     regions: [],
-    metrics: [],
-    dimensions: [],
+    extract: [],
     cookies: [],
     report: null,
     upload: null,

package/src/core/version.ts

@@ -1,2 +1,2 @@
-let version = "0.6.32";
+let version = "0.6.33";
 export default version;

package/src/data/encode.ts

@@ -9,6 +9,7 @@ import * as ping from "@src/data/ping";
 import * as summary from "@src/data/summary";
 import * as upgrade from "@src/data/upgrade";
 import * as variable from "@src/data/variable";
+import * as extract from "@src/data/extract";
 import { queue, track } from "./upload";
 
 export default function(event: Event): void {
@@ -105,5 +106,13 @@ export default function(event: Event): void {
                 queue(tokens, false);
             }
             break;
+        case Event.Extract:
+            let extractKeys = extract.keys;
+            for (let e of extractKeys) {
+                tokens.push(e);
+                tokens.push(extract.data[e]);
+            }
+            extract.reset();
+            queue(tokens, false);
     }
 }

package/src/data/extract.ts

@@ -0,0 +1,143 @@
+import { ExtractSource, Syntax, Type } from "@clarity-types/core";
+import { Event, Setting, ExtractData } from "@clarity-types/data";
+import config from "@src/core/config";
+import encode from "./encode";
+import * as internal from "@src/diagnostic/internal";
+import { Code, Constant, Severity } from "@clarity-types/data";
+
+export let data: ExtractData = {};
+export let keys: (number | string)[] = [];
+
+let variables : { [key: number]: Syntax[] } = {};
+let selectors : { [key: number]: string } = {};
+export let fragments: string[] = [];
+
+export function start(): void {
+    try {
+        let e = config.extract;
+        if (!e) { return; }
+        for (let i = 0; i < e.length; i+=3) {
+            let source = e[i] as ExtractSource;
+            let key = e[i+1] as number;
+            switch (source) {
+                case ExtractSource.Javascript:
+                    let variable = e[i+2] as string;
+                    variables[key] = parse(variable);
+                    break;
+                case ExtractSource.Cookie:
+                    /*Todo: Add cookie extract logic*/
+                    break;
+                case ExtractSource.Text:
+                    let match = e[i+2] as string;
+                    selectors[key] = match;
+                    break;
+                case ExtractSource.Fragment:
+                    fragments =  e[i+2] as string[];
+                    break;
+            }
+        }
+    }
+    catch(e) {
+        internal.log(Code.Config, Severity.Warning, e ? e.name : null);
+    }
+}
+
+export function clone(v: Syntax[]): Syntax[] {
+    return JSON.parse(JSON.stringify(v));
+}
+
+export function compute(): void {
+    try {
+        for (let v in variables) {
+            let value = str(evaluate(clone(variables[v])));
+            if (value) { update(v, value); }
+        }
+
+        for (let s in selectors) {
+            let node = document.querySelector(selectors[s] as string) as HTMLElement;
+            if (node) { update(s, node.innerText); }
+        }
+    }
+    catch (e) { internal.log(Code.Selector, Severity.Warning, e ? e.name : null); }
+
+    encode(Event.Extract);
+}
+
+export function reset(): void {
+    keys = [];
+}
+
+export function update(key: string, value: string | number, force: boolean = false): void {
+    if (!(key in data) || (key in data && data[key] !== value) || force ) {
+        data[key] = value;
+        keys.push(key);
+    }
+}
+
+export function stop(): void {
+    data = {};
+    keys = [];
+    variables = {};
+    selectors = {};
+}
+
+function parse(variable: string): Syntax[] {
+    let syntax: Syntax[] = [];
+    let parts = variable.split(Constant.Dot);
+    while (parts.length > 0) {
+        let part = parts.shift();
+        let arrayStart = part.indexOf(Constant.ArrayStart);
+        let conditionStart = part.indexOf(Constant.ConditionStart);
+        let conditionEnd = part.indexOf(Constant.ConditionEnd);
+        syntax.push({
+            name : arrayStart > 0 ? part.substring(0, arrayStart) : (conditionStart > 0 ? part.substring(0, conditionStart) : part),
+            type : arrayStart > 0 ? Type.Array : (conditionStart > 0 ? Type.Object : Type.Simple),
+            condition : conditionStart > 0 ? part.substring(conditionStart + 1, conditionEnd) : null
+        });
+    }
+
+    return syntax;
+}
+
+// The function below takes in a variable name in following format: "a.b.c" and safely evaluates its value in javascript context
+// For instance, for a.b.c, it will first check window["a"]. If it exists, it will recursively look at: window["a"]["b"] and finally,
+// return the value for window["a"]["b"]["c"].
+function evaluate(variable: Syntax[], base: Object = window): any {
+    if (variable.length == 0) { return base; }
+    let part = variable.shift();
+    let output;
+    if (base && base[part.name]) {
+        let obj = base[part.name];
+        if (part.type !== Type.Array && match(obj, part.condition)) {
+            output = evaluate(variable, obj);
+        }
+        else if (Array.isArray(obj)) {
+            let filtered = [];
+            for (var value of obj) {
+                if (match(value, part.condition)) {
+                    let op = evaluate(variable, value)
+                    if (op) { filtered.push(op); }
+                }
+            }
+            output = filtered;
+        }
+        
+        return output;
+    }
+
+    return null;
+}
+
+function str(input: string): string {
+    // Automatically trim string to max of Setting.ExtractLimit to avoid fetching long strings
+    return input ? JSON.stringify(input).substring(0, Setting.ExtractLimit) : input;
+}
+
+function match(base: Object, condition: string): boolean {
+    if (condition) {
+        let prop = condition.split(":");
+        return prop.length > 1 ? base[prop[0]] == prop[1] : base[prop[0]]
+    }
+
+    return true;
+}

package/src/data/index.ts

@@ -11,12 +11,13 @@ import * as summary from "@src/data/summary";
 import * as upgrade from "@src/data/upgrade";
 import * as upload from "@src/data/upload";
 import * as variable from "@src/data/variable";
+import * as extract from "@src/data/extract";
 export { event } from "@src/data/custom";
 export { consent, metadata } from "@src/data/metadata";
 export { upgrade } from "@src/data/upgrade";
 export { set, identify } from "@src/data/variable";
 
-const modules: Module[] = [baseline, dimension, variable, limit, summary, metadata, envelope, upload, ping, upgrade];
+const modules: Module[] = [baseline, dimension, variable, limit, summary, metadata, envelope, upload, ping, upgrade, extract];
 
 export function start(): void {
     // Metric needs to be initialized before we can start measuring. so metric is not wrapped in measure
@@ -40,4 +41,5 @@ export function compute(): void {
     metric.compute();
     summary.compute();
     limit.compute();
+    extract.compute();
 }

package/src/layout/dom.ts

@@ -4,10 +4,10 @@ import { Constant, NodeInfo, NodeValue, Selector, SelectorInput, Source } from "
 import config from "@src/core/config";
 import hash from "@src/core/hash";
 import * as internal from "@src/diagnostic/internal";
-import * as extract from "@src/layout/extract";
 import * as region from "@src/layout/region";
 import selector from "@src/layout/selector";
-
+import * as mutation from "@src/layout/mutation";
+import * as extract from "@src/data/extract";
 let index: number = 1;
 
 // Reference: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/Input#%3Cinput%3E_types
@@ -21,6 +21,7 @@ let updateMap: number[] = [];
 let hashMap: { [hash: string]: number } = {};
 let override = [];
 let unmask = [];
+let updatedFragments: { [fragment: number]: string } = {};
 
 // The WeakMap object is a collection of key/value pairs in which the keys are weakly referenced
 let idMap: WeakMap<Node, number> = null; // Maps node => id.
@@ -61,9 +62,7 @@ export function parse(root: ParentNode, init: boolean = false): void {
         // Since mutations may happen on leaf nodes too, e.g. text nodes, which may not support all selector APIs.
         // We ensure that the root note supports querySelectorAll API before executing the code below to identify new regions.
         if ("querySelectorAll" in root) {
-            extract.regions(root, config.regions);
-            extract.metrics(root, config.metrics);
-            extract.dimensions(root, config.dimensions);
+            config.regions.forEach(x => root.querySelectorAll(x[1]).forEach(e => region.observe(e, `${x[0]}`))); // Regions
             config.mask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.TextImage))); // Masked Elements
             unmask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.None))); // Unmasked Elements
         }
@@ -88,11 +87,13 @@ export function add(node: Node, parent: Node, data: NodeInfo, source: Source): v
     let privacy = config.content ? Privacy.Sensitive : Privacy.Text;
     let parentValue = null;
     let regionId = region.exists(node) ? id : null;
+    let fragmentId = null;
 
     if (parentId >= 0 && values[parentId]) {
         parentValue = values[parentId];
         parentValue.children.push(id);
         regionId = regionId === null ? parentValue.region : regionId;
+        fragmentId = parentValue.fragment;
         privacy = parentValue.metadata.privacy;
     }
 
@@ -115,12 +116,13 @@ export function add(node: Node, parent: Node, data: NodeInfo, source: Source): v
         selector: null,
         hash: null,
         region: regionId,
-        metadata: { active: true, suspend: false, privacy, position: null, size: null }
+        metadata: { active: true, suspend: false, privacy, position: null, size: null },
+        fragment: fragmentId,
     };
 
     updateSelector(values[id]);
     size(values[id], parentValue);
-    track(id, source);
+    track(id, source, values[id].fragment);
 }
 
 export function update(node: Node, parent: Node, data: NodeInfo, source: Source): void {
@@ -174,9 +176,14 @@ export function update(node: Node, parent: Node, data: NodeInfo, source: Source)
             }
         }
 
+        // track node if it is a part of scheduled fragment mutation
+        if(value.fragment && updatedFragments[value.fragment]) {
+            changed = true;
+        }
+
         // Update selector
         updateSelector(value);
-        track(id, source, changed, parentChanged);
+        track(id, source, values[id].fragment, changed, parentChanged);
     }
 }
 
@@ -296,6 +303,9 @@ function updateSelector(value: NodeValue): void {
     value.selector = [selector(s), selector(s, true)];
     value.hash = value.selector.map(x => x ? hash(x) : null) as [string, string];
     value.hash.forEach(h => hashMap[h] = value.id);
+    if (value.hash.some(h => extract.fragments.indexOf(h) !== -1)) {
+        value.fragment = value.id;
+    }
 }
 
 export function getNode(id: number): Node {
@@ -331,6 +341,11 @@ export function updates(): NodeValue[] {
         if (id in values) { output.push(values[id]); }
     }
     updateMap = [];
+    for (let id in updatedFragments) {
+        extract.update(updatedFragments[id], id, true)
+    }
+
+    updatedFragments = {}
     return output;
 }
 
@@ -368,7 +383,19 @@ function getPreviousId(node: Node): number {
     return id;
 }
 
-function track(id: number, source: Source, changed: boolean = true, parentChanged: boolean = false): void {
+function track(id: number, source: Source, fragment: number = null, changed: boolean = true, parentChanged: boolean = false): void {
+    // if updated node is a part of fragment and the fragment is not being tracked currently, schedule a mutation on the fragment node
+    if (fragment && !updatedFragments[fragment]) {
+        let node = getNode(fragment)
+        let value = getValue(fragment);
+        if (node && value) {
+            mutation.schedule(node, true);
+            value.hash.forEach(h => {
+                if(extract.fragments.indexOf(h) !== -1) { updatedFragments[fragment] = h;}
+            });
+        }
+    }
+
     // Keep track of the order in which mutations happened, they may not be sequential
     // Edge case: If an element is added later on, and pre-discovered element is moved as a child.
     // In that case, we need to reorder the pre-discovered element in the update list to keep visualization consistent.

package/src/layout/mutation.ts

@@ -218,7 +218,7 @@ async function processNodeList(list: NodeList, source: Source, timer: Timer): Pr
   }
 }
 
-function schedule(node: Node): Node {
+export function schedule(node: Node, fragment: boolean = false): Node {
   // Only schedule manual trigger for this node if it's not already in the queue
   if (queue.indexOf(node) < 0) { queue.push(node); }
 
@@ -226,19 +226,19 @@ function schedule(node: Node): Node {
   // It's common for a webpage to call multiple synchronous "insertRule" / "deleteRule" calls.
   // And in those cases we do not wish to monitor changes multiple times for the same node.
   if (timeout) { clearTimeout(timeout); }
-  timeout = setTimeout(trigger, Setting.LookAhead);
+  timeout = setTimeout(() => { trigger(fragment) }, Setting.LookAhead);
 
   return node;
 }
 
-function trigger(): void {
+function trigger(fragment: boolean): void {
   for (let node of queue) {
     // Generate a mutation for this node only if it still exists
     if (node) {
       let shadowRoot = node.nodeType === Node.DOCUMENT_FRAGMENT_NODE;
       // Skip re-processing shadowRoot if it was already discovered
       if (shadowRoot && dom.has(node)) { continue; }
-      generate(node, shadowRoot ? Constant.ChildList : Constant.CharacterData);
+      generate(node, shadowRoot || fragment ? Constant.ChildList : Constant.CharacterData);
     }
   }
   queue = [];

package/types/core.d.ts

@@ -3,9 +3,8 @@ import * as Data from "./data";
 type TaskFunction = () => Promise<void>;
 type TaskResolve = () => void;
 type UploadCallback = (data: string) => void;
-type Region = [number /* RegionId */, string /* Query Selector */, RegionFilter? /* Region Filter */, string? /* Filter Text */];
-type Metric = [Data.Metric /* MetricId */, Extract /* Extract Filter */, string /* Match Value */, number? /* Scale Factor */];
-type Dimension = [Data.Dimension /* DimensionId */, Extract /* Extract Filter */, string /* Match Value */];
+type Region = [number /* RegionId */, string /* Query Selector */];
+export type Extract = ExtractSource /* Extraction Source */ | number /* Extract Id */ | string  | string[] /* Hash or Query Selector or String Token */;
 
 /* Enum */
 
@@ -21,7 +20,6 @@ export const enum Time {
     Day = 24 * 60 * 60 * 1000
 }
 
-
 export const enum Task {
     Wait = 0,
     Run = 1,
@@ -32,15 +30,23 @@ export const enum Setting {
     LongTask = 30, // 30ms
 }
 
-export const enum RegionFilter {
-    Url = 0,
-    Javascript = 1
+export const enum ExtractSource {
+    Javascript = 0,
+    Cookie = 1,
+    Text = 2,
+    Fragment = 3
+}
+
+export const enum Type {
+    Array = 1, 
+    Object = 2,
+    Simple = 3
 }
 
-export const enum Extract {
-    Text = 0,
-    Javascript = 1,
-    Attribute = 2
+export type Syntax = {
+    name: string,
+    type: Type,
+    condition: string
 }
 
 export const enum Privacy {
@@ -119,8 +125,7 @@ export interface Config {
     mask?: string[];
     unmask?: string[];
     regions?: Region[];
-    metrics?: Metric[];
-    dimensions?: Dimension[];
+    extract?: Extract[];
     cookies?: string[];
     report?: string;
     upload?: string | UploadCallback;

package/types/data.d.ts

@@ -54,7 +54,8 @@ export const enum Event {
     Summary = 36,
     Box = 37,
     Clipboard = 38,
-    Submit = 39
+    Submit = 39,
+    Extract = 40
 }
 
 export const enum Metric {
@@ -138,7 +139,8 @@ export const enum Code {
 /**
  * @deprecated No longer support ContentSecurityPolicy
  */
-    ContentSecurityPolicy = 7
+    ContentSecurityPolicy = 7,
+    Config = 8
 }
 
 export const enum Severity {
@@ -187,7 +189,8 @@ export const enum Setting {
     MaxFirstPayloadBytes = 1 * 1024 * 1024, // 1MB: Cap the very first payload to a maximum of 1MB
     UploadFactor = 3, // Slow down sequence by specified factor
     MinUploadDelay = 100, // Minimum time before we are ready to flush events to the server
-    MaxUploadDelay = 30 * Time.Second // Do flush out payload once every 30s
+    MaxUploadDelay = 30 * Time.Second, // Do flush out payload once every 30s,
+    ExtractLimit = 10000 // Do not extract more than 10000 characters
 }
 
 export const enum Character {
@@ -248,6 +251,9 @@ export const enum Constant {
     Accept = "Accept",
     ClarityGzip = "application/x-clarity-gzip",
     Tilde = "~",
+    ArrayStart = "[",
+    ConditionStart = "{",
+    ConditionEnd = "}"
 }
 
 export const enum XMLReadyState {
@@ -366,6 +372,10 @@ export interface UpgradeData {
     key: string;
 }
 
+export interface ExtractData {
+    [key: string]: string | number;
+}
+
 export interface UploadData {
     sequence: number;
     attempts: number;

package/types/layout.d.ts

@@ -154,6 +154,7 @@ export interface NodeValue {
     hash: [string, string];
     region: number;
     metadata: NodeMeta;
+    fragment: number;
 }
 
 export interface NodeMeta {

package/src/layout/extract.ts

@@ -1,94 +0,0 @@
-import { Dimension, Extract, Metric, Region, RegionFilter } from "@clarity-types/core";
-import { Constant, Setting } from "@clarity-types/data";
-import * as dimension from "@src/data/dimension";
-import * as metric from "@src/data/metric";
-import * as region from "@src/layout/region";
-
-const formatRegex = /1/g;
-const digitsRegex = /[^0-9\.]/g;
-const digitsWithCommaRegex = /[^0-9\.,]/g;
-const regexCache: {[key: string]: RegExp} = {};
-
-export function regions(root: ParentNode, value: Region[]): void {
-    for (let v of value) {
-        const [regionId, selector, filter, match] = v;
-        let valid = true;
-        switch (filter) {
-            case RegionFilter.Url: valid = match && !!top.location.href.match(regex(match)); break;
-            case RegionFilter.Javascript: valid = match && !!evaluate(match); break;
-        }
-        if (valid) { root.querySelectorAll(selector).forEach(e => region.observe(e, regionId.toString())); }
-    }
-}
-
-export function metrics(root: ParentNode, value: Metric[]): void {
-    for (let v of value) {
-        const [metricId, source, match, scale] = v;
-        if (match) {
-            switch (source) {
-                case Extract.Text:  root.querySelectorAll(match).forEach(e => { metric.max(metricId, num((e as HTMLElement).innerText, scale)); }); break;
-                case Extract.Attribute: root.querySelectorAll(`[${match}]`).forEach(e => { metric.max(metricId, num(e.getAttribute(match), scale, false)); }); break;
-                case Extract.Javascript: metric.max(metricId, evaluate(match, Constant.Number) as number); break;
-            }
-        }
-    }
-}
-
-export function dimensions(root: ParentNode, value: Dimension[]): void {
-    for (let v of value) {
-        const [dimensionId, source, match] = v;
-        if (match) {
-            switch (source) {
-                case Extract.Text:  root.querySelectorAll(match).forEach(e => { dimension.log(dimensionId, str((e as HTMLElement).innerText)); }); break;
-                case Extract.Attribute: root.querySelectorAll(`[${match}]`).forEach(e => { dimension.log(dimensionId, str(e.getAttribute(match))); }); break;
-                case Extract.Javascript: dimension.log(dimensionId, str(evaluate(match, Constant.String))); break;
-            }
-        }
-    }
-}
-
-function regex(match: string): RegExp {
-    regexCache[match] = match in regexCache ? regexCache[match] : new RegExp(match);
-    return regexCache[match];
-}
-
-// The function below takes in a variable name in following format: "a.b.c" and safely evaluates its value in javascript context
-// For instance, for a.b.c, it will first check window["a"]. If it exists, it will recursively look at: window["a"]["b"] and finally,
-// return the value for window["a"]["b"]["c"].
-function evaluate(variable: string, type: string = null, base: Object = window): any {
-    let parts = variable.split(Constant.Dot);
-    let first = parts.shift();
-    if (base && base[first]) {
-        if (parts.length > 0) { return evaluate(parts.join(Constant.Dot), type, base[first]); } 
-        let output = type === null || type === typeof base[first] ? base[first] : null;
-        return output;
-    }
-    return null;
-}
-
-function str(input: string): string {
-    // Automatically trim string to max of Setting.DimensionLimit to avoid fetching long strings
-    return input ? input.substr(0, Setting.DimensionLimit) : input;
-}
-
-function num(text: string, scale: number, localize: boolean = true): number {
-    try {
-        scale = scale || 1; 
-        // Reference: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/NumberFormat
-        let lang = document.documentElement.lang;
-        if (Intl && Intl.NumberFormat && lang && localize) {
-            text = text.replace(digitsWithCommaRegex, Constant.Empty);
-            // Infer current group and decimal separator from current locale
-            let group = Intl.NumberFormat(lang).format(11111).replace(formatRegex, Constant.Empty);
-            let decimal = Intl.NumberFormat(lang).format(1.1).replace(formatRegex, Constant.Empty);
-            
-            // Parse number using inferred group and decimal separators
-            return Math.round(parseFloat(text
-                .replace(new RegExp('\\' + group, 'g'), Constant.Empty)
-                .replace(new RegExp('\\' + decimal), Constant.Dot)
-            ) * scale);
-        }
-        // Fallback to en locale
-        return Math.round(parseFloat(text.replace(digitsRegex, Constant.Empty)) * scale);
-    } catch { return null; }
-}