clarity-js 0.6.27 → 0.6.31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clarity-js",
-  "version": "0.6.27",
+  "version": "0.6.31",
   "description": "An analytics library that uses web page interactions to generate aggregated insights",
   "author": "Microsoft Corp.",
   "license": "MIT",

package/src/core/measure.ts

@@ -1,12 +1,13 @@
 import { Setting } from "@clarity-types/core";
 import { Metric } from "@clarity-types/data";
+import { report } from "@src/core/report";
 import * as metric from "@src/data/metric";
 
 // tslint:disable-next-line: ban-types
 export default function (method: Function): Function {
     return function (): void {
         let start = performance.now();
-        method.apply(this, arguments);
+        try { method.apply(this, arguments); } catch (ex) { throw report(ex); }
         let duration = performance.now() - start;
         metric.sum(Metric.TotalCost, duration);
         if (duration > Setting.LongTask) {

package/src/core/report.ts

@@ -1,7 +1,6 @@
 import { Report } from "@clarity-types/core";
-import { Check } from "@clarity-types/data";
 import config from "@src/core/config";
-import { data } from "@src/data/metadata";
+import { data } from "@src/data/envelope";
 
 let history: string[];
 
@@ -9,19 +8,21 @@ export function reset(): void {
     history = [];
 }
 
-export function report(check: Check, message: string = null): void {
+export function report(e: Error): Error {
     // Do not report the same message twice for the same page
-    if (history && history.indexOf(message) === -1) {
+    if (history && history.indexOf(e.message) === -1) {
         const url = config.report;
         if (url && url.length > 0) {
-            let payload: Report = {c: check, p: data.projectId, u: data.userId, s: data.sessionId, n: data.pageNum };
-            if (message) payload.m = message;
+            let payload: Report = {v: data.version, p: data.projectId, u: data.userId, s: data.sessionId, n: data.pageNum};
+            if (e.message) { payload.m = e.message; }
+            if (e.stack) { payload.e = e.stack; }
             // Using POST request instead of a GET request (img-src) to not violate existing CSP rules
             // Since, Clarity already uses XHR to upload data, we stick with similar POST mechanism for reporting too
             let xhr = new XMLHttpRequest();
             xhr.open("POST", url);
             xhr.send(JSON.stringify(payload));
-            history.push(message);
+            history.push(e.message);
         }
     }
+    return e;
 }

package/src/core/version.ts

@@ -1,2 +1,2 @@
-let version = "0.6.27";
+let version = "0.6.31";
 export default version;

package/src/data/limit.ts

@@ -1,6 +1,5 @@
 import { Check, Event, LimitData, Setting } from "@clarity-types/data";
 import * as clarity from "@src/clarity";
-import { report } from "@src/core/report";
 import { time } from "@src/core/time";
 import * as envelope from "@src/data/envelope";
 import * as metadata from "@src/data/metadata";
@@ -25,7 +24,6 @@ export function check(bytes: number): void {
 }
 
 export function trigger(reason: Check): void {
-    report(reason);
     data.check = reason;
     metadata.clear();
     clarity.stop();

package/src/data/metadata.ts

@@ -38,10 +38,10 @@ export function start(): void {
   dimension.log(Dimension.TabId, tab());
   dimension.log(Dimension.PageLanguage, document.documentElement.lang);
   dimension.log(Dimension.DocumentDirection, document.dir);
-
   if (navigator) {
     dimension.log(Dimension.Language, (<any>navigator).userLanguage || navigator.language);
     metric.max(Metric.Automation, navigator.webdriver ? BooleanFlag.True : BooleanFlag.False);
+    userAgentData();
   }
 
   // Metrics
@@ -64,13 +64,39 @@ export function start(): void {
   track(u);
 }
 
+export function userAgentData(): void {
+  if (navigator["userAgentData"] && navigator["userAgentData"].getHighEntropyValues) {
+    navigator["userAgentData"].getHighEntropyValues(
+      ["model",
+      "platform",
+      "platformVersion",
+      "uaFullVersion"])
+      .then(ua => { 
+        dimension.log(Dimension.Platform, ua.platform); 
+        dimension.log(Dimension.PlatformVersion, ua.platformVersion); 
+        ua.brands?.forEach(brand => {
+          dimension.log(Dimension.Brand, brand.name + Constant.Tilde + brand.version); 
+        });
+        dimension.log(Dimension.Model, ua.model); 
+        metric.max(Metric.Mobile, ua.mobile ? BooleanFlag.True : BooleanFlag.False); 
+      });
+  }
+}
+
 export function stop(): void {
   callback = null;
   rootDomain = null;
+  data = null;
 }
 
-export function metadata(cb: MetadataCallback): void {
-  callback = cb;
+export function metadata(cb: MetadataCallback, wait: boolean = true): void {
+  if (data && wait === false) {
+    // Immediately invoke the callback if the caller explicitly doesn't want to wait for the upgrade confirmation
+    cb(data, !config.lean);
+  } else {
+    // Save the callback for future reference; so we can inform the caller when page gets upgraded and we have a valid playback flag
+    callback = cb;
+  }
 }
 
 export function id(): string {

package/src/diagnostic/internal.ts

@@ -28,7 +28,8 @@ export function log(code: Code, severity: Severity, name: string = null, message
 
 function csp(e: SecurityPolicyViolationEvent): void {
     let upload = config.upload as string;
-    let parts = upload ? upload.substr(0, upload.indexOf("/", Constant.HTTPS.length)).split(Constant.Dot) : []; // Look for first "/" starting after initial "https://" string
+    // Look for first "/" starting after initial "https://" string
+    let parts = upload && typeof upload === Constant.String ? upload.substr(0, upload.indexOf("/", Constant.HTTPS.length)).split(Constant.Dot) : []; 
     let domain = parts.length >= 2 ? parts.splice(-2).join(Constant.Dot) : null;
     // Capture content security policy violation only if disposition value is not explicitly set to "report"
     if (domain && e.blockedURI && e.blockedURI.indexOf(domain) >= 0 && e["disposition"] !== Constant.Report) {

package/src/layout/dom.ts

@@ -1,6 +1,6 @@
 import { Privacy } from "@clarity-types/core";
 import { Code, Setting, Severity } from "@clarity-types/data";
-import { Constant, NodeInfo, NodeValue, SelectorInput, Source } from "@clarity-types/layout";
+import { Constant, NodeInfo, NodeValue, Selector, SelectorInput, Source } from "@clarity-types/layout";
 import config from "@src/core/config";
 import hash from "@src/core/hash";
 import * as internal from "@src/diagnostic/internal";
@@ -12,13 +12,15 @@ let index: number = 1;
 
 // Reference: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/Input#%3Cinput%3E_types
 const DISALLOWED_TYPES = ["password", "hidden", "email", "tel"];
-const DISALLOWED_NAMES = ["addr", "cell", "code", "dob", "email", "mob", "name", "phone", "secret", "social", "ssn", "tel", "zip", "pass"];
+const DISALLOWED_NAMES = ["addr", "cell", "code", "dob", "email", "mob", "name", "phone", "secret", "social", "ssn", "tel", "zip", "pass", "card", "account", "cvv", "ccv"];
 const DISALLOWED_MATCH = ["address", "password", "contact"];
 
 let nodes: Node[] = [];
 let values: NodeValue[] = [];
 let updateMap: number[] = [];
 let hashMap: { [hash: string]: number } = {};
+let override = [];
+let unmask = [];
 
 // The WeakMap object is a collection of key/value pairs in which the keys are weakly referenced
 let idMap: WeakMap<Node, number> = null; // Maps node => id.
@@ -27,7 +29,7 @@ let privacyMap: WeakMap<Node, Privacy> = null; // Maps node => Privacy (enum)
 
 export function start(): void {
     reset();
-    parse(document);
+    parse(document, true);
 }
 
 export function stop(): void {
@@ -40,6 +42,8 @@ function reset(): void {
     values = [];
     updateMap = [];
     hashMap = {};
+    override = [];
+    unmask = [];
     idMap = new WeakMap();
     iframeMap = new WeakMap();
     privacyMap = new WeakMap();
@@ -47,10 +51,13 @@ function reset(): void {
 
 // We parse new root nodes for any regions or masked nodes in the beginning (document) and
 // later whenever there are new additions or modifications to DOM (mutations)
-export function parse(root: ParentNode): void {
+export function parse(root: ParentNode, init: boolean = false): void {
     // Wrap selectors in a try / catch block.
     // It's possible for script to receive invalid selectors, e.g. "'#id'" with extra quotes, and cause the code below to fail
     try {
+        // Parse unmask configuration into separate query selectors and override tokens as part of initialization
+        if (init) { config.unmask.forEach(x => x.indexOf(Constant.Bang) < 0 ? unmask.push(x) : override.push(x.substr(1))); }   
+
         // Since mutations may happen on leaf nodes too, e.g. text nodes, which may not support all selector APIs.
         // We ensure that the root note supports querySelectorAll API before executing the code below to identify new regions.
         if ("querySelectorAll" in root) {
@@ -58,7 +65,7 @@ export function parse(root: ParentNode): void {
             extract.metrics(root, config.metrics);
             extract.dimensions(root, config.dimensions);
             config.mask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.TextImage))); // Masked Elements
-            config.unmask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.None))); // Unmasked Elements
+            unmask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.None))); // Unmasked Elements
         }
     } catch (e) { internal.log(Code.Selector, Severity.Warning, e ? e.name : null); }
 }
@@ -80,19 +87,17 @@ export function add(node: Node, parent: Node, data: NodeInfo, source: Source): v
     let previousId = getPreviousId(node);
     let privacy = config.content ? Privacy.Sensitive : Privacy.Text;
     let parentValue = null;
-    let parentTag = Constant.Empty;
     let regionId = region.exists(node) ? id : null;
 
     if (parentId >= 0 && values[parentId]) {
         parentValue = values[parentId];
-        parentTag = parentValue.data.tag;
         parentValue.children.push(id);
         regionId = regionId === null ? parentValue.region : regionId;
         privacy = parentValue.metadata.privacy;
     }
 
     // Check to see if this particular node should be masked or not
-    privacy = getPrivacy(node, data, parentTag, privacy);
+    privacy = getPrivacy(node, data, parentValue, privacy);
 
     // If there's an explicit region attribute set on the element, use it to mark a region on the page
     if (data.attributes && Constant.RegionData in data.attributes) {
@@ -198,13 +203,27 @@ export function iframe(node: Node): HTMLIFrameElement {
     return doc && iframeMap.has(doc) ? iframeMap.get(doc) : null;
 }
 
-function getPrivacy(node: Node, data: NodeInfo, parentTag: string, privacy: Privacy): Privacy {
+function getPrivacy(node: Node, data: NodeInfo, parent: NodeValue, privacy: Privacy): Privacy {
     let attributes = data.attributes;
     let tag = data.tag.toUpperCase();
 
     // If this node was explicitly configured to contain sensitive content, use that information and return the value
     if (privacyMap.has(node)) { return privacyMap.get(node); }
 
+    // If it's a text node belonging to a STYLE or TITLE tag; 
+    // Or, the text node belongs to one of SCRUB_EXCEPTIONS
+    // then reset the privacy setting to ensure we capture the content
+    if (tag === Constant.TextTag && parent && parent.data) {
+        let path = parent.selector ? parent.selector[Selector.Stable] : Constant.Empty;
+        privacy = parent.data.tag === Constant.StyleTag || parent.data.tag === Constant.TitleTag ? Privacy.None : privacy;
+        for (let entry of override) {
+            if (path.indexOf(entry) >= 0) {
+                privacy = Privacy.None;
+                break;
+            }
+        }
+    }
+
     // Do not proceed if attributes are missing for the node
     if (attributes === null || attributes === undefined) { return privacy; }
 
@@ -243,10 +262,6 @@ function getPrivacy(node: Node, data: NodeInfo, parentTag: string, privacy: Priv
     if (Constant.MaskData in attributes) { privacy = Privacy.TextImage; }
     if (Constant.UnmaskData in attributes) { privacy = Privacy.None; }
 
-    // If it's a text node belonging to a STYLE or TITLE tag; then reset the privacy setting to ensure we capture the content
-    let cTag = tag === Constant.TextTag ? parentTag : tag;
-    if (cTag === Constant.StyleTag || cTag === Constant.TitleTag) { privacy = Privacy.None; }
-
     return privacy;
 }
 

package/src/layout/mutation.ts

@@ -36,7 +36,7 @@ export function start(): void {
 
     if (insertRule === null) { insertRule = CSSStyleSheet.prototype.insertRule; }
     if (deleteRule === null) { deleteRule = CSSStyleSheet.prototype.deleteRule; }
-    if (attachShadow === null) { attachShadow = HTMLElement.prototype.attachShadow; }
+    if (attachShadow === null) { attachShadow = Element.prototype.attachShadow; }
 
     // Some popular open source libraries, like styled-components, optimize performance
     // by injecting CSS using insertRule API vs. appending text node. A side effect of
@@ -56,7 +56,7 @@ export function start(): void {
     // In case we are unable to add a hook and browser throws an exception,
     // reset attachShadow variable and resume processing like before
     try {
-      HTMLElement.prototype.attachShadow = function (): ShadowRoot {
+      Element.prototype.attachShadow = function (): ShadowRoot {
         return schedule(attachShadow.apply(this, arguments)) as ShadowRoot;
       }
     } catch { attachShadow = null; }
@@ -107,7 +107,7 @@ export function stop(): void {
 
   // Restoring original attachShadow
   if (attachShadow != null) {
-    HTMLElement.prototype.attachShadow = attachShadow;
+    Element.prototype.attachShadow = attachShadow;
     attachShadow = null;
   }
 
@@ -145,6 +145,7 @@ async function process(): Promise<void> {
       let target = mutation.target;
       let type = track(mutation, timer);
       if (type && target && target.ownerDocument) { dom.parse(target.ownerDocument); }
+      if (type && target && target.nodeType == Node.DOCUMENT_FRAGMENT_NODE && (target as ShadowRoot).host) { dom.parse(target as ShadowRoot); }
       switch (type) {
         case Constant.Attributes:
             processNode(target, Source.Attributes);

package/src/layout/node.ts

@@ -46,6 +46,7 @@ export default function (node: Node, source: Source): Node {
         case Node.DOCUMENT_FRAGMENT_NODE:
             let shadowRoot = (node as ShadowRoot);
             if (shadowRoot.host) {
+                dom.parse(shadowRoot);
                 let type = typeof (shadowRoot.constructor);
                 if (type === Constant.Function && shadowRoot.constructor.toString().indexOf(Constant.NativeCode) >= 0) {
                     observe(shadowRoot);

package/src/layout/selector.ts

@@ -24,12 +24,13 @@ export default function(input: SelectorInput, beta: boolean = false): string {
             let selector = `${prefix}${input.tag}${suffix}`;
             let classes = Constant.Class in a && a[Constant.Class].length > 0 ? a[Constant.Class].trim().split(/\s+/) : null;
             if (beta) {
-                // In beta mode, update selector to use "id" field when available
-                // The only exception is if "id" appears to be an auto generated string token, e.g. guid or a random id
+                // In beta mode, update selector to use "id" field when available. There are two exceptions:
+                // (1) if "id" appears to be an auto generated string token, e.g. guid or a random id containing digits
+                // (2) if "id" appears inside a shadow DOM, in which case we continue to prefix up to shadow DOM to prevent conflicts
                 let id = Constant.Id in a && a[Constant.Id].length > 0 ? a[Constant.Id] : null;
                 classes = input.tag !== Constant.BodyTag && classes ? classes.filter(c => !hasDigits(c)) : [];
                 selector = classes.length > 0 ? `${prefix}${input.tag}.${classes.join(".")}${suffix}` : selector;
-                selector = id && hasDigits(id) === false ? `#${id}` : selector;
+                selector = id && hasDigits(id) === false ? `${getDomPrefix(prefix)}#${id}` : selector;
             } else {
                 // Otherwise, fallback to stable mode, where we include class names as part of the selector
                 selector = classes ? `${prefix}${input.tag}.${classes.join(".")}${suffix}` : selector;
@@ -38,11 +39,24 @@ export default function(input: SelectorInput, beta: boolean = false): string {
     }
 }
 
+function getDomPrefix(prefix: string): string {
+  const shadowDomStart = prefix.lastIndexOf(Constant.ShadowDomTag);
+  const iframeDomStart = prefix.lastIndexOf(`${Constant.IFramePrefix}${Constant.HTML}`);
+  const domStart = Math.max(shadowDomStart, iframeDomStart);
+  
+  if (domStart < 0) {
+    return "";
+  }
+
+  const domEnd = prefix.indexOf(">", domStart) + 1;
+  return prefix.substr(0, domEnd);
+}
+
 // Check if the given input string has digits or not
 function hasDigits(value: string): boolean {
     for (let i = 0; i < value.length; i++) {
         let c = value.charCodeAt(i);
-        return c >= Character.Zero && c <= Character.Nine;
+        if (c >= Character.Zero && c <= Character.Nine) { return true };
     }
     return false;
 }

package/test/helper.ts

@@ -1,5 +1,6 @@
 import { Core, Data, Layout } from "clarity-decode";
 import * as fs from 'fs';
+import * as url from 'url';
 import * as path from 'path';
 import { Browser, Page, chromium } from 'playwright';
 
@@ -8,7 +9,13 @@ export async function launch(): Promise<Browser> {
 }
 
 export async function markup(page: Page, file: string, override: Core.Config = null): Promise<string[]> {
-    const html = fs.readFileSync(path.resolve(__dirname, `./html/${file}`), 'utf8');
+    const htmlPath = path.resolve(__dirname, `./html/${file}`);
+    const htmlFileUrl = url.pathToFileURL(htmlPath).toString();
+    const html = fs.readFileSync(htmlPath, 'utf8');
+    await Promise.all([
+        page.goto(htmlFileUrl),
+        page.waitForNavigation()
+    ]);
     await page.setContent(html.replace("</body>", `
         <script>
           window.payloads = [];
@@ -17,6 +24,7 @@ export async function markup(page: Page, file: string, override: Core.Config = n
         </script>
         </body>
     `));
+    await page.hover("#two");
     await page.waitForFunction("payloads && payloads.length > 1");
     return await page.evaluate('payloads');
 }
@@ -34,7 +42,7 @@ export function node(decoded: Data.DecodedPayload[], key: string, value: string
     }
 
     // Walking over the decoded payload to find the right match
-    for (let i = decoded.length - 1; i > 0; i--) {
+    for (let i = decoded.length - 1; i >= 0; i--) {
         if (decoded[i].dom) {
             for (let j = 0; j < decoded[i].dom.length; j++) {
                 if (decoded[i].dom[j].data) {

package/types/core.d.ts

@@ -101,12 +101,13 @@ export interface BrowserEvent {
 }
 
 export interface Report {
-    c: Data.Check; // Reporting code
+    v: string; // Version
     p: string; // Project Id
     u: string; // User Id
     s: string; // Session Id
     n: number; // Page Number
     m?: string; // Message, optional
+    e?: string; // Error Stack, optional
 }
 
 export interface Config {

package/types/data.d.ts

@@ -84,7 +84,8 @@ export const enum Metric {
     CartTax = 23,
     CartTotal = 24,
     EventCount = 25,
-    Automation = 26
+    Automation = 26,
+    Mobile = 27
 }
 
 export const enum Dimension {
@@ -109,7 +110,11 @@ export const enum Dimension {
     Headline = 18,
     MetaType = 19,
     MetaTitle = 20,
-    Generator = 21
+    Generator = 21,
+    Platform = 22,
+    PlatformVersion = 23,
+    Brand = 24,
+    Model = 25
 }
 
 export const enum Check {
@@ -238,7 +243,8 @@ export const enum Constant {
     HTTPS = "https://",
     CompressionStream = "CompressionStream",
     Accept = "Accept",
-    ClarityGzip = "application/x-clarity-gzip"
+    ClarityGzip = "application/x-clarity-gzip",
+    Tilde = "~",
 }
 
 export const enum XMLReadyState {

package/types/index.d.ts

@@ -15,7 +15,7 @@ interface Clarity {
   event: (name: string, value: string) => void;
   set: (variable: string, value: string | string[]) => void;
   identify: (userId: string, sessionId?: string, pageId?: string) => void;
-  metadata: (callback: Data.MetadataCallback) => void;
+  metadata: (callback: Data.MetadataCallback, wait?: boolean) => void;
 }
 
 interface Helper {

package/types/layout.d.ts

@@ -40,6 +40,7 @@ export const enum Constant {
     Src = "src",
     Srcset = "srcset",
     Box = "#",
+    Bang = "!",
     Period = ".",
     MaskData = "data-clarity-mask",
     UnmaskData = "data-clarity-unmask",