clarity-js 0.6.28 → 0.6.32

package/src/data/upload.ts

@@ -42,7 +42,7 @@ export function queue(tokens: Token[], transmit: boolean = true): void {
         let now = time();
         let type = tokens.length > 1 ? tokens[1] : null;
         let event = JSON.stringify(tokens);
-        
+
         switch (type) {
             case Event.Discover:
                 discoverBytes += event.length;
@@ -118,12 +118,12 @@ async function upload(final: boolean = false): Promise<void> {
 
     let p = sendPlaybackBytes ? `[${playback.join()}]` : Constant.Empty;
     let encoded: EncodedPayload = {e, a, p};
-    
+
     // Get the payload ready for sending over the wire
     // We also attempt to compress the payload if it is not the last payload and the browser supports it
     // In all other cases, we continue to send back string value
     let payload = stringify(encoded);
-    let zipped = last ? null : await compress(payload) 
+    let zipped = last ? null : await compress(payload)
     metric.sum(Metric.TotalBytes, zipped ? zipped.length : payload.length);
     send(payload, zipped, envelope.data.sequence, last);
 
@@ -151,8 +151,11 @@ function send(payload: string, zipped: Uint8Array, sequence: number, beacon: boo
         // However, we don't want to rely on it for every payload, since we have no ability to retry if the upload failed.
         // Also, in case of sendBeacon, we do not have a way to alter HTTP headers and therefore can't send compressed payload
         if (beacon && "sendBeacon" in navigator) {
-            dispatched = navigator.sendBeacon(url, payload);
-            if (dispatched) { done(sequence); }
+            try {
+                // Navigator needs to be bound to sendBeacon before it is used to avoid errors in some browsers
+                dispatched = navigator.sendBeacon.bind(navigator)(url, payload);
+                if (dispatched) { done(sequence); }
+            } catch { /* do nothing - and we will automatically fallback to XHR below */ }
         }
 
         // Before initiating XHR upload, we check if the data has already been uploaded using sendBeacon
@@ -189,7 +192,7 @@ function check(xhr: XMLHttpRequest, sequence: number): void {
     if (xhr && xhr.readyState === XMLReadyState.Done && transitData) {
         // Attempt send payload again (as configured in settings) if we do not receive a success (2XX) response code back from the server
         if ((xhr.status < 200 || xhr.status > 208) && transitData.attempts <= Setting.RetryLimit) {
-            // We re-attempt in all cases except when server explicitly rejects our request with 4XX error 
+            // We re-attempt in all cases except when server explicitly rejects our request with 4XX error
             if (xhr.status >= 400 && xhr.status < 500) {
                 // In case of a 4XX response from the server, we bail out instead of trying again
                 limit.trigger(Check.Server);
@@ -212,7 +215,7 @@ function check(xhr: XMLHttpRequest, sequence: number): void {
             // Handle response if it was a 200 response with a valid body
             if (xhr.status === 200 && xhr.responseText) { response(xhr.responseText); }
             // If we exhausted our retries then trigger Clarity's shutdown for this page since the data will be incomplete
-            if (xhr.status === 0) { 
+            if (xhr.status === 0) {
                 // And, right before we terminate the session, we will attempt one last time to see if we can use
                 // different transport option (sendBeacon vs. XHR) to get this data to the server for analysis purposes
                 send(transitData.data, null, sequence, true);
@@ -233,7 +236,7 @@ function done(sequence: number): void {
 
 function delay(): number {
     // Progressively increase delay as we continue to send more payloads from the client to the server
-    // If we are not uploading data to a server, and instead invoking UploadCallback, in that case keep returning configured value 
+    // If we are not uploading data to a server, and instead invoking UploadCallback, in that case keep returning configured value
     let gap = config.lean === false && discoverBytes > 0 ? Setting.MinUploadDelay : envelope.data.sequence * config.delay;
     return typeof config.upload === Constant.String ? Math.max(Math.min(gap, Setting.MaxUploadDelay), Setting.MinUploadDelay) : config.delay;
 }

package/src/diagnostic/internal.ts

@@ -1,7 +1,5 @@
-import { Code, Constant, Event, Severity } from "@clarity-types/data";
+import { Code, Event, Severity } from "@clarity-types/data";
 import { LogData } from "@clarity-types/diagnostic";
-import config from "@src/core/config";
-import { bind } from "@src/core/event";
 import encode from "./encode";
 
 let history: { [key: number]: string[] } = {};
@@ -9,7 +7,6 @@ export let data: LogData;
 
 export function start(): void {
     history = {};
-    bind(document, "securitypolicyviolation", csp);
 }
 
 export function log(code: Code, severity: Severity, name: string = null, message: string = null, stack: string = null): void {
@@ -26,16 +23,6 @@ export function log(code: Code, severity: Severity, name: string = null, message
     encode(Event.Log);
 }
 
-function csp(e: SecurityPolicyViolationEvent): void {
-    let upload = config.upload as string;
-    let parts = upload ? upload.substr(0, upload.indexOf("/", Constant.HTTPS.length)).split(Constant.Dot) : []; // Look for first "/" starting after initial "https://" string
-    let domain = parts.length >= 2 ? parts.splice(-2).join(Constant.Dot) : null;
-    // Capture content security policy violation only if disposition value is not explicitly set to "report"
-    if (domain && e.blockedURI && e.blockedURI.indexOf(domain) >= 0 && e["disposition"] !== Constant.Report) {
-        log(Code.ContentSecurityPolicy, Severity.Warning, e.blockedURI);
-    }
-}
-
 export function stop(): void {
     history = {};
 }

package/src/layout/dom.ts

@@ -1,6 +1,6 @@
 import { Privacy } from "@clarity-types/core";
 import { Code, Setting, Severity } from "@clarity-types/data";
-import { Constant, NodeInfo, NodeValue, SelectorInput, Source } from "@clarity-types/layout";
+import { Constant, NodeInfo, NodeValue, Selector, SelectorInput, Source } from "@clarity-types/layout";
 import config from "@src/core/config";
 import hash from "@src/core/hash";
 import * as internal from "@src/diagnostic/internal";
@@ -12,13 +12,15 @@ let index: number = 1;
 
 // Reference: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/Input#%3Cinput%3E_types
 const DISALLOWED_TYPES = ["password", "hidden", "email", "tel"];
-const DISALLOWED_NAMES = ["addr", "cell", "code", "dob", "email", "mob", "name", "phone", "secret", "social", "ssn", "tel", "zip", "pass"];
+const DISALLOWED_NAMES = ["addr", "cell", "code", "dob", "email", "mob", "name", "phone", "secret", "social", "ssn", "tel", "zip", "pass", "card", "account", "cvv", "ccv"];
 const DISALLOWED_MATCH = ["address", "password", "contact"];
 
 let nodes: Node[] = [];
 let values: NodeValue[] = [];
 let updateMap: number[] = [];
 let hashMap: { [hash: string]: number } = {};
+let override = [];
+let unmask = [];
 
 // The WeakMap object is a collection of key/value pairs in which the keys are weakly referenced
 let idMap: WeakMap<Node, number> = null; // Maps node => id.
@@ -27,7 +29,7 @@ let privacyMap: WeakMap<Node, Privacy> = null; // Maps node => Privacy (enum)
 
 export function start(): void {
     reset();
-    parse(document);
+    parse(document, true);
 }
 
 export function stop(): void {
@@ -40,6 +42,8 @@ function reset(): void {
     values = [];
     updateMap = [];
     hashMap = {};
+    override = [];
+    unmask = [];
     idMap = new WeakMap();
     iframeMap = new WeakMap();
     privacyMap = new WeakMap();
@@ -47,10 +51,13 @@ function reset(): void {
 
 // We parse new root nodes for any regions or masked nodes in the beginning (document) and
 // later whenever there are new additions or modifications to DOM (mutations)
-export function parse(root: ParentNode): void {
+export function parse(root: ParentNode, init: boolean = false): void {
     // Wrap selectors in a try / catch block.
     // It's possible for script to receive invalid selectors, e.g. "'#id'" with extra quotes, and cause the code below to fail
     try {
+        // Parse unmask configuration into separate query selectors and override tokens as part of initialization
+        if (init) { config.unmask.forEach(x => x.indexOf(Constant.Bang) < 0 ? unmask.push(x) : override.push(x.substr(1))); }   
+
         // Since mutations may happen on leaf nodes too, e.g. text nodes, which may not support all selector APIs.
         // We ensure that the root note supports querySelectorAll API before executing the code below to identify new regions.
         if ("querySelectorAll" in root) {
@@ -58,7 +65,7 @@ export function parse(root: ParentNode): void {
             extract.metrics(root, config.metrics);
             extract.dimensions(root, config.dimensions);
             config.mask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.TextImage))); // Masked Elements
-            config.unmask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.None))); // Unmasked Elements
+            unmask.forEach(x => root.querySelectorAll(x).forEach(e => privacyMap.set(e, Privacy.None))); // Unmasked Elements
         }
     } catch (e) { internal.log(Code.Selector, Severity.Warning, e ? e.name : null); }
 }
@@ -80,19 +87,17 @@ export function add(node: Node, parent: Node, data: NodeInfo, source: Source): v
     let previousId = getPreviousId(node);
     let privacy = config.content ? Privacy.Sensitive : Privacy.Text;
     let parentValue = null;
-    let parentTag = Constant.Empty;
     let regionId = region.exists(node) ? id : null;
 
     if (parentId >= 0 && values[parentId]) {
         parentValue = values[parentId];
-        parentTag = parentValue.data.tag;
         parentValue.children.push(id);
         regionId = regionId === null ? parentValue.region : regionId;
         privacy = parentValue.metadata.privacy;
     }
 
     // Check to see if this particular node should be masked or not
-    privacy = getPrivacy(node, data, parentTag, privacy);
+    privacy = getPrivacy(node, data, parentValue, privacy);
 
     // If there's an explicit region attribute set on the element, use it to mark a region on the page
     if (data.attributes && Constant.RegionData in data.attributes) {
@@ -198,13 +203,27 @@ export function iframe(node: Node): HTMLIFrameElement {
     return doc && iframeMap.has(doc) ? iframeMap.get(doc) : null;
 }
 
-function getPrivacy(node: Node, data: NodeInfo, parentTag: string, privacy: Privacy): Privacy {
+function getPrivacy(node: Node, data: NodeInfo, parent: NodeValue, privacy: Privacy): Privacy {
     let attributes = data.attributes;
     let tag = data.tag.toUpperCase();
 
     // If this node was explicitly configured to contain sensitive content, use that information and return the value
     if (privacyMap.has(node)) { return privacyMap.get(node); }
 
+    // If it's a text node belonging to a STYLE or TITLE tag; 
+    // Or, the text node belongs to one of SCRUB_EXCEPTIONS
+    // then reset the privacy setting to ensure we capture the content
+    if (tag === Constant.TextTag && parent && parent.data) {
+        let path = parent.selector ? parent.selector[Selector.Stable] : Constant.Empty;
+        privacy = parent.data.tag === Constant.StyleTag || parent.data.tag === Constant.TitleTag ? Privacy.None : privacy;
+        for (let entry of override) {
+            if (path.indexOf(entry) >= 0) {
+                privacy = Privacy.None;
+                break;
+            }
+        }
+    }
+
     // Do not proceed if attributes are missing for the node
     if (attributes === null || attributes === undefined) { return privacy; }
 
@@ -243,10 +262,6 @@ function getPrivacy(node: Node, data: NodeInfo, parentTag: string, privacy: Priv
     if (Constant.MaskData in attributes) { privacy = Privacy.TextImage; }
     if (Constant.UnmaskData in attributes) { privacy = Privacy.None; }
 
-    // If it's a text node belonging to a STYLE or TITLE tag; then reset the privacy setting to ensure we capture the content
-    let cTag = tag === Constant.TextTag ? parentTag : tag;
-    if (cTag === Constant.StyleTag || cTag === Constant.TitleTag) { privacy = Privacy.None; }
-
     return privacy;
 }
 

package/src/layout/mutation.ts

@@ -1,6 +1,7 @@
 import { Priority, Task, Timer } from "@clarity-types/core";
 import { Code, Event, Metric, Severity } from "@clarity-types/data";
 import { Constant, MutationHistory, MutationQueue, Setting, Source } from "@clarity-types/layout";
+import api from "@src/core/api";
 import { bind } from "@src/core/event";
 import measure from "@src/core/measure";
 import * as task from "@src/core/task";
@@ -36,7 +37,7 @@ export function start(): void {
 
     if (insertRule === null) { insertRule = CSSStyleSheet.prototype.insertRule; }
     if (deleteRule === null) { deleteRule = CSSStyleSheet.prototype.deleteRule; }
-    if (attachShadow === null) { attachShadow = HTMLElement.prototype.attachShadow; }
+    if (attachShadow === null) { attachShadow = Element.prototype.attachShadow; }
 
     // Some popular open source libraries, like styled-components, optimize performance
     // by injecting CSS using insertRule API vs. appending text node. A side effect of
@@ -56,7 +57,7 @@ export function start(): void {
     // In case we are unable to add a hook and browser throws an exception,
     // reset attachShadow variable and resume processing like before
     try {
-      HTMLElement.prototype.attachShadow = function (): ShadowRoot {
+      Element.prototype.attachShadow = function (): ShadowRoot {
         return schedule(attachShadow.apply(this, arguments)) as ShadowRoot;
       }
     } catch { attachShadow = null; }
@@ -68,11 +69,8 @@ export function observe(node: Node): void {
   // For this reason, we need to wire up mutations every time we see a new shadow dom.
   // Also, wrap it inside a try / catch. In certain browsers (e.g. legacy Edge), observer on shadow dom can throw errors
   try {
-    // In an edge case, it's possible to get stuck into infinite Mutation loop within Angular applications
-    // This appears to be an issue with Zone.js package, see: https://github.com/angular/angular/issues/31712
-    // As a temporary work around, ensuring Clarity can invoke MutationObserver outside of Zone (and use native implementation instead)
-    let api: string = window[Constant.Zone] && Constant.Symbol in window[Constant.Zone] ? window[Constant.Zone][Constant.Symbol](Constant.MutationObserver) : Constant.MutationObserver;
-    let observer =  api in window ? new window[api](measure(handle) as MutationCallback) : null;
+    let m = api(Constant.MutationObserver);
+    let observer =  m in window ? new window[m](measure(handle) as MutationCallback) : null;
     if (observer) {
       observer.observe(node, { attributes: true, childList: true, characterData: true, subtree: true });
       observers.push(observer);
@@ -107,7 +105,7 @@ export function stop(): void {
 
   // Restoring original attachShadow
   if (attachShadow != null) {
-    HTMLElement.prototype.attachShadow = attachShadow;
+    Element.prototype.attachShadow = attachShadow;
     attachShadow = null;
   }
 
@@ -145,6 +143,7 @@ async function process(): Promise<void> {
       let target = mutation.target;
       let type = track(mutation, timer);
       if (type && target && target.ownerDocument) { dom.parse(target.ownerDocument); }
+      if (type && target && target.nodeType == Node.DOCUMENT_FRAGMENT_NODE && (target as ShadowRoot).host) { dom.parse(target as ShadowRoot); }
       switch (type) {
         case Constant.Attributes:
             processNode(target, Source.Attributes);
@@ -234,10 +233,13 @@ function schedule(node: Node): Node {
 
 function trigger(): void {
   for (let node of queue) {
-    let shadowRoot = node && node.nodeType === Node.DOCUMENT_FRAGMENT_NODE;
-    // Skip re-processing shadowRoot if it was already discovered
-    if (shadowRoot && dom.has(node)) { continue; }
-    generate(node, shadowRoot ? Constant.ChildList : Constant.CharacterData);
+    // Generate a mutation for this node only if it still exists
+    if (node) {
+      let shadowRoot = node.nodeType === Node.DOCUMENT_FRAGMENT_NODE;
+      // Skip re-processing shadowRoot if it was already discovered
+      if (shadowRoot && dom.has(node)) { continue; }
+      generate(node, shadowRoot ? Constant.ChildList : Constant.CharacterData);
+    }
   }
   queue = [];
 }

package/src/layout/node.ts

@@ -46,6 +46,7 @@ export default function (node: Node, source: Source): Node {
         case Node.DOCUMENT_FRAGMENT_NODE:
             let shadowRoot = (node as ShadowRoot);
             if (shadowRoot.host) {
+                dom.parse(shadowRoot);
                 let type = typeof (shadowRoot.constructor);
                 if (type === Constant.Function && shadowRoot.constructor.toString().indexOf(Constant.NativeCode) >= 0) {
                     observe(shadowRoot);

package/src/layout/selector.ts

@@ -27,12 +27,10 @@ export default function(input: SelectorInput, beta: boolean = false): string {
                 // In beta mode, update selector to use "id" field when available. There are two exceptions:
                 // (1) if "id" appears to be an auto generated string token, e.g. guid or a random id containing digits
                 // (2) if "id" appears inside a shadow DOM, in which case we continue to prefix up to shadow DOM to prevent conflicts
-                let shadowStart = prefix.lastIndexOf(Constant.ShadowDomTag);
-                let shadowEnd = prefix.indexOf(">", shadowStart) + 1;
                 let id = Constant.Id in a && a[Constant.Id].length > 0 ? a[Constant.Id] : null;
                 classes = input.tag !== Constant.BodyTag && classes ? classes.filter(c => !hasDigits(c)) : [];
                 selector = classes.length > 0 ? `${prefix}${input.tag}.${classes.join(".")}${suffix}` : selector;
-                selector = id && hasDigits(id) === false ? (shadowStart >= 0 ? `${prefix.substr(0, shadowEnd)}#${id}` : `#${id}`) : selector;
+                selector = id && hasDigits(id) === false ? `${getDomPrefix(prefix)}#${id}` : selector;
             } else {
                 // Otherwise, fallback to stable mode, where we include class names as part of the selector
                 selector = classes ? `${prefix}${input.tag}.${classes.join(".")}${suffix}` : selector;
@@ -41,6 +39,19 @@ export default function(input: SelectorInput, beta: boolean = false): string {
     }
 }
 
+function getDomPrefix(prefix: string): string {
+  const shadowDomStart = prefix.lastIndexOf(Constant.ShadowDomTag);
+  const iframeDomStart = prefix.lastIndexOf(`${Constant.IFramePrefix}${Constant.HTML}`);
+  const domStart = Math.max(shadowDomStart, iframeDomStart);
+  
+  if (domStart < 0) {
+    return "";
+  }
+
+  const domEnd = prefix.indexOf(">", domStart) + 1;
+  return prefix.substr(0, domEnd);
+}
+
 // Check if the given input string has digits or not
 function hasDigits(value: string): boolean {
     for (let i = 0; i < value.length; i++) {

package/test/helper.ts

@@ -1,5 +1,6 @@
 import { Core, Data, Layout } from "clarity-decode";
 import * as fs from 'fs';
+import * as url from 'url';
 import * as path from 'path';
 import { Browser, Page, chromium } from 'playwright';
 
@@ -8,7 +9,13 @@ export async function launch(): Promise<Browser> {
 }
 
 export async function markup(page: Page, file: string, override: Core.Config = null): Promise<string[]> {
-    const html = fs.readFileSync(path.resolve(__dirname, `./html/${file}`), 'utf8');
+    const htmlPath = path.resolve(__dirname, `./html/${file}`);
+    const htmlFileUrl = url.pathToFileURL(htmlPath).toString();
+    const html = fs.readFileSync(htmlPath, 'utf8');
+    await Promise.all([
+        page.goto(htmlFileUrl),
+        page.waitForNavigation()
+    ]);
     await page.setContent(html.replace("</body>", `
         <script>
           window.payloads = [];
@@ -17,6 +24,7 @@ export async function markup(page: Page, file: string, override: Core.Config = n
         </script>
         </body>
     `));
+    await page.hover("#two");
     await page.waitForFunction("payloads && payloads.length > 1");
     return await page.evaluate('payloads');
 }
@@ -34,7 +42,7 @@ export function node(decoded: Data.DecodedPayload[], key: string, value: string
     }
 
     // Walking over the decoded payload to find the right match
-    for (let i = decoded.length - 1; i > 0; i--) {
+    for (let i = decoded.length - 1; i >= 0; i--) {
         if (decoded[i].dom) {
             for (let j = 0; j < decoded[i].dom.length; j++) {
                 if (decoded[i].dom[j].data) {

package/types/core.d.ts

@@ -127,3 +127,10 @@ export interface Config {
     fallback?: string;
     upgrade?: (key: string) => void;
 }
+
+export const enum Constant {
+    Zone = "Zone",
+    Symbol = "__symbol__",
+    AddEventListener = "addEventListener",
+    RemoveEventListener = "removeEventListener"
+}

package/types/data.d.ts

@@ -135,6 +135,9 @@ export const enum Code {
     CallStackDepth = 4,
     Selector = 5,
     Metric = 6,
+/**
+ * @deprecated No longer support ContentSecurityPolicy
+ */
     ContentSecurityPolicy = 7
 }
 

package/types/layout.d.ts

@@ -40,6 +40,7 @@ export const enum Constant {
     Src = "src",
     Srcset = "srcset",
     Box = "#",
+    Bang = "!",
     Period = ".",
     MaskData = "data-clarity-mask",
     UnmaskData = "data-clarity-unmask",
@@ -74,8 +75,6 @@ export const enum Constant {
     BorderBox = "border-box",
     Value = "value",
     MutationObserver = "MutationObserver",
-    Zone = "Zone",
-    Symbol = "__symbol__",
     JsonLD = "application/ld+json",
     String = "string",
     Number = "number",