@gemx-dev/clarity-js 0.8.39

package/src/data/upload.ts

@@ -0,0 +1,333 @@
+import { UploadCallback } from "@clarity-types/core";
+import { BooleanFlag, Check, Code, Constant, EncodedPayload, Event, Metric, Setting, Severity, Token, Transit, UploadData, XMLReadyState } from "@clarity-types/data";
+import * as clarity from "@src/clarity";
+import config from "@src/core/config";
+import measure from "@src/core/measure";
+import { time } from "@src/core/time";
+import { clearTimeout, setTimeout } from "@src/core/timeout";
+import compress from "@src/data/compress";
+import encode from "@src/data/encode";
+import * as envelope from "@src/data/envelope";
+import * as data from "@src/data/index";
+import * as limit from "@src/data/limit";
+import * as metadata from "@src/data/metadata";
+import * as metric from "@src/data/metric";
+import * as ping from "@src/data/ping";
+import * as internal from "@src/diagnostic/internal";
+import * as timeline from "@src/interaction/timeline";
+import * as region from "@src/layout/region";
+import * as extract from "@src/data/extract";
+import * as style from "@src/layout/style";
+import { report } from "@src/core/report";
+import { signalsEvent } from "@src/data/signal";
+import { snapshot } from "@src/insight/snapshot";
+import * as dynamic from "@src/core/dynamic";
+
+let discoverBytes: number = 0;
+let playbackBytes: number = 0;
+let playback: string[];
+let analysis: string[];
+let timeout: number = null;
+let transit: Transit;
+let active: boolean;
+let queuedTime: number = 0;
+let leanLimit = false;
+export let track: UploadData;
+
+export function start(): void {
+    active = true;
+    discoverBytes = 0;
+    playbackBytes = 0;
+    leanLimit = false;
+    queuedTime = 0;
+    playback = [];
+    analysis = [];
+    transit = {};
+    track = null;
+}
+
+export function queue(tokens: Token[], transmit: boolean = true): void {
+    if (!active) {
+        return;
+    }
+
+    let now = time();
+    let type = tokens.length > 1 ? tokens[1] : null;
+    let event = JSON.stringify(tokens);
+
+    if (!config.lean) {
+        leanLimit = false;
+    } else if (!leanLimit && playbackBytes + event.length > Setting.PlaybackBytesLimit) {
+        internal.log(Code.LeanLimit, Severity.Info);
+        leanLimit = true;
+    }
+
+    switch (type) {
+        case Event.Discover:
+            if (leanLimit) { break; }
+            discoverBytes += event.length;
+        case Event.Box:
+        case Event.Mutation:
+        case Event.Snapshot:
+        case Event.StyleSheetAdoption:
+        case Event.StyleSheetUpdate:
+        case Event.Animation:
+        case Event.CustomElement:
+            if (leanLimit) { break; }
+            playbackBytes += event.length;
+            playback.push(event);
+            break;
+        default:
+            analysis.push(event);
+            break;
+    }
+
+    // Increment event count metric
+    metric.count(Metric.EventCount);
+
+    // Following two checks are precautionary and act as a fail safe mechanism to get out of unexpected situations.
+    // Check 1: If for any reason the upload hasn't happened after waiting for 2x the config.delay time,
+    // reset the timer. This allows Clarity to attempt an upload again.
+    let gap = delay();
+    if (now - queuedTime > (gap * 2)) {
+        clearTimeout(timeout);
+        timeout = null;
+    }
+
+    // Transmit Check: When transmit is set to true (default), it indicates that we should schedule an upload
+    // However, in certain scenarios - like metric calculation - which are triggered as part of an existing upload
+    // We enrich the data going out with the existing upload. In these cases, call to upload comes with 'transmit' set to false.
+    if (transmit && timeout === null) {
+        if (type !== Event.Ping) { ping.reset(); }
+        timeout = setTimeout(upload, gap);
+        queuedTime = now;
+        limit.check(playbackBytes);
+    }
+}
+
+export function stop(): void {
+    clearTimeout(timeout);
+    upload(true);
+    discoverBytes = 0;
+    playbackBytes = 0;
+    leanLimit = false;
+    queuedTime = 0;
+    playback = [];
+    analysis = [];
+    transit = {};
+    track = null;
+    active = false;
+}
+
+async function upload(final: boolean = false): Promise<void> {
+    if (!active) {
+        return;
+    }
+
+    timeout = null;
+
+    // Check if we can send playback bytes over the wire or not
+    // For better instrumentation coverage, we send playback bytes from second sequence onwards
+    // And, we only send playback metric when we are able to send the playback bytes back to server
+    let sendPlaybackBytes = config.lean === false && playbackBytes > 0 && (playbackBytes < Setting.MaxFirstPayloadBytes || envelope.data.sequence > 0);
+    if (sendPlaybackBytes) { metric.max(Metric.Playback, BooleanFlag.True); }
+
+    // CAUTION: Ensure "transmit" is set to false in the queue function for following events
+    // Otherwise you run a risk of infinite loop.
+    region.compute();
+    timeline.compute();
+    data.compute();
+    style.compute();
+
+    // Treat this as the last payload only if final boolean was explicitly set to true.
+    // In real world tests, we noticed that certain third party scripts (e.g. https://www.npmjs.com/package/raven-js)
+    // could inject function arguments for internal tracking (likely stack traces for script errors).
+    // For these edge cases, we want to ensure that an injected object (e.g. {"key": "value"}) isn't mistaken to be true.
+    let last = final === true;
+    
+    // In some cases envelope has null data because it's part of the shutdown process while there's one upload call queued which might introduce runtime error
+    if(!envelope.data) return;
+
+    let e = JSON.stringify(envelope.envelope(last));
+    let a = `[${analysis.join()}]`;
+
+    let p = sendPlaybackBytes ? `[${playback.join()}]` : Constant.Empty;
+    
+    // For final (beacon) payloads, If size is too large, we need to remove playback data
+    if (last && p.length > 0 && (e.length + a.length + p.length > Setting.MaxBeaconPayloadBytes)) {
+        p = Constant.Empty;
+    }
+
+    let encoded: EncodedPayload = {e, a, p};
+
+    // Get the payload ready for sending over the wire
+    // We also attempt to compress the payload if it is not the last payload and the browser supports it
+    // In all other cases, we continue to send back string value
+    let payload = stringify(encoded);
+    let zipped = last ? null : await compress(payload);
+    metric.sum(Metric.TotalBytes, zipped ? zipped.length : payload.length);
+    send(payload, zipped, envelope.data.sequence, last);
+
+    // Clear out events now that payload has been dispatched
+    analysis = [];
+    if (sendPlaybackBytes) {
+        playback = [];
+        playbackBytes = 0;
+        discoverBytes = 0;
+        leanLimit = false;
+    }
+}
+
+function stringify(encoded: EncodedPayload): string {
+    return encoded.p.length > 0 ? `{"e":${encoded.e},"a":${encoded.a},"p":${encoded.p}}` : `{"e":${encoded.e},"a":${encoded.a}}`;
+}
+
+function send(payload: string, zipped: Uint8Array, sequence: number, beacon: boolean = false): void {
+    // Upload data if a valid URL is defined in the config
+    if (typeof config.upload === Constant.String) {
+        const url = config.upload as string;
+        let dispatched = false;
+
+        // If it's the last payload, attempt to upload using sendBeacon first.
+        // The advantage to using sendBeacon is that browser can decide to upload asynchronously, improving chances of success
+        // However, we don't want to rely on it for every payload, since we have no ability to retry if the upload failed.
+        // Also, in case of sendBeacon, we do not have a way to alter HTTP headers and therefore can't send compressed payload
+        if (beacon && navigator && navigator["sendBeacon"]) {
+            try {
+                // Navigator needs to be bound to sendBeacon before it is used to avoid errors in some browsers
+                dispatched = navigator.sendBeacon.bind(navigator)(url, payload);
+                if (dispatched) { 
+                    done(sequence); 
+                }
+            } catch(error) {
+                // If sendBeacon fails, we do nothing and continue with XHR upload
+            }
+        }
+
+        // Before initiating XHR upload, we check if the data has already been uploaded using sendBeacon
+        // There are two cases when dispatched could still be false:
+        //   a) It's not the last payload, and therefore we didn't attempt sending sendBeacon
+        //   b) It's the last payload, however, we failed to queue sendBeacon call and need to now fall back to XHR.
+        //      E.g. if data is over 64KB, several user agents (like Chrome) will reject to queue the sendBeacon call.
+        if (dispatched === false) {
+            // While tracking payload for retry, we only track string value of the payload to err on the safe side
+            // Not all browsers support compression API and the support for it in supported browsers is still experimental
+            if (sequence in transit) { transit[sequence].attempts++; } else { transit[sequence] = { data: payload, attempts: 1 }; }
+            let xhr = new XMLHttpRequest();
+            xhr.open("POST", url, true);
+            xhr.timeout = Setting.UploadTimeout;
+            xhr.ontimeout = () => { report(new Error(`${Constant.Timeout} : ${url}`)) };
+            if (sequence !== null) { xhr.onreadystatechange = (): void => { measure(check)(xhr, sequence); }; }
+            xhr.withCredentials = true;
+            if (zipped) {
+                // If we do have valid compressed array, send it with appropriate HTTP headers so server can decode it appropriately
+                xhr.setRequestHeader(Constant.Accept, Constant.ClarityGzip);
+                xhr.send(zipped);
+            } else {
+                // In all other cases, continue sending string back to the server
+                xhr.send(payload);
+            }
+        }
+    } else if (config.upload) {
+        const callback = config.upload as UploadCallback;
+        callback(payload);
+        done(sequence);
+    }
+}
+
+function check(xhr: XMLHttpRequest, sequence: number): void {
+    var transitData = transit[sequence];
+    if (xhr && xhr.readyState === XMLReadyState.Done && transitData) {
+        // Attempt send payload again (as configured in settings) if we do not receive a success (2XX) response code back from the server
+        if ((xhr.status < 200 || xhr.status > 208) && transitData.attempts <= Setting.RetryLimit) {
+            // We re-attempt in all cases except when server explicitly rejects our request with 4XX error
+            if (xhr.status >= 400 && xhr.status < 500) {
+                // In case of a 4XX response from the server, we bail out instead of trying again
+                limit.trigger(Check.Server);
+            } else {
+                // Browser will send status = 0 when it refuses to put network request over the wire
+                // This could happen for several reasons, couple of known ones are:
+                //    1: Browsers block upload because of content security policy violation
+                //    2: Safari will terminate pending XHR requests with status code 0 if the user navigates away from the page
+                // In any case, we switch the upload URL to fallback configuration (if available) before re-trying one more time
+                if (xhr.status === 0) { config.upload = config.fallback ? config.fallback : config.upload; }
+                // Capture the status code and number of attempts so we can report it back to the server
+                track = { sequence, attempts: transitData.attempts, status: xhr.status };
+                encode(Event.Upload);
+                // In all other cases, re-attempt sending the same data
+                // For retry we always fallback to string payload, even though we may have attempted
+                // sending zipped payload earlier
+                send(transitData.data, null, sequence);
+            }
+        } else {
+            track = { sequence, attempts: transitData.attempts, status: xhr.status };
+            // Send back an event only if we were not successful in our first attempt
+            if (transitData.attempts > 1) { encode(Event.Upload); }
+            // Handle response if it was a 200 response with a valid body
+            if (xhr.status === 200 && xhr.responseText) { response(xhr.responseText); }
+            // If we exhausted our retries then trigger Clarity's shutdown for this page since the data will be incomplete
+            if (xhr.status === 0) {
+                // And, right before we terminate the session, we will attempt one last time to see if we can use
+                // different transport option (sendBeacon vs. XHR) to get this data to the server for analysis purposes
+                send(transitData.data, null, sequence, true);
+                limit.trigger(Check.Retry);
+            }
+            // Signal that this request completed successfully
+            if (xhr.status >= 200 && xhr.status <= 208) { done(sequence); }
+            // Stop tracking this payload now that it's all done
+            delete transit[sequence];
+        }
+    }
+}
+
+function done(sequence: number): void {
+    // If we everything went successfully, and it is the first sequence, save this session for future reference
+    if (sequence === 1) {
+        metadata.save();
+        metadata.callback();
+    }
+}
+
+function delay(): number {
+    // Progressively increase delay as we continue to send more payloads from the client to the server
+    // If we are not uploading data to a server, and instead invoking UploadCallback, in that case keep returning configured value
+    let gap = config.lean === false && discoverBytes > 0 ? Setting.MinUploadDelay : envelope.data.sequence * config.delay;
+    return typeof config.upload === Constant.String ? Math.max(Math.min(gap, Setting.MaxUploadDelay), Setting.MinUploadDelay) : config.delay;
+}
+
+function response(payload: string): void {
+    let lines = payload && payload.length > 0 ? payload.split("\n") : [];
+    for (var line of lines)
+    {
+        let parts = line && line.length > 0 ? line.split(/ (.*)/) : [Constant.Empty];
+        switch (parts[0]) {
+            case Constant.End:
+                // Clear out session storage and end the session so we can start fresh the next time
+                limit.trigger(Check.Server);
+                break;
+            case Constant.Upgrade:
+                // Upgrade current session to send back playback information
+                clarity.upgrade(Constant.Auto);
+                break;
+            case Constant.Action:
+                // Invoke action callback, if configured and has a valid value
+                if (config.action && parts.length > 1) { config.action(parts[1]); }
+                break;
+            case Constant.Extract:
+                if (parts.length > 1) { extract.trigger(parts[1]); }
+                break;
+            case Constant.Signal:
+                if (parts.length > 1) { signalsEvent(parts[1]); }
+                break;
+            case Constant.Module:
+                if (parts.length > 1) {
+                    dynamic.event(parts[1]);
+                }
+                break;
+            case Constant.Snapshot:
+                config.lean = false; // Disable lean mode to ensure we can send playback information to server.
+                snapshot();
+                break;
+        }
+    }
+}

package/src/data/variable.ts

@@ -0,0 +1,84 @@
+import { Constant, Event, IdentityData, Setting, VariableData } from "@clarity-types/data";
+import * as core from "@src/core";
+import { scrub } from "@src/core/scrub";
+import encode from "./encode";
+
+export let data: VariableData = null;
+
+export function start(): void {
+    reset();
+}
+
+export function set(variable: string, value: string | string[]): void {
+    let values = typeof value === Constant.String ? [value as string] : value as string[];
+    log(variable, values);
+}
+
+export async function identify(userId: string, sessionId: string = null, pageId: string = null, userHint: string = null): Promise<IdentityData> {
+    let output: IdentityData = { userId: await sha256(userId), userHint: userHint || redact(userId) };
+
+    // By default, hash custom userId using SHA256 algorithm on the client to preserve privacy
+    log(Constant.UserId, [output.userId]);
+    
+    // Optional non-identifying name for the user
+    // If name is not explicitly provided, we automatically generate a redacted version of the userId
+    log(Constant.UserHint, [output.userHint]);
+    log(Constant.UserType, [detect(userId)]);    
+
+    // Log sessionId and pageId if provided
+    if (sessionId) { 
+        log(Constant.SessionId, [sessionId]);
+        output.sessionId = sessionId;
+    }
+    if (pageId) {
+        log(Constant.PageId, [pageId]);
+        output.pageId = pageId;
+    }
+
+    return output;
+}
+
+function log(variable: string, value: string[]): void {
+    if (core.active() &&
+        variable &&
+        value &&
+        typeof variable === Constant.String &&
+        variable.length < 255) {
+        let validValues = variable in data ? data[variable] : [];
+        for (let i = 0; i < value.length; i++) {
+            if (typeof value[i] === Constant.String && value[i].length < 255) { validValues.push(value[i]); }
+        }
+        data[variable] = validValues;
+    }
+}
+
+export function compute(): void {
+    encode(Event.Variable);
+}
+
+export function reset(): void {
+    data = {};
+}
+
+export function stop(): void {
+    reset();
+}
+
+function redact(input: string): string {
+    return input && input.length >= Setting.WordLength ? 
+        `${input.substring(0,2)}${scrub(input.substring(2), Constant.Asterix, Constant.Asterix)}` : scrub(input, Constant.Asterix, Constant.Asterix);
+}
+
+async function sha256(input: string): Promise<string> {
+    try {
+        if (crypto && input) {
+            // Reference: https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest#converting_a_digest_to_a_hex_string
+            const buffer = await crypto.subtle.digest(Constant.SHA256, new TextEncoder().encode(input));
+            return Array.prototype.map.call(new Uint8Array(buffer), (x: any) =>(('00'+x.toString(16)).slice(-2))).join('');
+        } else { return Constant.Empty; }
+    } catch { return Constant.Empty; }
+}
+
+function detect(input: string): string {
+    return input && input.indexOf(Constant.At) > 0 ? Constant.Email : Constant.String;
+}

package/src/diagnostic/encode.ts

@@ -0,0 +1,40 @@
+import { Event, Token } from "@clarity-types/data";
+import * as scrub from "@src/core/scrub";
+import { time } from "@src/core/time";
+import { queue } from "@src/data/upload";
+import * as fraud from "@src/diagnostic/fraud";
+import * as internal from "@src/diagnostic/internal";
+import * as script from "@src/diagnostic/script";
+
+export default async function (type: Event): Promise<void> {
+    let tokens: Token[] = [time(), type];
+
+    switch (type) {
+        case Event.ScriptError:
+            tokens.push(script.data.message);
+            tokens.push(script.data.line);
+            tokens.push(script.data.column);
+            tokens.push(script.data.stack);
+            tokens.push(scrub.url(script.data.source));
+            queue(tokens);
+            break;
+        case Event.Log:
+            if (internal.data) {
+                tokens.push(internal.data.code);
+                tokens.push(internal.data.name);
+                tokens.push(internal.data.message);
+                tokens.push(internal.data.stack);
+                tokens.push(internal.data.severity);
+                queue(tokens, false);
+            }
+            break;
+        case Event.Fraud:
+            if (fraud.data) {
+                tokens.push(fraud.data.id);
+                tokens.push(fraud.data.target);
+                tokens.push(fraud.data.checksum);
+                queue(tokens, false);
+            }
+            break;
+    }
+}

package/src/diagnostic/fraud.ts

@@ -0,0 +1,37 @@
+import { BooleanFlag, Event, IframeStatus, Metric, Setting } from "@clarity-types/data";
+import { FraudData } from "@clarity-types/diagnostic";
+import config from "@src/core/config";
+import hash from "@src/core/hash";
+import * as metric from "@src/data/metric";
+import encode from "./encode";
+
+let history = [];
+export let data: FraudData;
+
+export function start(): void {
+    history = [];
+    metric.max(Metric.Automation, navigator.webdriver ? BooleanFlag.True : BooleanFlag.False);
+    try {
+        // some sites (unintentionally) overwrite the window.self property, so we also check for the main window object
+        metric.max(Metric.Iframed, window.top == window.self || window.top == window ? IframeStatus.TopFrame : IframeStatus.Iframe);
+    } catch (ex) {
+        metric.max(Metric.Iframed, IframeStatus.Unknown);
+    }
+    
+}
+
+export function check(id: number, target: number, input: string): void {
+    // Compute hash for fraud detection, if enabled. Hash is computed only if input meets the minimum length criteria
+    if (config.fraud && id !== null && input && input.length >= Setting.WordLength) {
+        data = { id, target, checksum: hash(input, Setting.ChecksumPrecision) };
+        // Only encode this event if we haven't already reported this hash
+        if (history.indexOf(data.checksum) < 0) {
+            history.push(data.checksum);
+            encode(Event.Fraud);
+        }
+    }
+}
+
+export function stop(): void {
+    history = [];
+}

package/src/diagnostic/index.ts

@@ -0,0 +1,13 @@
+import * as fraud from "@src/diagnostic/fraud";
+import * as internal from "@src/diagnostic/internal";
+import * as script from "@src/diagnostic/script";
+
+export function start(): void {
+    fraud.start();
+    script.start();
+    internal.start();
+}
+
+export function stop(): void {
+    internal.stop();
+}

package/src/diagnostic/internal.ts

@@ -0,0 +1,28 @@
+import { Code, Event, Severity } from "@clarity-types/data";
+import { LogData } from "@clarity-types/diagnostic";
+import encode from "./encode";
+
+let history: { [key: number]: string[] } = {};
+export let data: LogData;
+
+export function start(): void {
+    history = {};
+}
+
+export function log(code: Code, severity: Severity, name: string = null, message: string = null, stack: string = null): void {
+    let key = name ? `${name}|${message}`: "";
+    // While rare, it's possible for code to fail repeatedly during the lifetime of the same page
+    // In those cases, we only want to log the failure once and not spam logs with redundant information.
+    if (code in history && history[code].indexOf(key) >= 0) { return; }
+
+    data = { code, name, message, stack, severity };
+
+    // Maintain history of errors in memory to avoid sending redundant information
+    if (code in history) { history[code].push(key); } else { history[code] = [key]; }
+
+    encode(Event.Log);
+}
+
+export function stop(): void {
+    history = {};
+}

package/src/diagnostic/script.ts

@@ -0,0 +1,35 @@
+import { Event, Setting } from "@clarity-types/data";
+import { ScriptErrorData } from "@clarity-types/diagnostic";
+import { bind } from "@src/core/event";
+import encode from "./encode";
+
+let history: { [key: string]: number } = {};
+export let data: ScriptErrorData;
+
+export function start(): void {
+    bind(window, "error", handler);
+    history = {};
+}
+
+function handler(error: ErrorEvent): boolean {
+    let e = error["error"] || error;
+    // While rare, it's possible for code to fail repeatedly during the lifetime of the same page
+    // In those cases, we only want to log the failure first few times and not spam logs with redundant information.
+    if (!(e.message in history)) { history[e.message] = 0; }
+    if (history[e.message]++ >= Setting.ScriptErrorLimit) { return true; }
+
+    // Send back information only if the handled error has valid information
+    if (e && e.message) {
+        data = {
+            message: e.message,
+            line: error["lineno"],
+            column: error["colno"],
+            stack: e.stack,
+            source: error["filename"]
+        };
+
+        encode(Event.ScriptError);
+    }
+
+    return true;
+}

package/src/dynamic/agent/blank.ts

@@ -0,0 +1,2 @@
+export function start(): void {}
+export function stop(): void {}

package/src/dynamic/agent/crisp.ts

@@ -0,0 +1,40 @@
+import { Action } from "@clarity-types/agent";
+import encode from "./encode";
+
+function open() {
+  encode(Action.AgentMaximized);
+}
+
+function close() {
+  encode(Action.AgentMinimized);
+}
+
+function human() {
+  encode(Action.HumanMessage);
+}
+
+function agent() {
+  encode(Action.AgentMessage);
+}
+
+export function start(): void {
+  if (window.$crisp) {
+    window.$crisp.push(["on", "chat:opened", open]);
+    window.$crisp.push(["on", "chat:closed", close]);
+    window.$crisp.push(["on", "message:sent", human]);
+    window.$crisp.push(["on", "message:received", agent]);
+  }
+  // Register stop callback with main Clarity
+  if (typeof window !== "undefined" && (window as any).clarity) {
+    (window as any).clarity("register", stop);
+  }
+}
+
+export function stop(): void {
+  if (window.$crisp) {
+    window.$crisp.push(["off", "chat:opened", open]);
+    window.$crisp.push(["off", "chat:closed", close]);
+    window.$crisp.push(["off", "message:sent", human]);
+    window.$crisp.push(["off", "message:received", agent]);
+  }
+}

package/src/dynamic/agent/encode.ts

@@ -0,0 +1,25 @@
+import { Event, Token } from "@clarity-types/data";
+import { Action } from "@clarity-types/agent";
+
+export default function (event: Action): void {
+  const t = (window as any).clarity("time");
+  let tokens: Token[] = [t, Event.Chat];
+  switch (event) {
+    case Action.AgentOnline:
+    case Action.AgentOffline:
+    case Action.AgentHidden:
+    case Action.AgentMaximized:
+    case Action.AgentMinimized:
+    case Action.AgentMessage:
+    case Action.HumanMessage:
+      tokens.push(event);
+      break;
+  }
+  queueTokens(tokens);
+}
+
+function queueTokens(tokens: Token[]) {
+  if (typeof window !== "undefined" && (window as any).clarity) {
+    (window as any).clarity("queue", tokens);
+  }
+}

package/src/dynamic/agent/index.ts

@@ -0,0 +1,8 @@
+import * as livechat from "@src/dynamic/agent/livechat";
+import * as tidio from "@src/dynamic/agent/tidio";
+import * as crisp from "@src/dynamic/agent/crisp";
+(function () {
+  livechat.start();
+  tidio.start();
+  crisp.start();
+})();