@kubesense/kubesense-browser-rum 1.8.0 → 2.0.0-beta.1

package/src/domain/record/trackers/trackCanvasCapture.ts

@@ -0,0 +1,137 @@
+import { clearTimeout, noop, setTimeout } from '@kubesense/kubesense-browser-core'
+import type { TimeoutId } from '@kubesense/kubesense-browser-core'
+import { getNodePrivacyLevel, NodePrivacyLevel } from '@kubesense/kubesense-browser-rum-core'
+import { ONE_SECOND } from '@kubesense/kubesense-js-core/time'
+import type { RecordingScope } from '../recordingScope'
+import type { CanvasCaptureAttempt } from '../canvas/canvasManager'
+import { CanvasStatus } from '../canvas/canvasManager'
+import { captureCanvasImage, createCanvasSnapshot } from '../canvas/canvasSnapshot'
+import { computeImageHash } from '../canvas/canvasHash'
+import type { Tracker } from './tracker.types'
+
+export const trackCanvasCapture = (scope: RecordingScope, notifyContentMutated: () => void): Tracker => {
+  const canvasManager = scope.canvasManager
+  const configuration = scope.configuration.sessionReplayCanvasRecording
+  const maxFramesPerSecond = configuration?.maxFramesPerSecond ?? 0
+
+  if (!configuration?.enable || maxFramesPerSecond === 0) {
+    return { stop: noop }
+  }
+
+  let stopped = false
+  let captureTimeoutId: TimeoutId | undefined
+  const captureInterval = ONE_SECOND / maxFramesPerSecond
+
+  function scheduleNextCapture(delay: number) {
+    if (!stopped) {
+      captureTimeoutId = setTimeout(() => {
+        void runCaptureTask()
+      }, delay)
+    }
+  }
+
+  async function runCaptureTask() {
+    const startTime = performance.now()
+
+    try {
+      await captureDirtyCanvases()
+    } finally {
+      const taskDuration = performance.now() - startTime
+      const nextDelay = Math.max(captureInterval - taskDuration, captureInterval * 0.5)
+
+      scheduleNextCapture(nextDelay)
+    }
+  }
+
+  async function captureDirtyCanvases() {
+    const capturableCanvases = canvasManager.takeCapturableCanvases()
+
+    if (stopped) {
+      return
+    }
+
+    for (const canvas of capturableCanvases) {
+      const nodeId = scope.nodeIds.get(canvas)
+      if (nodeId === undefined) {
+        canvasManager.forgetCanvas(canvas)
+        continue
+      }
+      if (getNodePrivacyLevel(canvas, scope.configuration.defaultPrivacyLevel) !== NodePrivacyLevel.ALLOW) {
+        canvasManager.markCanvas(canvas, CanvasStatus.Dirty)
+        continue
+      }
+      await captureCanvas(canvas)
+    }
+  }
+
+  function markDirtyIfCurrent(captureAttempt: CanvasCaptureAttempt, canvas: HTMLCanvasElement) {
+    if (captureAttempt.isCurrent()) {
+      canvasManager.markCanvas(canvas, CanvasStatus.Dirty)
+    }
+  }
+
+  async function captureCanvas(canvas: HTMLCanvasElement) {
+    const captureAttempt = canvasManager.startCaptureAttempt(canvas)
+    const cancelled = () => stopped || !captureAttempt.isCurrent()
+
+    if (cancelled()) {
+      return
+    }
+
+    try {
+      const snapshot = createCanvasSnapshot(canvas, configuration?.maxImageDimension ?? 1000)
+      if (!snapshot) {
+        markDirtyIfCurrent(captureAttempt, canvas)
+        canvasManager.discardCaptureAttempt(canvas, captureAttempt)
+        return // snapshot failed; leave it dirty
+      }
+
+      const hash = await computeImageHash(snapshot, configuration?.hashingMaxDimension ?? 100)
+
+      if (cancelled()) {
+        return
+      }
+      if (hash === undefined) {
+        markDirtyIfCurrent(captureAttempt, canvas)
+        canvasManager.discardCaptureAttempt(canvas, captureAttempt)
+        return // hashing failed; leave it dirty
+      }
+
+      if (hash === captureAttempt.lastChangeHash) {
+        canvasManager.discardCaptureAttempt(canvas, captureAttempt)
+        return // unchanged: no capture/output
+      }
+
+      const image = await captureCanvasImage(snapshot, configuration?.encodeQuality ?? 0.5)
+      if (cancelled()) {
+        return
+      }
+      if (!image) {
+        markDirtyIfCurrent(captureAttempt, canvas)
+        canvasManager.discardCaptureAttempt(canvas, captureAttempt)
+        return // encoding failed; leave it dirty
+      }
+
+      canvasManager.addCanvasContentMutation({ canvas, captureAttempt, hash, image })
+      notifyContentMutated()
+    } catch (error) {
+      if (!cancelled()) {
+        canvasManager.markCanvas(canvas, isSecurityError(error) ? CanvasStatus.Tainted : CanvasStatus.Dirty)
+        canvasManager.discardCaptureAttempt(canvas, captureAttempt)
+      }
+    }
+  }
+
+  scheduleNextCapture(captureInterval)
+
+  return {
+    stop: () => {
+      stopped = true
+      clearTimeout(captureTimeoutId)
+    },
+  }
+}
+
+function isSecurityError(error: unknown): boolean {
+  return error instanceof DOMException && error.name === 'SecurityError'
+}

package/src/domain/record/trackers/trackCanvasContent.ts

@@ -0,0 +1,46 @@
+import { instrumentMethod, noop } from '@kubesense/kubesense-browser-core'
+import type { RecordingScope } from '../recordingScope'
+import { CanvasStatus } from '../canvas/canvasManager'
+import type { Tracker } from './tracker.types'
+
+const CANVAS_2D_DRAWING_METHODS = [
+  'clearRect',
+  'fillRect',
+  'strokeRect',
+  'fill',
+  'stroke',
+  'fillText',
+  'strokeText',
+  'drawImage',
+  'putImageData',
+  'drawFocusIfNeeded',
+  'reset',
+] as const
+
+export function trackCanvasContent(scope: RecordingScope): Tracker {
+  if (
+    !scope.configuration.sessionReplayCanvasRecording?.enable ||
+    scope.configuration.sessionReplayCanvasRecording.maxFramesPerSecond === 0 ||
+    typeof CanvasRenderingContext2D === 'undefined'
+  ) {
+    return { stop: noop }
+  }
+
+  const instrumentationStoppers: Tracker[] = []
+
+  CANVAS_2D_DRAWING_METHODS.forEach((method) => {
+    instrumentationStoppers.push(
+      instrumentMethod(CanvasRenderingContext2D.prototype, method, ({ target: context, onPostCall }) => {
+        onPostCall(() => {
+          if (scope.nodeIds.get(context.canvas) !== undefined) {
+            scope.canvasManager.markCanvas(context.canvas, CanvasStatus.Dirty)
+          }
+        })
+      })
+    )
+  })
+
+  return {
+    stop: () => instrumentationStoppers.forEach((stopper) => stopper.stop()),
+  }
+}

package/src/domain/record/trackers/trackFocus.ts

@@ -1,12 +1,12 @@
-import { DOM_EVENT, addEventListeners, timeStampNow } from '@kubesense/kubesense-browser-core'
+import { DOM_EVENT, addEventListeners } from '@kubesense/kubesense-browser-core'
+import { timeStampNow } from '@kubesense/kubesense-js-core/time'
 import type { FocusRecord } from '../../../types'
 import { RecordType } from '../../../types'
-import type { RecordingScope } from '../recordingScope'
 import type { EmitRecordCallback } from '../record.types'
 import type { Tracker } from './tracker.types'
 
-export function trackFocus(emitRecord: EmitRecordCallback<FocusRecord>, scope: RecordingScope): Tracker {
-  return addEventListeners(scope.configuration, window, [DOM_EVENT.FOCUS, DOM_EVENT.BLUR], () => {
+export function trackFocus(emitRecord: EmitRecordCallback<FocusRecord>): Tracker {
+  return addEventListeners(window, [DOM_EVENT.FOCUS, DOM_EVENT.BLUR], () => {
     emitRecord({
       data: { has_focus: document.hasFocus() },
       type: RecordType.Focus,

package/src/domain/record/trackers/trackInput.ts

@@ -1,26 +1,53 @@
 import { instrumentSetter, DOM_EVENT, addEventListeners, noop } from '@kubesense/kubesense-browser-core'
+import { timeStampNow } from '@kubesense/kubesense-js-core/time'
 import { NodePrivacyLevel, getNodePrivacyLevel, shouldMaskNode } from '@kubesense/kubesense-browser-rum-core'
-import { IncrementalSource } from '../../../types'
-import type { InputData, InputState } from '../../../types'
+import { InputSelectionState, StringRole } from '../../../types'
 import { getEventTarget } from '../eventsUtils'
+import { createString } from '../encoding'
+import type { NodeId } from '../encoding'
 import type { RecordingScope } from '../recordingScope'
-import { getElementInputValue } from '../serialization'
-import { assembleIncrementalSnapshot } from '../assembly'
-import type { EmitRecordCallback } from '../record.types'
+import type { SerializationTransaction } from '../serialization'
+import { getElementInputValue, SerializationKind, serializeInTransaction } from '../serialization'
+import type { EmitRecordCallback, EmitStatsCallback } from '../record.types'
 import type { Tracker } from './tracker.types'
 
+const enum InputStateType {
+  VALUE,
+  SELECTION,
+}
+
+/**
+ * The state of a form element: either a value, for an element whose state can be
+ * reconstructed from its `value` property, or a selection state, for an element the user
+ * selects rather than fills in.
+ */
+type InputState =
+  { type: InputStateType.VALUE; value: string } | { type: InputStateType.SELECTION; selection: InputSelectionState }
+
+/** The state every radio button of a group but the selected one changes to. */
+const DESELECTED_INPUT_STATE: InputState = {
+  type: InputStateType.SELECTION,
+  selection: InputSelectionState.Deselected,
+}
+
 export function trackInput(
   target: Document | ShadowRoot,
   emitRecord: EmitRecordCallback,
+  emitStats: EmitStatsCallback,
   scope: RecordingScope
 ): Tracker {
   const defaultPrivacyLevel = scope.configuration.defaultPrivacyLevel
   const lastInputStateMap: WeakMap<Node, InputState> = new WeakMap()
 
-  const isShadowRoot = target !== document
+  const isShadowRoot = target.nodeType !== target.DOCUMENT_NODE
+
+  const targetGlobal = (isShadowRoot ? (target as ShadowRoot).ownerDocument : (target as Document)).defaultView
+  if (!targetGlobal) {
+    // A document with no global object, such as one created by DOMParser, produces no input events.
+    return { stop: noop }
+  }
 
   const { stop: stopEventListeners } = addEventListeners(
-    scope.configuration,
     target,
     // The 'input' event bubbles across shadow roots, so we don't have to listen for it on shadow
     // roots since it will be handled by the event listener that we did add to the document. Only
@@ -29,9 +56,9 @@ export function trackInput(
     (event) => {
       const target = getEventTarget(event)
       if (
-        target instanceof HTMLInputElement ||
-        target instanceof HTMLTextAreaElement ||
-        target instanceof HTMLSelectElement
+        target instanceof targetGlobal.HTMLInputElement ||
+        target instanceof targetGlobal.HTMLTextAreaElement ||
+        target instanceof targetGlobal.HTMLSelectElement
       ) {
         onElementChange(target)
       }
@@ -45,11 +72,11 @@ export function trackInput(
   let stopPropertySetterInstrumentation: () => void
   if (!isShadowRoot) {
     const instrumentationStoppers = [
-      instrumentSetter(HTMLInputElement.prototype, 'value', onElementChange),
-      instrumentSetter(HTMLInputElement.prototype, 'checked', onElementChange),
-      instrumentSetter(HTMLSelectElement.prototype, 'value', onElementChange),
-      instrumentSetter(HTMLTextAreaElement.prototype, 'value', onElementChange),
-      instrumentSetter(HTMLSelectElement.prototype, 'selectedIndex', onElementChange),
+      instrumentSetter(targetGlobal.HTMLInputElement.prototype, 'value', onElementChange),
+      instrumentSetter(targetGlobal.HTMLInputElement.prototype, 'checked', onElementChange),
+      instrumentSetter(targetGlobal.HTMLSelectElement.prototype, 'value', onElementChange),
+      instrumentSetter(targetGlobal.HTMLTextAreaElement.prototype, 'value', onElementChange),
+      instrumentSetter(targetGlobal.HTMLSelectElement.prototype, 'selectedIndex', onElementChange),
     ]
     stopPropertySetterInstrumentation = () => {
       instrumentationStoppers.forEach((stopper) => stopper.stop())
@@ -73,56 +100,134 @@ export function trackInput(
 
     const type = target.type
 
+    // The changed node. Usually the target; for <select>, the now-selected <option>.
+    let changedNode: Node = target
     let inputState: InputState
     if (type === 'radio' || type === 'checkbox') {
       if (shouldMaskNode(target, nodePrivacyLevel)) {
         return
       }
-      inputState = { isChecked: (target as HTMLInputElement).checked }
+      inputState = { type: InputStateType.SELECTION, selection: selectionStateOf(target as HTMLInputElement) }
+    } else if (isSelectElement(target)) {
+      if (shouldMaskNode(target, nodePrivacyLevel)) {
+        return
+      }
+      // TODO: Record every selected <option> of a <select multiple>, not just the first.
+      const selectedOption = target.options[target.selectedIndex]
+      if (!selectedOption) {
+        return
+      }
+      changedNode = selectedOption
+      inputState = { type: InputStateType.SELECTION, selection: InputSelectionState.Selected }
     } else {
       const value = getElementInputValue(target, nodePrivacyLevel)
       if (value === undefined) {
         return
       }
-      inputState = { text: value }
+      inputState = { type: InputStateType.VALUE, value }
     }
 
-    // Can be multiple changes on the same node within the same batched mutation observation.
-    createRecordIfStateChanged(target, inputState)
+    const changedNodeId = affectedNodeIdForChange(changedNode, inputState)
 
-    // If a radio was checked, other radios with the same name attribute will be unchecked.
-    const name = target.name
-    if (type === 'radio' && name && (target as HTMLInputElement).checked) {
-      document.querySelectorAll(`input[type="radio"][name="${CSS.escape(name)}"]`).forEach((el: Element) => {
-        if (el !== target) {
-          // TODO: Consider the privacy implications for various differing input privacy levels
-          createRecordIfStateChanged(el, { isChecked: false })
+    // Selecting a radio button deselects every other radio button in its group.
+    const deselectedNodeIds: NodeId[] = []
+    if (type === 'radio' && (target as HTMLInputElement).checked) {
+      forEachOtherRadioInGroup(target as HTMLInputElement, (radio: HTMLInputElement) => {
+        // TODO: Consider the privacy implications for various differing input privacy levels
+        const deselectedNodeId = affectedNodeIdForChange(radio, DESELECTED_INPUT_STATE)
+        if (deselectedNodeId !== undefined) {
+          deselectedNodeIds.push(deselectedNodeId)
         }
       })
     }
+
+    if (isSelectElement(target)) {
+      // Update the state of now-deselected options.
+      // TODO: Record the deselections once <select multiple> is supported.
+      for (let i = 0; i < target.options.length; i++) {
+        const option = target.options[i]
+        if (option !== changedNode) {
+          lastInputStateMap.delete(option)
+        }
+      }
+    }
+
+    if (changedNodeId === undefined && deselectedNodeIds.length === 0) {
+      return
+    }
+
+    serializeInTransaction(
+      SerializationKind.INCREMENTAL_SNAPSHOT,
+      emitRecord,
+      emitStats,
+      scope,
+      timeStampNow(),
+      (transaction: SerializationTransaction) => {
+        if (changedNodeId !== undefined) {
+          if (inputState.type === InputStateType.VALUE) {
+            transaction.setInputValue(changedNodeId, createString(StringRole.FormInput, inputState.value))
+          } else {
+            transaction.setInputSelection(inputState.selection, [changedNodeId])
+          }
+        }
+        if (deselectedNodeIds.length > 0) {
+          transaction.setInputSelection(InputSelectionState.Deselected, deselectedNodeIds)
+        }
+      }
+    )
   }
 
   /**
-   * There can be multiple changes on the same node within the same batched mutation observation.
+   * Given a form input change, returns the affected node id, if any. It's possible for
+   * there to be no affected node id if the change was redundant, or if the node has not
+   * yet been snapshotted (in which case a future snapshot will pick up the change
+   * instead), or if the node is not being recorded due to its privacy level.
    */
-  function createRecordIfStateChanged(target: Node, inputState: InputState) {
+  function affectedNodeIdForChange(target: Node, inputState: InputState): NodeId | undefined {
     const id = scope.nodeIds.get(target)
     if (id === undefined) {
-      return
+      return undefined
     }
     const lastInputState = lastInputStateMap.get(target)
-    if (
-      !lastInputState ||
-      (lastInputState as { text?: string }).text !== (inputState as { text?: string }).text ||
-      (lastInputState as { isChecked?: boolean }).isChecked !== (inputState as { isChecked?: boolean }).isChecked
-    ) {
-      lastInputStateMap.set(target, inputState)
-      emitRecord(
-        assembleIncrementalSnapshot<InputData>(IncrementalSource.Input, {
-          id,
-          ...inputState,
-        })
-      )
+    if (lastInputState && isSameInputState(lastInputState, inputState)) {
+      return undefined
     }
+    lastInputStateMap.set(target, inputState)
+    return id
+  }
+}
+
+function isSelectElement(element: Element): element is HTMLSelectElement {
+  return element.tagName === 'SELECT'
+}
+
+/** The selection state of a checkbox or radio button. */
+function selectionStateOf(input: HTMLInputElement): InputSelectionState {
+  // TODO: Record indeterminate checkboxes as InputSelectionState.Indeterminate.
+  return input.checked ? InputSelectionState.Selected : InputSelectionState.Deselected
+}
+
+/** Runs the given callback for every other radio button in the same group as the given one. */
+function forEachOtherRadioInGroup(radio: HTMLInputElement, callback: (radio: HTMLInputElement) => void): void {
+  if (!radio.name) {
+    // A radio button with no name isn't part of a group at all.
+    return
+  }
+
+  // TODO: Correctly scope by form and shadow tree.
+  radio.ownerDocument
+    .querySelectorAll(`input[type="radio"][name="${CSS.escape(radio.name)}"]`)
+    .forEach((element: Element) => {
+      const otherRadio = element as HTMLInputElement
+      if (otherRadio !== radio) {
+        callback(otherRadio)
+      }
+    })
+}
+
+function isSameInputState(a: InputState, b: InputState): boolean {
+  if (a.type === InputStateType.VALUE) {
+    return b.type === InputStateType.VALUE && a.value === b.value
   }
+  return b.type === InputStateType.SELECTION && a.selection === b.selection
 }

package/src/domain/record/trackers/trackMediaInteraction.ts

@@ -13,7 +13,6 @@ export function trackMediaInteraction(
   scope: RecordingScope
 ): Tracker {
   return addEventListeners(
-    scope.configuration,
     document,
     [DOM_EVENT.PLAY, DOM_EVENT.PAUSE],
     (event) => {

package/src/domain/record/trackers/trackMouseInteraction.ts

@@ -57,12 +57,11 @@ export function trackMouseInteraction(
     }
 
     emitRecord({
-      id: scope.eventIds.getIdForEvent(event),
+      id: scope.eventIds.getOrInsert(event),
       ...assembleIncrementalSnapshot<MouseInteractionData>(IncrementalSource.MouseInteraction, interaction),
     })
   }
   return addEventListeners(
-    scope.configuration,
     document,
     Object.keys(eventTypeToMouseInteraction) as Array<keyof typeof eventTypeToMouseInteraction>,
     handler,

package/src/domain/record/trackers/trackMove.ts

@@ -46,7 +46,6 @@ export function trackMove(
   )
 
   const { stop: removeListener } = addEventListeners(
-    scope.configuration,
     document,
     [DOM_EVENT.MOUSE_MOVE, DOM_EVENT.TOUCH_MOVE],
     updatePosition,