@kubesense/kubesense-browser-rum 1.8.0 → 2.0.0-beta.1

package/src/domain/record/encoding/itemIds.ts

@@ -0,0 +1,84 @@
+export type EventId = number & { __brand: 'EventId' }
+export type EventIds = ItemIds<Event, EventId>
+export const enum EventIdConstants {
+  FIRST_ID = 1,
+}
+export function createEventIds(): EventIds {
+  return createWeakIdMap(EventIdConstants.FIRST_ID as EventId)
+}
+
+export type NodeId = number & { __brand: 'NodeId' }
+export type NodeIds = ItemIds<Node, NodeId>
+export const enum NodeIdConstants {
+  FIRST_ID = 0,
+}
+export function createNodeIds(): NodeIds {
+  return createWeakIdMap(NodeIdConstants.FIRST_ID as NodeId)
+}
+
+export type StyleSheetId = number & { __brand: 'StyleSheetId' }
+export type StyleSheetIds = ItemIds<CSSStyleSheet, StyleSheetId>
+export const enum StyleSheetIdConstants {
+  FIRST_ID = 0,
+}
+export function createStyleSheetIds(): StyleSheetIds {
+  return createWeakIdMap(StyleSheetIdConstants.FIRST_ID as StyleSheetId)
+}
+
+export interface ItemIds<ItemType, ItemId extends number> {
+  clear(this: void): void
+  delete(this: void, item: ItemType): void
+  get(this: void, item: ItemType): ItemId | undefined
+  getOrInsert(this: void, item: ItemType): ItemId
+  get nextId(): ItemId
+  get size(): number
+}
+
+function createWeakIdMap<ItemType extends object, ItemId extends number>(firstId: ItemId): ItemIds<ItemType, ItemId> {
+  return createItemIds(() => new WeakMap<ItemType, ItemId>(), firstId)
+}
+
+interface MapLike<Key, Value> {
+  delete(key: Key): void
+  get(key: Key): Value | undefined
+  set(key: Key, value: Value): void
+}
+
+function createItemIds<ItemType, ItemId extends number>(
+  createMap: () => MapLike<ItemType, ItemId>,
+  firstId: ItemId
+): ItemIds<ItemType, ItemId> {
+  let map = createMap()
+  let nextId = firstId
+
+  const get = (object: ItemType): ItemId | undefined => map.get(object)
+
+  return {
+    clear(): void {
+      if (nextId === firstId) {
+        return
+      }
+      map = createMap()
+      nextId = firstId
+    },
+    delete(object: ItemType): void {
+      map.delete(object)
+    },
+    get,
+    getOrInsert(object: ItemType): ItemId {
+      // Try to reuse any existing id.
+      let id = get(object)
+      if (id === undefined) {
+        id = nextId++ as ItemId
+        map.set(object, id)
+      }
+      return id
+    },
+    get nextId(): ItemId {
+      return nextId
+    },
+    get size(): number {
+      return nextId - firstId
+    },
+  }
+}

package/src/domain/record/encoding/roles.ts

@@ -0,0 +1,136 @@
+/* eslint-disable id-denylist */
+
+import type {
+  AddNodeChange,
+  RoleAnnotatedStringLiteral,
+  StringLiteral,
+  StringOrStringReference,
+  StringReference,
+} from '../../../types'
+import { StringRole } from '../../../types'
+import type { NodeId } from './itemIds'
+
+/** The node name of an AddNode change operation, in its role-annotated form. */
+export type RoleAnnotatedAddNodeName<NodeChange extends AddNodeChange> = Exclude<
+  NodeChange[1],
+  StringLiteral | StringReference
+>
+
+/**
+ * The node-type-specific parameters of an AddNode change operation, with every string
+ * in its role-annotated form.
+ */
+export type RoleAnnotatedAddNodeParams<NodeChange extends AddNodeChange> = NodeChange extends [
+  any,
+  any,
+  ...infer Params,
+]
+  ? RoleAnnotatedParams<Params>
+  : never
+
+type RoleAnnotatedParams<Params> = { [Index in keyof Params]: RoleAnnotatedParam<Params[Index]> }
+
+type RoleAnnotatedParam<Param> = Param extends StringOrStringReference
+  ? RoleAnnotatedStringLiteral
+  : RoleAnnotatedParams<Param>
+
+/**
+ * An attribute change whose strings are all annotated with their roles. The node id is a number
+ * rather than a string, so it is unaffected.
+ */
+export type RoleAnnotatedAttributeChange = [NodeId, ...RoleAnnotatedAttributeAssignmentOrDeletion[]]
+
+/** An attribute assignment, annotated with string roles. */
+export type RoleAnnotatedAttributeAssignment = [RoleAnnotatedStringLiteral, RoleAnnotatedStringLiteral]
+
+/** An attribute assignment or deletion, annotated with string roles. */
+export type RoleAnnotatedAttributeAssignmentOrDeletion =
+  [RoleAnnotatedStringLiteral, RoleAnnotatedStringLiteral] | [RoleAnnotatedStringLiteral]
+
+/** The string literal portion of a role-annotated string literal type. */
+export type RoleAnnotatedStringLiteralValue<Literal> = Literal extends RoleAnnotatedStringLiteral
+  ? Literal['string']
+  : never
+
+/** The rules of a stylesheet, annotated with their roles. */
+export type RoleAnnotatedStyleSheetRules = RoleAnnotatedStringLiteral | RoleAnnotatedStringLiteral[]
+
+/** The media list of a stylesheet, annotated with their roles. */
+export type RoleAnnotatedStyleSheetMediaList = RoleAnnotatedStringLiteral[]
+
+/** Create a serialized string literal annotated with the given role. */
+export function createString<Role extends StringRole, Literal extends string>(
+  role: Role,
+  string: Literal
+): RoleAnnotatedStringLiteral & { role: Role; string: Literal } {
+  return { role, string }
+}
+
+/**
+ * Given a normalized tag name, an attribute name, and an attribute value, returns a
+ * serialized attribute assignment annotated with the appropriate roles.
+ */
+export function createAttributeAssignment(
+  tagName: string,
+  attributeName: string,
+  attributeValue: string
+): RoleAnnotatedAttributeAssignment {
+  return [
+    createString(StringRole.AttributeName, attributeName),
+    createString(attributeValueRole(tagName, attributeName), attributeValue),
+  ]
+}
+
+/**
+ * Given a normalized tag name, an attribute name, and an attribute value, returns a
+ * serialized attribute assignment or deletion annotated with the appropriate roles.
+ */
+export function createAttributeAssignmentOrDeletion(
+  tagName: string,
+  attributeName: string,
+  attributeValue: string | null
+): RoleAnnotatedAttributeAssignmentOrDeletion {
+  const name = createString(StringRole.AttributeName, attributeName)
+
+  if (attributeValue === null) {
+    return [name]
+  }
+
+  return [name, createString(attributeValueRole(tagName, attributeName), attributeValue)]
+}
+
+/**
+ * Elements whose `value` attribute holds form input. Every other element that carries a `value`
+ * attribute — `<li>`, `<button>`, `<meter>`, `<param>`, and the rest — holds an ordinary attribute
+ * value there, and masking it would change how the page renders rather than hide anything private.
+ */
+const FORM_ELEMENT_NAMES = new Set(['input', 'option', 'select', 'textarea'])
+
+/** Attributes whose value is a URL or a list of URLs. */
+const URL_ATTRIBUTE_NAMES = new Set([
+  'action',
+  'background',
+  'cite',
+  'data',
+  'formaction',
+  'href',
+  'longdesc',
+  'manifest',
+  'ping',
+  'poster',
+  'src',
+  'srcset',
+  'xlink:href',
+])
+
+/** The role that the value of the given attribute plays on the given element. */
+function attributeValueRole(tagName: string, attributeName: string): StringRole {
+  switch (attributeName) {
+    case 'value':
+      return FORM_ELEMENT_NAMES.has(tagName) ? StringRole.FormInput : StringRole.AttributeValue
+    case 'style':
+      return StringRole.Css
+    default:
+      return URL_ATTRIBUTE_NAMES.has(attributeName) ? StringRole.Url : StringRole.AttributeValue
+  }
+}

package/src/domain/record/encoding/stringIds.ts

@@ -0,0 +1,70 @@
+import type { RoleAnnotatedStringLiteral } from '../../../types'
+import { StringRole } from '../../../types'
+
+export type StringId = number & { __brand: 'StringId' }
+
+export interface StringIds {
+  clear(this: void): void
+  get(this: void, literal: RoleAnnotatedStringLiteral): StringId | undefined
+  getOrInsert(this: void, literal: RoleAnnotatedStringLiteral): StringId
+  get size(): number
+}
+
+export const enum StringIdConstants {
+  FIRST_ID = 0,
+
+  // An arbitrarily-chosen soft limit on the maximum size of the string id map.
+  SOFT_MAX_SIZE = 1000000,
+}
+
+type StringIdMap = Record<string, StringId | undefined>
+type StringIdMapsByRole = Record<StringRole, StringIdMap>
+
+export function createStringIds(): StringIds {
+  const firstId = StringIdConstants.FIRST_ID as StringId
+  let nextId = firstId
+  let mapsByRole = createMapsByRole()
+
+  const get = (literal: RoleAnnotatedStringLiteral): StringId | undefined => mapsByRole[literal.role][literal.string]
+
+  return {
+    clear(): void {
+      if (nextId === firstId) {
+        return
+      }
+      mapsByRole = createMapsByRole()
+      nextId = firstId
+    },
+    get,
+    getOrInsert(literal: RoleAnnotatedStringLiteral): StringId {
+      // Try to reuse any existing id.
+      let id = get(literal)
+      if (id === undefined) {
+        id = nextId++ as StringId
+        mapsByRole[literal.role][literal.string] = id
+      }
+      return id
+    },
+    get size(): number {
+      return nextId - firstId
+    },
+  }
+}
+
+function createMapsByRole(): StringIdMapsByRole {
+  return {
+    [StringRole.Default]: createStringIdMap(),
+    [StringRole.NodeName]: createStringIdMap(),
+    [StringRole.AttributeName]: createStringIdMap(),
+    [StringRole.AttributeValue]: createStringIdMap(),
+    [StringRole.TextContent]: createStringIdMap(),
+    [StringRole.FormInput]: createStringIdMap(),
+    [StringRole.Css]: createStringIdMap(),
+    [StringRole.Url]: createStringIdMap(),
+    [StringRole.ResourceId]: createStringIdMap(),
+  }
+}
+
+function createStringIdMap(): StringIdMap {
+  return Object.create(null) as StringIdMap
+}

package/src/domain/record/encoding/stringTable.ts

@@ -0,0 +1,68 @@
+import type { RoleAnnotatedStringLiteral, StringLiteral, StringOrStringReference } from '../../../types'
+import { StringRole } from '../../../types'
+import { createString } from './roles'
+import type { StringId } from './stringIds'
+
+export interface StringTable {
+  add(newString: string, role: StringRole): void
+
+  /** Remove every string from the table, so that the next string added gets the first id. */
+  clear(): void
+
+  /** The string that the given value holds, whichever form it takes. */
+  decode(value: StringOrStringReference): string
+
+  /**
+   * The given value in the form the decoded record should carry it: the string on its own, or the
+   * string together with the role it was defined in, depending on how the table was created.
+   */
+  decodeAnnotated(value: StringOrStringReference): StringLiteral | RoleAnnotatedStringLiteral
+}
+
+export function createStringTable(allowObsoleteStringRepresentations = false, keepRoles = false): StringTable {
+  const strings = new Map<StringId, RoleAnnotatedStringLiteral>()
+
+  // The string the given value holds, along with the role it holds it in.
+  const literalOf = (value: StringOrStringReference): RoleAnnotatedStringLiteral => {
+    if (typeof value === 'string') {
+      // A plain string literal. The encoder tags every literal it emits with a role, so one
+      // without a role means something built this change data outside the encoder.
+      if (!allowObsoleteStringRepresentations) {
+        throw new Error(`Obsolete untagged string literal: ${value}`)
+      }
+      // An untagged literal says nothing about what it holds, which is the default role.
+      return createString(StringRole.Default, value)
+    }
+
+    if (typeof value === 'object') {
+      // A role-annotated string literal. The encoder gives every string an entry in the
+      // string table, so something must have built this change data outside the encoder.
+      if (!allowObsoleteStringRepresentations) {
+        throw new Error(`Obsolete string literal outside the string table: ${value.string}`)
+      }
+      return value
+    }
+
+    const referencedString = strings.get(value as StringId)
+    if (referencedString === undefined) {
+      throw new Error(`Reference to unknown string: ${value}`)
+    }
+    return referencedString
+  }
+
+  return {
+    add(newString: string, role: StringRole): void {
+      strings.set(strings.size as StringId, createString(role, newString))
+    },
+    clear(): void {
+      strings.clear()
+    },
+    decode(value: StringOrStringReference): string {
+      return literalOf(value).string
+    },
+    decodeAnnotated(value: StringOrStringReference): StringLiteral | RoleAnnotatedStringLiteral {
+      const literal = literalOf(value)
+      return keepRoles ? literal : literal.string
+    },
+  }
+}

package/src/domain/record/index.ts

@@ -1,7 +1,10 @@
+export { MUTATION_PROCESS_MIN_DELAY } from './mutationBatch'
+export type { EmitResourceCallback } from './record.types'
 export { takeFullSnapshot, takeNodeSnapshot } from './internalApi'
 export { record } from './record'
+export type { ChangeDecoder, ChangeDecoderOptions } from './encoding'
+export { createChangeDecoder } from './encoding'
 export type { SerializationMetric, SerializationStats } from './serialization'
-export { createSerializationStats, aggregateSerializationStats } from './serialization'
-export { serializeNode, serializeDocument } from './serialization'
+export { aggregateSerializationStats, createSerializationStats } from './serialization'
 export { createElementsScrollPositions } from './elementsScrollPositions'
 export type { ShadowRootsController } from './shadowRootsController'

package/src/domain/record/internalApi.ts

@@ -1,23 +1,22 @@
-import { noop, timeStampNow } from '@kubesense/kubesense-browser-core'
+import { noop } from '@kubesense/kubesense-browser-core'
+import { timeStampNow } from '@kubesense/kubesense-js-core/time'
 import type { RumConfiguration } from '@kubesense/kubesense-browser-rum-core'
 import { getNodePrivacyLevel, NodePrivacyLevel } from '@kubesense/kubesense-browser-rum-core'
-import type { BrowserRecord, SerializedNodeWithId } from '../../types'
+import type { BrowserRecord } from '../../types'
 import { takeFullSnapshot as doTakeFullSnapshot } from './startFullSnapshots'
-import type { ShadowRootsController } from './shadowRootsController'
 import type { RecordingScope } from './recordingScope'
 import { createRecordingScope } from './recordingScope'
 import { createElementsScrollPositions } from './elementsScrollPositions'
-import { createEventIds } from './eventIds'
-import { createNodeIds } from './nodeIds'
 import type { EmitRecordCallback } from './record.types'
 import type { SerializationTransaction } from './serialization'
-import { SerializationKind, serializeInTransaction, serializeNode } from './serialization'
+import { createRootInsertionCursor, SerializationKind, serializeInTransaction, serializeNode } from './serialization'
+import { createCanvasManager } from './canvas/canvasManager'
 
 /**
- * Take a full snapshot of the document, generating the same records that the Kubesense browser SDK
+ * Take a full snapshot of the document, generating the same records that the browser SDK
  * would generate.
  *
- * This is an internal API function. Be sure to update kubesense-internal callers if you
+ * This is an internal API function. Be sure to update KubeSense-internal callers if you
  * change its signature or behavior.
  */
 export function takeFullSnapshot({
@@ -41,48 +40,52 @@ export function takeFullSnapshot({
 
 /**
  * Take a snapshot of a DOM node, generating the serialized representation that the
- * kubesense browser SDK would generate.
+ * browser SDK would generate.
  *
- * This is an internal API function. Be sure to update Kubesense-internal callers if you
+ * This is an internal API function. Be sure to update KubeSense-internal callers if you
  * change its signature or behavior.
  */
 export function takeNodeSnapshot(
   node: Node,
   { configuration }: { configuration?: Partial<RumConfiguration> } = {}
-): SerializedNodeWithId | null {
-  let serializedNode: SerializedNodeWithId | null = null
+): BrowserRecord | undefined {
+  let nodeSnapshotRecord: BrowserRecord | undefined
+  const emitRecord = (record: BrowserRecord) => {
+    nodeSnapshotRecord = record
+  }
 
   serializeInTransaction(
     SerializationKind.INITIAL_FULL_SNAPSHOT,
-    noop,
+    emitRecord,
     noop,
     createTemporaryRecordingScope(configuration),
+    timeStampNow(),
     (transaction: SerializationTransaction): void => {
       const privacyLevel = getNodePrivacyLevel(node, transaction.scope.configuration.defaultPrivacyLevel)
       if (privacyLevel === NodePrivacyLevel.HIDDEN || privacyLevel === NodePrivacyLevel.IGNORE) {
         return
       }
-      serializedNode = serializeNode(node, privacyLevel, transaction)
+      const cursor = createRootInsertionCursor(transaction.scope.nodeIds)
+      serializeNode(cursor, node, privacyLevel, transaction)
     }
   )
 
-  return serializedNode
+  return nodeSnapshotRecord
 }
 
 function createTemporaryRecordingScope(configuration?: Partial<RumConfiguration>): RecordingScope {
   return createRecordingScope(
+    createCanvasManager(),
     {
       defaultPrivacyLevel: NodePrivacyLevel.ALLOW,
       ...configuration,
     } as RumConfiguration,
     createElementsScrollPositions(),
-    createEventIds(),
-    createNodeIds(),
     {
       addShadowRoot: noop,
       removeShadowRoot: noop,
       flush: noop,
       stop: noop,
-    } as ShadowRootsController
+    }
   )
 }

package/src/domain/record/mutationBatch.ts

@@ -18,9 +18,18 @@ export const MUTATION_PROCESS_MIN_DELAY = 16
 export function createMutationBatch(processMutationBatch: (mutations: RumMutationRecord[]) => void) {
   let cancelScheduledFlush = noop
   let pendingMutations: RumMutationRecord[] = []
+  let flushScheduled = false
+
+  function scheduleFlush() {
+    if (!flushScheduled) {
+      flushScheduled = true
+      cancelScheduledFlush = requestIdleCallback(throttledFlush, { timeout: MUTATION_PROCESS_MAX_DELAY })
+    }
+  }
 
   function flush() {
     cancelScheduledFlush()
+    flushScheduled = false
     processMutationBatch(pendingMutations)
     pendingMutations = []
   }
@@ -31,12 +40,14 @@ export function createMutationBatch(processMutationBatch: (mutations: RumMutatio
 
   return {
     addMutations: (mutations: RumMutationRecord[]) => {
-      if (pendingMutations.length === 0) {
-        cancelScheduledFlush = requestIdleCallback(throttledFlush, { timeout: MUTATION_PROCESS_MAX_DELAY })
+      scheduleFlush()
+      for (const mutation of mutations) {
+        pendingMutations.push(mutation)
       }
-      pendingMutations.push(...mutations)
     },
 
+    notifyContentMutated: scheduleFlush,
+
     flush,
 
     stop: () => {

package/src/domain/record/record.ts

@@ -1,11 +1,11 @@
 import { sendToExtension } from '@kubesense/kubesense-browser-core'
+import { timeStampNow } from '@kubesense/kubesense-js-core/time'
 import type { LifeCycle, RumConfiguration, ViewHistory } from '@kubesense/kubesense-browser-rum-core'
 import * as replayStats from '../replayStats'
 import type { BrowserRecord } from '../../types'
 import type { Tracker } from './trackers'
 import {
   trackFocus,
-  trackFrustration,
   trackInput,
   trackMediaInteraction,
   trackMouseInteraction,
@@ -16,18 +16,21 @@ import {
   trackViewEnd,
   trackViewportResize,
   trackVisualViewportResize,
+  trackCanvasContent,
+  trackCanvasCapture,
 } from './trackers'
 import { createElementsScrollPositions } from './elementsScrollPositions'
 import type { ShadowRootsController } from './shadowRootsController'
 import { initShadowRootsController } from './shadowRootsController'
-import { startFullSnapshots } from './startFullSnapshots'
-import { createEventIds } from './eventIds'
-import { createNodeIds } from './nodeIds'
-import type { EmitRecordCallback, EmitStatsCallback } from './record.types'
+import { startFullSnapshots, takeFullSnapshot } from './startFullSnapshots'
+import type { EmitRecordCallback, EmitStatsCallback, EmitResourceCallback } from './record.types'
 import { createRecordingScope } from './recordingScope'
+import { createCanvasManager } from './canvas/canvasManager'
+import { SerializationKind } from './serialization'
 
 export interface RecordOptions {
   emitRecord: EmitRecordCallback
+  emitResource: EmitResourceCallback
   emitStats: EmitStatsCallback
   configuration: RumConfiguration
   lifeCycle: LifeCycle
@@ -38,12 +41,17 @@ export interface RecordAPI {
   stop: () => void
   flushMutations: () => void
   shadowRootsController: ShadowRootsController
+  /**
+   * Starts the record stream over with a new full snapshot, so replay can recover when records
+   * already emitted never reached the intake.
+   */
+  takeFullSnapshot: () => void
 }
 
 export function record(options: RecordOptions): RecordAPI {
-  const { emitRecord, emitStats, configuration, lifeCycle } = options
+  const { emitRecord, emitResource, emitStats, configuration, lifeCycle } = options
   // runtime checks for user options
-  if (!emitRecord || !emitStats) {
+  if (!emitRecord || !emitResource || !emitStats) {
     throw new Error('emit functions are required')
   }
 
@@ -54,12 +62,18 @@ export function record(options: RecordOptions): RecordAPI {
     replayStats.addRecord(view.id)
   }
 
-  const shadowRootsController = initShadowRootsController(processRecord, emitStats)
+  const processResource: EmitResourceCallback = (hash, content, onDiscard) => {
+    emitResource(hash, content, onDiscard)
+    const view = options.viewHistory.findView()!
+    replayStats.addResource(view.id)
+  }
+
+  const canvasManager = createCanvasManager()
+  const shadowRootsController = initShadowRootsController(processRecord, processResource, emitStats)
   const scope = createRecordingScope(
+    canvasManager,
     configuration,
     createElementsScrollPositions(),
-    createEventIds(),
-    createNodeIds(),
     shadowRootsController
   )
 
@@ -70,20 +84,21 @@ export function record(options: RecordOptions): RecordAPI {
     mutationTracker.flush()
   }
 
-  const mutationTracker = trackMutation(document, processRecord, emitStats, scope)
+  const mutationTracker = trackMutation(document, processRecord, processResource, emitStats, scope)
   const trackers: Tracker[] = [
     mutationTracker,
     trackMove(processRecord, scope),
     trackMouseInteraction(processRecord, scope),
     trackScroll(document, processRecord, scope),
-    trackViewportResize(processRecord, scope),
-    trackInput(document, processRecord, scope),
+    trackViewportResize(processRecord),
+    trackInput(document, processRecord, emitStats, scope),
     trackMediaInteraction(processRecord, scope),
     trackStyleSheet(processRecord, scope),
-    trackFocus(processRecord, scope),
-    trackVisualViewportResize(processRecord, scope),
-    trackFrustration(lifeCycle, processRecord, scope),
+    trackFocus(processRecord),
+    trackVisualViewportResize(processRecord),
     trackViewEnd(lifeCycle, processRecord, flushMutations),
+    trackCanvasContent(scope),
+    trackCanvasCapture(scope, mutationTracker.notifyContentMutated),
   ]
 
   return {
@@ -94,5 +109,9 @@ export function record(options: RecordOptions): RecordAPI {
     },
     flushMutations,
     shadowRootsController,
+    takeFullSnapshot: () => {
+      flushMutations()
+      takeFullSnapshot(timeStampNow(), SerializationKind.SUBSEQUENT_FULL_SNAPSHOT, processRecord, emitStats, scope)
+    },
   }
 }

package/src/domain/record/record.types.ts

@@ -2,4 +2,5 @@ import type { BrowserRecord } from '../../types'
 import type { SerializationStats } from './serialization'
 
 export type EmitRecordCallback<Record extends BrowserRecord = BrowserRecord> = (record: Record) => void
+export type EmitResourceCallback = (hash: string, content: Blob, onDiscard?: () => void) => void
 export type EmitStatsCallback = (stats: SerializationStats) => void