@multiplayer-app/session-recorder-react-native 1.0.1-beta.2 → 1.0.1-beta.4

package/src/recorder/gestureRecorder.ts

@@ -0,0 +1,681 @@
+import { Dimensions, Platform } from 'react-native'
+import { trace, SpanStatusCode, type Span } from '@opentelemetry/api'
+import { logger } from '../utils'
+import { type GestureEvent, type RecorderConfig, type EventRecorder } from '../types'
+import { MouseInteractions, type eventWithTime, EventType, IncrementalSource } from '@rrweb/types'
+import { type NativeGestureEvent, SessionRecorderNative, gestureEventEmitter } from '../native'
+// Force TypeScript recompilation
+
+
+export class GestureRecorder implements EventRecorder {
+  // @ts-ignore
+  private config?: RecorderConfig
+  private isRecording = false
+  private events: GestureEvent[] = []
+  private screenDimensions: { width: number; height: number } | null = null
+  private lastGestureTime: number = 0
+  private gestureThrottleMs: number = 50 // Throttle gestures to avoid spam
+  private lastTouchTime: number = 0
+  private touchThrottleMs: number = 100 // Throttle touch events to max 10 per second
+  private eventRecorder?: EventRecorder
+  private imageNodeId: number = 1 // ID of the image node for touch interactions
+  private screenRecorder?: any // Reference to screen recorder for force capture
+  private gestureEventListener?: any // Native event listener
+  private currentPanSpan?: Span // Aggregated span for pan gesture
+  private panMoveCount: number = 0
+
+  init(config: RecorderConfig, eventRecorder?: EventRecorder, screenRecorder?: any): void {
+    this.config = config
+    this.eventRecorder = eventRecorder
+    this.screenRecorder = screenRecorder
+    this._getScreenDimensions()
+  }
+
+  start(): void {
+    logger.info('GestureRecorder', 'Native gesture recording started')
+    this.isRecording = true
+    this.events = []
+
+    // Check if we're on web platform
+    if (Platform.OS === 'web') {
+      logger.warn('GestureRecorder', 'Native gesture recording not available on web platform')
+      return
+    }
+
+    // Start native gesture recording
+    SessionRecorderNative.startGestureRecording()
+      .then(() => {
+        logger.info('GestureRecorder', 'Native gesture recording started successfully')
+        this._setupGestureEventListener()
+      })
+      .catch((error) => {
+        logger.error('GestureRecorder', 'Failed to start native gesture recording', error)
+      })
+  }
+
+  stop(): void {
+    this.isRecording = false
+    this._removeGestureEventListener()
+
+    // Check if we're on web platform
+    if (Platform.OS === 'web') {
+      logger.warn('GestureRecorder', 'Native gesture recording not available on web platform')
+      return
+    }
+
+    // Stop native gesture recording
+    SessionRecorderNative.stopGestureRecording()
+      .then(() => {
+        logger.info('GestureRecorder', 'Native gesture recording stopped successfully')
+      })
+      .catch((error) => {
+        logger.error('GestureRecorder', 'Failed to stop native gesture recording', error)
+      })
+  }
+
+  private _getScreenDimensions(): void {
+    try {
+      this.screenDimensions = Dimensions.get('window')
+    } catch (error) {
+      // Failed to get screen dimensions - silently continue
+      this.screenDimensions = { width: 375, height: 667 } // Default fallback
+    }
+  }
+
+  private _setupGestureEventListener(): void {
+    if (!gestureEventEmitter) {
+      logger.warn('GestureRecorder', 'Gesture event emitter not available')
+      return
+    }
+
+    this.gestureEventListener = gestureEventEmitter.addListener(
+      'onGestureDetected',
+      (nativeGesture: any) => {
+        this._handleNativeGesture(nativeGesture as NativeGestureEvent)
+      }
+    )
+  }
+
+  private _removeGestureEventListener(): void {
+    if (this.gestureEventListener) {
+      this.gestureEventListener.remove()
+      this.gestureEventListener = undefined
+    }
+  }
+
+  private _handleNativeGesture(nativeGesture: NativeGestureEvent): void {
+    if (!this.isRecording) return
+
+    // Throttle gestures to avoid spam
+    const now = Date.now()
+    if (now - this.lastGestureTime < this.gestureThrottleMs) {
+      return
+    }
+    this.lastGestureTime = now
+
+    // Convert native gesture to our format
+    const gesture: GestureEvent = {
+      type: nativeGesture.type as any,
+      timestamp: nativeGesture.timestamp,
+      coordinates: { x: nativeGesture.x, y: nativeGesture.y },
+      target: nativeGesture.target,
+      targetInfo: nativeGesture.targetInfo, // Pass through targetInfo from native
+      metadata: {
+        ...nativeGesture.metadata,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this.events.push(gesture)
+    this._sendEvent(gesture)
+    this._recordOpenTelemetrySpan(gesture)
+
+    // Handle specific gesture types
+    switch (nativeGesture.type) {
+      case 'tap':
+        this._handleTapGesture(nativeGesture)
+        break
+      case 'pan_start':
+        this._handlePanStartGesture(nativeGesture)
+        break
+      case 'pan_move':
+        this._handlePanMoveGesture(nativeGesture)
+        break
+      case 'pan_end':
+        this._handlePanEndGesture(nativeGesture)
+        break
+      case 'long_press':
+        this._handleLongPressGesture(nativeGesture)
+        break
+      case 'pinch':
+        this._handlePinchGesture(nativeGesture)
+        break
+      case 'swipe':
+        this._handleSwipeGesture(nativeGesture)
+        break
+    }
+  }
+
+  private _handleTapGesture(nativeGesture: NativeGestureEvent): void {
+    this.recordTouchStart(nativeGesture.x, nativeGesture.y, nativeGesture.target)
+    this.recordTouchEnd(nativeGesture.x, nativeGesture.y, nativeGesture.target)
+  }
+
+  private _handlePanStartGesture(nativeGesture: NativeGestureEvent): void {
+    this.recordTouchStart(nativeGesture.x, nativeGesture.y, nativeGesture.target)
+  }
+
+  private _handlePanMoveGesture(nativeGesture: NativeGestureEvent): void {
+    this.recordTouchMove(nativeGesture.x, nativeGesture.y, nativeGesture.target)
+  }
+
+  private _handlePanEndGesture(nativeGesture: NativeGestureEvent): void {
+    this.recordTouchEnd(nativeGesture.x, nativeGesture.y, nativeGesture.target)
+  }
+
+  private _handleLongPressGesture(nativeGesture: NativeGestureEvent): void {
+    this.recordTouchStart(nativeGesture.x, nativeGesture.y, nativeGesture.target)
+    this.recordTouchEnd(nativeGesture.x, nativeGesture.y, nativeGesture.target)
+  }
+
+  private _handlePinchGesture(nativeGesture: NativeGestureEvent): void {
+    this.recordPinch(
+      nativeGesture.metadata?.scale || 1.0,
+      nativeGesture.target,
+      nativeGesture.metadata?.velocity || 0
+    )
+  }
+
+  private _handleSwipeGesture(nativeGesture: NativeGestureEvent): void {
+    this.recordSwipe(
+      nativeGesture.metadata?.direction || 'unknown',
+      nativeGesture.target,
+      nativeGesture.metadata?.velocity || 0
+    )
+  }
+
+  private _sendEvent(event: GestureEvent): void {
+    // Send event to backend or store locally
+    logger.debug('GestureRecorder', 'Gesture event recorded', { type: event.type, target: event.target })
+  }
+
+  private _recordOpenTelemetrySpan(event: GestureEvent): void {
+    try {
+      logger.debug('GestureRecorder', 'Creating OTEL span for native gesture', {
+        type: event.type,
+        target: event.target,
+        targetInfo: event.targetInfo,
+        hasTargetInfo: !!event.targetInfo
+      })
+      // Special handling to aggregate pan gestures into a single span
+      if (event.type === 'pan_start') {
+        // End any previously dangling pan span defensively
+        if (this.currentPanSpan) {
+          this.currentPanSpan.setStatus({ code: SpanStatusCode.OK })
+          this.currentPanSpan.end()
+        }
+        this.panMoveCount = 0
+        const panSpan = trace
+          .getTracer('@opentelemetry/instrumentation-user-interaction')
+          .startSpan('NativeGesture.pan', {
+            startTime: event.timestamp,
+          })
+
+        panSpan.setAttribute('gesture.type', 'pan')
+        panSpan.setAttribute('gesture.timestamp', event.timestamp)
+        panSpan.setAttribute('gesture.platform', 'react-native')
+        panSpan.setAttribute('gesture.source', 'native-module')
+
+        if (event.coordinates) {
+          panSpan.setAttribute('gesture.start.x', event.coordinates.x)
+          panSpan.setAttribute('gesture.start.y', event.coordinates.y)
+        }
+        if (event.target) {
+          panSpan.setAttribute('gesture.target', this._truncateText(event.target, 50))
+        }
+        // Enrich with target info if provided
+        const info = event.targetInfo
+        if (info) {
+          if (info.label) {
+            const truncatedLabel = this._truncateText(String(info.label), 50)
+            panSpan.setAttribute('gesture.target.label', truncatedLabel)
+          }
+          if (info.role) {
+            panSpan.setAttribute('gesture.target.role', String(info.role))
+          }
+          if (info.testId) {
+            panSpan.setAttribute('gesture.target.test_id', String(info.testId))
+          }
+          if (info.text) {
+            const truncatedText = this._truncateText(String(info.text), 50)
+            panSpan.setAttribute('gesture.target.text', truncatedText)
+          }
+        }
+        // Save the span for subsequent move/end events
+        this.currentPanSpan = panSpan
+        return
+      }
+
+      if (event.type === 'pan_move') {
+        if (this.currentPanSpan) {
+          this.panMoveCount += 1
+          this.currentPanSpan.setAttribute('gesture.pan.move_count', this.panMoveCount)
+          if (event.coordinates) {
+            this.currentPanSpan.setAttribute('gesture.last.x', event.coordinates.x)
+            this.currentPanSpan.setAttribute('gesture.last.y', event.coordinates.y)
+          }
+          // Don't end the span here; just update it
+          return
+        }
+        // If we received a move without a start, fall through to single-shot span below
+      }
+
+      if (event.type === 'pan_end') {
+        if (this.currentPanSpan) {
+          if (event.coordinates) {
+            this.currentPanSpan.setAttribute('gesture.end.x', event.coordinates.x)
+            this.currentPanSpan.setAttribute('gesture.end.y', event.coordinates.y)
+          }
+          this.currentPanSpan.setStatus({ code: SpanStatusCode.OK })
+          this.currentPanSpan.end()
+          this.currentPanSpan = undefined
+          this.panMoveCount = 0
+          return
+        }
+        // If no current span, fall through and create a single-shot span for the end event
+      }
+
+      // Default behavior: create a short-lived span per non-pan event
+      const span = trace
+        .getTracer('@opentelemetry/instrumentation-user-interaction')
+        .startSpan(`NativeGesture.${event.type}`, {
+          startTime: event.timestamp,
+          attributes: {
+            'gesture.type': event.type,
+            'gesture.timestamp': event.timestamp,
+            'gesture.platform': 'react-native',
+            'gesture.source': 'native-module',
+          },
+        })
+
+      if (event.coordinates) {
+        span.setAttribute('gesture.coordinates.x', event.coordinates.x)
+        span.setAttribute('gesture.coordinates.y', event.coordinates.y)
+
+        // Calculate relative position
+        if (this.screenDimensions) {
+          const relativeX = event.coordinates.x / this.screenDimensions.width
+          const relativeY = event.coordinates.y / this.screenDimensions.height
+          span.setAttribute('gesture.coordinates.relative_x', relativeX)
+          span.setAttribute('gesture.coordinates.relative_y', relativeY)
+        }
+      }
+
+      if (event.target) {
+        span.setAttribute('gesture.target', this._truncateText(event.target, 100))
+      }
+
+      // Enrich with target info if provided
+      const info = event.targetInfo
+      if (info) {
+        if (info.label) {
+          const truncatedLabel = this._truncateText(String(info.label), 100)
+          span.setAttribute('gesture.target.label', truncatedLabel)
+        }
+        if (info.role) {
+          span.setAttribute('gesture.target.role', String(info.role))
+        }
+        if (info.testId) {
+          span.setAttribute('gesture.target.test_id', String(info.testId))
+        }
+        if (info.text) {
+          const truncatedText = this._truncateText(String(info.text), 200)
+          span.setAttribute('gesture.target.text', truncatedText)
+        }
+      }
+
+      if (event.metadata) {
+        Object.entries(event.metadata).forEach(([key, value]) => {
+          span.setAttribute(`gesture.metadata.${key}`, String(value))
+        })
+      }
+
+      span.setStatus({ code: SpanStatusCode.OK })
+      span.end()
+      logger.debug('GestureRecorder', 'OTEL span created and ended successfully')
+    } catch (error) {
+      logger.error('GestureRecorder', 'Failed to record OpenTelemetry span for native gesture', error)
+    }
+  }
+
+  // Public methods for manual event recording (same as before)
+  recordTap(x: number, y: number, target?: string, pressure?: number, timestamp?: number): void {
+    const event: GestureEvent = {
+      type: 'tap',
+      timestamp: timestamp || Date.now(),
+      coordinates: { x, y },
+      target,
+      metadata: {
+        pressure: pressure || 1.0,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordSwipe(direction: string, target?: string, velocity?: number, distance?: number): void {
+    const event: GestureEvent = {
+      type: 'swipe',
+      timestamp: Date.now(),
+      target,
+      metadata: {
+        direction,
+        velocity: velocity || 0,
+        distance: distance || 0,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordPinch(scale: number, target?: string, velocity?: number): void {
+    const event: GestureEvent = {
+      type: 'pinch',
+      timestamp: Date.now(),
+      target,
+      metadata: {
+        scale,
+        velocity: velocity || 0,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordPan(deltaX: number, deltaY: number, target?: string, velocity?: number): void {
+    const event: GestureEvent = {
+      type: 'pan',
+      timestamp: Date.now(),
+      target,
+      metadata: {
+        deltaX,
+        deltaY,
+        velocity: velocity || 0,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordLongPress(duration: number, target?: string, pressure?: number): void {
+    const event: GestureEvent = {
+      type: 'longPress',
+      timestamp: Date.now(),
+      target,
+      metadata: {
+        duration,
+        pressure: pressure || 1.0,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordDoubleTap(x: number, y: number, target?: string): void {
+    const event: GestureEvent = {
+      type: 'doubleTap',
+      timestamp: Date.now(),
+      coordinates: { x, y },
+      target,
+      metadata: {
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordRotate(rotation: number, target?: string, velocity?: number): void {
+    const event: GestureEvent = {
+      type: 'rotate',
+      timestamp: Date.now(),
+      target,
+      metadata: {
+        rotation,
+        velocity: velocity || 0,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordFling(direction: string, velocity: number, target?: string): void {
+    const event: GestureEvent = {
+      type: 'fling',
+      timestamp: Date.now(),
+      target,
+      metadata: {
+        direction,
+        velocity,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordMultiTouch(touchCount: number, target?: string): void {
+    const event: GestureEvent = {
+      type: 'multiTouch',
+      timestamp: Date.now(),
+      target,
+      metadata: {
+        touchCount,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordScroll(direction: string, distance: number, velocity: number, target?: string): void {
+    const event: GestureEvent = {
+      type: 'scroll',
+      timestamp: Date.now(),
+      target,
+      metadata: {
+        direction,
+        distance,
+        velocity,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  recordZoom(scale: number, target?: string, velocity?: number): void {
+    const event: GestureEvent = {
+      type: 'zoom',
+      timestamp: Date.now(),
+      target,
+      metadata: {
+        scale,
+        velocity: velocity || 0,
+        screenWidth: this.screenDimensions?.width,
+        screenHeight: this.screenDimensions?.height,
+      },
+    }
+
+    this._recordEvent(event)
+  }
+
+  // Touch event methods (same as before)
+  recordTouchStart(x: number, y: number, target?: string, pressure?: number): void {
+    // Throttle touch events to prevent spam
+    const now = Date.now()
+    if (now - this.lastTouchTime < this.touchThrottleMs) {
+      logger.debug('GestureRecorder', `Touch start throttled (${now - this.lastTouchTime}ms < ${this.touchThrottleMs}ms)`)
+      return
+    }
+    this.lastTouchTime = now
+
+    logger.debug('GestureRecorder', 'Touch start recorded', { x, y, target, pressure })
+    this._createMouseInteractionEvent(x, y, MouseInteractions.TouchStart, now)
+  }
+
+  recordTouchMove(x: number, y: number, target?: string, pressure?: number): void {
+    // Throttle touch move events more aggressively
+    const now = Date.now()
+    if (now - this.lastTouchTime < this.touchThrottleMs * 2) { // 200ms throttle for move events
+      logger.debug('GestureRecorder', `Touch move throttled (${now - this.lastTouchTime}ms < ${this.touchThrottleMs * 2}ms)`)
+      return
+    }
+    this.lastTouchTime = now
+
+    logger.debug('GestureRecorder', 'Touch move recorded', { x, y, target, pressure })
+    this._createMouseMoveEvent(x, y, target)
+  }
+
+  recordTouchEnd(x: number, y: number, target?: string, pressure?: number): void {
+    const timestamp = Date.now()
+
+    logger.debug('GestureRecorder', 'Touch end recorded', { x, y, target, pressure, timestamp })
+
+    this.recordTap(x, y, target, pressure, timestamp)
+    this._createMouseInteractionEvent(x, y, MouseInteractions.TouchEnd, timestamp)
+
+    // Only force screen capture on touch end (not on every touch event)
+    this.screenRecorder?.forceCapture(timestamp)
+  }
+
+  recordTouchCancel(x: number, y: number, _target?: string): void {
+    this._createMouseInteractionEvent(x, y, MouseInteractions.TouchCancel)
+  }
+
+  setImageNodeId(nodeId: number): void {
+    this.imageNodeId = nodeId
+  }
+
+  private _recordEvent(event: GestureEvent): void {
+    if (!this.isRecording) return
+
+    // Throttle gestures to avoid spam
+    const now = Date.now()
+    if (now - this.lastGestureTime < this.gestureThrottleMs) {
+      return
+    }
+    this.lastGestureTime = now
+
+    this.events.push(event)
+    this._sendEvent(event)
+    this._recordOpenTelemetrySpan(event)
+  }
+
+  private _createMouseInteractionEvent(
+    x: number,
+    y: number,
+    interactionType: MouseInteractions,
+    timestamp?: number,
+  ): void {
+    const incrementalSnapshotEvent: eventWithTime = {
+      type: EventType.IncrementalSnapshot,
+      data: {
+        source: IncrementalSource.MouseInteraction,
+        type: interactionType,
+        id: this.imageNodeId, // Reference to the image node
+        x: x, // Preserve decimal precision like web rrweb
+        y: y, // Preserve decimal precision like web rrweb
+        pointerType: 2, // 2 = Touch for React Native (0=Mouse, 1=Pen, 2=Touch)
+      },
+      timestamp: timestamp || Date.now(),
+    }
+
+    this.recordEvent(incrementalSnapshotEvent)
+  }
+
+  private _createMouseMoveEvent(x: number, y: number, _target?: string): void {
+    const incrementalSnapshotEvent: eventWithTime = {
+      type: EventType.IncrementalSnapshot,
+      data: {
+        source: IncrementalSource.TouchMove, // Use MouseMove instead of MouseInteraction
+        positions: [
+          {
+            x: x, // Preserve decimal precision like web rrweb
+            y: y, // Preserve decimal precision like web rrweb
+            id: this.imageNodeId, // Reference to the image node
+            timeOffset: 0, // No time offset for single position
+          },
+        ],
+      },
+      timestamp: Date.now(),
+    }
+
+    this.recordEvent(incrementalSnapshotEvent)
+  }
+
+  recordEvent(event: any): void {
+    if (this.eventRecorder) {
+      this.eventRecorder.recordEvent(event)
+    }
+  }
+
+  // Get recorded events
+  getEvents(): GestureEvent[] {
+    return [...this.events]
+  }
+
+  // Clear events
+  clearEvents(): void {
+    this.events = []
+  }
+
+  // Get event statistics
+  getEventStats(): Record<string, number> {
+    const stats: Record<string, number> = {}
+    this.events.forEach(event => {
+      stats[event.type] = (stats[event.type] || 0) + 1
+    })
+    return stats
+  }
+
+  // Set gesture throttle
+  setGestureThrottle(throttleMs: number): void {
+    this.gestureThrottleMs = throttleMs
+  }
+
+  // Get recording status
+  isRecordingEnabled(): boolean {
+    return this.isRecording
+  }
+
+  /**
+   * Truncate text to prevent large span attributes
+   * @param text - Text to truncate
+   * @param maxLength - Maximum length (default: 100)
+   * @returns Truncated text with ellipsis if needed
+   */
+  private _truncateText(text: string, maxLength: number = 100): string {
+    if (!text || text.length <= maxLength) {
+      return text
+    }
+    return text.substring(0, maxLength - 3) + '...'
+  }
+}