@kubesense/kubesense-browser-rum-core 1.8.0 → 2.0.0-beta.1

package/src/domain/resource/resourceCollection.ts

@@ -1,78 +1,99 @@
-import type { ClocksState, Duration } from '@kubesense/kubesense-browser-core'
 import {
-  combine,
+  createTaskQueue,
+  display,
   generateUUID,
+  matchList,
+  mockable,
   RequestType,
   ResourceType,
-  toServerDuration,
-  relativeToClocks,
-  createTaskQueue,
+  runOnReadyState,
+  safeTruncate,
+  setTimeout,
 } from '@kubesense/kubesense-browser-core'
-import type { RumConfiguration } from '../configuration'
-import type { RumPerformanceResourceTiming } from '../../browser/performanceObservable'
-import { RumPerformanceEntryType, createPerformanceObservable } from '../../browser/performanceObservable'
-import type {
-  RumXhrResourceEventDomainContext,
-  RumFetchResourceEventDomainContext,
-  RumOtherResourceEventDomainContext,
-} from '../../domainContext.types'
-import type { RawRumResourceEvent } from '../../rawRumEvent.types'
+import type { MatchOption } from '@kubesense/kubesense-browser-core'
+import type { Duration } from '@kubesense/kubesense-js-core/time'
+import { elapsed, relativeToClocks, toServerDuration } from '@kubesense/kubesense-js-core/time'
+import { combine } from '@kubesense/kubesense-js-core/util'
+import { createPerformanceObservable, RumPerformanceEntryType } from '../../browser/performanceObservable'
+import { getNavigationEntry } from '../../browser/performanceUtils'
+import type { RumResourceEventDomainContext } from '../../domainContext.types'
+import type { NetworkHeaders, RawRumResourceEvent, ResourceRequest, ResourceResponse } from '../../rawRumEvent.types'
 import { RumEventType } from '../../rawRumEvent.types'
-import type { RawRumEventCollectedData, LifeCycle } from '../lifeCycle'
+import type { MatchHeader, RumConfiguration } from '../configuration'
+import { DEFAULT_TRACKED_RESOURCE_HEADERS } from '../configuration'
+import { startEventTracker } from '../eventTracker'
+import { extractRegexMatch } from '../extractRegexMatch'
+import type { LifeCycle, RawRumEventCollectedData } from '../lifeCycle'
 import { LifeCycleEventType } from '../lifeCycle'
 import type { RequestCompleteEvent } from '../requestCollection'
-import type { PageStateHistory } from '../contexts/pageStateHistory'
-import { PageState } from '../contexts/pageStateHistory'
+import { getDocumentTraceId } from '../tracing/getDocumentTraceId'
 import { createSpanIdentifier } from '../tracing/identifier'
-import { matchRequestResourceEntry } from './matchRequestResourceEntry'
+import type { WebSocketCompleteEvent } from './webSocketCollection'
+import type { GraphQlMetadata } from './graphql'
+import { extractGraphQlMetadata, findGraphQlConfiguration } from './graphql'
+import { createRequestRegistry } from './requestRegistry'
+import type { ResourceLikeEntry } from './resourceUtils'
 import {
+  computeResourceEntryDeliveryType,
+  computeResourceProvider,
   computeResourceEntryDetails,
   computeResourceEntryDuration,
-  computeResourceEntryType,
-  computeResourceEntrySize,
   computeResourceEntryProtocol,
-  computeResourceEntryDeliveryType,
-  computeResourceProvider,
+  computeResourceEntrySize,
+  computeResourceEntryType,
   isResourceEntryRequestType,
   sanitizeIfLongDataUrl,
 } from './resourceUtils'
-import { retrieveInitialDocumentResourceTiming } from './retrieveInitialDocumentResourceTiming'
-import type { RequestRegistry } from './requestRegistry'
-import { createRequestRegistry } from './requestRegistry'
-import type { GraphQlMetadata } from './graphql'
-import { extractGraphQlMetadata, findGraphQlConfiguration } from './graphql'
+import type { ManualResourceData } from './trackManualResources'
+import { trackManualResources } from './trackManualResources'
 
-export function startResourceCollection(
-  lifeCycle: LifeCycle,
-  configuration: RumConfiguration,
-  pageStateHistory: PageStateHistory,
-  taskQueue = createTaskQueue(),
-  retrieveInitialDocumentResourceTimingImpl = retrieveInitialDocumentResourceTiming
-) {
-  let requestRegistry: RequestRegistry | undefined
-  const isEarlyRequestCollectionEnabled = configuration.trackEarlyRequests
-
-  if (isEarlyRequestCollectionEnabled) {
-    requestRegistry = createRequestRegistry(lifeCycle)
-  } else {
-    lifeCycle.subscribe(LifeCycleEventType.REQUEST_COMPLETED, (request: RequestCompleteEvent) => {
-      handleResource(() => processRequest(request, configuration, pageStateHistory))
-    })
-  }
+// Delay before looking up the request matching a request-type performance entry. See the call
+// site in `startResourceCollection` for the rationale.
+export const REQUEST_MATCHING_DELAY = 50 as Duration
+
+export function startResourceCollection(lifeCycle: LifeCycle, configuration: RumConfiguration) {
+  const taskQueue = mockable(createTaskQueue)()
+  const requestRegistry = createRequestRegistry(lifeCycle)
 
-  const performanceResourceSubscription = createPerformanceObservable(configuration, {
+  lifeCycle.subscribe(LifeCycleEventType.WEBSOCKET_COMPLETED, (event: WebSocketCompleteEvent) => {
+    // Avoid taskQueue to ensure the websocket resource is reported before the session expires.
+    const rawEvent = assembleWebSocketResource(event, configuration)
+    if (rawEvent) {
+      lifeCycle.notify(LifeCycleEventType.RAW_RUM_EVENT_COLLECTED, rawEvent)
+    }
+  })
+
+  const performanceResourceSubscription = createPerformanceObservable({
     type: RumPerformanceEntryType.RESOURCE,
     buffered: true,
   }).subscribe((entries) => {
     for (const entry of entries) {
-      if (isEarlyRequestCollectionEnabled || !isResourceEntryRequestType(entry)) {
-        handleResource(() => processResourceEntry(entry, configuration, pageStateHistory, requestRegistry))
+      if (isResourceEntryRequestType(entry)) {
+        // The PerformanceObserver callback can fire before the fetch's resolve microtask runs
+        // (notably on Firefox), so the matching REQUEST_COMPLETED isn't in the registry yet.
+        // Defer the lookup to give the request time to land before we look it up.
+        //
+        // Publish synchronously after the delay rather than via the task queue: the queue's
+        // requestIdleCallback can fire after the click action's PAGE_ACTIVITY_END_DELAY window
+        // (notably on newer Chromium scheduling), leaving the request uncounted by its parent
+        // action.
+        //
+        // Note: we could clear the timeout on stop(), but this requires a bit of bookkeeping that
+        // is not necessary right now. We could reevaluate in the future.
+        setTimeout(() => {
+          const rawEvent = assembleResource(entry, requestRegistry.getMatchingRequest(entry), configuration)
+          if (rawEvent) {
+            lifeCycle.notify(LifeCycleEventType.RAW_RUM_EVENT_COLLECTED, rawEvent)
+          }
+        }, REQUEST_MATCHING_DELAY)
+      } else {
+        handleResource(() => assembleResource(entry, undefined, configuration))
       }
     }
   })
 
-  retrieveInitialDocumentResourceTimingImpl(configuration, (timing) => {
-    handleResource(() => processResourceEntry(timing, configuration, pageStateHistory, requestRegistry))
+  const { stop: stopRunOnReadyState } = runOnReadyState('interactive', () => {
+    handleResource(() => assembleResource(getNavigationEntry(), undefined, configuration))
   })
 
   function handleResource(computeRawEvent: () => RawRumEventCollectedData<RawRumResourceEvent> | undefined) {
@@ -84,58 +105,92 @@ export function startResourceCollection(
     })
   }
 
+  const resourceTracker = startEventTracker<ManualResourceData>(lifeCycle)
+  const manualResources = trackManualResources(lifeCycle, resourceTracker)
+
   return {
+    startResource: manualResources.startResource,
+    stopResource: manualResources.stopResource,
     stop: () => {
+      stopRunOnReadyState()
       taskQueue.stop()
       performanceResourceSubscription.unsubscribe()
+      resourceTracker.stopAll()
     },
   }
 }
 
-function processRequest(
-  request: RequestCompleteEvent,
-  configuration: RumConfiguration,
-  pageStateHistory: PageStateHistory
+function assembleWebSocketResource(
+  event: WebSocketCompleteEvent,
+  configuration: RumConfiguration
 ): RawRumEventCollectedData<RawRumResourceEvent> | undefined {
-  const matchingTiming = matchRequestResourceEntry(request)
-  return assembleResource(matchingTiming, request, pageStateHistory, configuration)
-}
+  const duration = elapsed(event.startClocks.timeStamp, event.endClocks.timeStamp)
 
-function processResourceEntry(
-  entry: RumPerformanceResourceTiming,
-  configuration: RumConfiguration,
-  pageStateHistory: PageStateHistory,
-  requestRegistry: RequestRegistry | undefined
-): RawRumEventCollectedData<RawRumResourceEvent> | undefined {
-  const matchingRequest =
-    isResourceEntryRequestType(entry) && requestRegistry ? requestRegistry.getMatchingRequest(entry) : undefined
-  return assembleResource(entry, matchingRequest, pageStateHistory, configuration)
+  const rawRumEvent: RawRumResourceEvent = {
+    date: event.startClocks.timeStamp,
+    type: RumEventType.RESOURCE,
+    resource: {
+      id: generateUUID(),
+      type: ResourceType.WEBSOCKET,
+      url: event.url,
+      provider: computeResourceProvider(event.url, configuration.firstPartyDomains),
+      duration: toServerDuration(duration),
+      websocket: {
+        connection_id: event.connectionId,
+        handshake_succeeded: event.handshakeSucceeded,
+        start_time: event.startClocks.timeStamp,
+        end_time: event.endClocks.timeStamp,
+        start_view_id: event.startViewId,
+        end_view_id: event.endViewId,
+        tracking_end_reason: event.trackingEndReason,
+        close_code: event.closeCode,
+        close_reason: event.closeReason,
+        was_clean: event.wasClean,
+        messages_in: event.messagesIn,
+        messages_out: event.messagesOut,
+        time_to_first_message_in: toServerDuration(event.firstMessageInOffset),
+        time_to_first_message_out: toServerDuration(event.firstMessageOutOffset),
+        last_message_in_at: event.lastMessageInAt,
+        longest_inbound_silence: toServerDuration(event.longestInboundSilence),
+        inbound_idle_duration_before_close: toServerDuration(event.inboundIdleDurationBeforeClose),
+        buffered_amount_max: event.bufferedAmountMax,
+        protocol: event.protocol,
+        setup_duration: toServerDuration(event.setupDuration),
+      },
+    },
+    _kubesense: {
+      discarded: !configuration.trackResources,
+    },
+  }
+
+  return {
+    startClocks: event.startClocks,
+    duration,
+    rawRumEvent,
+    domainContext: {
+      isWebSocket: true,
+      isManual: false,
+      webSocket: event.webSocket,
+    },
+  }
 }
 
-// TODO: In the future, the `entry` parameter should be required, making things simpler.
 function assembleResource(
-  entry: RumPerformanceResourceTiming | undefined,
+  entry: ResourceLikeEntry,
   request: RequestCompleteEvent | undefined,
-  pageStateHistory: PageStateHistory,
   configuration: RumConfiguration
 ): RawRumEventCollectedData<RawRumResourceEvent> | undefined {
-  if (!entry && !request) {
-    return
-  }
-
   const tracingInfo = request
     ? computeRequestTracingInfo(request, configuration)
-    : computeResourceEntryTracingInfo(entry!, configuration)
+    : computeResourceEntryTracingInfo(entry, configuration)
   if (!configuration.trackResources && !tracingInfo) {
     return
   }
 
-  const startClocks = entry ? relativeToClocks(entry.startTime) : request!.startClocks
-  const duration = entry
-    ? computeResourceEntryDuration(entry)
-    : computeRequestDuration(pageStateHistory, startClocks, request!.duration)
+  const startClocks = relativeToClocks(entry.startTime)
+  const duration = computeResourceEntryDuration(entry)
 
-  const graphql = request && computeGraphQlMetaData(request, configuration)
+  const networkHeaders = computeNetworkHeaders(request, configuration)
 
   const resourceEvent = combine(
     {
@@ -143,19 +198,17 @@ function assembleResource(
       resource: {
         id: generateUUID(),
         duration: toServerDuration(duration),
-        // TODO: in the future when `entry` is required, we can probably only rely on `computeResourceEntryType`
-        type: request
-          ? request.type === RequestType.XHR
-            ? ResourceType.XHR
-            : ResourceType.FETCH
-          : computeResourceEntryType(entry!),
-        method: request ? request.method : undefined,
-        status_code: request ? request.status : discardZeroStatus(entry!.responseStatus),
-        url: request ? sanitizeIfLongDataUrl(request.url) : entry!.name,
-        protocol: entry && computeResourceEntryProtocol(entry),
-        delivery_type: entry && computeResourceEntryDeliveryType(entry),
-        provider: computeResourceProvider(request ? request.url : entry!.name, configuration.firstPartyDomains),
-        graphql,
+        type: computeResourceEntryType(entry),
+        method: request?.method,
+        status_code: request ? request.status : discardZeroStatus(entry.responseStatus),
+        url: request ? sanitizeIfLongDataUrl(request.url) : entry.name,
+        protocol: computeResourceEntryProtocol(entry),
+        delivery_type: computeResourceEntryDeliveryType(entry),
+        provider: computeResourceProvider(request ? request.url : entry.name, configuration.firstPartyDomains),
+        graphql: request && computeGraphQlMetaData(request, configuration),
+        render_blocking_status: entry.renderBlockingStatus,
+        ...computeResourceEntrySize(entry),
+        ...computeResourceEntryDetails(entry),
       },
       type: RumEventType.RESOURCE,
       _kubesense: {
@@ -163,11 +216,12 @@ function assembleResource(
       },
     },
     tracingInfo,
-    entry && computeResourceEntryMetrics(entry)
+    computeContentTypeFromPerformanceEntry(entry),
+    networkHeaders
   )
 
   return {
-    startTime: startClocks.relative,
+    startClocks,
     duration,
     rawRumEvent: resourceEvent,
     domainContext: getResourceDomainContext(entry, request),
@@ -186,47 +240,41 @@ function computeGraphQlMetaData(
   return extractGraphQlMetadata(request, graphQlConfig)
 }
 
-function getResourceDomainContext(
-  entry: RumPerformanceResourceTiming | undefined,
-  request: RequestCompleteEvent | undefined
-): RumFetchResourceEventDomainContext | RumXhrResourceEventDomainContext | RumOtherResourceEventDomainContext {
-  if (request) {
-    const baseDomainContext = {
-      performanceEntry: entry,
-      isAborted: request.isAborted,
-      handlingStack: request.handlingStack,
-    }
+function computeContentTypeFromPerformanceEntry(
+  entry: ResourceLikeEntry
+): { resource: Pick<RawRumResourceEvent['resource'], 'response'> } | undefined {
+  const contentType = entry.contentType
 
-    if (request.type === RequestType.XHR) {
-      return {
-        xhr: request.xhr!,
-        ...baseDomainContext,
-      }
-    }
+  if (contentType) {
     return {
-      requestInput: request.input as RequestInfo,
-      requestInit: request.init,
-      response: request.response,
-      error: request.error,
-      ...baseDomainContext,
+      resource: {
+        response: {
+          headers: {
+            'content-type': contentType,
+          },
+        },
+      },
     }
   }
-  return {
-    // Currently, at least one of `entry` or `request` must be defined when calling this function.
-    // So `entry` is guaranteed to be defined here. In the future, when `entry` is required, we can
-    // remove the `!` assertion.
-    performanceEntry: entry!,
-  }
+
+  return undefined
 }
 
-function computeResourceEntryMetrics(entry: RumPerformanceResourceTiming) {
-  const { renderBlockingStatus } = entry
+function getResourceDomainContext(
+  entry: ResourceLikeEntry,
+  request: RequestCompleteEvent | undefined
+): RumResourceEventDomainContext {
   return {
-    resource: {
-      render_blocking_status: renderBlockingStatus,
-      ...computeResourceEntrySize(entry),
-      ...computeResourceEntryDetails(entry),
-    },
+    performanceEntry: entry as unknown as PerformanceResourceTiming | PerformanceNavigationTiming,
+    isManual: false,
+    isAborted: request ? request.isAborted : false,
+    isWebSocket: false,
+    handlingStack: request?.handlingStack,
+    requestInit: request?.init,
+    requestInput: request?.input as RequestInfo | undefined,
+    response: request?.response,
+    error: request?.error,
+    xhr: request?.xhr,
   }
 }
 
@@ -244,26 +292,23 @@ function computeRequestTracingInfo(request: RequestCompleteEvent, configuration:
   }
 }
 
-function computeResourceEntryTracingInfo(entry: RumPerformanceResourceTiming, configuration: RumConfiguration) {
-  const hasBeenTraced = entry.traceId
-  if (!hasBeenTraced) {
+function computeResourceEntryTracingInfo(entry: ResourceLikeEntry, configuration: RumConfiguration) {
+  if (entry.initiatorType !== 'navigation') {
+    return undefined
+  }
+  const traceId = mockable(getDocumentTraceId)(document)
+  if (!traceId) {
     return undefined
   }
   return {
     _kubesense: {
-      trace_id: entry.traceId,
+      trace_id: traceId,
       span_id: createSpanIdentifier().toString(),
       rule_psr: configuration.rulePsr,
     },
   }
 }
 
-function computeRequestDuration(pageStateHistory: PageStateHistory, startClocks: ClocksState, duration: Duration) {
-  return !pageStateHistory.wasInPageStateDuringPeriod(PageState.FROZEN, startClocks.relative, duration)
-    ? duration
-    : undefined
-}
-
 /**
  * The status is 0 for cross-origin resources without CORS headers, so the status is meaningless, and we shouldn't report it
  * https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming/responseStatus#cross-origin_response_status_codes
@@ -271,3 +316,140 @@ function computeRequestDuration(pageStateHistory: PageStateHistory, startClocks:
 function discardZeroStatus(statusCode: number | undefined): number | undefined {
   return statusCode === 0 ? undefined : statusCode
 }
+
+function computeNetworkHeaders(
+  request: RequestCompleteEvent | undefined,
+  configuration: RumConfiguration
+): { resource: { request?: ResourceRequest; response?: ResourceResponse } } | undefined {
+  const matchers = configuration.trackResourceHeaders
+  if (matchers.length === 0 || !request) {
+    return undefined
+  }
+
+  const urlMatchers = matchers.filter((m) => (m.url !== undefined ? matchList([m.url], request.url, true) : true))
+  if (urlMatchers.length === 0) {
+    return undefined
+  }
+
+  const responseMatchers = urlMatchers.filter(
+    (m) => m.location === undefined || m.location === 'any' || m.location === 'response'
+  )
+  const requestMatchers = urlMatchers.filter(
+    (m) => m.location === undefined || m.location === 'any' || m.location === 'request'
+  )
+
+  const responseHeaders = responseMatchers.length > 0 ? getResponseHeaders(request, responseMatchers) : undefined
+  const requestHeaders = requestMatchers.length > 0 ? getRequestHeaders(request, requestMatchers) : undefined
+
+  if (!responseHeaders && !requestHeaders) {
+    return undefined
+  }
+
+  return {
+    resource: {
+      request: requestHeaders ? { headers: requestHeaders } : undefined,
+      response: responseHeaders ? { headers: responseHeaders } : undefined,
+    },
+  }
+}
+
+function getResponseHeaders(request: RequestCompleteEvent, matchers: MatchHeader[]): NetworkHeaders | undefined {
+  if (request.type === RequestType.FETCH && request.response) {
+    return filterHeaders(request.response.headers, matchers)
+  }
+
+  if (request.type === RequestType.XHR && request.xhr) {
+    const rawXhrHeaders = request.xhr.getAllResponseHeaders()
+    if (rawXhrHeaders) {
+      try {
+        return filterHeaders(new Headers(parseRawXhrHeaders(rawXhrHeaders)), matchers)
+      } catch {
+        // Ignore parsing errors
+      }
+    }
+  }
+
+  return undefined
+}
+
+function getRequestHeaders(request: RequestCompleteEvent, matchers: MatchHeader[]): NetworkHeaders | undefined {
+  if (request.type !== RequestType.FETCH) {
+    return undefined
+  }
+
+  let headers: Headers | undefined
+
+  if (request.init?.headers) {
+    headers = new Headers(request.init.headers)
+  } else if (request.input instanceof Request) {
+    headers = request.input.headers
+  }
+
+  return headers ? filterHeaders(headers, matchers) : undefined
+}
+
+const FORBIDDEN_HEADER_PATTERN =
+  /(token|cookie|secret|authorization|(api|secret|access|app).?key|(client|connecting|real).?ip|forwarded)/
+const MAX_HEADER_COUNT = 100
+const MAX_HEADER_VALUE_LENGTH = 128
+const defaultHeadersMatchOption: MatchOption = (headerName) =>
+  (DEFAULT_TRACKED_RESOURCE_HEADERS as readonly string[]).includes(headerName)
+
+function filterHeaders(headers: Headers, matchers: MatchHeader[]): NetworkHeaders | undefined {
+  const result: NetworkHeaders = {}
+  let collectedHeaderCount = 0
+  let hasReachedMaxHeaderCount = false
+
+  headers.forEach((value, name) => {
+    if (collectedHeaderCount >= MAX_HEADER_COUNT) {
+      if (!hasReachedMaxHeaderCount) {
+        display.warn(`Maximum number of headers (${MAX_HEADER_COUNT}) has been reached. Further headers are dropped.`)
+        hasReachedMaxHeaderCount = true
+      }
+
+      return
+    }
+
+    const lowerName = name.toLowerCase()
+
+    if (FORBIDDEN_HEADER_PATTERN.test(lowerName)) {
+      return
+    }
+
+    const matchHeader = matchers.find((m) => matchList([m.name ?? defaultHeadersMatchOption], lowerName))
+    if (!matchHeader) {
+      return
+    }
+
+    const { extractor } = matchHeader
+    const capturedValue = extractor ? extractRegexMatch(value, extractor) : value
+    if (capturedValue === undefined) {
+      return
+    }
+
+    if (capturedValue.length > MAX_HEADER_VALUE_LENGTH) {
+      display.warn(
+        `Header "${lowerName}" value was truncated from ${capturedValue.length} to ${MAX_HEADER_VALUE_LENGTH} characters.`
+      )
+    }
+
+    result[lowerName] = safeTruncate(capturedValue, MAX_HEADER_VALUE_LENGTH)
+    collectedHeaderCount++
+  })
+
+  return collectedHeaderCount > 0 ? result : undefined
+}
+
+// Input:  "content-type: application/json\r\ncache-control: no-cache"
+// Output: [["content-type", "application/json"], ["cache-control", "no-cache"]]
+function parseRawXhrHeaders(rawXhrheaders: string): Array<[string, string]> {
+  const pairs: Array<[string, string]> = []
+  const lines = rawXhrheaders.trim().split(/\r\n/)
+  for (const line of lines) {
+    const colonIndex = line.indexOf(':')
+    if (colonIndex > 0) {
+      pairs.push([line.substring(0, colonIndex).trim(), line.substring(colonIndex + 1).trim()])
+    }
+  }
+  return pairs
+}