@openobserve/browser-core 0.2.11 → 0.2.12-beta.13

package/src/domain/telemetry/telemetry.ts

@@ -1,26 +1,48 @@
 import type { Context } from '../../tools/serialisation/context'
 import { ConsoleApiName } from '../../tools/display'
-import { toStackTraceString, NO_ERROR_STACK_PRESENT_MESSAGE } from '../error/error'
+import { NO_ERROR_STACK_PRESENT_MESSAGE, isError } from '../error/error'
+import { toStackTraceString } from '../../tools/stackTrace/handlingStack'
 import { getExperimentalFeatures } from '../../tools/experimentalFeatures'
 import type { Configuration } from '../configuration'
-import { INTAKE_SITE_STAGING, INTAKE_SITE_US1_FED } from '../configuration'
-import type { StackTrace } from '../tracekit'
-import { computeStackTrace } from '../tracekit'
-import { Observable } from '../../tools/observable'
-import { timeStampNow } from '../../tools/utils/timeUtils'
+import { buildTags } from '../tags'
+import { INTAKE_SITE_STAGING, INTAKE_SITE_US1_FED } from '../intakeSites'
+import { BufferedObservable, Observable } from '../../tools/observable'
+import { clocksNow } from '../../tools/utils/timeUtils'
 import { displayIfDebugEnabled, startMonitorErrorCollection } from '../../tools/monitor'
 import { sendToExtension } from '../../tools/sendToExtension'
-import { startsWith, arrayFrom, includes, assign } from '../../tools/utils/polyfills'
 import { performDraw } from '../../tools/utils/numberUtils'
 import { jsonStringify } from '../../tools/serialisation/jsonStringify'
 import { combine } from '../../tools/mergeInto'
+import type { RawError } from '../error/error.types'
 import { NonErrorPrefix } from '../error/error.types'
+import type { StackTrace } from '../../tools/stackTrace/computeStackTrace'
+import { computeStackTrace } from '../../tools/stackTrace/computeStackTrace'
+import { getConnectivity } from '../connectivity'
+import {
+  canUseEventBridge,
+  createFlushController,
+  createHttpRequest,
+  getEventBridge,
+  createBatch,
+} from '../../transport'
+import type { Encoder } from '../../tools/encoder'
+import type { PageMayExitEvent } from '../../browser/pageMayExitObservable'
+import { DeflateEncoderStreamId } from '../deflate'
+import type { AbstractHooks, RecursivePartial } from '../../tools/abstractHooks'
+import { HookNames, DISCARDED } from '../../tools/abstractHooks'
+import { globalObject, isWorkerEnvironment } from '../../tools/globalObject'
 import type { TelemetryEvent } from './telemetryEvent.types'
-import type { RawTelemetryConfiguration, RawTelemetryEvent } from './rawTelemetryEvent.types'
+import type {
+  RawTelemetryConfiguration,
+  RawTelemetryEvent,
+  RuntimeEnvInfo,
+  RawTelemetryUsage,
+} from './rawTelemetryEvent.types'
 import { StatusType, TelemetryType } from './rawTelemetryEvent.types'
 
 // replaced at build time
 declare const __BUILD_ENV__SDK_VERSION__: string
+declare const __BUILD_ENV__SDK_SETUP__: string
 
 const ALLOWED_FRAME_URLS = [
   'http://localhost',
@@ -33,143 +55,265 @@ export const enum TelemetryService {
 }
 
 export interface Telemetry {
-  setContextProvider: (provider: () => Context) => void
-  observable: Observable<TelemetryEvent & Context>
+  stop: () => void
   enabled: boolean
+  metricsEnabled: boolean
 }
 
+export const enum TelemetryMetrics {
+  CUSTOMER_DATA_METRIC_NAME = 'Customer data measures',
+  REMOTE_CONFIGURATION_METRIC_NAME = 'remote configuration metrics',
+  RECORDER_INIT_METRICS_TELEMETRY_NAME = 'Recorder init metrics',
+  SEGMENT_METRICS_TELEMETRY_NAME = 'Segment network request metrics',
+  INITIAL_VIEW_METRICS_TELEMETRY_NAME = 'Initial view metrics',
+}
+
+const METRIC_SAMPLE_RATE = 1
+
 const TELEMETRY_EXCLUDED_SITES: string[] = [INTAKE_SITE_US1_FED]
+const MAX_TELEMETRY_EVENTS_PER_PAGE = 15
 
-const telemetryConfiguration: {
-  maxEventsPerPage: number
-  sentEventCount: number
-  telemetryEnabled: boolean
-  telemetryConfigurationEnabled: boolean
-} = { maxEventsPerPage: 0, sentEventCount: 0, telemetryEnabled: false, telemetryConfigurationEnabled: false }
+let telemetryObservable: BufferedObservable<{ rawEvent: RawTelemetryEvent; metricName?: string }> | undefined
 
-let onRawTelemetryEventCollected: ((event: RawTelemetryEvent) => void) | undefined
+export function getTelemetryObservable() {
+  if (!telemetryObservable) {
+    telemetryObservable = new BufferedObservable(100)
+  }
+  return telemetryObservable
+}
 
-export function startTelemetry(telemetryService: TelemetryService, configuration: Configuration): Telemetry {
-  let contextProvider: () => Context
+export function startTelemetry(
+  telemetryService: TelemetryService,
+  configuration: Configuration,
+  hooks: AbstractHooks,
+  reportError: (error: RawError) => void,
+  pageMayExitObservable: Observable<PageMayExitEvent>,
+  createEncoder: (streamId: DeflateEncoderStreamId) => Encoder
+): Telemetry {
   const observable = new Observable<TelemetryEvent & Context>()
 
-  telemetryConfiguration.telemetryEnabled =
-    !includes(TELEMETRY_EXCLUDED_SITES, configuration.site) && performDraw(configuration.telemetrySampleRate)
-  telemetryConfiguration.telemetryConfigurationEnabled =
-    telemetryConfiguration.telemetryEnabled && performDraw(configuration.telemetryConfigurationSampleRate)
+  const { stop } = startTelemetryTransport(configuration, reportError, pageMayExitObservable, createEncoder, observable)
 
-  onRawTelemetryEventCollected = (rawEvent: RawTelemetryEvent) => {
-    if (telemetryConfiguration.telemetryEnabled) {
-      const event = toTelemetryEvent(telemetryService, rawEvent)
-      observable.notify(event)
-      sendToExtension('telemetry', event)
-    }
+  const { enabled, metricsEnabled } = startTelemetryCollection(telemetryService, configuration, hooks, observable)
+
+  return {
+    stop,
+    enabled,
+    metricsEnabled,
   }
-  startMonitorErrorCollection(addTelemetryError)
+}
 
-  assign(telemetryConfiguration, {
-    maxEventsPerPage: configuration.maxTelemetryEventsPerPage,
-    sentEventCount: 0,
-  })
+export function startTelemetryCollection(
+  telemetryService: TelemetryService,
+  configuration: Configuration,
+  hooks: AbstractHooks,
+  observable: Observable<TelemetryEvent & Context>,
+  metricSampleRate = METRIC_SAMPLE_RATE,
+  maxTelemetryEventsPerPage = MAX_TELEMETRY_EVENTS_PER_PAGE
+) {
+  const alreadySentEventsByKind: Record<string, Set<string>> = {}
 
-  function toTelemetryEvent(telemetryService: TelemetryService, event: RawTelemetryEvent): TelemetryEvent & Context {
-    return combine(
-      {
-        type: 'telemetry' as const,
-        date: timeStampNow(),
-        service: telemetryService,
-        version: __BUILD_ENV__SDK_VERSION__,
-        source: 'browser' as const,
-        _oo: {
-          format_version: 2 as const,
-        },
-        telemetry: event as any, // https://github.com/microsoft/TypeScript/issues/48457
-        experimental_features: arrayFrom(getExperimentalFeatures()),
-      },
-      contextProvider !== undefined ? contextProvider() : {}
-    )
+  const telemetryEnabled =
+    !TELEMETRY_EXCLUDED_SITES.includes(configuration.site) && performDraw(configuration.telemetrySampleRate)
+
+  const telemetryEnabledPerType = {
+    [TelemetryType.LOG]: telemetryEnabled,
+    [TelemetryType.CONFIGURATION]: telemetryEnabled && performDraw(configuration.telemetryConfigurationSampleRate),
+    [TelemetryType.USAGE]: telemetryEnabled && performDraw(configuration.telemetryUsageSampleRate),
+    // not an actual "type" but using a single draw for all metrics
+    metric: telemetryEnabled && performDraw(metricSampleRate),
   }
 
+  const runtimeEnvInfo = getRuntimeEnvInfo()
+  const telemetryObservable = getTelemetryObservable()
+  telemetryObservable.subscribe(({ rawEvent, metricName }) => {
+    if ((metricName && !telemetryEnabledPerType['metric']) || !telemetryEnabledPerType[rawEvent.type!]) {
+      return
+    }
+
+    const kind = metricName || (rawEvent.status as string | undefined) || rawEvent.type!
+    let alreadySentEvents = alreadySentEventsByKind[kind]
+    if (!alreadySentEvents) {
+      alreadySentEvents = alreadySentEventsByKind[kind] = new Set()
+    }
+
+    if (alreadySentEvents.size >= maxTelemetryEventsPerPage) {
+      return
+    }
+
+    const stringifiedEvent = jsonStringify(rawEvent)!
+    if (alreadySentEvents.has(stringifiedEvent)) {
+      return
+    }
+
+    const defaultTelemetryEventAttributes = hooks.triggerHook(HookNames.AssembleTelemetry, {
+      startTime: clocksNow().relative,
+    })
+
+    if (defaultTelemetryEventAttributes === DISCARDED) {
+      return
+    }
+    const event = toTelemetryEvent(
+      defaultTelemetryEventAttributes as RecursivePartial<TelemetryEvent>,
+      telemetryService,
+      rawEvent,
+      runtimeEnvInfo
+    )
+    observable.notify(event)
+    sendToExtension('telemetry', event)
+    alreadySentEvents.add(stringifiedEvent)
+  })
+  telemetryObservable.unbuffer()
+
+  startMonitorErrorCollection(addTelemetryError)
+
   return {
-    setContextProvider: (provider: () => Context) => {
-      contextProvider = provider
-    },
-    observable,
-    enabled: telemetryConfiguration.telemetryEnabled,
+    enabled: telemetryEnabled,
+    metricsEnabled: telemetryEnabledPerType['metric'],
+  }
+
+  function toTelemetryEvent(
+    defaultTelemetryEventAttributes: RecursivePartial<TelemetryEvent>,
+    telemetryService: TelemetryService,
+    rawEvent: RawTelemetryEvent,
+    runtimeEnvInfo: RuntimeEnvInfo
+  ): TelemetryEvent & Context {
+    const clockNow = clocksNow()
+
+    const event = {
+      type: 'telemetry' as const,
+      date: clockNow.timeStamp,
+      service: telemetryService,
+      version: __BUILD_ENV__SDK_VERSION__,
+      source: 'browser' as const,
+      _oo: {
+        format_version: 2 as const,
+      },
+      telemetry: combine(rawEvent, {
+        runtime_env: runtimeEnvInfo,
+        connectivity: getConnectivity(),
+        sdk_setup: __BUILD_ENV__SDK_SETUP__,
+      }) as TelemetryEvent['telemetry'],
+      ootags: buildTags(configuration).join(','),
+      experimental_features: Array.from(getExperimentalFeatures()),
+    }
+
+    return combine(event, defaultTelemetryEventAttributes) as TelemetryEvent & Context
   }
 }
 
-export function startFakeTelemetry() {
-  const events: RawTelemetryEvent[] = []
-  assign(telemetryConfiguration, {
-    maxEventsPerPage: Infinity,
-    sentEventCount: 0,
-  })
+function startTelemetryTransport(
+  configuration: Configuration,
+  reportError: (error: RawError) => void,
+  pageMayExitObservable: Observable<PageMayExitEvent>,
+  createEncoder: (streamId: DeflateEncoderStreamId) => Encoder,
+  telemetryObservable: Observable<TelemetryEvent & Context>
+) {
+  const cleanupTasks: Array<() => void> = []
+  if (canUseEventBridge()) {
+    const bridge = getEventBridge<'internal_telemetry', TelemetryEvent>()!
+    const telemetrySubscription = telemetryObservable.subscribe((event) => bridge.send('internal_telemetry', event))
+    cleanupTasks.push(telemetrySubscription.unsubscribe)
+  } else {
+    const endpoints = [configuration.rumEndpointBuilder]
+    if (configuration.replica && isTelemetryReplicationAllowed(configuration)) {
+      endpoints.push(configuration.replica.rumEndpointBuilder)
+    }
+    const telemetryBatch = createBatch({
+      encoder: createEncoder(DeflateEncoderStreamId.TELEMETRY),
+      request: createHttpRequest(endpoints, reportError),
+      flushController: createFlushController({
+        pageMayExitObservable,
 
-  onRawTelemetryEventCollected = (event: RawTelemetryEvent) => {
-    events.push(event)
+        // We don't use an actual session expire observable here, to make telemetry collection
+        // independent of the session. This allows to start and send telemetry events earlier.
+        sessionExpireObservable: new Observable(),
+      }),
+    })
+    cleanupTasks.push(telemetryBatch.stop)
+    const telemetrySubscription = telemetryObservable.subscribe(telemetryBatch.add)
+    cleanupTasks.push(telemetrySubscription.unsubscribe)
   }
 
-  return events
+  return {
+    stop: () => cleanupTasks.forEach((task) => task()),
+  }
+}
+
+function getRuntimeEnvInfo(): RuntimeEnvInfo {
+  return {
+    is_local_file: globalObject.location?.protocol === 'file:',
+    is_worker: isWorkerEnvironment,
+  }
 }
 
 export function resetTelemetry() {
-  onRawTelemetryEventCollected = undefined
+  telemetryObservable = undefined
 }
 
 /**
  * Avoid mixing telemetry events from different data centers
  * but keep replicating staging events for reliability
  */
-export function isTelemetryReplicationAllowed(configuration: Configuration) {
+function isTelemetryReplicationAllowed(configuration: Configuration) {
   return configuration.site === INTAKE_SITE_STAGING
 }
 
 export function addTelemetryDebug(message: string, context?: Context) {
   displayIfDebugEnabled(ConsoleApiName.debug, message, context)
-  addTelemetry(
-    assign(
-      {
-        type: TelemetryType.log,
-        message,
-        status: StatusType.debug,
-      },
-      context
-    )
-  )
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.LOG,
+      message,
+      status: StatusType.debug,
+      ...context,
+    },
+  })
 }
 
 export function addTelemetryError(e: unknown, context?: Context) {
-  addTelemetry(
-    assign(
-      {
-        type: TelemetryType.log,
-        status: StatusType.error,
-      },
-      formatError(e),
-      context
-    )
-  )
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.LOG,
+      status: StatusType.error,
+      ...formatError(e),
+      ...context,
+    },
+  })
 }
 
 export function addTelemetryConfiguration(configuration: RawTelemetryConfiguration) {
-  if (telemetryConfiguration.telemetryConfigurationEnabled) {
-    addTelemetry({
-      type: TelemetryType.configuration,
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.CONFIGURATION,
       configuration,
-    })
-  }
+    },
+  })
 }
 
-function addTelemetry(event: RawTelemetryEvent) {
-  if (onRawTelemetryEventCollected && telemetryConfiguration.sentEventCount < telemetryConfiguration.maxEventsPerPage) {
-    telemetryConfiguration.sentEventCount += 1
-    onRawTelemetryEventCollected(event)
-  }
+export function addTelemetryMetrics(metricName: TelemetryMetrics, context?: Context) {
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.LOG,
+      message: metricName,
+      status: StatusType.debug,
+      ...context,
+    },
+    metricName,
+  })
+}
+
+export function addTelemetryUsage(usage: RawTelemetryUsage) {
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.USAGE,
+      usage,
+    },
+  })
 }
 
 export function formatError(e: unknown) {
-  if (e instanceof Error) {
+  if (isError(e)) {
     const stackTrace = computeStackTrace(e)
     return {
       error: {
@@ -189,7 +333,7 @@ export function formatError(e: unknown) {
 
 export function scrubCustomerFrames(stackTrace: StackTrace) {
   stackTrace.stack = stackTrace.stack.filter(
-    (frame) => !frame.url || ALLOWED_FRAME_URLS.some((allowedFrameUrl) => startsWith(frame.url!, allowedFrameUrl))
+    (frame) => !frame.url || ALLOWED_FRAME_URLS.some((allowedFrameUrl) => frame.url!.startsWith(allowedFrameUrl))
   )
   return stackTrace
 }