@openobserve/browser-core 0.2.11-rc1 → 0.2.12-beta.1

package/src/domain/synthetics/syntheticsWorkerValues.ts

@@ -1,4 +1,5 @@
 import { getInitCookie } from '../../browser/cookie'
+import { globalObject, isWorkerEnvironment } from '../../tools/globalObject'
 
 export const SYNTHETICS_TEST_ID_COOKIE_NAME = 'openobserve-synthetics-public-id'
 export const SYNTHETICS_RESULT_ID_COOKIE_NAME = 'openobserve-synthetics-result-id'
@@ -11,8 +12,13 @@ export interface BrowserWindow extends Window {
 }
 
 export function willSyntheticsInjectRum(): boolean {
+  if (isWorkerEnvironment) {
+    // We don't expect to run synthetics tests in a worker environment
+    return false
+  }
+
   return Boolean(
-    (window as BrowserWindow)._OO_SYNTHETICS_INJECTS_RUM || getInitCookie(SYNTHETICS_INJECTS_RUM_COOKIE_NAME)
+    (globalObject as BrowserWindow)._OO_SYNTHETICS_INJECTS_RUM || getInitCookie(SYNTHETICS_INJECTS_RUM_COOKIE_NAME)
   )
 }
 
@@ -22,6 +28,11 @@ export function getSyntheticsTestId(): string | undefined {
 }
 
 export function getSyntheticsResultId(): string | undefined {
-  const value = (window as BrowserWindow)._OO_SYNTHETICS_RESULT_ID || getInitCookie(SYNTHETICS_RESULT_ID_COOKIE_NAME)
+  const value =
+    (window as BrowserWindow)._OO_SYNTHETICS_RESULT_ID || getInitCookie(SYNTHETICS_RESULT_ID_COOKIE_NAME)
   return typeof value === 'string' ? value : undefined
 }
+
+export function isSyntheticsTest(): boolean {
+  return Boolean(getSyntheticsTestId() && getSyntheticsResultId())
+}

package/src/domain/tags.ts

@@ -0,0 +1,76 @@
+import { DOCS_ORIGIN, MORE_DETAILS, display } from '../tools/display'
+import type { Configuration } from './configuration'
+
+export const TAG_SIZE_LIMIT = 200
+
+// replaced at build time
+declare const __BUILD_ENV__SDK_VERSION__: string
+
+export function buildTags(configuration: Configuration): string[] {
+  const { env, service, version, datacenter, sdkVersion, variant } = configuration
+  const tags = [buildTag('sdk_version', sdkVersion ?? __BUILD_ENV__SDK_VERSION__)]
+
+  if (env) {
+    tags.push(buildTag('env', env))
+  }
+  if (service) {
+    tags.push(buildTag('service', service))
+  }
+  if (version) {
+    tags.push(buildTag('version', version))
+  }
+  if (datacenter) {
+    tags.push(buildTag('datacenter', datacenter))
+  }
+
+  if (variant) {
+    tags.push(buildTag('variant', variant))
+  }
+
+  return tags
+}
+
+export function buildTag(key: string, rawValue?: string) {
+  // See https://docs.datadoghq.com/getting_started/tagging/#defining-tags for tags syntax. Note
+  // that the backend may not follow the exact same rules, so we only want to display an informal
+  // warning.
+  const tag = rawValue ? `${key}:${rawValue}` : key
+
+  if (tag.length > TAG_SIZE_LIMIT || hasForbiddenCharacters(tag)) {
+    display.warn(
+      `Tag ${tag} doesn't meet tag requirements and will be sanitized. ${MORE_DETAILS} ${DOCS_ORIGIN}/getting_started/tagging/#defining-tags`
+    )
+  }
+
+  // Let the backend do most of the sanitization, but still make sure multiple tags can't be crafted
+  // by forging a value containing commas.
+  return sanitizeTag(tag)
+}
+
+export function sanitizeTag(tag: string) {
+  return tag.replace(/,/g, '_')
+}
+
+function hasForbiddenCharacters(rawValue: string) {
+  // Unicode property escapes is not supported in all browsers, so we use a try/catch.
+  // Todo: Remove the try/catch when dropping support for Chrome 63 and Firefox 67
+  // see: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions/Unicode_character_class_escape#browser_compatibility
+  if (!supportUnicodePropertyEscapes()) {
+    return false
+  }
+
+  // We use the Unicode property escapes to match any character that is a letter including other languages like Chinese, Japanese, etc.
+  // p{Ll} matches a lowercase letter.
+  // p{Lo} matches a letter that is neither uppercase nor lowercase (ex: Japanese characters).
+  // See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions/Unicode_character_class_escape#unicode_property_escapes_vs._character_classes
+  return new RegExp('[^\\p{Ll}\\p{Lo}0-9_:./-]', 'u').test(rawValue)
+}
+
+export function supportUnicodePropertyEscapes() {
+  try {
+    new RegExp('[\\p{Ll}]', 'u')
+    return true
+  } catch {
+    return false
+  }
+}

package/src/domain/telemetry/index.ts

@@ -1,14 +1,22 @@
+export type { Telemetry } from './telemetry'
 export {
-  Telemetry,
   TelemetryService,
+  TelemetryMetrics,
   addTelemetryDebug,
   addTelemetryError,
-  startFakeTelemetry,
   resetTelemetry,
   startTelemetry,
-  isTelemetryReplicationAllowed,
   addTelemetryConfiguration,
+  addTelemetryUsage,
+  addTelemetryMetrics,
+  getTelemetryObservable,
 } from './telemetry'
 
 export * from './rawTelemetryEvent.types'
-export * from './telemetryEvent.types'
+export type {
+  TelemetryEvent,
+  TelemetryErrorEvent,
+  TelemetryDebugEvent,
+  TelemetryConfigurationEvent,
+  TelemetryUsageEvent,
+} from './telemetryEvent.types'

package/src/domain/telemetry/rawTelemetryEvent.types.ts

@@ -1,8 +1,9 @@
-import type { TelemetryEvent, TelemetryConfigurationEvent } from './telemetryEvent.types'
+import type { TelemetryEvent, TelemetryConfigurationEvent, TelemetryUsageEvent } from './telemetryEvent.types'
 
 export const TelemetryType = {
-  log: 'log',
-  configuration: 'configuration',
+  LOG: 'log',
+  CONFIGURATION: 'configuration',
+  USAGE: 'usage',
 } as const
 
 export const enum StatusType {
@@ -17,3 +18,5 @@ export interface RuntimeEnvInfo {
 
 export type RawTelemetryEvent = TelemetryEvent['telemetry']
 export type RawTelemetryConfiguration = TelemetryConfigurationEvent['telemetry']['configuration']
+export type RawTelemetryUsage = TelemetryUsageEvent['telemetry']['usage']
+export type RawTelemetryUsageFeature = TelemetryUsageEvent['telemetry']['usage']['feature']

package/src/domain/telemetry/telemetry.ts

@@ -1,27 +1,48 @@
 import type { Context } from '../../tools/serialisation/context'
 import { ConsoleApiName } from '../../tools/display'
-import { toStackTraceString, NO_ERROR_STACK_PRESENT_MESSAGE } from '../error/error'
+import { NO_ERROR_STACK_PRESENT_MESSAGE, isError } from '../error/error'
+import { toStackTraceString } from '../../tools/stackTrace/handlingStack'
 import { getExperimentalFeatures } from '../../tools/experimentalFeatures'
 import type { Configuration } from '../configuration'
-import { INTAKE_SITE_STAGING, INTAKE_SITE_US1_FED } from '../configuration'
-import { Observable } from '../../tools/observable'
-import { timeStampNow } from '../../tools/utils/timeUtils'
+import { buildTags } from '../tags'
+import { INTAKE_SITE_STAGING, INTAKE_SITE_US1_FED } from '../intakeSites'
+import { BufferedObservable, Observable } from '../../tools/observable'
+import { clocksNow } from '../../tools/utils/timeUtils'
 import { displayIfDebugEnabled, startMonitorErrorCollection } from '../../tools/monitor'
 import { sendToExtension } from '../../tools/sendToExtension'
-import { startsWith, arrayFrom, includes, assign } from '../../tools/utils/polyfills'
 import { performDraw } from '../../tools/utils/numberUtils'
 import { jsonStringify } from '../../tools/serialisation/jsonStringify'
 import { combine } from '../../tools/mergeInto'
+import type { RawError } from '../error/error.types'
 import { NonErrorPrefix } from '../error/error.types'
-import type { StackTrace } from '../error/computeStackTrace'
-import { computeStackTrace } from '../error/computeStackTrace'
+import type { StackTrace } from '../../tools/stackTrace/computeStackTrace'
+import { computeStackTrace } from '../../tools/stackTrace/computeStackTrace'
 import { getConnectivity } from '../connectivity'
+import {
+  canUseEventBridge,
+  createFlushController,
+  createHttpRequest,
+  getEventBridge,
+  createBatch,
+} from '../../transport'
+import type { Encoder } from '../../tools/encoder'
+import type { PageMayExitEvent } from '../../browser/pageMayExitObservable'
+import { DeflateEncoderStreamId } from '../deflate'
+import type { AbstractHooks, RecursivePartial } from '../../tools/abstractHooks'
+import { HookNames, DISCARDED } from '../../tools/abstractHooks'
+import { globalObject, isWorkerEnvironment } from '../../tools/globalObject'
 import type { TelemetryEvent } from './telemetryEvent.types'
-import type { RawTelemetryConfiguration, RawTelemetryEvent, RuntimeEnvInfo } from './rawTelemetryEvent.types'
+import type {
+  RawTelemetryConfiguration,
+  RawTelemetryEvent,
+  RuntimeEnvInfo,
+  RawTelemetryUsage,
+} from './rawTelemetryEvent.types'
 import { StatusType, TelemetryType } from './rawTelemetryEvent.types'
 
 // replaced at build time
 declare const __BUILD_ENV__SDK_VERSION__: string
+declare const __BUILD_ENV__SDK_SETUP__: string
 
 const ALLOWED_FRAME_URLS = [
   'http://localhost',
@@ -34,157 +55,265 @@ export const enum TelemetryService {
 }
 
 export interface Telemetry {
-  setContextProvider: (provider: () => Context) => void
-  observable: Observable<TelemetryEvent & Context>
+  stop: () => void
   enabled: boolean
+  metricsEnabled: boolean
 }
 
+export const enum TelemetryMetrics {
+  CUSTOMER_DATA_METRIC_NAME = 'Customer data measures',
+  REMOTE_CONFIGURATION_METRIC_NAME = 'remote configuration metrics',
+  RECORDER_INIT_METRICS_TELEMETRY_NAME = 'Recorder init metrics',
+  SEGMENT_METRICS_TELEMETRY_NAME = 'Segment network request metrics',
+  INITIAL_VIEW_METRICS_TELEMETRY_NAME = 'Initial view metrics',
+}
+
+const METRIC_SAMPLE_RATE = 1
+
 const TELEMETRY_EXCLUDED_SITES: string[] = [INTAKE_SITE_US1_FED]
+const MAX_TELEMETRY_EVENTS_PER_PAGE = 15
 
-const telemetryConfiguration: {
-  maxEventsPerPage: number
-  sentEventCount: number
-  telemetryEnabled: boolean
-  telemetryConfigurationEnabled: boolean
-} = { maxEventsPerPage: 0, sentEventCount: 0, telemetryEnabled: false, telemetryConfigurationEnabled: false }
+let telemetryObservable: BufferedObservable<{ rawEvent: RawTelemetryEvent; metricName?: string }> | undefined
 
-let onRawTelemetryEventCollected: ((event: RawTelemetryEvent) => void) | undefined
+export function getTelemetryObservable() {
+  if (!telemetryObservable) {
+    telemetryObservable = new BufferedObservable(100)
+  }
+  return telemetryObservable
+}
 
-export function startTelemetry(telemetryService: TelemetryService, configuration: Configuration): Telemetry {
-  let contextProvider: () => Context
+export function startTelemetry(
+  telemetryService: TelemetryService,
+  configuration: Configuration,
+  hooks: AbstractHooks,
+  reportError: (error: RawError) => void,
+  pageMayExitObservable: Observable<PageMayExitEvent>,
+  createEncoder: (streamId: DeflateEncoderStreamId) => Encoder
+): Telemetry {
   const observable = new Observable<TelemetryEvent & Context>()
 
-  telemetryConfiguration.telemetryEnabled =
-    !includes(TELEMETRY_EXCLUDED_SITES, configuration.site) && performDraw(configuration.telemetrySampleRate)
-  telemetryConfiguration.telemetryConfigurationEnabled =
-    telemetryConfiguration.telemetryEnabled && performDraw(configuration.telemetryConfigurationSampleRate)
+  const { stop } = startTelemetryTransport(configuration, reportError, pageMayExitObservable, createEncoder, observable)
+
+  const { enabled, metricsEnabled } = startTelemetryCollection(telemetryService, configuration, hooks, observable)
+
+  return {
+    stop,
+    enabled,
+    metricsEnabled,
+  }
+}
+
+export function startTelemetryCollection(
+  telemetryService: TelemetryService,
+  configuration: Configuration,
+  hooks: AbstractHooks,
+  observable: Observable<TelemetryEvent & Context>,
+  metricSampleRate = METRIC_SAMPLE_RATE,
+  maxTelemetryEventsPerPage = MAX_TELEMETRY_EVENTS_PER_PAGE
+) {
+  const alreadySentEventsByKind: Record<string, Set<string>> = {}
+
+  const telemetryEnabled =
+    !TELEMETRY_EXCLUDED_SITES.includes(configuration.site) && performDraw(configuration.telemetrySampleRate)
+
+  const telemetryEnabledPerType = {
+    [TelemetryType.LOG]: telemetryEnabled,
+    [TelemetryType.CONFIGURATION]: telemetryEnabled && performDraw(configuration.telemetryConfigurationSampleRate),
+    [TelemetryType.USAGE]: telemetryEnabled && performDraw(configuration.telemetryUsageSampleRate),
+    // not an actual "type" but using a single draw for all metrics
+    metric: telemetryEnabled && performDraw(metricSampleRate),
+  }
 
   const runtimeEnvInfo = getRuntimeEnvInfo()
-  onRawTelemetryEventCollected = (rawEvent: RawTelemetryEvent) => {
-    if (telemetryConfiguration.telemetryEnabled) {
-      const event = toTelemetryEvent(telemetryService, rawEvent, runtimeEnvInfo)
-      observable.notify(event)
-      sendToExtension('telemetry', event)
+  const telemetryObservable = getTelemetryObservable()
+  telemetryObservable.subscribe(({ rawEvent, metricName }) => {
+    if ((metricName && !telemetryEnabledPerType['metric']) || !telemetryEnabledPerType[rawEvent.type!]) {
+      return
+    }
+
+    const kind = metricName || (rawEvent.status as string | undefined) || rawEvent.type!
+    let alreadySentEvents = alreadySentEventsByKind[kind]
+    if (!alreadySentEvents) {
+      alreadySentEvents = alreadySentEventsByKind[kind] = new Set()
     }
-  }
-  startMonitorErrorCollection(addTelemetryError)
 
-  assign(telemetryConfiguration, {
-    maxEventsPerPage: configuration.maxTelemetryEventsPerPage,
-    sentEventCount: 0,
+    if (alreadySentEvents.size >= maxTelemetryEventsPerPage) {
+      return
+    }
+
+    const stringifiedEvent = jsonStringify(rawEvent)!
+    if (alreadySentEvents.has(stringifiedEvent)) {
+      return
+    }
+
+    const defaultTelemetryEventAttributes = hooks.triggerHook(HookNames.AssembleTelemetry, {
+      startTime: clocksNow().relative,
+    })
+
+    if (defaultTelemetryEventAttributes === DISCARDED) {
+      return
+    }
+    const event = toTelemetryEvent(
+      defaultTelemetryEventAttributes as RecursivePartial<TelemetryEvent>,
+      telemetryService,
+      rawEvent,
+      runtimeEnvInfo
+    )
+    observable.notify(event)
+    sendToExtension('telemetry', event)
+    alreadySentEvents.add(stringifiedEvent)
   })
+  telemetryObservable.unbuffer()
+
+  startMonitorErrorCollection(addTelemetryError)
+
+  return {
+    enabled: telemetryEnabled,
+    metricsEnabled: telemetryEnabledPerType['metric'],
+  }
 
   function toTelemetryEvent(
+    defaultTelemetryEventAttributes: RecursivePartial<TelemetryEvent>,
     telemetryService: TelemetryService,
-    event: RawTelemetryEvent,
+    rawEvent: RawTelemetryEvent,
     runtimeEnvInfo: RuntimeEnvInfo
   ): TelemetryEvent & Context {
-    return combine(
-      {
-        type: 'telemetry' as const,
-        date: timeStampNow(),
-        service: telemetryService,
-        version: __BUILD_ENV__SDK_VERSION__,
-        source: 'browser' as const,
-        _oo: {
-          format_version: 2 as const,
-        },
-        telemetry: combine(event, {
-          runtime_env: runtimeEnvInfo,
-          connectivity: getConnectivity(),
-        }),
-        experimental_features: arrayFrom(getExperimentalFeatures()),
+    const clockNow = clocksNow()
+
+    const event = {
+      type: 'telemetry' as const,
+      date: clockNow.timeStamp,
+      service: telemetryService,
+      version: __BUILD_ENV__SDK_VERSION__,
+      source: 'browser' as const,
+      _oo: {
+        format_version: 2 as const,
       },
-      contextProvider !== undefined ? contextProvider() : {}
-    ) as TelemetryEvent & Context
-  }
+      telemetry: combine(rawEvent, {
+        runtime_env: runtimeEnvInfo,
+        connectivity: getConnectivity(),
+        sdk_setup: __BUILD_ENV__SDK_SETUP__,
+      }) as TelemetryEvent['telemetry'],
+      ootags: buildTags(configuration).join(','),
+      experimental_features: Array.from(getExperimentalFeatures()),
+    }
 
-  return {
-    setContextProvider: (provider: () => Context) => {
-      contextProvider = provider
-    },
-    observable,
-    enabled: telemetryConfiguration.telemetryEnabled,
+    return combine(event, defaultTelemetryEventAttributes) as TelemetryEvent & Context
   }
 }
-function getRuntimeEnvInfo(): RuntimeEnvInfo {
+
+function startTelemetryTransport(
+  configuration: Configuration,
+  reportError: (error: RawError) => void,
+  pageMayExitObservable: Observable<PageMayExitEvent>,
+  createEncoder: (streamId: DeflateEncoderStreamId) => Encoder,
+  telemetryObservable: Observable<TelemetryEvent & Context>
+) {
+  const cleanupTasks: Array<() => void> = []
+  if (canUseEventBridge()) {
+    const bridge = getEventBridge<'internal_telemetry', TelemetryEvent>()!
+    const telemetrySubscription = telemetryObservable.subscribe((event) => bridge.send('internal_telemetry', event))
+    cleanupTasks.push(telemetrySubscription.unsubscribe)
+  } else {
+    const endpoints = [configuration.rumEndpointBuilder]
+    if (configuration.replica && isTelemetryReplicationAllowed(configuration)) {
+      endpoints.push(configuration.replica.rumEndpointBuilder)
+    }
+    const telemetryBatch = createBatch({
+      encoder: createEncoder(DeflateEncoderStreamId.TELEMETRY),
+      request: createHttpRequest(endpoints, reportError),
+      flushController: createFlushController({
+        pageMayExitObservable,
+
+        // We don't use an actual session expire observable here, to make telemetry collection
+        // independent of the session. This allows to start and send telemetry events earlier.
+        sessionExpireObservable: new Observable(),
+      }),
+    })
+    cleanupTasks.push(telemetryBatch.stop)
+    const telemetrySubscription = telemetryObservable.subscribe(telemetryBatch.add)
+    cleanupTasks.push(telemetrySubscription.unsubscribe)
+  }
+
   return {
-    is_local_file: window.location.protocol === 'file:',
-    is_worker: 'WorkerGlobalScope' in self,
+    stop: () => cleanupTasks.forEach((task) => task()),
   }
 }
 
-export function startFakeTelemetry() {
-  const events: RawTelemetryEvent[] = []
-  assign(telemetryConfiguration, {
-    maxEventsPerPage: Infinity,
-    sentEventCount: 0,
-  })
-
-  onRawTelemetryEventCollected = (event: RawTelemetryEvent) => {
-    events.push(event)
+function getRuntimeEnvInfo(): RuntimeEnvInfo {
+  return {
+    is_local_file: globalObject.location?.protocol === 'file:',
+    is_worker: isWorkerEnvironment,
   }
-
-  return events
 }
 
 export function resetTelemetry() {
-  onRawTelemetryEventCollected = undefined
+  telemetryObservable = undefined
 }
 
 /**
  * Avoid mixing telemetry events from different data centers
  * but keep replicating staging events for reliability
  */
-export function isTelemetryReplicationAllowed(configuration: Configuration) {
+function isTelemetryReplicationAllowed(configuration: Configuration) {
   return configuration.site === INTAKE_SITE_STAGING
 }
 
 export function addTelemetryDebug(message: string, context?: Context) {
   displayIfDebugEnabled(ConsoleApiName.debug, message, context)
-  addTelemetry(
-    assign(
-      {
-        type: TelemetryType.log,
-        message,
-        status: StatusType.debug,
-      },
-      context
-    )
-  )
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.LOG,
+      message,
+      status: StatusType.debug,
+      ...context,
+    },
+  })
 }
 
 export function addTelemetryError(e: unknown, context?: Context) {
-  addTelemetry(
-    assign(
-      {
-        type: TelemetryType.log,
-        status: StatusType.error,
-      },
-      formatError(e),
-      context
-    )
-  )
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.LOG,
+      status: StatusType.error,
+      ...formatError(e),
+      ...context,
+    },
+  })
 }
 
 export function addTelemetryConfiguration(configuration: RawTelemetryConfiguration) {
-  if (telemetryConfiguration.telemetryConfigurationEnabled) {
-    addTelemetry({
-      type: TelemetryType.configuration,
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.CONFIGURATION,
       configuration,
-    })
-  }
+    },
+  })
 }
 
-function addTelemetry(event: RawTelemetryEvent) {
-  if (onRawTelemetryEventCollected && telemetryConfiguration.sentEventCount < telemetryConfiguration.maxEventsPerPage) {
-    telemetryConfiguration.sentEventCount += 1
-    onRawTelemetryEventCollected(event)
-  }
+export function addTelemetryMetrics(metricName: TelemetryMetrics, context?: Context) {
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.LOG,
+      message: metricName,
+      status: StatusType.debug,
+      ...context,
+    },
+    metricName,
+  })
+}
+
+export function addTelemetryUsage(usage: RawTelemetryUsage) {
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.USAGE,
+      usage,
+    },
+  })
 }
 
 export function formatError(e: unknown) {
-  if (e instanceof Error) {
+  if (isError(e)) {
     const stackTrace = computeStackTrace(e)
     return {
       error: {
@@ -204,7 +333,7 @@ export function formatError(e: unknown) {
 
 export function scrubCustomerFrames(stackTrace: StackTrace) {
   stackTrace.stack = stackTrace.stack.filter(
-    (frame) => !frame.url || ALLOWED_FRAME_URLS.some((allowedFrameUrl) => startsWith(frame.url!, allowedFrameUrl))
+    (frame) => !frame.url || ALLOWED_FRAME_URLS.some((allowedFrameUrl) => frame.url!.startsWith(allowedFrameUrl))
   )
   return stackTrace
 }