@kubesense/kubesense-browser-core 1.0.0

package/src/domain/tags.ts

@@ -0,0 +1,75 @@
+import { DOCS_ORIGIN, MORE_DETAILS, display } from '../tools/display'
+import type { Configuration } from './configuration/configuration'
+
+export const TAG_SIZE_LIMIT = 200
+
+// replaced at build time
+declare const __BUILD_ENV__SDK_VERSION__: string
+
+export function buildTags(configuration: Configuration): string[] {
+  const { env, service, version, datacenter, sdkVersion, variant } = configuration
+  const tags = [buildTag('sdk_version', sdkVersion ?? __BUILD_ENV__SDK_VERSION__)]
+
+  if (env) {
+    tags.push(buildTag('env', env))
+  }
+  if (service) {
+    tags.push(buildTag('service', service))
+  }
+  if (version) {
+    tags.push(buildTag('version', version))
+  }
+  if (datacenter) {
+    tags.push(buildTag('datacenter', datacenter))
+  }
+
+  if (variant) {
+    tags.push(buildTag('variant', variant))
+  }
+
+  return tags
+}
+
+export function buildTag(key: string, rawValue?: string) {
+  // Note that the backend may not follow the exact same rules, so we only want to display an informal
+  // warning.
+  const tag = rawValue ? `${key}:${rawValue}` : key
+
+  if (tag.length > TAG_SIZE_LIMIT || hasForbiddenCharacters(tag)) {
+    display.warn(
+      `Tag ${tag} doesn't meet tag requirements and will be sanitized. ${MORE_DETAILS} ${DOCS_ORIGIN}/getting_started/tagging/#defining-tags`
+    )
+  }
+
+  // Let the backend do most of the sanitization, but still make sure multiple tags can't be crafted
+  // by forging a value containing commas.
+  return sanitizeTag(tag)
+}
+
+export function sanitizeTag(tag: string) {
+  return tag.replace(/,/g, '_')
+}
+
+function hasForbiddenCharacters(rawValue: string) {
+  // Unicode property escapes is not supported in all browsers, so we use a try/catch.
+  // Todo: Remove the try/catch when dropping support for Chrome 63 and Firefox 67
+  // see: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions/Unicode_character_class_escape#browser_compatibility
+  if (!supportUnicodePropertyEscapes()) {
+    return false
+  }
+
+  // We use the Unicode property escapes to match any character that is a letter including other languages like Chinese, Japanese, etc.
+  // p{Ll} matches a lowercase letter.
+  // p{Lo} matches a letter that is neither uppercase nor lowercase (ex: Japanese characters).
+  // See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions/Unicode_character_class_escape#unicode_property_escapes_vs._character_classes
+  return new RegExp('[^\\p{Ll}\\p{Lo}0-9_:./-]', 'u').test(rawValue)
+}
+
+export function supportUnicodePropertyEscapes() {
+  try {
+    new RegExp('[\\p{Ll}]', 'u')
+    return true
+  } catch {
+    return false
+  }
+}

package/src/domain/telemetry/index.ts

@@ -0,0 +1,22 @@
+export type { Telemetry } from './telemetry'
+export {
+  TelemetryService,
+  TelemetryMetrics,
+  addTelemetryDebug,
+  addTelemetryError,
+  resetTelemetry,
+  startTelemetry,
+  addTelemetryConfiguration,
+  addTelemetryUsage,
+  addTelemetryMetrics,
+  getTelemetryObservable,
+} from './telemetry'
+
+export * from './rawTelemetryEvent.types'
+export type {
+  TelemetryEvent,
+  TelemetryErrorEvent,
+  TelemetryDebugEvent,
+  TelemetryConfigurationEvent,
+  TelemetryUsageEvent,
+} from './telemetryEvent.types'

package/src/domain/telemetry/rawTelemetryEvent.types.ts

@@ -0,0 +1,22 @@
+import type { TelemetryEvent, TelemetryConfigurationEvent, TelemetryUsageEvent } from './telemetryEvent.types'
+
+export const TelemetryType = {
+  LOG: 'log',
+  CONFIGURATION: 'configuration',
+  USAGE: 'usage'
+} as const
+
+export const enum StatusType {
+  debug = 'debug',
+  error = 'error',
+}
+
+export interface RuntimeEnvInfo {
+  is_local_file: boolean
+  is_worker: boolean
+}
+
+export type RawTelemetryEvent = TelemetryEvent['telemetry']
+export type RawTelemetryConfiguration = TelemetryConfigurationEvent['telemetry']['configuration']
+export type RawTelemetryUsage = TelemetryUsageEvent['telemetry']['usage']
+export type RawTelemetryUsageFeature = TelemetryUsageEvent['telemetry']['usage']['feature']

package/src/domain/telemetry/telemetry.ts

@@ -0,0 +1,344 @@
+import type { Context } from '../../tools/serialisation/context'
+import { ConsoleApiName } from '../../tools/display'
+import { NO_ERROR_STACK_PRESENT_MESSAGE, isError } from '../error/error'
+import { toStackTraceString } from '../../tools/stackTrace/handlingStack'
+import { getExperimentalFeatures } from '../../tools/experimentalFeatures'
+import type { Configuration } from '../configuration'
+import { buildTags } from '../tags'
+import { INTAKE_SITE_STAGING, INTAKE_SITE_PROD } from '../intakeSites'
+import { BufferedObservable, Observable } from '../../tools/observable'
+import { clocksNow } from '../../tools/utils/timeUtils'
+import { displayIfDebugEnabled, startMonitorErrorCollection } from '../../tools/monitor'
+import { sendToExtension } from '../../tools/sendToExtension'
+import { performDraw } from '../../tools/utils/numberUtils'
+import { jsonStringify } from '../../tools/serialisation/jsonStringify'
+import { combine } from '../../tools/mergeInto'
+import type { RawError } from '../error/error.types'
+import { NonErrorPrefix } from '../error/error.types'
+import type { StackTrace } from '../../tools/stackTrace/computeStackTrace'
+import { computeStackTrace } from '../../tools/stackTrace/computeStackTrace'
+import { getConnectivity } from '../connectivity'
+import {
+  canUseEventBridge,
+  createFlushController,
+  createHttpRequest,
+  getEventBridge,
+  createBatch,
+} from '../../transport'
+import type { Encoder } from '../../tools/encoder'
+import type { PageMayExitEvent } from '../../browser/pageMayExitObservable'
+import { DeflateEncoderStreamId } from '../deflate'
+import type { AbstractHooks, RecursivePartial } from '../../tools/abstractHooks'
+import { HookNames, DISCARDED } from '../../tools/abstractHooks'
+import { globalObject, isWorkerEnvironment } from '../../tools/globalObject'
+import type { TelemetryEvent } from './telemetryEvent.types'
+import type {
+  RawTelemetryConfiguration,
+  RawTelemetryEvent,
+  RuntimeEnvInfo,
+  RawTelemetryUsage,
+} from './rawTelemetryEvent.types'
+import { StatusType, TelemetryType } from './rawTelemetryEvent.types'
+
+// replaced at build time
+declare const __BUILD_ENV__SDK_VERSION__: string
+declare const __BUILD_ENV__SDK_SETUP__: string
+
+const ALLOWED_FRAME_URLS = [
+  'https://rum.kubesense.ai',
+  'https://rum.tyke.ai',
+  'https://d3uc069fcn7uxw.cloudfront.net',
+  'https://d20xtzwzcl0ceb.cloudfront.net',
+  'http://localhost',
+  '<anonymous>',
+]
+
+export const enum TelemetryService {
+  LOGS = 'kubesense-browser-logs-sdk',
+  RUM = 'kubesense-browser-rum-sdk',
+}
+
+export interface Telemetry {
+  stop: () => void
+  enabled: boolean
+  metricsEnabled: boolean
+}
+
+export const enum TelemetryMetrics {
+  CUSTOMER_DATA_METRIC_NAME = 'Customer data measures',
+  REMOTE_CONFIGURATION_METRIC_NAME = 'remote configuration metrics',
+  RECORDER_INIT_METRICS_TELEMETRY_NAME = 'Recorder init metrics',
+  SEGMENT_METRICS_TELEMETRY_NAME = 'Segment network request metrics',
+  INITIAL_VIEW_METRICS_TELEMETRY_NAME = 'Initial view metrics',
+}
+
+const METRIC_SAMPLE_RATE = 1
+
+const TELEMETRY_EXCLUDED_SITES: string[] = [INTAKE_SITE_STAGING]
+
+const MAX_TELEMETRY_EVENTS_PER_PAGE = 15
+
+let telemetryObservable: BufferedObservable<{ rawEvent: RawTelemetryEvent; metricName?: string }> | undefined
+
+export function getTelemetryObservable() {
+  if (!telemetryObservable) {
+    telemetryObservable = new BufferedObservable(100)
+  }
+  return telemetryObservable
+}
+
+export function startTelemetry(
+  telemetryService: TelemetryService,
+  configuration: Configuration,
+  hooks: AbstractHooks,
+  reportError: (error: RawError) => void,
+  pageMayExitObservable: Observable<PageMayExitEvent>,
+  createEncoder: (streamId: DeflateEncoderStreamId) => Encoder
+): Telemetry {
+  const observable = new Observable<TelemetryEvent & Context>()
+
+  const { stop } = startTelemetryTransport(configuration, reportError, pageMayExitObservable, createEncoder, observable)
+
+  const { enabled, metricsEnabled } = startTelemetryCollection(telemetryService, configuration, hooks, observable)
+
+  return {
+    stop,
+    enabled,
+    metricsEnabled,
+  }
+}
+
+export function startTelemetryCollection(
+  telemetryService: TelemetryService,
+  configuration: Configuration,
+  hooks: AbstractHooks,
+  observable: Observable<TelemetryEvent & Context>,
+  metricSampleRate = METRIC_SAMPLE_RATE,
+  maxTelemetryEventsPerPage = MAX_TELEMETRY_EVENTS_PER_PAGE
+) {
+  const alreadySentEventsByKind: Record<string, Set<string>> = {}
+
+  const telemetryEnabled =
+    !TELEMETRY_EXCLUDED_SITES.includes(configuration.kubesenseRumEndpoint) && performDraw(configuration.telemetrySampleRate)
+
+  const telemetryEnabledPerType = {
+    [TelemetryType.LOG]: telemetryEnabled,
+    [TelemetryType.CONFIGURATION]: telemetryEnabled && performDraw(configuration.telemetryConfigurationSampleRate),
+    [TelemetryType.USAGE]: telemetryEnabled && performDraw(configuration.telemetryUsageSampleRate),
+    // not an actual "type" but using a single draw for all metrics
+    metric: telemetryEnabled && performDraw(metricSampleRate),
+  }
+
+  const runtimeEnvInfo = getRuntimeEnvInfo()
+  const telemetryObservable = getTelemetryObservable()
+  telemetryObservable.subscribe(({ rawEvent, metricName }) => {
+    if ((metricName && !telemetryEnabledPerType['metric']) || !telemetryEnabledPerType[rawEvent.type!]) {
+      return
+    }
+
+    const kind = metricName || (rawEvent.status as string | undefined) || rawEvent.type!
+    let alreadySentEvents = alreadySentEventsByKind[kind]
+    if (!alreadySentEvents) {
+      alreadySentEvents = alreadySentEventsByKind[kind] = new Set()
+    }
+
+    if (alreadySentEvents.size >= maxTelemetryEventsPerPage) {
+      return
+    }
+
+    const stringifiedEvent = jsonStringify(rawEvent)!
+    if (alreadySentEvents.has(stringifiedEvent)) {
+      return
+    }
+
+    const defaultTelemetryEventAttributes = hooks.triggerHook(HookNames.AssembleTelemetry, {
+      startTime: clocksNow().relative,
+    })
+
+    if (defaultTelemetryEventAttributes === DISCARDED) {
+      return
+    }
+    const event = toTelemetryEvent(
+      defaultTelemetryEventAttributes as RecursivePartial<TelemetryEvent>,
+      telemetryService,
+      rawEvent,
+      runtimeEnvInfo
+    )
+    observable.notify(event)
+    sendToExtension('telemetry', event)
+    alreadySentEvents.add(stringifiedEvent)
+  })
+  telemetryObservable.unbuffer()
+
+  startMonitorErrorCollection(addTelemetryError)
+
+  return {
+    enabled: telemetryEnabled,
+    metricsEnabled: telemetryEnabledPerType['metric'],
+  }
+
+  function toTelemetryEvent(
+    defaultTelemetryEventAttributes: RecursivePartial<TelemetryEvent>,
+    telemetryService: TelemetryService,
+    rawEvent: RawTelemetryEvent,
+    runtimeEnvInfo: RuntimeEnvInfo
+  ): TelemetryEvent & Context {
+    const clockNow = clocksNow()
+
+    const event = {
+      type: 'telemetry' as const,
+      date: clockNow.timeStamp,
+      service: telemetryService,
+      version: __BUILD_ENV__SDK_VERSION__,
+      source: 'browser' as const,
+      _kubesense: {
+        format_version: 2 as const,
+      },
+      telemetry: combine(rawEvent, {
+        runtime_env: runtimeEnvInfo,
+        connectivity: getConnectivity(),
+        sdk_setup: __BUILD_ENV__SDK_SETUP__,
+      }) as TelemetryEvent['telemetry'],
+      ktags: buildTags(configuration).join(','),
+      experimental_features: Array.from(getExperimentalFeatures()),
+    }
+
+    return combine(event, defaultTelemetryEventAttributes) as TelemetryEvent & Context
+  }
+}
+
+function startTelemetryTransport(
+  configuration: Configuration,
+  reportError: (error: RawError) => void,
+  pageMayExitObservable: Observable<PageMayExitEvent>,
+  createEncoder: (streamId: DeflateEncoderStreamId) => Encoder,
+  telemetryObservable: Observable<TelemetryEvent & Context>
+) {
+  const cleanupTasks: Array<() => void> = []
+  if (canUseEventBridge()) {
+    const bridge = getEventBridge<'internal_telemetry', TelemetryEvent>()!
+    const telemetrySubscription = telemetryObservable.subscribe((event) => bridge.send('internal_telemetry', event))
+    cleanupTasks.push(telemetrySubscription.unsubscribe)
+  } else {
+    const endpoints = [configuration.rumEndpointBuilder]
+    if (configuration.replica && isTelemetryReplicationAllowed(configuration)) {
+      endpoints.push(configuration.replica.rumEndpointBuilder)
+    }
+    const telemetryBatch = createBatch({
+      encoder: createEncoder(DeflateEncoderStreamId.TELEMETRY),
+      request: createHttpRequest(endpoints, reportError),
+      flushController: createFlushController({
+        pageMayExitObservable,
+
+        // We don't use an actual session expire observable here, to make telemetry collection
+        // independent of the session. This allows to start and send telemetry events earlier.
+        sessionExpireObservable: new Observable(),
+      }),
+    })
+    cleanupTasks.push(telemetryBatch.stop)
+    const telemetrySubscription = telemetryObservable.subscribe(telemetryBatch.add)
+    cleanupTasks.push(telemetrySubscription.unsubscribe)
+  }
+
+  return {
+    stop: () => cleanupTasks.forEach((task) => task()),
+  }
+}
+
+function getRuntimeEnvInfo(): RuntimeEnvInfo {
+  return {
+    is_local_file: globalObject.location?.protocol === 'file:',
+    is_worker: isWorkerEnvironment,
+  }
+}
+
+export function resetTelemetry() {
+  telemetryObservable = undefined
+}
+
+/**
+ * Avoid mixing telemetry events from different data centers
+ * but keep replicating staging events for reliability
+ */
+function isTelemetryReplicationAllowed(configuration: Configuration) {
+  return configuration.kubesenseRumEndpoint === INTAKE_SITE_STAGING
+}
+
+export function addTelemetryDebug(message: string, context?: Context) {
+  displayIfDebugEnabled(ConsoleApiName.debug, message, context)
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.LOG,
+      message,
+      status: StatusType.debug,
+      ...context,
+    },
+  })
+}
+
+export function addTelemetryError(e: unknown, context?: Context) {
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.LOG,
+      status: StatusType.error,
+      ...formatError(e),
+      ...context,
+    },
+  })
+}
+
+export function addTelemetryConfiguration(configuration: RawTelemetryConfiguration) {
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.CONFIGURATION,
+      configuration,
+    },
+  })
+}
+
+export function addTelemetryMetrics(metricName: TelemetryMetrics, context?: Context) {
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.LOG,
+      message: metricName,
+      status: StatusType.debug,
+      ...context,
+    },
+    metricName,
+  })
+}
+
+export function addTelemetryUsage(usage: RawTelemetryUsage) {
+  getTelemetryObservable().notify({
+    rawEvent: {
+      type: TelemetryType.USAGE,
+      usage,
+    },
+  })
+}
+
+export function formatError(e: unknown) {
+  if (isError(e)) {
+    const stackTrace = computeStackTrace(e)
+    return {
+      error: {
+        kind: stackTrace.name,
+        stack: toStackTraceString(scrubCustomerFrames(stackTrace)),
+      },
+      message: stackTrace.message!,
+    }
+  }
+  return {
+    error: {
+      stack: NO_ERROR_STACK_PRESENT_MESSAGE,
+    },
+    message: `${NonErrorPrefix.UNCAUGHT} ${jsonStringify(e)!}`,
+  }
+}
+
+export function scrubCustomerFrames(stackTrace: StackTrace) {
+  stackTrace.stack = stackTrace.stack.filter(
+    (frame) => !frame.url || ALLOWED_FRAME_URLS.some((allowedFrameUrl) => frame.url!.startsWith(allowedFrameUrl))
+  )
+  return stackTrace
+}