@openobserve/browser-core 0.2.8

package/src/domain/configuration/configuration.ts

@@ -0,0 +1,191 @@
+import { catchUserErrors } from '../../tools/catchUserErrors'
+import { display } from '../../tools/display'
+import type { RawTelemetryConfiguration } from '../telemetry'
+import { ExperimentalFeature, addExperimentalFeatures } from '../../tools/experimentalFeatures'
+import type { Duration } from '../../tools/utils/timeUtils'
+import { ONE_SECOND } from '../../tools/utils/timeUtils'
+import { isPercentage } from '../../tools/utils/numberUtils'
+import { ONE_KIBI_BYTE } from '../../tools/utils/byteUtils'
+import { objectHasValue } from '../../tools/utils/objectUtils'
+import { assign } from '../../tools/utils/polyfills'
+import { selectSessionStoreStrategyType } from '../session/sessionStore'
+import type { SessionStoreStrategyType } from '../session/storeStrategies/sessionStoreStrategy'
+import type { TransportConfiguration } from './transportConfiguration'
+import { computeTransportConfiguration } from './transportConfiguration'
+
+export const DefaultPrivacyLevel = {
+  ALLOW: 'allow',
+  MASK: 'mask',
+  MASK_USER_INPUT: 'mask-user-input',
+} as const
+export type DefaultPrivacyLevel = (typeof DefaultPrivacyLevel)[keyof typeof DefaultPrivacyLevel]
+
+export interface InitConfiguration {
+  // global options
+  clientToken: string
+  beforeSend?: GenericBeforeSendCallback | undefined
+  /**
+   * @deprecated use sessionSampleRate instead
+   */
+  sampleRate?: number | undefined
+  sessionSampleRate?: number | undefined
+  telemetrySampleRate?: number | undefined
+  silentMultipleInit?: boolean | undefined
+  trackResources?: boolean | undefined
+  trackLongTasks?: boolean | undefined
+
+  // transport options
+  proxy?: string | undefined
+  /**
+   * @deprecated use `proxy` instead
+   */
+  proxyUrl?: string | undefined
+  site?: string | undefined
+
+  // tag and context options
+  service: string | undefined
+  env: string | undefined
+  version: string | undefined
+
+  // cookie options
+  useCrossSiteSessionCookie?: boolean | undefined
+  useSecureSessionCookie?: boolean | undefined
+  trackSessionAcrossSubdomains?: boolean | undefined
+
+  // alternate storage option
+  allowFallbackToLocalStorage?: boolean | undefined
+
+  // internal options
+  enableExperimentalFeatures?: string[] | undefined
+  replica?: ReplicaUserConfiguration | undefined
+  datacenter?: string
+  internalAnalyticsSubdomain?: string
+
+  telemetryConfigurationSampleRate?: number
+
+  apiVersion: string
+  organizationIdentifier: string
+  insecureHTTP: boolean
+}
+
+// This type is only used to build the core configuration. Logs and RUM SDKs are using a proper type
+// for this option.
+type GenericBeforeSendCallback = (event: any, context?: any) => unknown
+
+interface ReplicaUserConfiguration {
+  applicationId?: string
+  clientToken: string
+}
+
+export interface Configuration extends TransportConfiguration {
+  // Built from init configuration
+  beforeSend: GenericBeforeSendCallback | undefined
+  sessionStoreStrategyType: SessionStoreStrategyType | undefined
+  sessionSampleRate: number
+  telemetrySampleRate: number
+  telemetryConfigurationSampleRate: number
+  service: string | undefined
+  silentMultipleInit: boolean
+
+  // Event limits
+  eventRateLimiterThreshold: number // Limit the maximum number of actions, errors and logs per minutes
+  maxTelemetryEventsPerPage: number
+
+  // Batch configuration
+  batchBytesLimit: number
+  flushTimeout: Duration
+  batchMessagesLimit: number
+  messageBytesLimit: number
+}
+
+export function validateAndBuildConfiguration(initConfiguration: InitConfiguration): Configuration | undefined {
+  if (!initConfiguration || !initConfiguration.clientToken) {
+    display.error('Client Token is not configured, we will not send any data.')
+    return
+  }
+
+  const sessionSampleRate = initConfiguration.sessionSampleRate ?? initConfiguration.sampleRate
+  if (sessionSampleRate !== undefined && !isPercentage(sessionSampleRate)) {
+    display.error('Session Sample Rate should be a number between 0 and 100')
+    return
+  }
+
+  if (initConfiguration.telemetrySampleRate !== undefined && !isPercentage(initConfiguration.telemetrySampleRate)) {
+    display.error('Telemetry Sample Rate should be a number between 0 and 100')
+    return
+  }
+
+  if (
+    initConfiguration.telemetryConfigurationSampleRate !== undefined &&
+    !isPercentage(initConfiguration.telemetryConfigurationSampleRate)
+  ) {
+    display.error('Telemetry Configuration Sample Rate should be a number between 0 and 100')
+    return
+  }
+
+  // Set the experimental feature flags as early as possible, so we can use them in most places
+  if (Array.isArray(initConfiguration.enableExperimentalFeatures)) {
+    addExperimentalFeatures(
+      initConfiguration.enableExperimentalFeatures.filter((flag): flag is ExperimentalFeature =>
+        objectHasValue(ExperimentalFeature, flag)
+      )
+    )
+  }
+
+  return assign(
+    {
+      beforeSend:
+        initConfiguration.beforeSend && catchUserErrors(initConfiguration.beforeSend, 'beforeSend threw an error:'),
+      sessionStoreStrategyType: selectSessionStoreStrategyType(initConfiguration),
+      sessionSampleRate: sessionSampleRate ?? 100,
+      telemetrySampleRate: initConfiguration.telemetrySampleRate ?? 20,
+      telemetryConfigurationSampleRate: initConfiguration.telemetryConfigurationSampleRate ?? 5,
+      service: initConfiguration.service,
+      silentMultipleInit: !!initConfiguration.silentMultipleInit,
+
+      /**
+       * beacon payload max queue size implementation is 64kb
+       * ensure that we leave room for logs, rum and potential other users
+       */
+      batchBytesLimit: 16 * ONE_KIBI_BYTE,
+
+      eventRateLimiterThreshold: 3000,
+      maxTelemetryEventsPerPage: 15,
+
+      /**
+       * flush automatically, aim to be lower than ALB connection timeout
+       * to maximize connection reuse.
+       */
+      flushTimeout: (30 * ONE_SECOND) as Duration,
+
+      /**
+       * Logs intake limit
+       */
+      batchMessagesLimit: 50,
+      messageBytesLimit: 256 * ONE_KIBI_BYTE,
+
+      apiVersion: initConfiguration.apiVersion ?? 'v1',
+      organizationIdentifier: initConfiguration.organizationIdentifier,
+      insecureHTTP: initConfiguration.insecureHTTP,
+    },
+    computeTransportConfiguration(initConfiguration)
+  )
+}
+
+export function serializeConfiguration(initConfiguration: InitConfiguration): Partial<RawTelemetryConfiguration> {
+  const proxy = initConfiguration.proxy ?? initConfiguration.proxyUrl
+  return {
+    session_sample_rate: initConfiguration.sessionSampleRate ?? initConfiguration.sampleRate,
+    telemetry_sample_rate: initConfiguration.telemetrySampleRate,
+    telemetry_configuration_sample_rate: initConfiguration.telemetryConfigurationSampleRate,
+    use_before_send: !!initConfiguration.beforeSend,
+    use_cross_site_session_cookie: initConfiguration.useCrossSiteSessionCookie,
+    use_secure_session_cookie: initConfiguration.useSecureSessionCookie,
+    use_proxy: proxy !== undefined ? !!proxy : undefined,
+    silent_multiple_init: initConfiguration.silentMultipleInit,
+    track_session_across_subdomains: initConfiguration.trackSessionAcrossSubdomains,
+    track_resources: initConfiguration.trackResources,
+    track_long_task: initConfiguration.trackLongTasks,
+    allow_fallback_to_local_storage: !!initConfiguration.allowFallbackToLocalStorage,
+  }
+}

package/src/domain/configuration/endpointBuilder.ts

@@ -0,0 +1,134 @@
+import type { RetryInfo, FlushReason } from '../../transport'
+import { timeStampNow } from '../../tools/utils/timeUtils'
+import { normalizeUrl } from '../../tools/utils/urlPolyfill'
+import { ExperimentalFeature, isExperimentalFeatureEnabled } from '../../tools/experimentalFeatures'
+import { generateUUID } from '../../tools/utils/stringUtils'
+import type { InitConfiguration } from './configuration'
+import { INTAKE_SITE_AP1, INTAKE_SITE_US1 } from './intakeSites'
+
+// replaced at build time
+declare const __BUILD_ENV__SDK_VERSION__: string
+
+export const ENDPOINTS = {
+  logs: 'logs',
+  rum: 'rum',
+  sessionReplay: 'session-replay',
+} as const
+
+const INTAKE_TRACKS = {
+  logs: 'logs',
+  rum: 'rum',
+  sessionReplay: 'replay',
+}
+
+export type EndpointType = keyof typeof ENDPOINTS
+
+export type EndpointBuilder = ReturnType<typeof createEndpointBuilder>
+
+export function createEndpointBuilder(
+  initConfiguration: InitConfiguration,
+  endpointType: EndpointType,
+  configurationTags: string[]
+) {
+  const buildUrlWithParameters = createEndpointUrlWithParametersBuilder(initConfiguration, endpointType)
+
+  return {
+    build(api: 'xhr' | 'fetch' | 'beacon', flushReason?: FlushReason, retry?: RetryInfo) {
+      const parameters = buildEndpointParameters(
+        initConfiguration,
+        endpointType,
+        configurationTags,
+        api,
+        flushReason,
+        retry
+      )
+      return buildUrlWithParameters(parameters)
+    },
+    urlPrefix: buildUrlWithParameters(''),
+    endpointType,
+  }
+}
+
+/**
+ * Create a function used to build a full endpoint url from provided parameters. The goal of this
+ * function is to pre-compute some parts of the URL to avoid re-computing everything on every
+ * request, as only parameters are changing.
+ */
+function createEndpointUrlWithParametersBuilder(
+  initConfiguration: InitConfiguration,
+  endpointType: EndpointType
+): (parameters: string) => string {
+  const { proxy, proxyUrl, apiVersion, organizationIdentifier, insecureHTTP } = initConfiguration
+
+  const path = `/rum/${apiVersion}/${organizationIdentifier}/${INTAKE_TRACKS[endpointType]}`
+
+  if (proxy) {
+    const normalizedProxyUrl = normalizeUrl(proxy)
+    return (parameters) => `${normalizedProxyUrl}?ooforward=${encodeURIComponent(`${path}?${parameters}`)}`
+  }
+
+  const host = buildEndpointHost(initConfiguration, endpointType)
+
+  if (proxy === undefined && proxyUrl) {
+    // TODO: remove this in a future major.
+    const normalizedProxyUrl = normalizeUrl(proxyUrl)
+    return (parameters) =>
+      `${normalizedProxyUrl}?ooforward=${encodeURIComponent(`https://${host}${path}?${parameters}`)}`
+  }
+
+  const protocol = insecureHTTP ? 'http' : 'https'
+
+  return (parameters) => `${protocol}://${host}${path}?${parameters}`
+}
+
+function buildEndpointHost(initConfiguration: InitConfiguration, endpointType: EndpointType) {
+  const { site = INTAKE_SITE_US1, internalAnalyticsSubdomain } = initConfiguration
+
+  return site;
+  if (internalAnalyticsSubdomain && site === INTAKE_SITE_US1) {
+    return `${internalAnalyticsSubdomain}.${INTAKE_SITE_US1}`
+  }
+
+  const domainParts = site.split('.')
+  const extension = domainParts.pop()
+  const subdomain = site !== INTAKE_SITE_AP1 ? `${ENDPOINTS[endpointType]}.` : ''
+  return `${subdomain}browser-intake-${domainParts.join('-')}.${extension!}`
+}
+
+/**
+ * Build parameters to be used for an intake request. Parameters should be re-built for each
+ * request, as they change randomly.
+ */
+function buildEndpointParameters(
+  { clientToken, internalAnalyticsSubdomain }: InitConfiguration,
+  endpointType: EndpointType,
+  configurationTags: string[],
+  api: 'xhr' | 'fetch' | 'beacon',
+  flushReason: FlushReason | undefined,
+  retry: RetryInfo | undefined
+) {
+  const tags = [`sdk_version:${__BUILD_ENV__SDK_VERSION__}`, `api:${api}`].concat(configurationTags)
+  if (flushReason && isExperimentalFeatureEnabled(ExperimentalFeature.COLLECT_FLUSH_REASON)) {
+    tags.push(`flush_reason:${flushReason}`)
+  }
+  if (retry) {
+    tags.push(`retry_count:${retry.count}`, `retry_after:${retry.lastFailureStatus}`)
+  }
+  const parameters = [
+    'oosource=browser',
+    `ootags=${encodeURIComponent(tags.join(','))}`,
+    `oo-api-key=${clientToken}`,
+    `oo-evp-origin-version=${encodeURIComponent(__BUILD_ENV__SDK_VERSION__)}`,
+    'oo-evp-origin=browser',
+    `oo-request-id=${generateUUID()}`,
+  ]
+
+  if (endpointType === 'rum') {
+    parameters.push(`batch_time=${timeStampNow()}`)
+  }
+  if (internalAnalyticsSubdomain) {
+    parameters.reverse()
+  }
+
+  return parameters.join('&')
+}

package/src/domain/configuration/index.ts

@@ -0,0 +1,9 @@
+export {
+  Configuration,
+  InitConfiguration,
+  DefaultPrivacyLevel,
+  validateAndBuildConfiguration,
+  serializeConfiguration,
+} from './configuration'
+export { createEndpointBuilder, EndpointBuilder, EndpointType } from './endpointBuilder'
+export * from './intakeSites'

package/src/domain/configuration/intakeSites.ts

@@ -0,0 +1,5 @@
+export const INTAKE_SITE_STAGING = 'alpha1-api.dev.zinclabs.dev'
+export const INTAKE_SITE_US1 = 'api.openobserve.ai'
+export const INTAKE_SITE_EU1 = 'api.openobserve.ai'
+export const INTAKE_SITE_AP1 = 'api.openobserve.ai'
+export const INTAKE_SITE_US1_FED = 'api.openobserve.ai'

package/src/domain/configuration/tags.ts

@@ -0,0 +1,43 @@
+import { display } from '../../tools/display'
+import type { InitConfiguration } from './configuration'
+
+export const TAG_SIZE_LIMIT = 200
+
+export function buildTags(configuration: InitConfiguration): string[] {
+  const { env, service, version, datacenter } = configuration
+  const tags = []
+
+  if (env) {
+    tags.push(buildTag('env', env))
+  }
+  if (service) {
+    tags.push(buildTag('service', service))
+  }
+  if (version) {
+    tags.push(buildTag('version', version))
+  }
+  if (datacenter) {
+    tags.push(buildTag('datacenter', datacenter))
+  }
+
+  return tags
+}
+
+const FORBIDDEN_CHARACTERS = /[^a-z0-9_:./-]/
+
+export function buildTag(key: string, rawValue: string) {
+  // See https://docs.datadoghq.com/getting_started/tagging/#defining-tags for tags syntax. Note
+  // that the backend may not follow the exact same rules, so we only want to display an informal
+  // warning.
+  const valueSizeLimit = TAG_SIZE_LIMIT - key.length - 1
+
+  if (rawValue.length > valueSizeLimit || FORBIDDEN_CHARACTERS.test(rawValue)) {
+    display.warn(`${key} value doesn't meet tag requirements and will be sanitized`)
+  }
+
+  // Let the backend do most of the sanitization, but still make sure multiple tags can't be crafted
+  // by forging a value containing commas.
+  const sanitizedValue = rawValue.replace(/,/g, '_')
+
+  return `${key}:${sanitizedValue}`
+}

package/src/domain/configuration/transportConfiguration.ts

@@ -0,0 +1,71 @@
+import { objectValues, assign } from '../../tools/utils/polyfills'
+import type { InitConfiguration } from './configuration'
+import type { EndpointBuilder } from './endpointBuilder'
+import { createEndpointBuilder } from './endpointBuilder'
+import { buildTags } from './tags'
+import { INTAKE_SITE_US1 } from './intakeSites'
+
+export interface TransportConfiguration {
+  logsEndpointBuilder: EndpointBuilder
+  rumEndpointBuilder: EndpointBuilder
+  sessionReplayEndpointBuilder: EndpointBuilder
+  isIntakeUrl: (url: string) => boolean
+  replica?: ReplicaConfiguration
+  site: string
+}
+
+export interface ReplicaConfiguration {
+  applicationId?: string
+  logsEndpointBuilder: EndpointBuilder
+  rumEndpointBuilder: EndpointBuilder
+}
+
+export function computeTransportConfiguration(initConfiguration: InitConfiguration): TransportConfiguration {
+  const tags = buildTags(initConfiguration)
+
+  const endpointBuilders = computeEndpointBuilders(initConfiguration, tags)
+  const intakeUrlPrefixes = objectValues(endpointBuilders).map((builder) => builder.urlPrefix)
+
+  const replicaConfiguration = computeReplicaConfiguration(initConfiguration, intakeUrlPrefixes, tags)
+
+  return assign(
+    {
+      isIntakeUrl: (url: string) => intakeUrlPrefixes.some((intakeEndpoint) => url.indexOf(intakeEndpoint) === 0),
+      replica: replicaConfiguration,
+      site: initConfiguration.site || INTAKE_SITE_US1,
+    },
+    endpointBuilders
+  )
+}
+
+function computeEndpointBuilders(initConfiguration: InitConfiguration, tags: string[]) {
+  return {
+    logsEndpointBuilder: createEndpointBuilder(initConfiguration, 'logs', tags),
+    rumEndpointBuilder: createEndpointBuilder(initConfiguration, 'rum', tags),
+    sessionReplayEndpointBuilder: createEndpointBuilder(initConfiguration, 'sessionReplay', tags),
+  }
+}
+
+function computeReplicaConfiguration(
+  initConfiguration: InitConfiguration,
+  intakeUrlPrefixes: string[],
+  tags: string[]
+): ReplicaConfiguration | undefined {
+  if (!initConfiguration.replica) {
+    return
+  }
+
+  const replicaConfiguration: InitConfiguration = assign({}, initConfiguration, {
+    site: INTAKE_SITE_US1,
+    clientToken: initConfiguration.replica.clientToken,
+  })
+
+  const replicaEndpointBuilders = {
+    logsEndpointBuilder: createEndpointBuilder(replicaConfiguration, 'logs', tags),
+    rumEndpointBuilder: createEndpointBuilder(replicaConfiguration, 'rum', tags),
+  }
+
+  intakeUrlPrefixes.push(...objectValues(replicaEndpointBuilders).map((builder) => builder.urlPrefix))
+
+  return assign({ applicationId: initConfiguration.replica.applicationId }, replicaEndpointBuilders)
+}

package/src/domain/console/consoleObservable.ts

@@ -0,0 +1,86 @@
+import { computeStackTrace } from '../tracekit'
+import { createHandlingStack, formatErrorMessage, toStackTraceString, tryToGetFingerprint } from '../error/error'
+import { mergeObservables, Observable } from '../../tools/observable'
+import { ConsoleApiName, globalConsole } from '../../tools/display'
+import { callMonitored } from '../../tools/monitor'
+import { sanitize } from '../../tools/serialisation/sanitize'
+import { find } from '../../tools/utils/polyfills'
+import { jsonStringify } from '../../tools/serialisation/jsonStringify'
+
+export interface ConsoleLog {
+  message: string
+  api: ConsoleApiName
+  stack?: string
+  handlingStack?: string
+  fingerprint?: string
+}
+
+let consoleObservablesByApi: { [k in ConsoleApiName]?: Observable<ConsoleLog> } = {}
+
+export function initConsoleObservable(apis: ConsoleApiName[]) {
+  const consoleObservables = apis.map((api) => {
+    if (!consoleObservablesByApi[api]) {
+      consoleObservablesByApi[api] = createConsoleObservable(api)
+    }
+    return consoleObservablesByApi[api]!
+  })
+
+  return mergeObservables<ConsoleLog>(...consoleObservables)
+}
+
+export function resetConsoleObservable() {
+  consoleObservablesByApi = {}
+}
+
+function createConsoleObservable(api: ConsoleApiName) {
+  const observable = new Observable<ConsoleLog>(() => {
+    const originalConsoleApi = globalConsole[api]
+
+    globalConsole[api] = (...params: unknown[]) => {
+      originalConsoleApi.apply(console, params)
+      const handlingStack = createHandlingStack()
+
+      callMonitored(() => {
+        observable.notify(buildConsoleLog(params, api, handlingStack))
+      })
+    }
+
+    return () => {
+      globalConsole[api] = originalConsoleApi
+    }
+  })
+
+  return observable
+}
+
+function buildConsoleLog(params: unknown[], api: ConsoleApiName, handlingStack: string): ConsoleLog {
+  // Todo: remove console error prefix in the next major version
+  let message = params.map((param) => formatConsoleParameters(param)).join(' ')
+  let stack
+  let fingerprint
+
+  if (api === ConsoleApiName.error) {
+    const firstErrorParam = find(params, (param: unknown): param is Error => param instanceof Error)
+    stack = firstErrorParam ? toStackTraceString(computeStackTrace(firstErrorParam)) : undefined
+    fingerprint = tryToGetFingerprint(firstErrorParam)
+    message = `console error: ${message}`
+  }
+
+  return {
+    api,
+    message,
+    stack,
+    handlingStack,
+    fingerprint,
+  }
+}
+
+function formatConsoleParameters(param: unknown) {
+  if (typeof param === 'string') {
+    return sanitize(param)
+  }
+  if (param instanceof Error) {
+    return formatErrorMessage(computeStackTrace(param))
+  }
+  return jsonStringify(sanitize(param), undefined, 2)
+}

package/src/domain/deflate/index.ts

@@ -0,0 +1 @@
+export * from './types'

package/src/domain/deflate/types.ts

@@ -0,0 +1,41 @@
+export type DeflateWorkerAction =
+  // Action to send when creating the worker to check if the communication is working correctly.
+  // The worker should respond with a 'initialized' response.
+  | {
+      action: 'init'
+    }
+  // Action to send when writing some unfinished data. The worker will respond with a 'wrote'
+  // response, with the same id and measurements of the wrote data bytes count.
+  | {
+      action: 'write'
+      id: number
+      streamId: number
+      data: string
+    }
+  // Action to send when all data has been written and the state of the stream needs to be reset.
+  | {
+      action: 'reset'
+      streamId: number
+    }
+
+export type DeflateWorkerResponse =
+  // Response to 'init' action
+  | {
+      type: 'initialized'
+      version: string
+    }
+  // Response to 'write' action
+  | {
+      type: 'wrote'
+      id: number
+      streamId: number
+      result: Uint8Array
+      trailer: Uint8Array
+      additionalBytesCount: number
+    }
+  // Could happen at any time when something goes wrong in the worker
+  | {
+      type: 'errored'
+      streamId?: number
+      error: Error | string
+    }