@openobserve/browser-core 0.2.8

package/src/transport/httpRequest.ts

@@ -0,0 +1,150 @@
+import type { EndpointBuilder, Configuration } from '../domain/configuration'
+import { addTelemetryError } from '../domain/telemetry'
+import type { Context } from '../tools/serialisation/context'
+import { monitor } from '../tools/monitor'
+import { addEventListener } from '../browser/addEventListener'
+import type { RawError } from '../domain/error/error.types'
+import { newRetryState, sendWithRetryStrategy } from './sendWithRetryStrategy'
+import type { FlushReason } from './flushController'
+
+/**
+ * Use POST request without content type to:
+ * - avoid CORS preflight requests
+ * - allow usage of sendBeacon
+ *
+ * multiple elements are sent separated by \n in order
+ * to be parsed correctly without content type header
+ */
+
+export type HttpRequest = ReturnType<typeof createHttpRequest>
+
+export interface HttpResponse extends Context {
+  status: number
+  type?: ResponseType
+}
+
+export interface Payload {
+  data: string | FormData
+  bytesCount: number
+  retry?: RetryInfo
+  flushReason?: FlushReason
+}
+
+export interface RetryInfo {
+  count: number
+  lastFailureStatus: number
+}
+
+export function createHttpRequest(
+  configuration: Configuration,
+  endpointBuilder: EndpointBuilder,
+  bytesLimit: number,
+  reportError: (error: RawError) => void
+) {
+  const retryState = newRetryState()
+  const sendStrategyForRetry = (payload: Payload, onResponse: (r: HttpResponse) => void) =>
+    fetchKeepAliveStrategy(configuration, endpointBuilder, bytesLimit, payload, onResponse)
+
+  return {
+    send: (payload: Payload) => {
+      sendWithRetryStrategy(payload, retryState, sendStrategyForRetry, endpointBuilder.endpointType, reportError)
+    },
+    /**
+     * Since fetch keepalive behaves like regular fetch on Firefox,
+     * keep using sendBeaconStrategy on exit
+     */
+    sendOnExit: (payload: Payload) => {
+      sendBeaconStrategy(configuration, endpointBuilder, bytesLimit, payload)
+    },
+  }
+}
+
+function sendBeaconStrategy(
+  configuration: Configuration,
+  endpointBuilder: EndpointBuilder,
+  bytesLimit: number,
+  { data, bytesCount, flushReason }: Payload
+) {
+  const canUseBeacon = !!navigator.sendBeacon && bytesCount < bytesLimit
+  if (canUseBeacon) {
+    try {
+      const beaconUrl = endpointBuilder.build('beacon', flushReason)
+      const isQueued = navigator.sendBeacon(beaconUrl, data)
+
+      if (isQueued) {
+        return
+      }
+    } catch (e) {
+      reportBeaconError(e)
+    }
+  }
+
+  const xhrUrl = endpointBuilder.build('xhr', flushReason)
+  sendXHR(configuration, xhrUrl, data)
+}
+
+let hasReportedBeaconError = false
+
+function reportBeaconError(e: unknown) {
+  if (!hasReportedBeaconError) {
+    hasReportedBeaconError = true
+    addTelemetryError(e)
+  }
+}
+
+export function fetchKeepAliveStrategy(
+  configuration: Configuration,
+  endpointBuilder: EndpointBuilder,
+  bytesLimit: number,
+  { data, bytesCount, flushReason, retry }: Payload,
+  onResponse?: (r: HttpResponse) => void
+) {
+  const canUseKeepAlive = isKeepAliveSupported() && bytesCount < bytesLimit
+  if (canUseKeepAlive) {
+    const fetchUrl = endpointBuilder.build('fetch', flushReason, retry)
+    fetch(fetchUrl, { method: 'POST', body: data, keepalive: true, mode: 'cors' }).then(
+      monitor((response: Response) => onResponse?.({ status: response.status, type: response.type })),
+      monitor(() => {
+        const xhrUrl = endpointBuilder.build('xhr', flushReason, retry)
+        // failed to queue the request
+        sendXHR(configuration, xhrUrl, data, onResponse)
+      })
+    )
+  } else {
+    const xhrUrl = endpointBuilder.build('xhr', flushReason, retry)
+    sendXHR(configuration, xhrUrl, data, onResponse)
+  }
+}
+
+function isKeepAliveSupported() {
+  // Request can throw, cf https://developer.mozilla.org/en-US/docs/Web/API/Request/Request#errors
+  try {
+    return window.Request && 'keepalive' in new Request('http://a')
+  } catch {
+    return false
+  }
+}
+
+export function sendXHR(
+  configuration: Configuration,
+  url: string,
+  data: Payload['data'],
+  onResponse?: (r: HttpResponse) => void
+) {
+  const request = new XMLHttpRequest()
+  request.open('POST', url, true)
+  addEventListener(
+    configuration,
+    request,
+    'loadend',
+    () => {
+      onResponse?.({ status: request.status })
+    },
+    {
+      // prevent multiple onResponse callbacks
+      // if the xhr instance is reused by a third party
+      once: true,
+    }
+  )
+  request.send(data)
+}

package/src/transport/index.ts

@@ -0,0 +1,4 @@
+export { HttpRequest, createHttpRequest, Payload, RetryInfo } from './httpRequest'
+export { canUseEventBridge, getEventBridge, BrowserWindowWithEventBridge } from './eventBridge'
+export { startBatchWithReplica } from './startBatchWithReplica'
+export { createFlushController, FlushController, FlushEvent, FlushReason } from './flushController'

package/src/transport/sendWithRetryStrategy.ts

@@ -0,0 +1,203 @@
+import type { EndpointType } from '../domain/configuration'
+import { setTimeout } from '../tools/timer'
+import { clocksNow, ONE_MINUTE, ONE_SECOND } from '../tools/utils/timeUtils'
+import { ONE_MEBI_BYTE, ONE_KIBI_BYTE } from '../tools/utils/byteUtils'
+import { isServerError } from '../tools/utils/responseUtils'
+import type { RawError } from '../domain/error/error.types'
+import { ErrorSource } from '../domain/error/error.types'
+import type { Payload, HttpResponse } from './httpRequest'
+
+export const MAX_ONGOING_BYTES_COUNT = 80 * ONE_KIBI_BYTE
+export const MAX_ONGOING_REQUESTS = 32
+export const MAX_QUEUE_BYTES_COUNT = 3 * ONE_MEBI_BYTE
+export const MAX_BACKOFF_TIME = ONE_MINUTE
+export const INITIAL_BACKOFF_TIME = ONE_SECOND
+
+const enum TransportStatus {
+  UP,
+  FAILURE_DETECTED,
+  DOWN,
+}
+
+const enum RetryReason {
+  AFTER_SUCCESS,
+  AFTER_RESUME,
+}
+
+export interface RetryState {
+  transportStatus: TransportStatus
+  currentBackoffTime: number
+  bandwidthMonitor: ReturnType<typeof newBandwidthMonitor>
+  queuedPayloads: ReturnType<typeof newPayloadQueue>
+  queueFullReported: boolean
+}
+
+type SendStrategy = (payload: Payload, onResponse: (r: HttpResponse) => void) => void
+
+export function sendWithRetryStrategy(
+  payload: Payload,
+  state: RetryState,
+  sendStrategy: SendStrategy,
+  endpointType: EndpointType,
+  reportError: (error: RawError) => void
+) {
+  if (
+    state.transportStatus === TransportStatus.UP &&
+    state.queuedPayloads.size() === 0 &&
+    state.bandwidthMonitor.canHandle(payload)
+  ) {
+    send(payload, state, sendStrategy, {
+      onSuccess: () => retryQueuedPayloads(RetryReason.AFTER_SUCCESS, state, sendStrategy, endpointType, reportError),
+      onFailure: () => {
+        state.queuedPayloads.enqueue(payload)
+        scheduleRetry(state, sendStrategy, endpointType, reportError)
+      },
+    })
+  } else {
+    state.queuedPayloads.enqueue(payload)
+  }
+}
+
+function scheduleRetry(
+  state: RetryState,
+  sendStrategy: SendStrategy,
+  endpointType: EndpointType,
+  reportError: (error: RawError) => void
+) {
+  if (state.transportStatus !== TransportStatus.DOWN) {
+    return
+  }
+  setTimeout(() => {
+    const payload = state.queuedPayloads.first()
+    send(payload, state, sendStrategy, {
+      onSuccess: () => {
+        state.queuedPayloads.dequeue()
+        state.currentBackoffTime = INITIAL_BACKOFF_TIME
+        retryQueuedPayloads(RetryReason.AFTER_RESUME, state, sendStrategy, endpointType, reportError)
+      },
+      onFailure: () => {
+        state.currentBackoffTime = Math.min(MAX_BACKOFF_TIME, state.currentBackoffTime * 2)
+        scheduleRetry(state, sendStrategy, endpointType, reportError)
+      },
+    })
+  }, state.currentBackoffTime)
+}
+
+function send(
+  payload: Payload,
+  state: RetryState,
+  sendStrategy: SendStrategy,
+  { onSuccess, onFailure }: { onSuccess: () => void; onFailure: () => void }
+) {
+  state.bandwidthMonitor.add(payload)
+  sendStrategy(payload, (response) => {
+    state.bandwidthMonitor.remove(payload)
+    if (!shouldRetryRequest(response)) {
+      state.transportStatus = TransportStatus.UP
+      onSuccess()
+    } else {
+      // do not consider transport down if another ongoing request could succeed
+      state.transportStatus =
+        state.bandwidthMonitor.ongoingRequestCount > 0 ? TransportStatus.FAILURE_DETECTED : TransportStatus.DOWN
+      payload.retry = {
+        count: payload.retry ? payload.retry.count + 1 : 1,
+        lastFailureStatus: response.status,
+      }
+      onFailure()
+    }
+  })
+}
+
+function retryQueuedPayloads(
+  reason: RetryReason,
+  state: RetryState,
+  sendStrategy: SendStrategy,
+  endpointType: EndpointType,
+  reportError: (error: RawError) => void
+) {
+  if (reason === RetryReason.AFTER_SUCCESS && state.queuedPayloads.isFull() && !state.queueFullReported) {
+    reportError({
+      message: `Reached max ${endpointType} events size queued for upload: ${MAX_QUEUE_BYTES_COUNT / ONE_MEBI_BYTE}MiB`,
+      source: ErrorSource.AGENT,
+      startClocks: clocksNow(),
+    })
+    state.queueFullReported = true
+  }
+  const previousQueue = state.queuedPayloads
+  state.queuedPayloads = newPayloadQueue()
+  while (previousQueue.size() > 0) {
+    sendWithRetryStrategy(previousQueue.dequeue()!, state, sendStrategy, endpointType, reportError)
+  }
+}
+
+function shouldRetryRequest(response: HttpResponse) {
+  return (
+    response.type !== 'opaque' &&
+    ((response.status === 0 && !navigator.onLine) ||
+      response.status === 408 ||
+      response.status === 429 ||
+      isServerError(response.status))
+  )
+}
+
+export function newRetryState(): RetryState {
+  return {
+    transportStatus: TransportStatus.UP,
+    currentBackoffTime: INITIAL_BACKOFF_TIME,
+    bandwidthMonitor: newBandwidthMonitor(),
+    queuedPayloads: newPayloadQueue(),
+    queueFullReported: false,
+  }
+}
+
+function newPayloadQueue() {
+  const queue: Payload[] = []
+  return {
+    bytesCount: 0,
+    enqueue(payload: Payload) {
+      if (this.isFull()) {
+        return
+      }
+      queue.push(payload)
+      this.bytesCount += payload.bytesCount
+    },
+    first() {
+      return queue[0]
+    },
+    dequeue() {
+      const payload = queue.shift()
+      if (payload) {
+        this.bytesCount -= payload.bytesCount
+      }
+      return payload
+    },
+    size() {
+      return queue.length
+    },
+    isFull() {
+      return this.bytesCount >= MAX_QUEUE_BYTES_COUNT
+    },
+  }
+}
+
+function newBandwidthMonitor() {
+  return {
+    ongoingRequestCount: 0,
+    ongoingByteCount: 0,
+    canHandle(payload: Payload) {
+      return (
+        this.ongoingRequestCount === 0 ||
+        (this.ongoingByteCount + payload.bytesCount <= MAX_ONGOING_BYTES_COUNT &&
+          this.ongoingRequestCount < MAX_ONGOING_REQUESTS)
+      )
+    },
+    add(payload: Payload) {
+      this.ongoingRequestCount += 1
+      this.ongoingByteCount += payload.bytesCount
+    },
+    remove(payload: Payload) {
+      this.ongoingRequestCount -= 1
+      this.ongoingByteCount -= payload.bytesCount
+    },
+  }
+}

package/src/transport/startBatchWithReplica.ts

@@ -0,0 +1,52 @@
+import type { Configuration, EndpointBuilder } from '../domain/configuration'
+import type { Context } from '../tools/serialisation/context'
+import type { Observable } from '../tools/observable'
+import type { PageExitEvent } from '../browser/pageExitObservable'
+import type { RawError } from '../domain/error/error.types'
+import { Batch } from './batch'
+import { createHttpRequest } from './httpRequest'
+import { createFlushController } from './flushController'
+
+export function startBatchWithReplica<T extends Context>(
+  configuration: Configuration,
+  primary: { endpoint: EndpointBuilder },
+  replica: { endpoint: EndpointBuilder; transformMessage?: (message: T) => T } | undefined,
+  reportError: (error: RawError) => void,
+  pageExitObservable: Observable<PageExitEvent>,
+  sessionExpireObservable: Observable<void>
+) {
+  const primaryBatch = createBatch(configuration, primary.endpoint)
+  const replicaBatch = replica && createBatch(configuration, replica.endpoint)
+
+  function createBatch(configuration: Configuration, endpointBuilder: EndpointBuilder) {
+    return new Batch(
+      createHttpRequest(configuration, endpointBuilder, configuration.batchBytesLimit, reportError),
+      createFlushController({
+        messagesLimit: configuration.batchMessagesLimit,
+        bytesLimit: configuration.batchBytesLimit,
+        durationLimit: configuration.flushTimeout,
+        pageExitObservable,
+        sessionExpireObservable,
+      }),
+      configuration.messageBytesLimit
+    )
+  }
+
+  return {
+    flushObservable: primaryBatch.flushController.flushObservable,
+
+    add(message: T, replicated = true) {
+      primaryBatch.add(message)
+      if (replicaBatch && replicated) {
+        replicaBatch.add(replica.transformMessage ? replica.transformMessage(message) : message)
+      }
+    },
+
+    upsert: (message: T, key: string) => {
+      primaryBatch.upsert(message, key)
+      if (replicaBatch) {
+        replicaBatch.upsert(replica.transformMessage ? replica.transformMessage(message) : message, key)
+      }
+    },
+  }
+}