@customerio/cdp-analytics-node 0.0.1

package/src/lib/env.ts

@@ -0,0 +1,32 @@
+export type RuntimeEnv =
+  | 'node'
+  | 'browser'
+  | 'web-worker'
+  | 'cloudflare-worker'
+  | 'unknown'
+
+export const detectRuntime = (): RuntimeEnv => {
+  if (typeof process === 'object' && process && process.env) {
+    return 'node'
+  }
+
+  if (typeof window === 'object') {
+    return 'browser'
+  }
+
+  // @ts-ignore
+  if (typeof WebSocketPair !== 'undefined') {
+    return 'cloudflare-worker'
+  }
+
+  if (
+    // @ts-ignore
+    typeof WorkerGlobalScope !== 'undefined' &&
+    // @ts-ignore
+    typeof importScripts === 'function'
+  ) {
+    return 'web-worker'
+  }
+
+  return 'unknown'
+}

package/src/lib/extract-promise-parts.ts

@@ -0,0 +1,21 @@
+/**
+ * Returns a promise and its associated `resolve` and `reject` methods.
+ */
+export function extractPromiseParts<T = unknown>(): {
+  promise: Promise<T>
+  resolve: (value: T) => void
+  reject: (reason?: unknown) => void
+} {
+  let resolver: (value: T) => void
+  let rejecter: (reason?: unknown) => void
+  const promise = new Promise<T>((resolve, reject) => {
+    resolver = resolve
+    rejecter = reject
+  })
+
+  return {
+    promise,
+    resolve: resolver!,
+    reject: rejecter!,
+  }
+}

package/src/lib/fetch.ts

@@ -0,0 +1,3 @@
+import { default as _fetch } from 'node-fetch'
+
+export const fetch = globalThis.fetch || _fetch

package/src/lib/get-message-id.ts

@@ -0,0 +1,10 @@
+import { uuid } from './uuid'
+
+/**
+ * get a unique messageId with a very low chance of collisions
+ * using @lukeed/uuid/secure uses the node crypto module, which is the fastest
+ * @example "node-next-1668208232027-743be593-7789-4b74-8078-cbcc8894c586"
+ */
+export const createMessageId = (): string => {
+  return `node-next-${Date.now()}-${uuid()}`
+}

package/src/lib/uuid.ts

@@ -0,0 +1 @@
+export { v4 as uuid } from 'uuid'

package/src/plugins/customerio/context-batch.ts

@@ -0,0 +1,71 @@
+import { v4 as uuid } from '@lukeed/uuid'
+import type { Context } from '../../app/context'
+import { CustomerioEvent } from '../../app/types'
+
+const MAX_EVENT_SIZE_IN_KB = 32
+const MAX_BATCH_SIZE_IN_KB = 480 //  (500 KB is the limit, leaving some padding)
+
+interface PendingItem {
+  resolver: (ctx: Context) => void
+  context: Context
+}
+
+export class ContextBatch {
+  public id = uuid()
+  private items: PendingItem[] = []
+  private sizeInBytes = 0
+  private maxEventCount: number
+
+  constructor(maxEventCount: number) {
+    this.maxEventCount = Math.max(1, maxEventCount)
+  }
+  public tryAdd(
+    item: PendingItem
+  ): { success: true } | { success: false; message: string } {
+    if (this.length === this.maxEventCount)
+      return {
+        success: false,
+        message: `Event limit of ${this.maxEventCount} has been exceeded.`,
+      }
+
+    const eventSize = this.calculateSize(item.context)
+    if (eventSize > MAX_EVENT_SIZE_IN_KB * 1024) {
+      return {
+        success: false,
+        message: `Event exceeds maximum event size of ${MAX_EVENT_SIZE_IN_KB} KB`,
+      }
+    }
+
+    if (this.sizeInBytes + eventSize > MAX_BATCH_SIZE_IN_KB * 1024) {
+      return {
+        success: false,
+        message: `Event has caused batch size to exceed ${MAX_BATCH_SIZE_IN_KB} KB`,
+      }
+    }
+
+    this.items.push(item)
+    this.sizeInBytes += eventSize
+    return { success: true }
+  }
+
+  get length(): number {
+    return this.items.length
+  }
+
+  private calculateSize(ctx: Context): number {
+    return encodeURI(JSON.stringify(ctx.event)).split(/%..|i/).length
+  }
+
+  getEvents(): CustomerioEvent[] {
+    const events = this.items.map(({ context }) => context.event)
+    return events
+  }
+
+  getContexts(): Context[] {
+    return this.items.map((item) => item.context)
+  }
+
+  resolveEvents(): void {
+    this.items.forEach(({ resolver, context }) => resolver(context))
+  }
+}

package/src/plugins/customerio/index.ts

@@ -0,0 +1,65 @@
+import { Publisher, PublisherProps } from './publisher'
+import { version } from '../../generated/version'
+import { detectRuntime } from '../../lib/env'
+import { Plugin } from '../../app/types'
+import { Context } from '../../app/context'
+import { NodeEmitter } from '../../app/emitter'
+
+function normalizeEvent(ctx: Context) {
+  ctx.updateEvent('context.library.name', '@customerio/cdp-analytics-node')
+  ctx.updateEvent('context.library.version', version)
+  const runtime = detectRuntime()
+  if (runtime === 'node') {
+    ctx.updateEvent('_metadata.nodeVersion', process.versions.node)
+  }
+  ctx.updateEvent('_metadata.jsRuntime', runtime)
+}
+
+type DefinedPluginFields =
+  | 'name'
+  | 'type'
+  | 'version'
+  | 'isLoaded'
+  | 'load'
+  | 'alias'
+  | 'group'
+  | 'identify'
+  | 'page'
+  | 'screen'
+  | 'track'
+
+type CustomerioNodePlugin = Plugin & Required<Pick<Plugin, DefinedPluginFields>>
+
+export type ConfigureNodePluginProps = PublisherProps
+
+export function createNodePlugin(publisher: Publisher): CustomerioNodePlugin {
+  function action(ctx: Context): Promise<Context> {
+    normalizeEvent(ctx)
+    return publisher.enqueue(ctx)
+  }
+
+  return {
+    name: 'Customer.io Data Pipelines',
+    type: 'after',
+    version: '1.0.0',
+    isLoaded: () => true,
+    load: () => Promise.resolve(),
+    alias: action,
+    group: action,
+    identify: action,
+    page: action,
+    screen: action,
+    track: action,
+  }
+}
+
+export const createConfiguredNodePlugin = (
+  props: ConfigureNodePluginProps,
+  emitter: NodeEmitter
+) => {
+  const publisher = new Publisher(props, emitter)
+  return {
+    publisher: publisher,
+    plugin: createNodePlugin(publisher),
+  }
+}

package/src/plugins/customerio/publisher.ts

@@ -0,0 +1,258 @@
+import { backoff } from '@customerio/cdp-analytics-core'
+import { abortSignalAfterTimeout } from '../../lib/abort'
+import type { Context } from '../../app/context'
+import { tryCreateFormattedUrl } from '../../lib/create-url'
+import { extractPromiseParts } from '../../lib/extract-promise-parts'
+import { fetch } from '../../lib/fetch'
+import { ContextBatch } from './context-batch'
+import { NodeEmitter } from '../../app/emitter'
+import { b64encode } from '../../lib/base-64-encode'
+
+function sleep(timeoutInMs: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, timeoutInMs))
+}
+
+function noop() { }
+
+interface PendingItem {
+  resolver: (ctx: Context) => void
+  context: Context
+}
+
+export interface PublisherProps {
+  host?: string
+  path?: string
+  flushInterval: number
+  maxEventsInBatch: number
+  maxRetries: number
+  writeKey: string
+  httpRequestTimeout?: number
+}
+
+/**
+ * The Publisher is responsible for batching events and sending them to the API.
+ */
+export class Publisher {
+  private pendingFlushTimeout?: ReturnType<typeof setTimeout>
+  private _batch?: ContextBatch
+
+  private _flushInterval: number
+  private _maxEventsInBatch: number
+  private _maxRetries: number
+  private _auth: string
+  private _url: string
+  private _closeAndFlushPendingItemsCount?: number
+  private _httpRequestTimeout: number
+  private _emitter: NodeEmitter
+  constructor(
+    {
+      host,
+      path,
+      maxRetries,
+      maxEventsInBatch,
+      flushInterval,
+      writeKey,
+      httpRequestTimeout,
+    }: PublisherProps,
+    emitter: NodeEmitter
+  ) {
+    this._emitter = emitter
+    this._maxRetries = maxRetries
+    this._maxEventsInBatch = Math.max(maxEventsInBatch, 1)
+    this._flushInterval = flushInterval
+    this._auth = b64encode(`${writeKey}:`)
+    this._url = tryCreateFormattedUrl(
+      host ?? 'https://cdp.customer.io',
+      path ?? '/v1/batch'
+    )
+    this._httpRequestTimeout = httpRequestTimeout ?? 10000
+  }
+
+  private createBatch(): ContextBatch {
+    this.pendingFlushTimeout && clearTimeout(this.pendingFlushTimeout)
+    const batch = new ContextBatch(this._maxEventsInBatch)
+    this._batch = batch
+    this.pendingFlushTimeout = setTimeout(() => {
+      if (batch === this._batch) {
+        this._batch = undefined
+      }
+      this.pendingFlushTimeout = undefined
+      if (batch.length) {
+        this.send(batch).catch(noop)
+      }
+    }, this._flushInterval)
+    return batch
+  }
+
+  private clearBatch() {
+    this.pendingFlushTimeout && clearTimeout(this.pendingFlushTimeout)
+    this._batch = undefined
+  }
+
+  flushAfterClose(pendingItemsCount: number) {
+    if (!pendingItemsCount) {
+      // if number of pending items is 0, there will never be anything else entering the batch, since the app is closed.
+      return
+    }
+
+    this._closeAndFlushPendingItemsCount = pendingItemsCount
+
+    // if batch is empty, there's nothing to flush, and when things come in, enqueue will handle them.
+    if (!this._batch) return
+
+    // the number of globally pending items will always be larger or the same as batch size.
+    // Any mismatch is because some globally pending items are in plugins.
+    const isExpectingNoMoreItems = this._batch.length === pendingItemsCount
+    if (isExpectingNoMoreItems) {
+      this.send(this._batch).catch(noop)
+      this.clearBatch()
+    }
+  }
+
+  /**
+   * Enqueues the context for future delivery.
+   * @param ctx - Context containing an event.
+   * @returns a promise that resolves with the context after the event has been delivered.
+   */
+  enqueue(ctx: Context): Promise<Context> {
+    const batch = this._batch ?? this.createBatch()
+
+    const { promise: ctxPromise, resolve } = extractPromiseParts<Context>()
+
+    const pendingItem: PendingItem = {
+      context: ctx,
+      resolver: resolve,
+    }
+
+    /*
+      The following logic ensures that a batch is never orphaned,
+      and is always sent before a new batch is created.
+
+      Add an event to the existing batch.
+        Success: Check if batch is full or no more items are expected to come in (i.e. closing). If so, send batch.
+        Failure: Assume event is too big to fit in current batch - send existing batch.
+          Add an event to the new batch.
+            Success: Check if batch is full and send if it is.
+            Failure: Event exceeds maximum size (it will never fit), fail the event.
+    */
+    const addStatus = batch.tryAdd(pendingItem)
+    if (addStatus.success) {
+      const isExpectingNoMoreItems =
+        batch.length === this._closeAndFlushPendingItemsCount
+      const isFull = batch.length === this._maxEventsInBatch
+      if (isFull || isExpectingNoMoreItems) {
+        this.send(batch).catch(noop)
+        this.clearBatch()
+      }
+      return ctxPromise
+    }
+
+    // If the new item causes the maximimum event size to be exceeded, send the current batch and create a new one.
+    if (batch.length) {
+      this.send(batch).catch(noop)
+      this.clearBatch()
+    }
+
+    const fallbackBatch = this.createBatch()
+
+    const fbAddStatus = fallbackBatch.tryAdd(pendingItem)
+
+    if (fbAddStatus.success) {
+      const isExpectingNoMoreItems =
+        fallbackBatch.length === this._closeAndFlushPendingItemsCount
+      if (isExpectingNoMoreItems) {
+        this.send(fallbackBatch).catch(noop)
+        this.clearBatch()
+      }
+      return ctxPromise
+    } else {
+      // this should only occur if max event size is exceeded
+      ctx.setFailedDelivery({
+        reason: new Error(fbAddStatus.message),
+      })
+      return Promise.resolve(ctx)
+    }
+  }
+
+  private async send(batch: ContextBatch) {
+    if (this._closeAndFlushPendingItemsCount) {
+      this._closeAndFlushPendingItemsCount -= batch.length
+    }
+    const events = batch.getEvents()
+    const payload = JSON.stringify({ batch: events })
+    const maxAttempts = this._maxRetries + 1
+
+    let currentAttempt = 0
+    while (currentAttempt < maxAttempts) {
+      currentAttempt++
+
+      let failureReason: unknown
+      const [signal, timeoutId] = abortSignalAfterTimeout(
+        this._httpRequestTimeout
+      )
+      try {
+        const requestInit = {
+          signal: signal,
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Basic ${this._auth}`,
+            'User-Agent': 'cdp-analytics-node/latest',
+          },
+          body: payload,
+        }
+
+        this._emitter.emit('http_request', {
+          url: this._url,
+          method: requestInit.method,
+          headers: requestInit.headers,
+          body: requestInit.body,
+        })
+        const response = await fetch(this._url, requestInit)
+
+        clearTimeout(timeoutId)
+
+        if (response.ok) {
+          // Successfully sent events, so exit!
+          batch.resolveEvents()
+          return
+        } else if (response.status === 400) {
+          // Request either malformed or size exceeded - don't retry.
+          resolveFailedBatch(
+            batch,
+            new Error(`[${response.status}] ${response.statusText}`)
+          )
+          return
+        } else {
+          // Treat other errors as transient and retry.
+          failureReason = new Error(
+            `[${response.status}] ${response.statusText}`
+          )
+        }
+      } catch (err) {
+        // Network errors get thrown, retry them.
+        failureReason = err
+      }
+
+      // Final attempt failed, update context and resolve events.
+      if (currentAttempt === maxAttempts) {
+        resolveFailedBatch(batch, failureReason)
+        return
+      }
+
+      // Retry after attempt-based backoff.
+      await sleep(
+        backoff({
+          attempt: currentAttempt,
+          minTimeout: 25,
+          maxTimeout: 1000,
+        })
+      )
+    }
+  }
+}
+
+function resolveFailedBatch(batch: ContextBatch, reason: unknown) {
+  batch.getContexts().forEach((ctx) => ctx.setFailedDelivery({ reason }))
+  batch.resolveEvents()
+}