@customerio/cdp-analytics-core 0.0.0-test-oidc

package/src/index.ts

@@ -0,0 +1,18 @@
+export * from './emitter'
+export * from './emitter/interface'
+export * from './plugins'
+export * from './events/interfaces'
+export * from './events'
+export * from './callback'
+export * from './priority-queue'
+export { backoff } from './priority-queue/backoff'
+export * from './context'
+export * from './queue/event-queue'
+export * from './analytics'
+export * from './analytics/dispatch'
+export * from './validation/helpers'
+export * from './validation/assertions'
+export * from './utils/bind-all'
+export * from './stats'
+export { CoreLogger } from './logger'
+export * from './queue/delivery'

package/src/logger/index.ts

@@ -0,0 +1,74 @@
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error'
+export type LogMessage = {
+  level: LogLevel
+  message: string
+  time?: Date
+  extras?: Record<string, any>
+}
+
+export interface GenericLogger {
+  log(level: LogLevel, message: string, extras?: object): void
+  flush(): void
+  logs: LogMessage[]
+}
+
+export class CoreLogger implements GenericLogger {
+  private _logs: LogMessage[] = []
+
+  log(level: LogLevel, message: string, extras?: object) {
+    const time = new Date()
+    this._logs.push({
+      level,
+      message,
+      time,
+      extras,
+    })
+  }
+
+  public get logs(): LogMessage[] {
+    return this._logs
+  }
+
+  public flush(): void {
+    if (this.logs.length > 1) {
+      const formatted = this._logs.reduce((logs, log) => {
+        const line = {
+          ...log,
+          json: JSON.stringify(log.extras, null, ' '),
+          extras: log.extras,
+        }
+
+        delete line['time']
+
+        let key = log.time?.toISOString() ?? ''
+        if (logs[key]) {
+          key = `${key}-${Math.random()}`
+        }
+
+        return {
+          ...logs,
+          [key]: line,
+        }
+      }, {} as Record<string, LogMessage>)
+
+      // ie doesn't like console.table
+      if (console.table) {
+        console.table(formatted)
+      } else {
+        console.log(formatted)
+      }
+    } else {
+      this.logs.forEach((logEntry) => {
+        const { level, message, extras } = logEntry
+
+        if (level === 'info' || level === 'debug') {
+          console.log(message, extras ?? '')
+        } else {
+          console[level](message, extras ?? '')
+        }
+      })
+    }
+
+    this._logs = []
+  }
+}

package/src/plugins/index.ts

@@ -0,0 +1,43 @@
+import type { CoreAnalytics } from '../analytics'
+import type { CoreContext } from '../context'
+
+interface CorePluginConfig {
+  options: any
+  priority: 'critical' | 'non-critical' // whether AJS should expect this plugin to be loaded before starting event delivery
+}
+
+export type PluginType =
+  | 'before'
+  | 'after'
+  | 'destination'
+  | 'enrichment'
+  | 'utility'
+
+// enrichment - modifies the event. Enrichment can happen in parallel, by reducing all changes in the final event. Failures in this stage could halt event delivery.
+// destination - runs in parallel at the end of the lifecycle. Cannot modify the event, can fail and not halt execution.
+// utility - do not affect lifecycle. Should be run and executed once. Their `track/identify` calls don't really do anything. example
+
+export interface CorePlugin<
+  Ctx extends CoreContext = CoreContext,
+  Analytics extends CoreAnalytics = any
+> {
+  name: string
+  alternativeNames?: string[]
+  version: string
+  type: PluginType
+  isLoaded: () => boolean
+  load: (
+    ctx: Ctx,
+    instance: Analytics,
+    config?: CorePluginConfig
+  ) => Promise<unknown>
+
+  unload?: (ctx: Ctx, instance: Analytics) => Promise<unknown> | unknown
+  ready?: () => Promise<unknown>
+  track?: (ctx: Ctx) => Promise<Ctx> | Ctx
+  identify?: (ctx: Ctx) => Promise<Ctx> | Ctx
+  page?: (ctx: Ctx) => Promise<Ctx> | Ctx
+  group?: (ctx: Ctx) => Promise<Ctx> | Ctx
+  alias?: (ctx: Ctx) => Promise<Ctx> | Ctx
+  screen?: (ctx: Ctx) => Promise<Ctx> | Ctx
+}

package/src/priority-queue/backoff.ts

@@ -0,0 +1,24 @@
+type BackoffParams = {
+  /** The number of milliseconds before starting the first retry. Default is 500 */
+  minTimeout?: number
+
+  /** The maximum number of milliseconds between two retries. Default is Infinity */
+  maxTimeout?: number
+
+  /** The exponential factor to use. Default is 2. */
+  factor?: number
+
+  /** The current attempt */
+  attempt: number
+}
+
+export function backoff(params: BackoffParams): number {
+  const random = Math.random() + 1
+  const {
+    minTimeout = 500,
+    factor = 2,
+    attempt,
+    maxTimeout = Infinity,
+  } = params
+  return Math.min(random * minTimeout * Math.pow(factor, attempt), maxTimeout)
+}

package/src/priority-queue/index.ts

@@ -0,0 +1,103 @@
+import { Emitter } from '../emitter'
+import { backoff } from './backoff'
+
+/**
+ * @internal
+ */
+export const ON_REMOVE_FROM_FUTURE = 'onRemoveFromFuture'
+
+interface QueueItem {
+  id: string
+}
+
+export class PriorityQueue<Item extends QueueItem = QueueItem> extends Emitter {
+  protected future: Item[] = []
+  protected queue: Item[]
+  protected seen: Record<string, number>
+
+  public maxAttempts: number
+
+  constructor(
+    maxAttempts: number,
+    queue: Item[],
+    seen?: Record<string, number>
+  ) {
+    super()
+    this.maxAttempts = maxAttempts
+    this.queue = queue
+    this.seen = seen ?? {}
+  }
+
+  push(...items: Item[]): boolean[] {
+    const accepted = items.map((operation) => {
+      const attempts = this.updateAttempts(operation)
+
+      if (attempts > this.maxAttempts || this.includes(operation)) {
+        return false
+      }
+
+      this.queue.push(operation)
+      return true
+    })
+
+    this.queue = this.queue.sort(
+      (a, b) => this.getAttempts(a) - this.getAttempts(b)
+    )
+    return accepted
+  }
+
+  pushWithBackoff(item: Item): boolean {
+    if (this.getAttempts(item) === 0) {
+      return this.push(item)[0]
+    }
+
+    const attempt = this.updateAttempts(item)
+
+    if (attempt > this.maxAttempts || this.includes(item)) {
+      return false
+    }
+
+    const timeout = backoff({ attempt: attempt - 1 })
+
+    setTimeout(() => {
+      this.queue.push(item)
+      // remove from future list
+      this.future = this.future.filter((f) => f.id !== item.id)
+      // Lets listeners know that a 'future' message is now available in the queue
+      this.emit(ON_REMOVE_FROM_FUTURE)
+    }, timeout)
+
+    this.future.push(item)
+    return true
+  }
+
+  public getAttempts(item: Item): number {
+    return this.seen[item.id] ?? 0
+  }
+
+  public updateAttempts(item: Item): number {
+    this.seen[item.id] = this.getAttempts(item) + 1
+    return this.getAttempts(item)
+  }
+
+  includes(item: Item): boolean {
+    return (
+      this.queue.includes(item) ||
+      this.future.includes(item) ||
+      Boolean(this.queue.find((i) => i.id === item.id)) ||
+      Boolean(this.future.find((i) => i.id === item.id))
+    )
+  }
+
+  pop(): Item | undefined {
+    return this.queue.shift()
+  }
+
+  public get length(): number {
+    return this.queue.length
+  }
+
+  public get todo(): number {
+    return this.queue.length + this.future.length
+  }
+}

package/src/queue/delivery.ts

@@ -0,0 +1,73 @@
+import { CoreContext, ContextCancelation } from '../context'
+import { CorePlugin } from '../plugins'
+
+async function tryAsync<T>(fn: () => T | Promise<T>): Promise<T> {
+  try {
+    return await fn()
+  } catch (err) {
+    return Promise.reject(err)
+  }
+}
+
+export function attempt<Ctx extends CoreContext = CoreContext>(
+  ctx: Ctx,
+  plugin: CorePlugin<Ctx>
+): Promise<Ctx | ContextCancelation | Error> {
+  ctx.log('debug', 'plugin', { plugin: plugin.name })
+  const start = new Date().getTime()
+
+  const hook = plugin[ctx.event.type]
+  if (hook === undefined) {
+    return Promise.resolve(ctx)
+  }
+
+  const newCtx = tryAsync(() => hook.apply(plugin, [ctx]))
+    .then((ctx) => {
+      const done = new Date().getTime() - start
+      ctx.stats.gauge('plugin_time', done, [`plugin:${plugin.name}`])
+
+      return ctx
+    })
+    .catch((err: Error | ContextCancelation) => {
+      if (
+        err instanceof ContextCancelation &&
+        err.type === 'middleware_cancellation'
+      ) {
+        throw err
+      }
+
+      if (err instanceof ContextCancelation) {
+        ctx.log('warn', err.type, {
+          plugin: plugin.name,
+          error: err,
+        })
+
+        return err
+      }
+
+      ctx.log('error', 'plugin Error', {
+        plugin: plugin.name,
+        error: err,
+      })
+      ctx.stats.increment('plugin_error', 1, [`plugin:${plugin.name}`])
+
+      return err
+    })
+
+  return newCtx
+}
+
+export function ensure<Ctx extends CoreContext = CoreContext>(
+  ctx: Ctx,
+  plugin: CorePlugin<Ctx>
+): Promise<Ctx | undefined> {
+  return attempt(ctx, plugin).then((newContext) => {
+    if (newContext instanceof CoreContext) {
+      return newContext
+    }
+
+    ctx.log('debug', 'Context canceled')
+    ctx.stats.increment('context_canceled')
+    ctx.cancel(newContext)
+  })
+}

package/src/queue/event-queue.ts

@@ -0,0 +1,320 @@
+import { CoreAnalytics } from '../analytics'
+import { groupBy } from '../utils/group-by'
+import { ON_REMOVE_FROM_FUTURE, PriorityQueue } from '../priority-queue'
+
+import { CoreContext, ContextCancelation } from '../context'
+import { Emitter } from '../emitter'
+import { Integrations, JSONObject } from '../events/interfaces'
+import { CorePlugin } from '../plugins'
+import { createTaskGroup, TaskGroup } from '../task/task-group'
+import { attempt, ensure } from './delivery'
+import { isOffline } from '../connection'
+
+export type EventQueueEmitterContract<Ctx extends CoreContext> = {
+  message_delivered: [ctx: Ctx]
+  message_enriched: [ctx: Ctx, plugin: CorePlugin<Ctx>]
+  delivery_success: [ctx: Ctx]
+  delivery_retry: [ctx: Ctx]
+  delivery_failure: [ctx: Ctx, err: Ctx | Error | ContextCancelation]
+  flush: [ctx: Ctx, delivered: boolean]
+}
+
+export abstract class CoreEventQueue<
+  Ctx extends CoreContext = CoreContext,
+  Plugin extends CorePlugin<Ctx> = CorePlugin<Ctx>
+> extends Emitter<EventQueueEmitterContract<Ctx>> {
+  /**
+   * All event deliveries get suspended until all the tasks in this task group are complete.
+   * For example: a middleware that augments the event object should be loaded safely as a
+   * critical task, this way, event queue will wait for it to be ready before sending events.
+   *
+   * This applies to all the events already in the queue, and the upcoming ones
+   */
+  criticalTasks: TaskGroup = createTaskGroup()
+  queue: PriorityQueue<Ctx>
+  plugins: Plugin[] = []
+  failedInitializations: string[] = []
+  private flushing = false
+
+  constructor(priorityQueue: PriorityQueue<Ctx>) {
+    super()
+
+    this.queue = priorityQueue
+    this.queue.on(ON_REMOVE_FROM_FUTURE, () => {
+      this.scheduleFlush(0)
+    })
+  }
+
+  async register(
+    ctx: Ctx,
+    plugin: Plugin,
+    instance: CoreAnalytics
+  ): Promise<void> {
+    await Promise.resolve(plugin.load(ctx, instance))
+      .then(() => {
+        this.plugins.push(plugin)
+      })
+      .catch((err) => {
+        if (plugin.type === 'destination') {
+          this.failedInitializations.push(plugin.name)
+          console.warn(plugin.name, err)
+
+          ctx.log('warn', 'Failed to load destination', {
+            plugin: plugin.name,
+            error: err,
+          })
+
+          return
+        }
+
+        throw err
+      })
+  }
+
+  async deregister(
+    ctx: Ctx,
+    plugin: CorePlugin<Ctx>,
+    instance: CoreAnalytics
+  ): Promise<void> {
+    try {
+      if (plugin.unload) {
+        await Promise.resolve(plugin.unload(ctx, instance))
+      }
+
+      this.plugins = this.plugins.filter((p) => p.name !== plugin.name)
+    } catch (e) {
+      ctx.log('warn', 'Failed to unload destination', {
+        plugin: plugin.name,
+        error: e,
+      })
+    }
+  }
+
+  async dispatch(ctx: Ctx): Promise<Ctx> {
+    ctx.log('debug', 'Dispatching')
+    ctx.stats.increment('message_dispatched')
+
+    this.queue.push(ctx)
+    const willDeliver = this.subscribeToDelivery(ctx)
+    this.scheduleFlush(0)
+    return willDeliver
+  }
+
+  private async subscribeToDelivery(ctx: Ctx): Promise<Ctx> {
+    return new Promise((resolve) => {
+      const onDeliver = (flushed: Ctx, delivered: boolean): void => {
+        if (flushed.isSame(ctx)) {
+          this.off('flush', onDeliver)
+          if (delivered) {
+            resolve(flushed)
+          } else {
+            resolve(flushed)
+          }
+        }
+      }
+
+      this.on('flush', onDeliver)
+    })
+  }
+
+  async dispatchSingle(ctx: Ctx): Promise<Ctx> {
+    ctx.log('debug', 'Dispatching')
+    ctx.stats.increment('message_dispatched')
+
+    this.queue.updateAttempts(ctx)
+    ctx.attempts = 1
+
+    return this.deliver(ctx).catch((err) => {
+      const accepted = this.enqueuRetry(err, ctx)
+      if (!accepted) {
+        ctx.setFailedDelivery({ reason: err })
+        return ctx
+      }
+
+      return this.subscribeToDelivery(ctx)
+    })
+  }
+
+  isEmpty(): boolean {
+    return this.queue.length === 0
+  }
+
+  private scheduleFlush(timeout = 500): void {
+    if (this.flushing) {
+      return
+    }
+
+    this.flushing = true
+
+    setTimeout(() => {
+      // eslint-disable-next-line @typescript-eslint/no-floating-promises
+      this.flush().then(() => {
+        setTimeout(() => {
+          this.flushing = false
+
+          if (this.queue.length) {
+            this.scheduleFlush(0)
+          }
+        }, 0)
+      })
+    }, timeout)
+  }
+
+  private async deliver(ctx: Ctx): Promise<Ctx> {
+    await this.criticalTasks.done()
+
+    const start = Date.now()
+    try {
+      ctx = await this.flushOne(ctx)
+      const done = Date.now() - start
+      this.emit('delivery_success', ctx)
+      ctx.stats.gauge('delivered', done)
+      ctx.log('debug', 'Delivered', ctx.event)
+      return ctx
+    } catch (err: any) {
+      const error = err as Ctx | Error | ContextCancelation
+      ctx.log('error', 'Failed to deliver', error)
+      this.emit('delivery_failure', ctx, error)
+      ctx.stats.increment('delivery_failed')
+      throw err
+    }
+  }
+
+  private enqueuRetry(err: Error, ctx: Ctx): boolean {
+    const retriable = !(err instanceof ContextCancelation) || err.retry
+    if (!retriable) {
+      return false
+    }
+
+    return this.queue.pushWithBackoff(ctx)
+  }
+
+  async flush(): Promise<Ctx[]> {
+    if (this.queue.length === 0 || isOffline()) {
+      return []
+    }
+
+    let ctx = this.queue.pop()
+    if (!ctx) {
+      return []
+    }
+
+    ctx.attempts = this.queue.getAttempts(ctx)
+
+    try {
+      ctx = await this.deliver(ctx)
+      this.emit('flush', ctx, true)
+    } catch (err: any) {
+      const accepted = this.enqueuRetry(err, ctx)
+
+      if (!accepted) {
+        ctx.setFailedDelivery({ reason: err })
+        this.emit('flush', ctx, false)
+      }
+
+      return []
+    }
+
+    return [ctx]
+  }
+
+  private isReady(): boolean {
+    // return this.plugins.every((p) => p.isLoaded())
+    // should we wait for every plugin to load?
+    return true
+  }
+
+  private availableExtensions(denyList: Integrations) {
+    const available = this.plugins.filter((p) => {
+      // Only filter out destination plugins or the Customer.io Data Pipeline plugin
+      if (p.type !== 'destination' && p.name !== 'Customer.io Data Pipelines') {
+        return true
+      }
+
+      let alternativeNameMatch: boolean | JSONObject | undefined = undefined
+      p.alternativeNames?.forEach((name) => {
+        if (denyList[name] !== undefined) {
+          alternativeNameMatch = denyList[name]
+        }
+      })
+
+      // Explicit integration option takes precedence, `All: false` does not apply to Customer.io Data Pipelines
+      return (
+        denyList[p.name] ??
+        alternativeNameMatch ??
+        (p.name === 'Customer.io Data Pipelines' ? true : denyList.All) !==
+        false
+      )
+    })
+
+    const {
+      before = [],
+      enrichment = [],
+      destination = [],
+      after = [],
+    } = groupBy(available, 'type')
+
+    return {
+      before,
+      enrichment,
+      destinations: destination,
+      after,
+    }
+  }
+
+  private async flushOne(ctx: Ctx): Promise<Ctx> {
+    if (!this.isReady()) {
+      throw new Error('Not ready')
+    }
+
+    if (ctx.attempts > 1) {
+      this.emit('delivery_retry', ctx)
+    }
+
+    const { before, enrichment } = this.availableExtensions(
+      ctx.event.integrations ?? {}
+    )
+
+    for (const beforeWare of before) {
+      const temp = await ensure(ctx, beforeWare)
+      if (temp instanceof CoreContext) {
+        ctx = temp
+      }
+
+      this.emit('message_enriched', ctx, beforeWare)
+    }
+
+    for (const enrichmentWare of enrichment) {
+      const temp = await attempt(ctx, enrichmentWare)
+      if (temp instanceof CoreContext) {
+        ctx = temp
+      }
+
+      this.emit('message_enriched', ctx, enrichmentWare)
+    }
+
+    // Enrichment and before plugins can re-arrange the deny list dynamically
+    // so we need to pluck them at the end
+    const { destinations, after } = this.availableExtensions(
+      ctx.event.integrations ?? {}
+    )
+
+    await new Promise((resolve, reject) => {
+      setTimeout(() => {
+        const attempts = destinations.map((destination) =>
+          attempt(ctx, destination)
+        )
+        Promise.all(attempts).then(resolve).catch(reject)
+      }, 0)
+    })
+
+    ctx.stats.increment('message_delivered')
+
+    this.emit('message_delivered', ctx)
+
+    const afterCalls = after.map((after) => attempt(ctx, after))
+    await Promise.all(afterCalls)
+
+    return ctx
+  }
+}