@customerio/cdp-analytics-core 0.0.1

package/src/queue/delivery.ts

@@ -0,0 +1,73 @@
+import { CoreContext, ContextCancelation } from '../context'
+import { CorePlugin } from '../plugins'
+
+async function tryAsync<T>(fn: () => T | Promise<T>): Promise<T> {
+  try {
+    return await fn()
+  } catch (err) {
+    return Promise.reject(err)
+  }
+}
+
+export function attempt<Ctx extends CoreContext = CoreContext>(
+  ctx: Ctx,
+  plugin: CorePlugin<Ctx>
+): Promise<Ctx | ContextCancelation | Error> {
+  ctx.log('debug', 'plugin', { plugin: plugin.name })
+  const start = new Date().getTime()
+
+  const hook = plugin[ctx.event.type]
+  if (hook === undefined) {
+    return Promise.resolve(ctx)
+  }
+
+  const newCtx = tryAsync(() => hook.apply(plugin, [ctx]))
+    .then((ctx) => {
+      const done = new Date().getTime() - start
+      ctx.stats.gauge('plugin_time', done, [`plugin:${plugin.name}`])
+
+      return ctx
+    })
+    .catch((err: Error | ContextCancelation) => {
+      if (
+        err instanceof ContextCancelation &&
+        err.type === 'middleware_cancellation'
+      ) {
+        throw err
+      }
+
+      if (err instanceof ContextCancelation) {
+        ctx.log('warn', err.type, {
+          plugin: plugin.name,
+          error: err,
+        })
+
+        return err
+      }
+
+      ctx.log('error', 'plugin Error', {
+        plugin: plugin.name,
+        error: err,
+      })
+      ctx.stats.increment('plugin_error', 1, [`plugin:${plugin.name}`])
+
+      return err
+    })
+
+  return newCtx
+}
+
+export function ensure<Ctx extends CoreContext = CoreContext>(
+  ctx: Ctx,
+  plugin: CorePlugin<Ctx>
+): Promise<Ctx | undefined> {
+  return attempt(ctx, plugin).then((newContext) => {
+    if (newContext instanceof CoreContext) {
+      return newContext
+    }
+
+    ctx.log('debug', 'Context canceled')
+    ctx.stats.increment('context_canceled')
+    ctx.cancel(newContext)
+  })
+}

package/src/queue/event-queue.ts

@@ -0,0 +1,320 @@
+import { CoreAnalytics } from '../analytics'
+import { groupBy } from '../utils/group-by'
+import { ON_REMOVE_FROM_FUTURE, PriorityQueue } from '../priority-queue'
+
+import { CoreContext, ContextCancelation } from '../context'
+import { Emitter } from '../emitter'
+import { Integrations, JSONObject } from '../events/interfaces'
+import { CorePlugin } from '../plugins'
+import { createTaskGroup, TaskGroup } from '../task/task-group'
+import { attempt, ensure } from './delivery'
+import { isOffline } from '../connection'
+
+export type EventQueueEmitterContract<Ctx extends CoreContext> = {
+  message_delivered: [ctx: Ctx]
+  message_enriched: [ctx: Ctx, plugin: CorePlugin<Ctx>]
+  delivery_success: [ctx: Ctx]
+  delivery_retry: [ctx: Ctx]
+  delivery_failure: [ctx: Ctx, err: Ctx | Error | ContextCancelation]
+  flush: [ctx: Ctx, delivered: boolean]
+}
+
+export abstract class CoreEventQueue<
+  Ctx extends CoreContext = CoreContext,
+  Plugin extends CorePlugin<Ctx> = CorePlugin<Ctx>
+> extends Emitter<EventQueueEmitterContract<Ctx>> {
+  /**
+   * All event deliveries get suspended until all the tasks in this task group are complete.
+   * For example: a middleware that augments the event object should be loaded safely as a
+   * critical task, this way, event queue will wait for it to be ready before sending events.
+   *
+   * This applies to all the events already in the queue, and the upcoming ones
+   */
+  criticalTasks: TaskGroup = createTaskGroup()
+  queue: PriorityQueue<Ctx>
+  plugins: Plugin[] = []
+  failedInitializations: string[] = []
+  private flushing = false
+
+  constructor(priorityQueue: PriorityQueue<Ctx>) {
+    super()
+
+    this.queue = priorityQueue
+    this.queue.on(ON_REMOVE_FROM_FUTURE, () => {
+      this.scheduleFlush(0)
+    })
+  }
+
+  async register(
+    ctx: Ctx,
+    plugin: Plugin,
+    instance: CoreAnalytics
+  ): Promise<void> {
+    await Promise.resolve(plugin.load(ctx, instance))
+      .then(() => {
+        this.plugins.push(plugin)
+      })
+      .catch((err) => {
+        if (plugin.type === 'destination') {
+          this.failedInitializations.push(plugin.name)
+          console.warn(plugin.name, err)
+
+          ctx.log('warn', 'Failed to load destination', {
+            plugin: plugin.name,
+            error: err,
+          })
+
+          return
+        }
+
+        throw err
+      })
+  }
+
+  async deregister(
+    ctx: Ctx,
+    plugin: CorePlugin<Ctx>,
+    instance: CoreAnalytics
+  ): Promise<void> {
+    try {
+      if (plugin.unload) {
+        await Promise.resolve(plugin.unload(ctx, instance))
+      }
+
+      this.plugins = this.plugins.filter((p) => p.name !== plugin.name)
+    } catch (e) {
+      ctx.log('warn', 'Failed to unload destination', {
+        plugin: plugin.name,
+        error: e,
+      })
+    }
+  }
+
+  async dispatch(ctx: Ctx): Promise<Ctx> {
+    ctx.log('debug', 'Dispatching')
+    ctx.stats.increment('message_dispatched')
+
+    this.queue.push(ctx)
+    const willDeliver = this.subscribeToDelivery(ctx)
+    this.scheduleFlush(0)
+    return willDeliver
+  }
+
+  private async subscribeToDelivery(ctx: Ctx): Promise<Ctx> {
+    return new Promise((resolve) => {
+      const onDeliver = (flushed: Ctx, delivered: boolean): void => {
+        if (flushed.isSame(ctx)) {
+          this.off('flush', onDeliver)
+          if (delivered) {
+            resolve(flushed)
+          } else {
+            resolve(flushed)
+          }
+        }
+      }
+
+      this.on('flush', onDeliver)
+    })
+  }
+
+  async dispatchSingle(ctx: Ctx): Promise<Ctx> {
+    ctx.log('debug', 'Dispatching')
+    ctx.stats.increment('message_dispatched')
+
+    this.queue.updateAttempts(ctx)
+    ctx.attempts = 1
+
+    return this.deliver(ctx).catch((err) => {
+      const accepted = this.enqueuRetry(err, ctx)
+      if (!accepted) {
+        ctx.setFailedDelivery({ reason: err })
+        return ctx
+      }
+
+      return this.subscribeToDelivery(ctx)
+    })
+  }
+
+  isEmpty(): boolean {
+    return this.queue.length === 0
+  }
+
+  private scheduleFlush(timeout = 500): void {
+    if (this.flushing) {
+      return
+    }
+
+    this.flushing = true
+
+    setTimeout(() => {
+      // eslint-disable-next-line @typescript-eslint/no-floating-promises
+      this.flush().then(() => {
+        setTimeout(() => {
+          this.flushing = false
+
+          if (this.queue.length) {
+            this.scheduleFlush(0)
+          }
+        }, 0)
+      })
+    }, timeout)
+  }
+
+  private async deliver(ctx: Ctx): Promise<Ctx> {
+    await this.criticalTasks.done()
+
+    const start = Date.now()
+    try {
+      ctx = await this.flushOne(ctx)
+      const done = Date.now() - start
+      this.emit('delivery_success', ctx)
+      ctx.stats.gauge('delivered', done)
+      ctx.log('debug', 'Delivered', ctx.event)
+      return ctx
+    } catch (err: any) {
+      const error = err as Ctx | Error | ContextCancelation
+      ctx.log('error', 'Failed to deliver', error)
+      this.emit('delivery_failure', ctx, error)
+      ctx.stats.increment('delivery_failed')
+      throw err
+    }
+  }
+
+  private enqueuRetry(err: Error, ctx: Ctx): boolean {
+    const retriable = !(err instanceof ContextCancelation) || err.retry
+    if (!retriable) {
+      return false
+    }
+
+    return this.queue.pushWithBackoff(ctx)
+  }
+
+  async flush(): Promise<Ctx[]> {
+    if (this.queue.length === 0 || isOffline()) {
+      return []
+    }
+
+    let ctx = this.queue.pop()
+    if (!ctx) {
+      return []
+    }
+
+    ctx.attempts = this.queue.getAttempts(ctx)
+
+    try {
+      ctx = await this.deliver(ctx)
+      this.emit('flush', ctx, true)
+    } catch (err: any) {
+      const accepted = this.enqueuRetry(err, ctx)
+
+      if (!accepted) {
+        ctx.setFailedDelivery({ reason: err })
+        this.emit('flush', ctx, false)
+      }
+
+      return []
+    }
+
+    return [ctx]
+  }
+
+  private isReady(): boolean {
+    // return this.plugins.every((p) => p.isLoaded())
+    // should we wait for every plugin to load?
+    return true
+  }
+
+  private availableExtensions(denyList: Integrations) {
+    const available = this.plugins.filter((p) => {
+      // Only filter out destination plugins or the Customer.io Data Pipeline plugin
+      if (p.type !== 'destination' && p.name !== 'Customer.io Data Pipelines') {
+        return true
+      }
+
+      let alternativeNameMatch: boolean | JSONObject | undefined = undefined
+      p.alternativeNames?.forEach((name) => {
+        if (denyList[name] !== undefined) {
+          alternativeNameMatch = denyList[name]
+        }
+      })
+
+      // Explicit integration option takes precedence, `All: false` does not apply to Customer.io Data Pipelines
+      return (
+        denyList[p.name] ??
+        alternativeNameMatch ??
+        (p.name === 'Customer.io Data Pipelines' ? true : denyList.All) !==
+        false
+      )
+    })
+
+    const {
+      before = [],
+      enrichment = [],
+      destination = [],
+      after = [],
+    } = groupBy(available, 'type')
+
+    return {
+      before,
+      enrichment,
+      destinations: destination,
+      after,
+    }
+  }
+
+  private async flushOne(ctx: Ctx): Promise<Ctx> {
+    if (!this.isReady()) {
+      throw new Error('Not ready')
+    }
+
+    if (ctx.attempts > 1) {
+      this.emit('delivery_retry', ctx)
+    }
+
+    const { before, enrichment } = this.availableExtensions(
+      ctx.event.integrations ?? {}
+    )
+
+    for (const beforeWare of before) {
+      const temp = await ensure(ctx, beforeWare)
+      if (temp instanceof CoreContext) {
+        ctx = temp
+      }
+
+      this.emit('message_enriched', ctx, beforeWare)
+    }
+
+    for (const enrichmentWare of enrichment) {
+      const temp = await attempt(ctx, enrichmentWare)
+      if (temp instanceof CoreContext) {
+        ctx = temp
+      }
+
+      this.emit('message_enriched', ctx, enrichmentWare)
+    }
+
+    // Enrichment and before plugins can re-arrange the deny list dynamically
+    // so we need to pluck them at the end
+    const { destinations, after } = this.availableExtensions(
+      ctx.event.integrations ?? {}
+    )
+
+    await new Promise((resolve, reject) => {
+      setTimeout(() => {
+        const attempts = destinations.map((destination) =>
+          attempt(ctx, destination)
+        )
+        Promise.all(attempts).then(resolve).catch(reject)
+      }, 0)
+    })
+
+    ctx.stats.increment('message_delivered')
+
+    this.emit('message_delivered', ctx)
+
+    const afterCalls = after.map((after) => attempt(ctx, after))
+    await Promise.all(afterCalls)
+
+    return ctx
+  }
+}

package/src/stats/index.ts

@@ -0,0 +1,88 @@
+type CompactMetricType = 'g' | 'c'
+
+export type CoreMetricType = 'gauge' | 'counter'
+
+export interface CoreMetric {
+  metric: string
+  value: number
+  type: CoreMetricType
+  tags: string[]
+  timestamp: number // unit milliseconds
+}
+
+export interface CompactMetric {
+  m: string // metric name
+  v: number // value
+  k: CompactMetricType
+  t: string[] // tags
+  e: number // timestamp in unit milliseconds
+}
+
+const compactMetricType = (type: CoreMetricType): CompactMetricType => {
+  const enums: Record<CoreMetricType, CompactMetricType> = {
+    gauge: 'g',
+    counter: 'c',
+  }
+  return enums[type]
+}
+
+export abstract class CoreStats {
+  metrics: CoreMetric[] = []
+  increment(metric: string, by = 1, tags?: string[]): void {
+    this.metrics.push({
+      metric,
+      value: by,
+      tags: tags ?? [],
+      type: 'counter',
+      timestamp: Date.now(),
+    })
+  }
+
+  gauge(metric: string, value: number, tags?: string[]): void {
+    this.metrics.push({
+      metric,
+      value,
+      tags: tags ?? [],
+      type: 'gauge',
+      timestamp: Date.now(),
+    })
+  }
+
+  flush(): void {
+    const formatted = this.metrics.map((m) => ({
+      ...m,
+      tags: m.tags.join(','),
+    }))
+    // ie doesn't like console.table
+    if (console.table) {
+      console.table(formatted)
+    } else {
+      console.log(formatted)
+    }
+    this.metrics = []
+  }
+
+  /**
+   * compact keys for smaller payload
+   */
+  serialize(): CompactMetric[] {
+    return this.metrics.map((m) => {
+      return {
+        m: m.metric,
+        v: m.value,
+        t: m.tags,
+        k: compactMetricType(m.type),
+        e: m.timestamp,
+      }
+    })
+  }
+}
+
+export class NullStats extends CoreStats {
+  override gauge(..._args: Parameters<CoreStats['gauge']>) {}
+  override increment(..._args: Parameters<CoreStats['increment']>) {}
+  override flush(..._args: Parameters<CoreStats['flush']>) {}
+  override serialize(..._args: Parameters<CoreStats['serialize']>) {
+    return []
+  }
+}

package/src/task/task-group.ts

@@ -0,0 +1,31 @@
+import { isThenable } from '../utils/is-thenable'
+
+export type TaskGroup = {
+  done: () => Promise<void>
+  run: <Operation extends (...args: any[]) => any>(
+    op: Operation
+  ) => ReturnType<Operation>
+}
+
+export const createTaskGroup = (): TaskGroup => {
+  let taskCompletionPromise: Promise<void>
+  let resolvePromise: () => void
+  let count = 0
+
+  return {
+    done: () => taskCompletionPromise,
+    run: (op) => {
+      const returnValue = op()
+
+      if (isThenable(returnValue)) {
+        if (++count === 1) {
+          taskCompletionPromise = new Promise((res) => (resolvePromise = res))
+        }
+
+        returnValue.finally(() => --count === 0 && resolvePromise())
+      }
+
+      return returnValue
+    },
+  }
+}

package/src/user/index.ts

@@ -0,0 +1,7 @@
+export type ID = string | null | undefined
+
+// TODO: this is a base user
+export interface User {
+  id(): ID
+  anonymousId(): ID
+}

package/src/utils/bind-all.ts

@@ -0,0 +1,19 @@
+export function bindAll<
+  ObjType extends { [key: string]: any },
+  KeyType extends keyof ObjType
+>(obj: ObjType): ObjType {
+  const proto = obj.constructor.prototype
+  for (const key of Object.getOwnPropertyNames(proto)) {
+    if (key !== 'constructor') {
+      const desc = Object.getOwnPropertyDescriptor(
+        obj.constructor.prototype,
+        key
+      )
+      if (!!desc && typeof desc.value === 'function') {
+        obj[key as KeyType] = obj[key].bind(obj)
+      }
+    }
+  }
+
+  return obj
+}

package/src/utils/environment.ts

@@ -0,0 +1,7 @@
+export function isBrowser(): boolean {
+  return typeof window !== 'undefined'
+}
+
+export function isServer(): boolean {
+  return !isBrowser()
+}

package/src/utils/get-global.ts

@@ -0,0 +1,16 @@
+// This an imperfect polyfill for globalThis
+export const getGlobal = () => {
+  if (typeof globalThis !== 'undefined') {
+    return globalThis
+  }
+  if (typeof self !== 'undefined') {
+    return self
+  }
+  if (typeof window !== 'undefined') {
+    return window
+  }
+  if (typeof global !== 'undefined') {
+    return global
+  }
+  return null
+}

package/src/utils/group-by.ts

@@ -0,0 +1,30 @@
+type Grouper<T> = (obj: T) => string | number
+
+export function groupBy<T>(
+  collection: T[],
+  grouper: keyof T | Grouper<T>
+): Record<string, T[]> {
+  const results: Record<string, T[]> = {}
+
+  collection.forEach((item) => {
+    let key: string | number | undefined = undefined
+
+    if (typeof grouper === 'string') {
+      const suggestedKey = item[grouper]
+      key =
+        typeof suggestedKey !== 'string'
+          ? JSON.stringify(suggestedKey)
+          : suggestedKey
+    } else if (grouper instanceof Function) {
+      key = grouper(item)
+    }
+
+    if (key === undefined) {
+      return
+    }
+
+    results[key] = [...(results[key] ?? []), item]
+  })
+
+  return results
+}

package/src/utils/has-properties.ts

@@ -0,0 +1,7 @@
+export function hasProperties<T extends object, K extends string>(
+  obj: T,
+  ...keys: K[]
+): obj is T & { [J in K]: unknown } {
+  // eslint-disable-next-line no-prototype-builtins
+  return !!obj && keys.every((key) => obj.hasOwnProperty(key))
+}

package/src/utils/is-plain-object.ts

@@ -0,0 +1,26 @@
+// Code derived from https://github.com/jonschlinkert/is-plain-object/blob/master/is-plain-object.js
+
+function isObject(o: unknown): o is Object {
+  return Object.prototype.toString.call(o) === '[object Object]'
+}
+
+export function isPlainObject(o: unknown): o is Record<PropertyKey, unknown> {
+  if (isObject(o) === false) return false
+
+  // If has modified constructor
+  const ctor = (o as any).constructor
+  if (ctor === undefined) return true
+
+  // If has modified prototype
+  const prot = ctor.prototype
+  if (isObject(prot) === false) return false
+
+  // If constructor does not have an Object-specific method
+  // eslint-disable-next-line no-prototype-builtins
+  if ((prot as Object).hasOwnProperty('isPrototypeOf') === false) {
+    return false
+  }
+
+  // Most likely a plain Object
+  return true
+}

package/src/utils/is-thenable.ts

@@ -0,0 +1,9 @@
+/**
+ *  Check if  thenable
+ *  (instanceof Promise doesn't respect realms)
+ */
+export const isThenable = (value: unknown): boolean =>
+  typeof value === 'object' &&
+  value !== null &&
+  'then' in value &&
+  typeof (value as any).then === 'function'

package/src/utils/p-while.ts

@@ -0,0 +1,12 @@
+export const pWhile = async <T>(
+  condition: (value: T | undefined) => boolean,
+  action: () => T | PromiseLike<T>
+): Promise<void> => {
+  const loop = async (actionResult: T | undefined): Promise<void> => {
+    if (condition(actionResult)) {
+      return loop(await action())
+    }
+  }
+
+  return loop(undefined)
+}

package/src/utils/pick.ts

@@ -0,0 +1,8 @@
+export const pickBy = <T extends object, K extends keyof T>(
+  obj: T,
+  fn: (key: K, v: T[K]) => boolean
+) => {
+  return (Object.keys(obj) as K[])
+    .filter((k) => fn(k, obj[k]))
+    .reduce((acc, key) => ((acc[key] = obj[key]), acc), {} as Partial<T>)
+}

package/src/utils/ts-helpers.ts

@@ -0,0 +1,13 @@
+/**
+ * Remove Index Signature
+ */
+export type RemoveIndexSignature<T> = {
+  [K in keyof T as {} extends Record<K, 1> ? never : K]: T[K]
+}
+
+/**
+ * Recursively make all object properties nullable
+ */
+export type DeepNullable<T> = {
+  [K in keyof T]: T[K] extends object ? DeepNullable<T[K]> | null : T[K] | null
+}