@customerio/cdp-analytics-browser 0.0.1

package/src/plugins/customerio/batched-dispatcher.ts

@@ -0,0 +1,115 @@
+import { CustomerioEvent } from '../../core/events'
+import { fetch } from '../../lib/fetch'
+import { onPageLeave } from '../../lib/on-page-leave'
+
+export type BatchingDispatchConfig = {
+  size?: number
+  timeout?: number
+}
+
+const MAX_PAYLOAD_SIZE = 500
+
+function kilobytes(buffer: unknown): number {
+  const size = encodeURI(JSON.stringify(buffer)).split(/%..|./).length - 1
+  return size / 1024
+}
+
+/**
+ * Checks if the payload is over or close to
+ * the maximum payload size allowed by tracking
+ * API.
+ */
+function approachingTrackingAPILimit(buffer: unknown): boolean {
+  return kilobytes(buffer) >= MAX_PAYLOAD_SIZE - 50
+}
+
+function chunks(batch: object[]): Array<object[]> {
+  const result: object[][] = []
+  let index = 0
+
+  batch.forEach((item) => {
+    const size = kilobytes(result[index])
+    if (size >= 64) {
+      index++
+    }
+
+    if (result[index]) {
+      result[index].push(item)
+    } else {
+      result[index] = [item]
+    }
+  })
+
+  return result
+}
+
+export default function batch(
+  apiHost: string,
+  config?: BatchingDispatchConfig
+) {
+  let buffer: object[] = []
+  let pageUnloaded = false
+
+  const limit = config?.size ?? 10
+  const timeout = config?.timeout ?? 5000
+
+  function sendBatch(batch: object[]) {
+    if (batch.length === 0) {
+      return
+    }
+
+    const writeKey = (batch[0] as CustomerioEvent)?.writeKey
+
+    return fetch(`https://${apiHost}/b`, {
+      keepalive: pageUnloaded,
+      headers: {
+        'Content-Type': 'text/plain',
+      },
+      method: 'post',
+      body: JSON.stringify({ batch, writeKey }),
+    })
+  }
+
+  async function flush(): Promise<unknown> {
+    if (buffer.length) {
+      const batch = buffer
+      buffer = []
+      return sendBatch(batch)
+    }
+  }
+
+  let schedule: NodeJS.Timeout | undefined
+
+  function scheduleFlush(): void {
+    if (schedule) {
+      return
+    }
+
+    schedule = setTimeout(() => {
+      schedule = undefined
+      flush().catch(console.error)
+    }, timeout)
+  }
+
+  onPageLeave(() => {
+    pageUnloaded = true
+
+    if (buffer.length) {
+      const reqs = chunks(buffer).map(sendBatch)
+      Promise.all(reqs).catch(console.error)
+    }
+  })
+
+  async function dispatch(_url: string, body: object): Promise<unknown> {
+    buffer.push(body)
+
+    const bufferOverflow =
+      buffer.length >= limit || approachingTrackingAPILimit(buffer)
+
+    return bufferOverflow || pageUnloaded ? flush() : scheduleFlush()
+  }
+
+  return {
+    dispatch,
+  }
+}

package/src/plugins/customerio/fetch-dispatcher.ts

@@ -0,0 +1,24 @@
+import { fetch } from '../../lib/fetch'
+
+export type Dispatcher = (url: string, body: object) => Promise<unknown>
+
+export type StandardDispatcherConfig = {
+  keepalive?: boolean
+}
+
+export default function (config?: StandardDispatcherConfig): {
+  dispatch: Dispatcher
+} {
+  function dispatch(url: string, body: object): Promise<unknown> {
+    return fetch(url, {
+      keepalive: config?.keepalive,
+      headers: { 'Content-Type': 'text/plain' },
+      method: 'post',
+      body: JSON.stringify(body),
+    })
+  }
+
+  return {
+    dispatch,
+  }
+}

package/src/plugins/customerio/index.ts

@@ -0,0 +1,142 @@
+import { Facade } from '@segment/facade'
+import { Analytics } from '../../core/analytics'
+import { LegacySettings } from '../../browser'
+import { isOffline } from '../../core/connection'
+import { Context } from '../../core/context'
+import { Plugin } from '../../core/plugin'
+import { PriorityQueue } from '../../lib/priority-queue'
+import { PersistedPriorityQueue } from '../../lib/priority-queue/persisted'
+import { toFacade } from '../../lib/to-facade'
+import batch, { BatchingDispatchConfig } from './batched-dispatcher'
+import standard, { StandardDispatcherConfig } from './fetch-dispatcher'
+import { normalize } from './normalize'
+import { scheduleFlush } from './schedule-flush'
+
+type DeliveryStrategy =
+  | {
+    strategy?: 'standard'
+    config?: StandardDispatcherConfig
+  }
+  | {
+    strategy?: 'batching'
+    config?: BatchingDispatchConfig
+  }
+
+export type CustomerioSettings = {
+  apiKey: string
+  apiHost?: string
+  protocol?: 'http' | 'https'
+
+  addBundledMetadata?: boolean
+  unbundledIntegrations?: string[]
+  bundledConfigIds?: string[]
+  unbundledConfigIds?: string[]
+
+  maybeBundledConfigIds?: Record<string, string[]>
+
+  deliveryStrategy?: DeliveryStrategy
+}
+
+type JSON = ReturnType<Facade['json']>
+
+function onAlias(analytics: Analytics, json: JSON): JSON {
+  const user = analytics.user()
+  json.previousId =
+    json.previousId ?? json.from ?? user.id() ?? user.anonymousId()
+  json.userId = json.userId ?? json.to
+  delete json.from
+  delete json.to
+  return json
+}
+
+export function customerio(
+  analytics: Analytics,
+  settings?: CustomerioSettings,
+  integrations?: LegacySettings['integrations']
+): Plugin {
+  // Attach `pagehide` before buffer is created so that inflight events are added
+  // to the buffer before the buffer persists events in its own `pagehide` handler.
+  window.addEventListener('pagehide', () => {
+    buffer.push(...Array.from(inflightEvents))
+    inflightEvents.clear()
+  })
+
+  const buffer = analytics.options.disableClientPersistence
+    ? new PriorityQueue<Context>(analytics.queue.queue.maxAttempts, [])
+    : new PersistedPriorityQueue(
+      analytics.queue.queue.maxAttempts,
+      `dest-Customer.io-Data-Pipelines`
+    )
+
+  const inflightEvents = new Set<Context>()
+  const flushing = false
+
+  const apiHost = settings?.apiHost ?? 'cdp.customer.io/v1'
+  const protocol = settings?.protocol ?? 'https'
+  const remote = `${protocol}://${apiHost}`
+
+  const deliveryStrategy = settings?.deliveryStrategy
+  const client =
+    deliveryStrategy?.strategy === 'batching'
+      ? batch(apiHost, deliveryStrategy.config)
+      : standard(deliveryStrategy?.config as StandardDispatcherConfig)
+
+  async function send(ctx: Context): Promise<Context> {
+    if (isOffline()) {
+      buffer.push(ctx)
+      // eslint-disable-next-line @typescript-eslint/no-use-before-define
+      scheduleFlush(flushing, buffer, customerio, scheduleFlush)
+      return ctx
+    }
+
+    inflightEvents.add(ctx)
+
+    const path = ctx.event.type.charAt(0)
+    let json = toFacade(ctx.event).json()
+
+    if (ctx.event.type === 'track') {
+      delete json.traits
+    }
+
+    if (ctx.event.type === 'alias') {
+      json = onAlias(analytics, json)
+    }
+
+    return client
+      .dispatch(
+        `${remote}/${path}`,
+        normalize(analytics, json, settings, integrations)
+      )
+      .then(() => ctx)
+      .catch(() => {
+        buffer.pushWithBackoff(ctx)
+        // eslint-disable-next-line @typescript-eslint/no-use-before-define
+        scheduleFlush(flushing, buffer, customerio, scheduleFlush)
+        return ctx
+      })
+      .finally(() => {
+        inflightEvents.delete(ctx)
+      })
+  }
+
+  const customerio: Plugin = {
+    name: 'Customer.io Data Pipelines',
+    type: 'after',
+    version: '0.1.0',
+    isLoaded: (): boolean => true,
+    load: (): Promise<void> => Promise.resolve(),
+    track: send,
+    identify: send,
+    page: send,
+    alias: send,
+    group: send,
+  }
+
+  // Buffer may already have items if they were previously stored in localStorage.
+  // Start flushing them immediately.
+  if (buffer.todo) {
+    scheduleFlush(flushing, buffer, customerio, scheduleFlush)
+  }
+
+  return customerio
+}

package/src/plugins/customerio/normalize.ts

@@ -0,0 +1,223 @@
+import jar from 'js-cookie'
+import { Analytics } from '../../core/analytics'
+import { LegacySettings } from '../../browser'
+import { CustomerioEvent } from '../../core/events'
+import { gracefulDecodeURIComponent } from '../../core/query-string/gracefulDecodeURIComponent'
+import { tld } from '../../core/user/tld'
+import { CustomerioFacade } from '../../lib/to-facade'
+import { CustomerioSettings } from './index'
+import { version } from '../../generated/version'
+import { getAvailableStorageOptions, UniversalStorage } from '../../core/user'
+
+let cookieOptions: jar.CookieAttributes | undefined
+function getCookieOptions(): jar.CookieAttributes {
+  if (cookieOptions) {
+    return cookieOptions
+  }
+
+  const domain = tld(window.location.href)
+  cookieOptions = {
+    expires: 31536000000, // 1 year
+    secure: false,
+    path: '/',
+  }
+  if (domain) {
+    cookieOptions.domain = domain
+  }
+
+  return cookieOptions
+}
+
+// Default value will be updated to 'web' in `bundle-umd.ts` for web build.
+let _version: 'web' | 'npm' = 'npm'
+
+export function setVersionType(version: typeof _version) {
+  _version = version
+}
+
+export function getVersionType(): typeof _version {
+  return _version
+}
+
+type Ad = { id: string; type: string }
+
+export function ampId(): string | undefined {
+  const ampId = jar.get('_ga')
+  if (ampId && ampId.startsWith('amp')) {
+    return ampId
+  }
+}
+
+export function utm(query: string): Record<string, string> {
+  if (query.startsWith('?')) {
+    query = query.substring(1)
+  }
+  query = query.replace(/\?/g, '&')
+
+  return query.split('&').reduce((acc, str) => {
+    const [k, v = ''] = str.split('=')
+    if (k.includes('utm_') && k.length > 4) {
+      let utmParam = k.substr(4)
+      if (utmParam === 'campaign') {
+        utmParam = 'name'
+      }
+      acc[utmParam] = gracefulDecodeURIComponent(v)
+    }
+    return acc
+  }, {} as Record<string, string>)
+}
+
+function ads(query: string): Ad | undefined {
+  const queryIds: Record<string, string> = {
+    btid: 'dataxu',
+    urid: 'millennial-media',
+  }
+
+  if (query.startsWith('?')) {
+    query = query.substring(1)
+  }
+  query = query.replace(/\?/g, '&')
+  const parts = query.split('&')
+
+  for (const part of parts) {
+    const [k, v] = part.split('=')
+    if (queryIds[k]) {
+      return {
+        id: v,
+        type: queryIds[k],
+      }
+    }
+  }
+}
+
+function referrerId(
+  query: string,
+  ctx: CustomerioEvent['context'],
+  disablePersistance: boolean
+): void {
+  const storage = new UniversalStorage<{
+    's:context.referrer': Ad
+  }>(
+    disablePersistance ? [] : ['cookie'],
+    getAvailableStorageOptions(getCookieOptions())
+  )
+
+  const stored = storage.get('s:context.referrer')
+  let ad: Ad | undefined | null = ads(query)
+
+  ad = ad ?? stored
+
+  if (!ad) {
+    return
+  }
+
+  if (ctx) {
+    ctx.referrer = { ...ctx.referrer, ...ad }
+  }
+
+  storage.set('s:context.referrer', ad)
+}
+
+export function normalize(
+  analytics: Analytics,
+  json: ReturnType<CustomerioFacade['json']>,
+  settings?: CustomerioSettings,
+  integrations?: LegacySettings['integrations']
+): object {
+  const user = analytics.user()
+  const query = window.location.search
+
+  json.context = json.context ?? json.options ?? {}
+  const ctx = json.context
+
+  delete json.options
+  json.writeKey = settings?.apiKey
+  ctx.userAgent = window.navigator.userAgent
+
+  // @ts-ignore
+  const locale = navigator.userLanguage || navigator.language
+
+  if (typeof ctx.locale === 'undefined' && typeof locale !== 'undefined') {
+    ctx.locale = locale
+  }
+
+  if (!ctx.library) {
+    const type = getVersionType()
+    if (type === 'web') {
+      ctx.library = {
+        name: 'analytics.js',
+        version: `next-${version}`,
+      }
+    } else {
+      ctx.library = {
+        name: 'analytics.js',
+        version: `npm:next-${version}`,
+      }
+    }
+  }
+
+  if (query && !ctx.campaign) {
+    ctx.campaign = utm(query)
+  }
+
+  referrerId(query, ctx, analytics.options.disableClientPersistence ?? false)
+
+  json.userId = json.userId || user.id()
+  json.anonymousId = json.anonymousId || user.anonymousId()
+
+  json.sentAt = new Date()
+
+  const failed = analytics.queue.failedInitializations || []
+  if (failed.length > 0) {
+    json._metadata = { failedInitializations: failed }
+  }
+
+  const bundled: string[] = []
+  const unbundled: string[] = []
+
+  for (const key in integrations) {
+    const integration = integrations[key]
+    if (key === 'Customer.io Data Pipelines') {
+      bundled.push(key)
+    }
+    if (integration.bundlingStatus === 'bundled') {
+      bundled.push(key)
+    }
+    if (integration.bundlingStatus === 'unbundled') {
+      unbundled.push(key)
+    }
+  }
+
+  // This will make sure that the disabled cloud mode destinations will be
+  // included in the unbundled list.
+  for (const settingsUnbundled of settings?.unbundledIntegrations || []) {
+    if (!unbundled.includes(settingsUnbundled)) {
+      unbundled.push(settingsUnbundled)
+    }
+  }
+
+  const configIds = settings?.maybeBundledConfigIds ?? {}
+  const bundledConfigIds: string[] = []
+
+  bundled.sort().forEach((name) => {
+    ; (configIds[name] ?? []).forEach((id) => {
+      bundledConfigIds.push(id)
+    })
+  })
+
+  if (settings?.addBundledMetadata !== false) {
+    json._metadata = {
+      ...json._metadata,
+      bundled: bundled.sort(),
+      unbundled: unbundled.sort(),
+      bundledIds: bundledConfigIds,
+    }
+  }
+
+  const amp = ampId()
+  if (amp) {
+    ctx.amp = { id: amp }
+  }
+
+  return json
+}

package/src/plugins/customerio/schedule-flush.ts

@@ -0,0 +1,58 @@
+import { isOffline } from '../../core/connection'
+import { Context } from '../../core/context'
+import { Plugin } from '../../core/plugin'
+import { attempt } from '@customerio/cdp-analytics-core'
+import { pWhile } from '../../lib/p-while'
+import { PriorityQueue } from '../../lib/priority-queue'
+
+async function flushQueue(
+  xt: Plugin,
+  queue: PriorityQueue<Context>
+): Promise<PriorityQueue<Context>> {
+  const failedQueue: Context[] = []
+  if (isOffline()) {
+    return queue
+  }
+
+  await pWhile(
+    () => queue.length > 0 && !isOffline(),
+    async () => {
+      const ctx = queue.pop()
+      if (!ctx) {
+        return
+      }
+
+      const result = await attempt(ctx, xt)
+      const success = result instanceof Context
+      if (!success) {
+        failedQueue.push(ctx)
+      }
+    }
+  )
+  // re-add failed tasks
+  failedQueue.map((failed) => queue.pushWithBackoff(failed))
+  return queue
+}
+
+export function scheduleFlush(
+  flushing: boolean,
+  buffer: PriorityQueue<Context>,
+  xt: Plugin,
+  scheduleFlush: Function
+): void {
+  if (flushing) {
+    return
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-misused-promises
+  setTimeout(async () => {
+    let isFlushing = true
+    // eslint-disable-next-line @typescript-eslint/no-use-before-define
+    const newBuffer = await flushQueue(xt, buffer)
+    isFlushing = false
+
+    if (buffer.todo > 0) {
+      scheduleFlush(isFlushing, newBuffer, xt, scheduleFlush)
+    }
+  }, Math.random() * 5000)
+}

package/src/plugins/legacy-video-plugins/index.ts

@@ -0,0 +1,16 @@
+import { Analytics } from '../../core/analytics'
+
+export async function loadLegacyVideoPlugins(
+  analytics: Analytics
+): Promise<void> {
+  const plugins = await import(
+    // @ts-expect-error
+    '@segment/analytics.js-video-plugins/dist/index.umd.js'
+  )
+
+  // This is super gross, but we need to support the `window.analytics.plugins` namespace
+  // that is linked in the docs in order to be backwards compatible with ajs-classic
+
+  // @ts-expect-error
+  analytics._plugins = plugins
+}

package/src/plugins/middleware/index.ts

@@ -0,0 +1,131 @@
+import { Context, ContextCancelation } from '../../core/context'
+import { CustomerioEvent } from '../../core/events'
+import { Plugin } from '../../core/plugin'
+import { CustomerioFacade, toFacade } from '../../lib/to-facade'
+
+export interface MiddlewareParams {
+  payload: CustomerioFacade
+
+  integrations?: CustomerioEvent['integrations']
+  next: (payload: MiddlewareParams['payload'] | null) => void
+}
+
+export interface DestinationMiddlewareParams {
+  payload: CustomerioFacade
+  integration: string
+  next: (payload: MiddlewareParams['payload'] | null) => void
+}
+
+export type MiddlewareFunction = (
+  middleware: MiddlewareParams
+) => void | Promise<void>
+
+export type DestinationMiddlewareFunction = (
+  middleware: DestinationMiddlewareParams
+) => void | Promise<void>
+
+export async function applyDestinationMiddleware(
+  destination: string,
+  evt: CustomerioEvent,
+  middleware: DestinationMiddlewareFunction[]
+): Promise<CustomerioEvent | null> {
+  // Clone the event so mutations are localized to a single destination.
+  let modifiedEvent = toFacade(evt, {
+    clone: true,
+    traverse: false,
+  }).rawEvent() as CustomerioEvent
+  async function applyMiddleware(
+    event: CustomerioEvent,
+    fn: DestinationMiddlewareFunction
+  ): Promise<CustomerioEvent | null> {
+    let nextCalled = false
+    let returnedEvent: CustomerioEvent | null = null
+
+    await fn({
+      payload: toFacade(event, {
+        clone: true,
+        traverse: false,
+      }),
+      integration: destination,
+      next(evt) {
+        nextCalled = true
+
+        if (evt === null) {
+          returnedEvent = null
+        }
+
+        if (evt) {
+          returnedEvent = evt.obj
+        }
+      },
+    })
+
+    if (!nextCalled && returnedEvent !== null) {
+      returnedEvent = returnedEvent as CustomerioEvent
+      returnedEvent.integrations = {
+        ...event.integrations,
+        [destination]: false,
+      }
+    }
+
+    return returnedEvent
+  }
+
+  for (const md of middleware) {
+    const result = await applyMiddleware(modifiedEvent, md)
+    if (result === null) {
+      return null
+    }
+    modifiedEvent = result
+  }
+
+  return modifiedEvent
+}
+
+export function sourceMiddlewarePlugin(
+  fn: MiddlewareFunction,
+  integrations: CustomerioEvent['integrations']
+): Plugin {
+  async function apply(ctx: Context): Promise<Context> {
+    let nextCalled = false
+
+    await fn({
+      payload: toFacade(ctx.event, {
+        clone: true,
+        traverse: false,
+      }),
+      integrations: integrations ?? {},
+      next(evt) {
+        nextCalled = true
+        if (evt) {
+          ctx.event = evt.obj
+        }
+      },
+    })
+
+    if (!nextCalled) {
+      throw new ContextCancelation({
+        retry: false,
+        type: 'middleware_cancellation',
+        reason: 'Middleware `next` function skipped',
+      })
+    }
+
+    return ctx
+  }
+
+  return {
+    name: `Source Middleware ${fn.name}`,
+    type: 'before',
+    version: '0.1.0',
+
+    isLoaded: (): boolean => true,
+    load: (ctx): Promise<Context> => Promise.resolve(ctx),
+
+    track: apply,
+    page: apply,
+    identify: apply,
+    alias: apply,
+    group: apply,
+  }
+}