ai-experiments 0.1.0 → 2.0.2

package/src/tracking.ts

@@ -0,0 +1,339 @@
+/**
+ * Event tracking for experiments
+ */
+
+import type { TrackingEvent, TrackingBackend, TrackingOptions } from './types.js'
+
+/**
+ * Default tracking configuration
+ */
+let trackingConfig: Required<TrackingOptions> = {
+  backend: createConsoleBackend(),
+  enabled: true,
+  metadata: {},
+}
+
+/**
+ * Configure tracking
+ *
+ * @example
+ * ```ts
+ * import { configureTracking } from 'ai-experiments'
+ *
+ * // Use console backend (default)
+ * configureTracking({
+ *   enabled: true,
+ *   metadata: { projectId: 'my-project' },
+ * })
+ *
+ * // Use custom backend
+ * configureTracking({
+ *   backend: {
+ *     track: async (event) => {
+ *       await fetch('/api/analytics', {
+ *         method: 'POST',
+ *         body: JSON.stringify(event),
+ *       })
+ *     },
+ *   },
+ * })
+ *
+ * // Disable tracking
+ * configureTracking({ enabled: false })
+ * ```
+ */
+export function configureTracking(options: TrackingOptions): void {
+  trackingConfig = {
+    backend: options.backend ?? trackingConfig.backend,
+    enabled: options.enabled ?? trackingConfig.enabled,
+    metadata: options.metadata ?? trackingConfig.metadata,
+  }
+}
+
+/**
+ * Track an event
+ *
+ * @example
+ * ```ts
+ * import { track } from 'ai-experiments'
+ *
+ * track({
+ *   type: 'experiment.start',
+ *   timestamp: new Date(),
+ *   data: {
+ *     experimentId: 'my-experiment',
+ *     variantCount: 3,
+ *   },
+ * })
+ * ```
+ */
+export function track(event: TrackingEvent): void {
+  if (!trackingConfig.enabled) {
+    return
+  }
+
+  // Merge global metadata
+  const enrichedEvent: TrackingEvent = {
+    ...event,
+    data: {
+      ...event.data,
+      ...trackingConfig.metadata,
+    },
+  }
+
+  // Track via backend (handle both sync and async)
+  const result = trackingConfig.backend.track(enrichedEvent)
+  if (result instanceof Promise) {
+    // Don't await - fire and forget
+    result.catch((error) => {
+      console.error('Error tracking event:', error)
+    })
+  }
+}
+
+/**
+ * Flush pending events
+ *
+ * Call this before the process exits to ensure all events are sent.
+ *
+ * @example
+ * ```ts
+ * import { flush } from 'ai-experiments'
+ *
+ * process.on('SIGINT', async () => {
+ *   await flush()
+ *   process.exit(0)
+ * })
+ * ```
+ */
+export async function flush(): Promise<void> {
+  if (trackingConfig.backend.flush) {
+    await trackingConfig.backend.flush()
+  }
+}
+
+/**
+ * Create a console-based tracking backend
+ *
+ * Logs events to console.log in a human-readable format.
+ */
+export function createConsoleBackend(options?: {
+  /** Whether to include full event data (default: false) */
+  verbose?: boolean
+}): TrackingBackend {
+  const { verbose = false } = options ?? {}
+
+  return {
+    track: (event: TrackingEvent) => {
+      const timestamp = event.timestamp.toISOString()
+
+      if (verbose) {
+        console.log(`[${timestamp}] ${event.type}`, event.data)
+      } else {
+        // Condensed format
+        const key = extractKey(event)
+        console.log(`[${timestamp}] ${event.type} ${key}`)
+      }
+    },
+  }
+}
+
+/**
+ * Create an in-memory tracking backend that stores events
+ *
+ * Useful for testing or collecting events for batch processing.
+ *
+ * @example
+ * ```ts
+ * import { createMemoryBackend } from 'ai-experiments'
+ *
+ * const backend = createMemoryBackend()
+ * configureTracking({ backend })
+ *
+ * // Run experiments...
+ *
+ * // Get all events
+ * const events = backend.getEvents()
+ * console.log(`Tracked ${events.length} events`)
+ *
+ * // Clear events
+ * backend.clear()
+ * ```
+ */
+export function createMemoryBackend(): TrackingBackend & {
+  getEvents: () => TrackingEvent[]
+  clear: () => void
+} {
+  const events: TrackingEvent[] = []
+
+  return {
+    track: (event: TrackingEvent) => {
+      events.push(event)
+    },
+    getEvents: () => [...events],
+    clear: () => {
+      events.length = 0
+    },
+  }
+}
+
+/**
+ * Create a batching tracking backend
+ *
+ * Batches events and sends them in groups to reduce network overhead.
+ *
+ * @example
+ * ```ts
+ * import { createBatchBackend } from 'ai-experiments'
+ *
+ * const backend = createBatchBackend({
+ *   batchSize: 10,
+ *   flushInterval: 5000, // 5 seconds
+ *   send: async (events) => {
+ *     await fetch('/api/analytics/batch', {
+ *       method: 'POST',
+ *       body: JSON.stringify({ events }),
+ *     })
+ *   },
+ * })
+ *
+ * configureTracking({ backend })
+ * ```
+ */
+export function createBatchBackend(options: {
+  /** Maximum batch size before auto-flush */
+  batchSize: number
+  /** Interval in ms to auto-flush (default: no auto-flush) */
+  flushInterval?: number
+  /** Function to send batched events */
+  send: (events: TrackingEvent[]) => Promise<void>
+}): TrackingBackend {
+  const { batchSize, flushInterval, send } = options
+  const batch: TrackingEvent[] = []
+  let flushTimer: NodeJS.Timeout | null = null
+
+  const flush = async () => {
+    if (batch.length === 0) return
+
+    const eventsToSend = [...batch]
+    batch.length = 0
+
+    try {
+      await send(eventsToSend)
+    } catch (error) {
+      console.error('Error sending batch:', error)
+      // Re-add failed events to batch (simple retry strategy)
+      batch.unshift(...eventsToSend)
+    }
+  }
+
+  const scheduleFlush = () => {
+    if (flushTimer) {
+      clearTimeout(flushTimer)
+    }
+    if (flushInterval) {
+      flushTimer = setTimeout(() => {
+        flush().catch(console.error)
+      }, flushInterval)
+    }
+  }
+
+  return {
+    track: (event: TrackingEvent) => {
+      batch.push(event)
+
+      // Auto-flush if batch is full
+      if (batch.length >= batchSize) {
+        flush().catch(console.error)
+      } else {
+        scheduleFlush()
+      }
+    },
+    flush: async () => {
+      if (flushTimer) {
+        clearTimeout(flushTimer)
+        flushTimer = null
+      }
+      await flush()
+    },
+  }
+}
+
+/**
+ * Create a file-based tracking backend
+ *
+ * Writes events to a file (JSONL format).
+ *
+ * @example
+ * ```ts
+ * import { createFileBackend } from 'ai-experiments'
+ *
+ * const backend = createFileBackend({
+ *   path: './experiments.jsonl',
+ * })
+ *
+ * configureTracking({ backend })
+ * ```
+ */
+export function createFileBackend(options: {
+  /** Path to the file */
+  path: string
+}): TrackingBackend {
+  // Note: This requires Node.js fs module
+  // Import dynamically to avoid breaking in non-Node environments
+  let fs: typeof import('fs') | null = null
+  let writeStream: ReturnType<typeof import('fs').createWriteStream> | null = null
+
+  const ensureStream = async () => {
+    if (!writeStream) {
+      try {
+        fs = await import('fs')
+        writeStream = fs.createWriteStream(options.path, { flags: 'a' })
+      } catch (error) {
+        console.error('Failed to create file stream:', error)
+        throw error
+      }
+    }
+    return writeStream
+  }
+
+  return {
+    track: async (event: TrackingEvent) => {
+      try {
+        const stream = await ensureStream()
+        const line = JSON.stringify(event) + '\n'
+        stream.write(line)
+      } catch (error) {
+        console.error('Failed to write event to file:', error)
+      }
+    },
+    flush: async () => {
+      if (writeStream) {
+        return new Promise<void>((resolve, reject) => {
+          writeStream!.end((error?: Error) => {
+            if (error) reject(error)
+            else resolve()
+          })
+        })
+      }
+    },
+  }
+}
+
+/**
+ * Extract a key identifier from event data for logging
+ */
+function extractKey(event: TrackingEvent): string {
+  const data = event.data
+  if ('experimentId' in data) return `exp=${data.experimentId}`
+  if ('variantId' in data) return `variant=${data.variantId}`
+  if ('runId' in data) return `run=${data.runId}`
+  return ''
+}
+
+/**
+ * Get current tracking configuration
+ */
+export function getTrackingConfig(): Readonly<Required<TrackingOptions>> {
+  return { ...trackingConfig }
+}

package/src/types.ts

@@ -0,0 +1,215 @@
+/**
+ * Core types for AI experimentation
+ */
+
+/**
+ * A variant within an experiment
+ */
+export interface ExperimentVariant<TConfig = unknown> {
+  /** Unique identifier for the variant */
+  id: string
+  /** Human-readable name */
+  name: string
+  /** Variant configuration */
+  config: TConfig
+  /** Weight for weighted random selection (default: 1) */
+  weight?: number
+  /** Optional description */
+  description?: string
+}
+
+/**
+ * Configuration for an experiment
+ */
+export interface ExperimentConfig<TConfig = unknown, TResult = unknown> {
+  /** Unique experiment identifier */
+  id: string
+  /** Human-readable name */
+  name: string
+  /** Experiment description */
+  description?: string
+  /** List of variants to test */
+  variants: ExperimentVariant<TConfig>[]
+  /** Function to execute for each variant */
+  execute: (config: TConfig, context?: ExperimentContext) => Promise<TResult> | TResult
+  /** Optional success metric function */
+  metric?: (result: TResult) => number | Promise<number>
+  /** Metadata for the experiment */
+  metadata?: Record<string, unknown>
+}
+
+/**
+ * Context passed to experiment execution
+ */
+export interface ExperimentContext {
+  /** Experiment ID */
+  experimentId: string
+  /** Variant ID */
+  variantId: string
+  /** Run ID (unique per execution) */
+  runId: string
+  /** Timestamp when execution started */
+  startedAt: Date
+  /** Additional context data */
+  data?: Record<string, unknown>
+}
+
+/**
+ * Result of executing an experiment variant
+ */
+export interface ExperimentResult<TResult = unknown> {
+  /** Experiment ID */
+  experimentId: string
+  /** Variant ID */
+  variantId: string
+  /** Variant name */
+  variantName: string
+  /** Run ID */
+  runId: string
+  /** Execution result */
+  result: TResult
+  /** Computed metric value (if metric function provided) */
+  metricValue?: number
+  /** Execution duration in milliseconds */
+  duration: number
+  /** Timestamp when execution started */
+  startedAt: Date
+  /** Timestamp when execution completed */
+  completedAt: Date
+  /** Error if execution failed */
+  error?: Error
+  /** Success flag */
+  success: boolean
+  /** Additional metadata */
+  metadata?: Record<string, unknown>
+}
+
+/**
+ * Summary of experiment results across all variants
+ */
+export interface ExperimentSummary<TResult = unknown> {
+  /** Experiment ID */
+  experimentId: string
+  /** Experiment name */
+  experimentName: string
+  /** Results for each variant */
+  results: ExperimentResult<TResult>[]
+  /** Best performing variant (by metric) */
+  bestVariant?: {
+    variantId: string
+    variantName: string
+    metricValue: number
+  }
+  /** Total execution duration */
+  totalDuration: number
+  /** Number of successful runs */
+  successCount: number
+  /** Number of failed runs */
+  failureCount: number
+  /** Timestamp when experiment started */
+  startedAt: Date
+  /** Timestamp when experiment completed */
+  completedAt: Date
+}
+
+/**
+ * Options for running an experiment
+ */
+export interface RunExperimentOptions {
+  /** Run variants in parallel (default: true) */
+  parallel?: boolean
+  /** Maximum concurrent executions (default: unlimited) */
+  maxConcurrency?: number
+  /** Stop on first error (default: false) */
+  stopOnError?: boolean
+  /** Custom context data */
+  context?: Record<string, unknown>
+  /** Event callbacks */
+  onVariantStart?: (variantId: string, variantName: string) => void
+  onVariantComplete?: (result: ExperimentResult) => void
+  onVariantError?: (variantId: string, error: Error) => void
+}
+
+/**
+ * Parameters for cartesian product generation
+ */
+export type CartesianParams = Record<string, unknown[]>
+
+/**
+ * Result of cartesian product - array of parameter combinations
+ */
+export type CartesianResult<T extends CartesianParams> = Array<{
+  [K in keyof T]: T[K][number]
+}>
+
+/**
+ * Decision options
+ */
+export interface DecideOptions<T> {
+  /** Options to choose from */
+  options: T[]
+  /** Scoring function for each option */
+  score: (option: T) => number | Promise<number>
+  /** Context or prompt for decision making */
+  context?: string
+  /** Whether to return all options sorted by score (default: false) */
+  returnAll?: boolean
+}
+
+/**
+ * Result of a decision
+ */
+export interface DecisionResult<T> {
+  /** The selected option */
+  selected: T
+  /** Score of the selected option */
+  score: number
+  /** All options with their scores (if returnAll was true) */
+  allOptions?: Array<{ option: T; score: number }>
+}
+
+/**
+ * Tracking event types
+ */
+export type TrackingEventType =
+  | 'experiment.start'
+  | 'experiment.complete'
+  | 'variant.start'
+  | 'variant.complete'
+  | 'variant.error'
+  | 'metric.computed'
+  | 'decision.made'
+
+/**
+ * Tracking event
+ */
+export interface TrackingEvent {
+  /** Event type */
+  type: TrackingEventType
+  /** Timestamp */
+  timestamp: Date
+  /** Event data */
+  data: Record<string, unknown>
+}
+
+/**
+ * Tracking backend interface
+ */
+export interface TrackingBackend {
+  /** Track an event */
+  track(event: TrackingEvent): void | Promise<void>
+  /** Flush pending events */
+  flush?(): void | Promise<void>
+}
+
+/**
+ * Options for tracking configuration
+ */
+export interface TrackingOptions {
+  /** Custom tracking backend */
+  backend?: TrackingBackend
+  /** Whether tracking is enabled (default: true) */
+  enabled?: boolean
+  /** Additional metadata to include with all events */
+  metadata?: Record<string, unknown>
+}

package/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "dist"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.spec.ts"]
+}