ai-experiments 2.1.1 → 2.3.0

package/test/tracking.test.ts

@@ -0,0 +1,347 @@
+/**
+ * Tests for tracking utilities
+ *
+ * These tests verify the event tracking functionality including:
+ * - Track function basics
+ * - Configuration options
+ * - Backend implementations
+ * - Memory backend
+ * - Batch backend
+ */
+
+import { describe, it, expect, beforeEach, vi } from 'vitest'
+import {
+  track,
+  flush,
+  configureTracking,
+  getTrackingConfig,
+  createConsoleBackend,
+  createMemoryBackend,
+  createBatchBackend,
+} from '../src/index.js'
+
+describe('tracking configuration', () => {
+  beforeEach(() => {
+    // Reset to default state
+    configureTracking({
+      backend: createMemoryBackend(),
+      enabled: true,
+      metadata: {},
+    })
+  })
+
+  it('configures tracking with custom backend', () => {
+    const customBackend = createMemoryBackend()
+    configureTracking({ backend: customBackend })
+
+    const config = getTrackingConfig()
+    expect(config.backend).toBe(customBackend)
+  })
+
+  it('configures tracking enabled state', () => {
+    configureTracking({ enabled: false })
+    expect(getTrackingConfig().enabled).toBe(false)
+
+    configureTracking({ enabled: true })
+    expect(getTrackingConfig().enabled).toBe(true)
+  })
+
+  it('configures global metadata', () => {
+    configureTracking({ metadata: { project: 'test', env: 'development' } })
+
+    const config = getTrackingConfig()
+    expect(config.metadata).toEqual({ project: 'test', env: 'development' })
+  })
+
+  it('preserves existing config when partial update', () => {
+    const backend = createMemoryBackend()
+    configureTracking({ backend, enabled: true, metadata: { key: 'value' } })
+
+    // Update only enabled
+    configureTracking({ enabled: false })
+
+    const config = getTrackingConfig()
+    expect(config.backend).toBe(backend)
+    expect(config.enabled).toBe(false)
+    expect(config.metadata).toEqual({ key: 'value' })
+  })
+})
+
+describe('track function', () => {
+  let backend: ReturnType<typeof createMemoryBackend>
+
+  beforeEach(() => {
+    backend = createMemoryBackend()
+    configureTracking({ backend, enabled: true, metadata: {} })
+  })
+
+  it('tracks events to backend', () => {
+    track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: { experimentId: 'test-exp' },
+    })
+
+    const events = backend.getEvents()
+    expect(events).toHaveLength(1)
+    expect(events[0].type).toBe('experiment.start')
+    expect(events[0].data.experimentId).toBe('test-exp')
+  })
+
+  it('does not track when disabled', () => {
+    configureTracking({ enabled: false })
+
+    track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: {},
+    })
+
+    const events = backend.getEvents()
+    expect(events).toHaveLength(0)
+  })
+
+  it('merges global metadata with event data', () => {
+    configureTracking({
+      backend,
+      metadata: { projectId: 'proj-123', env: 'test' },
+    })
+
+    track({
+      type: 'variant.start',
+      timestamp: new Date(),
+      data: { variantId: 'v1' },
+    })
+
+    const events = backend.getEvents()
+    expect(events[0].data.variantId).toBe('v1')
+    expect(events[0].data.projectId).toBe('proj-123')
+    expect(events[0].data.env).toBe('test')
+  })
+
+  it('tracks all event types', () => {
+    const eventTypes = [
+      'experiment.start',
+      'experiment.complete',
+      'variant.start',
+      'variant.complete',
+      'variant.error',
+      'metric.computed',
+      'decision.made',
+    ] as const
+
+    for (const type of eventTypes) {
+      track({
+        type,
+        timestamp: new Date(),
+        data: { type },
+      })
+    }
+
+    const events = backend.getEvents()
+    expect(events).toHaveLength(eventTypes.length)
+  })
+})
+
+describe('flush function', () => {
+  it('calls backend flush when available', async () => {
+    const flushMock = vi.fn().mockResolvedValue(undefined)
+    const customBackend = {
+      track: vi.fn(),
+      flush: flushMock,
+    }
+
+    configureTracking({ backend: customBackend })
+
+    await flush()
+
+    expect(flushMock).toHaveBeenCalled()
+  })
+
+  it('handles backend without flush method', async () => {
+    const customBackend = {
+      track: vi.fn(),
+      // No flush method
+    }
+
+    configureTracking({ backend: customBackend })
+
+    // Should not throw
+    await expect(flush()).resolves.not.toThrow()
+  })
+})
+
+describe('createConsoleBackend', () => {
+  it('creates a tracking backend', () => {
+    const backend = createConsoleBackend()
+    expect(backend).toHaveProperty('track')
+    expect(typeof backend.track).toBe('function')
+  })
+
+  it('accepts verbose option', () => {
+    const verboseBackend = createConsoleBackend({ verbose: true })
+    expect(verboseBackend).toHaveProperty('track')
+  })
+
+  it('logs events without throwing', () => {
+    const backend = createConsoleBackend()
+    const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+
+    backend.track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: { experimentId: 'test' },
+    })
+
+    expect(consoleSpy).toHaveBeenCalled()
+    consoleSpy.mockRestore()
+  })
+})
+
+describe('createMemoryBackend', () => {
+  it('creates backend with getEvents and clear methods', () => {
+    const backend = createMemoryBackend()
+
+    expect(backend).toHaveProperty('track')
+    expect(backend).toHaveProperty('getEvents')
+    expect(backend).toHaveProperty('clear')
+  })
+
+  it('stores tracked events', () => {
+    const backend = createMemoryBackend()
+
+    backend.track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: { id: 1 },
+    })
+
+    backend.track({
+      type: 'experiment.complete',
+      timestamp: new Date(),
+      data: { id: 2 },
+    })
+
+    const events = backend.getEvents()
+    expect(events).toHaveLength(2)
+    expect(events[0].data.id).toBe(1)
+    expect(events[1].data.id).toBe(2)
+  })
+
+  it('clears all events', () => {
+    const backend = createMemoryBackend()
+
+    backend.track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: {},
+    })
+
+    expect(backend.getEvents()).toHaveLength(1)
+
+    backend.clear()
+
+    expect(backend.getEvents()).toHaveLength(0)
+  })
+
+  it('returns copy of events array', () => {
+    const backend = createMemoryBackend()
+
+    backend.track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: {},
+    })
+
+    const events1 = backend.getEvents()
+    const events2 = backend.getEvents()
+
+    // Should be equal but not same reference
+    expect(events1).toEqual(events2)
+    expect(events1).not.toBe(events2)
+  })
+})
+
+describe('createBatchBackend', () => {
+  it('creates backend with track and flush methods', () => {
+    const backend = createBatchBackend({
+      batchSize: 10,
+      send: async () => {},
+    })
+
+    expect(backend).toHaveProperty('track')
+    expect(backend).toHaveProperty('flush')
+  })
+
+  it('batches events until batchSize reached', async () => {
+    const sendMock = vi.fn().mockResolvedValue(undefined)
+    const backend = createBatchBackend({
+      batchSize: 3,
+      send: sendMock,
+    })
+
+    // Track 2 events - should not trigger send yet
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+
+    expect(sendMock).not.toHaveBeenCalled()
+
+    // Track 3rd event - should trigger batch send
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+
+    // Allow async processing
+    await new Promise((r) => setTimeout(r, 10))
+
+    expect(sendMock).toHaveBeenCalled()
+    expect(sendMock.mock.calls[0][0]).toHaveLength(3)
+  })
+
+  it('flushes remaining events on flush call', async () => {
+    const sendMock = vi.fn().mockResolvedValue(undefined)
+    const backend = createBatchBackend({
+      batchSize: 10,
+      send: sendMock,
+    })
+
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+
+    expect(sendMock).not.toHaveBeenCalled()
+
+    await backend.flush!()
+
+    expect(sendMock).toHaveBeenCalledTimes(1)
+    expect(sendMock.mock.calls[0][0]).toHaveLength(2)
+  })
+
+  it('does not send on flush when batch is empty', async () => {
+    const sendMock = vi.fn().mockResolvedValue(undefined)
+    const backend = createBatchBackend({
+      batchSize: 10,
+      send: sendMock,
+    })
+
+    await backend.flush!()
+
+    expect(sendMock).not.toHaveBeenCalled()
+  })
+
+  it('supports flush interval', async () => {
+    const sendMock = vi.fn().mockResolvedValue(undefined)
+    const backend = createBatchBackend({
+      batchSize: 100,
+      flushInterval: 50,
+      send: sendMock,
+    })
+
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+
+    // Wait for flush interval
+    await new Promise((r) => setTimeout(r, 100))
+
+    expect(sendMock).toHaveBeenCalled()
+
+    // Clean up timer
+    await backend.flush!()
+  })
+})

package/vitest.config.ts

@@ -0,0 +1,34 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    // CRITICAL: Limit concurrency to prevent resource exhaustion
+    maxConcurrency: 1,
+    maxWorkers: 1,
+    minWorkers: 1,
+    fileParallelism: false,
+
+    globals: false,
+    environment: 'node',
+    include: ['test/**/*.test.ts'],
+    exclude: ['node_modules/**', 'dist/**'],
+    testTimeout: 30000,
+    hookTimeout: 10000,
+
+    // Run tests sequentially
+    pool: 'forks',
+    poolOptions: {
+      forks: {
+        singleFork: true,
+      },
+    },
+
+    // Coverage configuration
+    coverage: {
+      provider: 'v8',
+      reporter: ['text', 'json', 'html'],
+      include: ['src/**/*.ts'],
+      exclude: ['**/*.test.ts', '**/__tests__/**', '**/node_modules/**'],
+    },
+  },
+})

package/.turbo/turbo-build.log

@@ -1,5 +0,0 @@
-
-
-> ai-experiments@2.0.1 build /Users/nathanclevenger/projects/primitives.org.ai/packages/ai-experiments
-> tsc
-

package/dist/cartesian.d.ts

@@ -1,140 +0,0 @@
-/**
- * Cartesian product utilities for parameter exploration
- */
-import type { CartesianParams, CartesianResult } from './types.js';
-/**
- * Generate cartesian product of parameter sets
- *
- * Takes an object where each key maps to an array of possible values,
- * and returns all possible combinations as an array of objects.
- *
- * @example
- * ```ts
- * import { cartesian } from 'ai-experiments'
- *
- * const combinations = cartesian({
- *   model: ['sonnet', 'opus', 'gpt-4o'],
- *   temperature: [0.3, 0.7, 1.0],
- *   maxTokens: [100, 500, 1000],
- * })
- *
- * // Returns 27 combinations (3 * 3 * 3):
- * // [
- * //   { model: 'sonnet', temperature: 0.3, maxTokens: 100 },
- * //   { model: 'sonnet', temperature: 0.3, maxTokens: 500 },
- * //   { model: 'sonnet', temperature: 0.3, maxTokens: 1000 },
- * //   { model: 'sonnet', temperature: 0.7, maxTokens: 100 },
- * //   ...
- * // ]
- *
- * // Use with experiments:
- * const variants = combinations.map((config, i) => ({
- *   id: `variant-${i}`,
- *   name: `${config.model} T=${config.temperature} max=${config.maxTokens}`,
- *   config,
- * }))
- * ```
- */
-export declare function cartesian<T extends CartesianParams>(params: T): CartesianResult<T>;
-/**
- * Generate a grid of parameter combinations with filtering
- *
- * Similar to cartesian(), but allows filtering out invalid combinations.
- *
- * @example
- * ```ts
- * import { cartesianFilter } from 'ai-experiments'
- *
- * const combinations = cartesianFilter(
- *   {
- *     model: ['sonnet', 'opus'],
- *     temperature: [0.3, 0.7, 1.0],
- *     maxTokens: [100, 500],
- *   },
- *   // Filter out combinations where opus uses high temperature
- *   (combo) => !(combo.model === 'opus' && combo.temperature > 0.7)
- * )
- * ```
- */
-export declare function cartesianFilter<T extends CartesianParams>(params: T, filter: (combo: {
-    [K in keyof T]: T[K][number];
-}) => boolean): CartesianResult<T>;
-/**
- * Generate a random sample from the cartesian product
- *
- * Useful when the full cartesian product is too large to test all combinations.
- *
- * @example
- * ```ts
- * import { cartesianSample } from 'ai-experiments'
- *
- * // Full product would be 1000 combinations (10 * 10 * 10)
- * // Sample just 20 random combinations
- * const sample = cartesianSample(
- *   {
- *     param1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
- *     param2: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
- *     param3: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
- *   },
- *   20
- * )
- * ```
- */
-export declare function cartesianSample<T extends CartesianParams>(params: T, sampleSize: number, options?: {
-    /** Random seed for reproducibility */
-    seed?: number;
-    /** Whether to sample without replacement (default: true) */
-    unique?: boolean;
-}): CartesianResult<T>;
-/**
- * Count the total number of combinations without generating them
- *
- * Useful for checking if cartesian product is feasible before generating.
- *
- * @example
- * ```ts
- * import { cartesianCount } from 'ai-experiments'
- *
- * const count = cartesianCount({
- *   model: ['sonnet', 'opus', 'gpt-4o'],
- *   temperature: [0.3, 0.5, 0.7, 0.9],
- *   maxTokens: [100, 500, 1000, 2000],
- * })
- * // Returns 48 (3 * 4 * 4)
- *
- * if (count > 100) {
- *   console.log('Too many combinations, use cartesianSample instead')
- * }
- * ```
- */
-export declare function cartesianCount<T extends CartesianParams>(params: T): number;
-/**
- * Generate cartesian product with labels for each dimension
- *
- * Returns combinations with additional metadata about which dimension each value came from.
- *
- * @example
- * ```ts
- * import { cartesianWithLabels } from 'ai-experiments'
- *
- * const labeled = cartesianWithLabels({
- *   model: ['sonnet', 'opus'],
- *   temperature: [0.3, 0.7],
- * })
- * // [
- * //   { values: { model: 'sonnet', temperature: 0.3 }, labels: { model: 0, temperature: 0 } },
- * //   { values: { model: 'sonnet', temperature: 0.7 }, labels: { model: 0, temperature: 1 } },
- * //   { values: { model: 'opus', temperature: 0.3 }, labels: { model: 1, temperature: 0 } },
- * //   { values: { model: 'opus', temperature: 0.7 }, labels: { model: 1, temperature: 1 } },
- * // ]
- * ```
- */
-export declare function cartesianWithLabels<T extends CartesianParams>(params: T): Array<{
-    values: {
-        [K in keyof T]: T[K][number];
-    };
-    labels: {
-        [K in keyof T]: number;
-    };
-}>;
-//# sourceMappingURL=cartesian.d.ts.map

package/dist/cartesian.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"cartesian.d.ts","sourceRoot":"","sources":["../src/cartesian.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAElE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,SAAS,CAAC,CAAC,SAAS,eAAe,EAAE,MAAM,EAAE,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,CA0BlF;AA4BD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,eAAe,CAAC,CAAC,SAAS,eAAe,EACvD,MAAM,EAAE,CAAC,EACT,MAAM,EAAE,CAAC,KAAK,EAAE;KAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;CAAE,KAAK,OAAO,GAC3D,eAAe,CAAC,CAAC,CAAC,CAGpB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,eAAe,CAAC,CAAC,SAAS,eAAe,EACvD,MAAM,EAAE,CAAC,EACT,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE;IACP,sCAAsC;IACtC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,4DAA4D;IAC5D,MAAM,CAAC,EAAE,OAAO,CAAA;CACZ,GACL,eAAe,CAAC,CAAC,CAAC,CAqBpB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,eAAe,EAAE,MAAM,EAAE,CAAC,GAAG,MAAM,CAS3E;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,SAAS,eAAe,EAC3D,MAAM,EAAE,CAAC,GACR,KAAK,CAAC;IACP,MAAM,EAAE;SAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;KAAE,CAAA;IACxC,MAAM,EAAE;SAAG,CAAC,IAAI,MAAM,CAAC,GAAG,MAAM;KAAE,CAAA;CACnC,CAAC,CA0BD"}