ai-experiments 2.1.3 → 2.4.0

package/test/tracking.test.ts

@@ -0,0 +1,347 @@
+/**
+ * Tests for tracking utilities
+ *
+ * These tests verify the event tracking functionality including:
+ * - Track function basics
+ * - Configuration options
+ * - Backend implementations
+ * - Memory backend
+ * - Batch backend
+ */
+
+import { describe, it, expect, beforeEach, vi } from 'vitest'
+import {
+  track,
+  flush,
+  configureTracking,
+  getTrackingConfig,
+  createConsoleBackend,
+  createMemoryBackend,
+  createBatchBackend,
+} from '../src/index.js'
+
+describe('tracking configuration', () => {
+  beforeEach(() => {
+    // Reset to default state
+    configureTracking({
+      backend: createMemoryBackend(),
+      enabled: true,
+      metadata: {},
+    })
+  })
+
+  it('configures tracking with custom backend', () => {
+    const customBackend = createMemoryBackend()
+    configureTracking({ backend: customBackend })
+
+    const config = getTrackingConfig()
+    expect(config.backend).toBe(customBackend)
+  })
+
+  it('configures tracking enabled state', () => {
+    configureTracking({ enabled: false })
+    expect(getTrackingConfig().enabled).toBe(false)
+
+    configureTracking({ enabled: true })
+    expect(getTrackingConfig().enabled).toBe(true)
+  })
+
+  it('configures global metadata', () => {
+    configureTracking({ metadata: { project: 'test', env: 'development' } })
+
+    const config = getTrackingConfig()
+    expect(config.metadata).toEqual({ project: 'test', env: 'development' })
+  })
+
+  it('preserves existing config when partial update', () => {
+    const backend = createMemoryBackend()
+    configureTracking({ backend, enabled: true, metadata: { key: 'value' } })
+
+    // Update only enabled
+    configureTracking({ enabled: false })
+
+    const config = getTrackingConfig()
+    expect(config.backend).toBe(backend)
+    expect(config.enabled).toBe(false)
+    expect(config.metadata).toEqual({ key: 'value' })
+  })
+})
+
+describe('track function', () => {
+  let backend: ReturnType<typeof createMemoryBackend>
+
+  beforeEach(() => {
+    backend = createMemoryBackend()
+    configureTracking({ backend, enabled: true, metadata: {} })
+  })
+
+  it('tracks events to backend', () => {
+    track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: { experimentId: 'test-exp' },
+    })
+
+    const events = backend.getEvents()
+    expect(events).toHaveLength(1)
+    expect(events[0].type).toBe('experiment.start')
+    expect(events[0].data.experimentId).toBe('test-exp')
+  })
+
+  it('does not track when disabled', () => {
+    configureTracking({ enabled: false })
+
+    track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: {},
+    })
+
+    const events = backend.getEvents()
+    expect(events).toHaveLength(0)
+  })
+
+  it('merges global metadata with event data', () => {
+    configureTracking({
+      backend,
+      metadata: { projectId: 'proj-123', env: 'test' },
+    })
+
+    track({
+      type: 'variant.start',
+      timestamp: new Date(),
+      data: { variantId: 'v1' },
+    })
+
+    const events = backend.getEvents()
+    expect(events[0].data.variantId).toBe('v1')
+    expect(events[0].data.projectId).toBe('proj-123')
+    expect(events[0].data.env).toBe('test')
+  })
+
+  it('tracks all event types', () => {
+    const eventTypes = [
+      'experiment.start',
+      'experiment.complete',
+      'variant.start',
+      'variant.complete',
+      'variant.error',
+      'metric.computed',
+      'decision.made',
+    ] as const
+
+    for (const type of eventTypes) {
+      track({
+        type,
+        timestamp: new Date(),
+        data: { type },
+      })
+    }
+
+    const events = backend.getEvents()
+    expect(events).toHaveLength(eventTypes.length)
+  })
+})
+
+describe('flush function', () => {
+  it('calls backend flush when available', async () => {
+    const flushMock = vi.fn().mockResolvedValue(undefined)
+    const customBackend = {
+      track: vi.fn(),
+      flush: flushMock,
+    }
+
+    configureTracking({ backend: customBackend })
+
+    await flush()
+
+    expect(flushMock).toHaveBeenCalled()
+  })
+
+  it('handles backend without flush method', async () => {
+    const customBackend = {
+      track: vi.fn(),
+      // No flush method
+    }
+
+    configureTracking({ backend: customBackend })
+
+    // Should not throw
+    await expect(flush()).resolves.not.toThrow()
+  })
+})
+
+describe('createConsoleBackend', () => {
+  it('creates a tracking backend', () => {
+    const backend = createConsoleBackend()
+    expect(backend).toHaveProperty('track')
+    expect(typeof backend.track).toBe('function')
+  })
+
+  it('accepts verbose option', () => {
+    const verboseBackend = createConsoleBackend({ verbose: true })
+    expect(verboseBackend).toHaveProperty('track')
+  })
+
+  it('logs events without throwing', () => {
+    const backend = createConsoleBackend()
+    const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
+
+    backend.track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: { experimentId: 'test' },
+    })
+
+    expect(consoleSpy).toHaveBeenCalled()
+    consoleSpy.mockRestore()
+  })
+})
+
+describe('createMemoryBackend', () => {
+  it('creates backend with getEvents and clear methods', () => {
+    const backend = createMemoryBackend()
+
+    expect(backend).toHaveProperty('track')
+    expect(backend).toHaveProperty('getEvents')
+    expect(backend).toHaveProperty('clear')
+  })
+
+  it('stores tracked events', () => {
+    const backend = createMemoryBackend()
+
+    backend.track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: { id: 1 },
+    })
+
+    backend.track({
+      type: 'experiment.complete',
+      timestamp: new Date(),
+      data: { id: 2 },
+    })
+
+    const events = backend.getEvents()
+    expect(events).toHaveLength(2)
+    expect(events[0].data.id).toBe(1)
+    expect(events[1].data.id).toBe(2)
+  })
+
+  it('clears all events', () => {
+    const backend = createMemoryBackend()
+
+    backend.track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: {},
+    })
+
+    expect(backend.getEvents()).toHaveLength(1)
+
+    backend.clear()
+
+    expect(backend.getEvents()).toHaveLength(0)
+  })
+
+  it('returns copy of events array', () => {
+    const backend = createMemoryBackend()
+
+    backend.track({
+      type: 'experiment.start',
+      timestamp: new Date(),
+      data: {},
+    })
+
+    const events1 = backend.getEvents()
+    const events2 = backend.getEvents()
+
+    // Should be equal but not same reference
+    expect(events1).toEqual(events2)
+    expect(events1).not.toBe(events2)
+  })
+})
+
+describe('createBatchBackend', () => {
+  it('creates backend with track and flush methods', () => {
+    const backend = createBatchBackend({
+      batchSize: 10,
+      send: async () => {},
+    })
+
+    expect(backend).toHaveProperty('track')
+    expect(backend).toHaveProperty('flush')
+  })
+
+  it('batches events until batchSize reached', async () => {
+    const sendMock = vi.fn().mockResolvedValue(undefined)
+    const backend = createBatchBackend({
+      batchSize: 3,
+      send: sendMock,
+    })
+
+    // Track 2 events - should not trigger send yet
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+
+    expect(sendMock).not.toHaveBeenCalled()
+
+    // Track 3rd event - should trigger batch send
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+
+    // Allow async processing
+    await new Promise((r) => setTimeout(r, 10))
+
+    expect(sendMock).toHaveBeenCalled()
+    expect(sendMock.mock.calls[0][0]).toHaveLength(3)
+  })
+
+  it('flushes remaining events on flush call', async () => {
+    const sendMock = vi.fn().mockResolvedValue(undefined)
+    const backend = createBatchBackend({
+      batchSize: 10,
+      send: sendMock,
+    })
+
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+
+    expect(sendMock).not.toHaveBeenCalled()
+
+    await backend.flush!()
+
+    expect(sendMock).toHaveBeenCalledTimes(1)
+    expect(sendMock.mock.calls[0][0]).toHaveLength(2)
+  })
+
+  it('does not send on flush when batch is empty', async () => {
+    const sendMock = vi.fn().mockResolvedValue(undefined)
+    const backend = createBatchBackend({
+      batchSize: 10,
+      send: sendMock,
+    })
+
+    await backend.flush!()
+
+    expect(sendMock).not.toHaveBeenCalled()
+  })
+
+  it('supports flush interval', async () => {
+    const sendMock = vi.fn().mockResolvedValue(undefined)
+    const backend = createBatchBackend({
+      batchSize: 100,
+      flushInterval: 50,
+      send: sendMock,
+    })
+
+    backend.track({ type: 'experiment.start', timestamp: new Date(), data: {} })
+
+    // Wait for flush interval
+    await new Promise((r) => setTimeout(r, 100))
+
+    expect(sendMock).toHaveBeenCalled()
+
+    // Clean up timer
+    await backend.flush!()
+  })
+})

package/vitest.config.ts

@@ -0,0 +1,34 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    // CRITICAL: Limit concurrency to prevent resource exhaustion
+    maxConcurrency: 1,
+    maxWorkers: 1,
+    minWorkers: 1,
+    fileParallelism: false,
+
+    globals: false,
+    environment: 'node',
+    include: ['test/**/*.test.ts'],
+    exclude: ['node_modules/**', 'dist/**'],
+    testTimeout: 30000,
+    hookTimeout: 10000,
+
+    // Run tests sequentially
+    pool: 'forks',
+    poolOptions: {
+      forks: {
+        singleFork: true,
+      },
+    },
+
+    // Coverage configuration
+    coverage: {
+      provider: 'v8',
+      reporter: ['text', 'json', 'html'],
+      include: ['src/**/*.ts'],
+      exclude: ['**/*.test.ts', '**/__tests__/**', '**/node_modules/**'],
+    },
+  },
+})

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2025 .org.ai
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/dist/chdb-storage.d.ts

@@ -1,135 +0,0 @@
-/**
- * ClickHouse storage backend for AI experiments using chdb (embedded ClickHouse)
- *
- * Design: Single flat table where experiment events are the atomic unit.
- * Cartesian product results, variant comparisons, and winner analysis
- * are all derived via aggregation queries.
- *
- * This maximizes compression and enables fast analytical queries across
- * thousands of experiment runs.
- */
-import type { TrackingBackend, TrackingEvent, ExperimentResult } from './types.js';
-export interface ChdbStorageOptions {
-    /** Path to store the chdb database (default: ./.experiments-chdb) */
-    dataPath?: string;
-    /** Whether to create tables on init (default: true) */
-    autoInit?: boolean;
-}
-export declare class ChdbStorage implements TrackingBackend {
-    private options;
-    private session;
-    private initialized;
-    private eventCounter;
-    constructor(options?: ChdbStorageOptions);
-    private init;
-    private escapeString;
-    private toJson;
-    private parseJson;
-    private nextEventId;
-    track(event: TrackingEvent): Promise<void>;
-    flush(): Promise<void>;
-    storeResult(result: ExperimentResult): Promise<void>;
-    storeResults(results: ExperimentResult[]): Promise<void>;
-    /**
-     * Get all experiments
-     */
-    getExperiments(): Promise<Array<{
-        experimentId: string;
-        experimentName: string;
-        variantCount: number;
-        runCount: number;
-        firstRun: string;
-        lastRun: string;
-    }>>;
-    /**
-     * Get variant performance for an experiment
-     */
-    getVariantStats(experimentId: string): Promise<Array<{
-        variantId: string;
-        variantName: string;
-        runCount: number;
-        successCount: number;
-        successRate: number;
-        avgDuration: number;
-        avgMetric: number;
-        minMetric: number;
-        maxMetric: number;
-    }>>;
-    /**
-     * Get the best performing variant for an experiment
-     */
-    getBestVariant(experimentId: string, options?: {
-        metric?: 'avgMetric' | 'successRate' | 'avgDuration';
-        minimumRuns?: number;
-    }): Promise<{
-        variantId: string;
-        variantName: string;
-        metricValue: number;
-        runCount: number;
-    } | null>;
-    /**
-     * Get cartesian product analysis - performance by dimension values
-     */
-    getCartesianAnalysis(experimentId: string, dimension: string): Promise<Array<{
-        dimensionValue: string;
-        runCount: number;
-        avgMetric: number;
-        successRate: number;
-    }>>;
-    /**
-     * Get multi-dimensional cartesian analysis
-     */
-    getCartesianGrid(experimentId: string, dimensions: string[]): Promise<Array<{
-        dimensions: Record<string, string>;
-        runCount: number;
-        avgMetric: number;
-        successRate: number;
-    }>>;
-    /**
-     * Get time series of experiment metrics
-     */
-    getTimeSeries(experimentId: string, options?: {
-        interval?: 'hour' | 'day' | 'week';
-        variantId?: string;
-    }): Promise<Array<{
-        period: string;
-        runCount: number;
-        avgMetric: number;
-        successRate: number;
-    }>>;
-    /**
-     * Get raw events for an experiment
-     */
-    getEvents(experimentId: string, options?: {
-        eventType?: string;
-        variantId?: string;
-        limit?: number;
-    }): Promise<TrackingEvent[]>;
-    /**
-     * Raw SQL query access for custom analytics
-     */
-    query(sql: string, format?: string): string;
-    /**
-     * Close the storage (cleanup)
-     */
-    close(): void;
-}
-/**
- * Create a chdb storage backend for experiments
- *
- * @example
- * ```ts
- * import { configureTracking } from 'ai-experiments'
- * import { createChdbBackend } from 'ai-experiments/storage'
- *
- * const storage = createChdbBackend({ dataPath: './.my-experiments' })
- *
- * configureTracking({ backend: storage })
- *
- * // Later: analyze results
- * const best = await storage.getBestVariant('my-experiment')
- * console.log(`Best variant: ${best.variantName} (${best.metricValue})`)
- * ```
- */
-export declare function createChdbBackend(options?: ChdbStorageOptions): ChdbStorage;
-//# sourceMappingURL=chdb-storage.d.ts.map

package/dist/chdb-storage.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"chdb-storage.d.ts","sourceRoot":"","sources":["../src/chdb-storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AA0ClF,MAAM,WAAW,kBAAkB;IACjC,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,uDAAuD;IACvD,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,qBAAa,WAAY,YAAW,eAAe;IAKrC,OAAO,CAAC,OAAO;IAJ3B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,YAAY,CAAI;gBAEJ,OAAO,GAAE,kBAAuB;YAItC,IAAI;IAQlB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,WAAW;IAQb,KAAK,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IA+C1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQtB,WAAW,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBpD,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAU9D;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,KAAK,CAAC;QACpC,YAAY,EAAE,MAAM,CAAA;QACpB,cAAc,EAAE,MAAM,CAAA;QACtB,YAAY,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,OAAO,EAAE,MAAM,CAAA;KAChB,CAAC,CAAC;IAwBH;;OAEG;IACG,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QACzD,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,MAAM,CAAA;QAChB,YAAY,EAAE,MAAM,CAAA;QACpB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,SAAS,EAAE,MAAM,CAAA;QACjB,SAAS,EAAE,MAAM,CAAA;QACjB,SAAS,EAAE,MAAM,CAAA;KAClB,CAAC,CAAC;IAwCH;;OAEG;IACG,cAAc,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE;QAClD,MAAM,CAAC,EAAE,WAAW,GAAG,aAAa,GAAG,aAAa,CAAA;QACpD,WAAW,CAAC,EAAE,MAAM,CAAA;KAChB,GAAG,OAAO,CAAC;QACf,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,MAAM,CAAA;KACjB,GAAG,IAAI,CAAC;IAuCT;;OAEG;IACG,oBAAoB,CAAC,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QACjF,cAAc,EAAE,MAAM,CAAA;QACtB,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAC,CAAC;IAgCH;;OAEG;IACG,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;QAChF,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAClC,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAC,CAAC;IAyCH;;OAEG;IACG,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE;QACjD,QAAQ,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAAA;QAClC,SAAS,CAAC,EAAE,MAAM,CAAA;KACd,GAAG,OAAO,CAAC,KAAK,CAAC;QACrB,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAC,CAAC;IA2CH;;OAEG;IACG,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE;QAC7C,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,KAAK,CAAC,EAAE,MAAM,CAAA;KACV,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA2DjC;;OAEG;IACH,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,MAAsB,GAAG,MAAM;IAI1D;;OAEG;IACH,KAAK,IAAI,IAAI;CAGd;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,WAAW,CAE3E"}