ai-experiments 2.1.3 → 2.4.0

package/src/cartesian.js

@@ -1,215 +0,0 @@
-/**
- * Cartesian product utilities for parameter exploration
- */
-/**
- * Generate cartesian product of parameter sets
- *
- * Takes an object where each key maps to an array of possible values,
- * and returns all possible combinations as an array of objects.
- *
- * @example
- * ```ts
- * import { cartesian } from 'ai-experiments'
- *
- * const combinations = cartesian({
- *   model: ['sonnet', 'opus', 'gpt-4o'],
- *   temperature: [0.3, 0.7, 1.0],
- *   maxTokens: [100, 500, 1000],
- * })
- *
- * // Returns 27 combinations (3 * 3 * 3):
- * // [
- * //   { model: 'sonnet', temperature: 0.3, maxTokens: 100 },
- * //   { model: 'sonnet', temperature: 0.3, maxTokens: 500 },
- * //   { model: 'sonnet', temperature: 0.3, maxTokens: 1000 },
- * //   { model: 'sonnet', temperature: 0.7, maxTokens: 100 },
- * //   ...
- * // ]
- *
- * // Use with experiments:
- * const variants = combinations.map((config, i) => ({
- *   id: `variant-${i}`,
- *   name: `${config.model} T=${config.temperature} max=${config.maxTokens}`,
- *   config,
- * }))
- * ```
- */
-export function cartesian(params) {
-    const keys = Object.keys(params);
-    const values = keys.map((k) => params[k]);
-    // Handle empty input
-    if (keys.length === 0) {
-        return [];
-    }
-    // Handle single parameter
-    if (keys.length === 1) {
-        const key = keys[0];
-        return values[0].map((val) => ({ [key]: val }));
-    }
-    // Generate all combinations using recursive helper
-    const combinations = cartesianProduct(values);
-    // Map back to objects
-    return combinations.map((combo) => {
-        const obj = {};
-        keys.forEach((key, i) => {
-            obj[key] = combo[i];
-        });
-        return obj;
-    });
-}
-/**
- * Recursive cartesian product implementation
- */
-function cartesianProduct(arrays) {
-    if (arrays.length === 0)
-        return [[]];
-    const [first, ...rest] = arrays;
-    // Base case: single array
-    if (rest.length === 0) {
-        return first.map((x) => [x]);
-    }
-    // Recursive case
-    const restProduct = cartesianProduct(rest);
-    const result = [];
-    for (const x of first) {
-        for (const restCombo of restProduct) {
-            result.push([x, ...restCombo]);
-        }
-    }
-    return result;
-}
-/**
- * Generate a grid of parameter combinations with filtering
- *
- * Similar to cartesian(), but allows filtering out invalid combinations.
- *
- * @example
- * ```ts
- * import { cartesianFilter } from 'ai-experiments'
- *
- * const combinations = cartesianFilter(
- *   {
- *     model: ['sonnet', 'opus'],
- *     temperature: [0.3, 0.7, 1.0],
- *     maxTokens: [100, 500],
- *   },
- *   // Filter out combinations where opus uses high temperature
- *   (combo) => !(combo.model === 'opus' && combo.temperature > 0.7)
- * )
- * ```
- */
-export function cartesianFilter(params, filter) {
-    const allCombinations = cartesian(params);
-    return allCombinations.filter(filter);
-}
-/**
- * Generate a random sample from the cartesian product
- *
- * Useful when the full cartesian product is too large to test all combinations.
- *
- * @example
- * ```ts
- * import { cartesianSample } from 'ai-experiments'
- *
- * // Full product would be 1000 combinations (10 * 10 * 10)
- * // Sample just 20 random combinations
- * const sample = cartesianSample(
- *   {
- *     param1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
- *     param2: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
- *     param3: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
- *   },
- *   20
- * )
- * ```
- */
-export function cartesianSample(params, sampleSize, options = {}) {
-    const { unique = true } = options;
-    // Generate all combinations first
-    const allCombinations = cartesian(params);
-    // If sample size is larger than available combinations, return all
-    if (sampleSize >= allCombinations.length) {
-        return allCombinations;
-    }
-    // Shuffle and take first n items
-    const shuffled = [...allCombinations];
-    // Simple Fisher-Yates shuffle
-    for (let i = shuffled.length - 1; i > 0; i--) {
-        const j = Math.floor(Math.random() * (i + 1));
-        [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
-    }
-    return shuffled.slice(0, sampleSize);
-}
-/**
- * Count the total number of combinations without generating them
- *
- * Useful for checking if cartesian product is feasible before generating.
- *
- * @example
- * ```ts
- * import { cartesianCount } from 'ai-experiments'
- *
- * const count = cartesianCount({
- *   model: ['sonnet', 'opus', 'gpt-4o'],
- *   temperature: [0.3, 0.5, 0.7, 0.9],
- *   maxTokens: [100, 500, 1000, 2000],
- * })
- * // Returns 48 (3 * 4 * 4)
- *
- * if (count > 100) {
- *   console.log('Too many combinations, use cartesianSample instead')
- * }
- * ```
- */
-export function cartesianCount(params) {
-    const keys = Object.keys(params);
-    if (keys.length === 0)
-        return 0;
-    return keys.reduce((total, key) => {
-        const arr = params[key];
-        return total * (arr?.length ?? 0);
-    }, 1);
-}
-/**
- * Generate cartesian product with labels for each dimension
- *
- * Returns combinations with additional metadata about which dimension each value came from.
- *
- * @example
- * ```ts
- * import { cartesianWithLabels } from 'ai-experiments'
- *
- * const labeled = cartesianWithLabels({
- *   model: ['sonnet', 'opus'],
- *   temperature: [0.3, 0.7],
- * })
- * // [
- * //   { values: { model: 'sonnet', temperature: 0.3 }, labels: { model: 0, temperature: 0 } },
- * //   { values: { model: 'sonnet', temperature: 0.7 }, labels: { model: 0, temperature: 1 } },
- * //   { values: { model: 'opus', temperature: 0.3 }, labels: { model: 1, temperature: 0 } },
- * //   { values: { model: 'opus', temperature: 0.7 }, labels: { model: 1, temperature: 1 } },
- * // ]
- * ```
- */
-export function cartesianWithLabels(params) {
-    const keys = Object.keys(params);
-    const values = keys.map((k) => params[k]);
-    if (keys.length === 0) {
-        return [];
-    }
-    const combinations = cartesianProduct(values);
-    return combinations.map((combo) => {
-        const valuesObj = {};
-        const labelsObj = {};
-        keys.forEach((key, i) => {
-            const value = combo[i];
-            valuesObj[key] = value;
-            const arr = params[key];
-            labelsObj[key] = arr ? arr.indexOf(value) : -1;
-        });
-        return {
-            values: valuesObj,
-            labels: labelsObj,
-        };
-    });
-}

package/src/chdb-storage.js

@@ -1,464 +0,0 @@
-/**
- * ClickHouse storage backend for AI experiments using chdb (embedded ClickHouse)
- *
- * Design: Single flat table where experiment events are the atomic unit.
- * Cartesian product results, variant comparisons, and winner analysis
- * are all derived via aggregation queries.
- *
- * This maximizes compression and enables fast analytical queries across
- * thousands of experiment runs.
- */
-import { Session } from 'chdb';
-const CREATE_TABLE_SQL = `
-CREATE TABLE IF NOT EXISTS experiments (
-    -- Event identity
-    eventId String,
-    eventType LowCardinality(String),
-    timestamp DateTime64(3),
-
-    -- Experiment context
-    experimentId String,
-    experimentName String DEFAULT '',
-
-    -- Variant context
-    variantId String DEFAULT '',
-    variantName String DEFAULT '',
-
-    -- Cartesian product dimensions (stored as JSON for flexibility)
-    dimensions String DEFAULT '{}',
-
-    -- Execution data
-    runId String DEFAULT '',
-    success UInt8 DEFAULT 1,
-    durationMs UInt32 DEFAULT 0,
-
-    -- Result data
-    result String DEFAULT '{}',
-    metricName LowCardinality(String) DEFAULT '',
-    metricValue Float64 DEFAULT 0,
-
-    -- Error tracking
-    errorMessage String DEFAULT '',
-    errorStack String DEFAULT '',
-
-    -- Metadata
-    metadata String DEFAULT '{}'
-)
-ENGINE = MergeTree()
-ORDER BY (experimentId, variantId, timestamp, eventId)
-SETTINGS index_granularity = 8192
-`;
-export class ChdbStorage {
-    options;
-    session;
-    initialized = false;
-    eventCounter = 0;
-    constructor(options = {}) {
-        this.options = options;
-        this.session = new Session(options.dataPath ?? './.experiments-chdb');
-    }
-    async init() {
-        if (this.initialized)
-            return;
-        if (this.options.autoInit !== false) {
-            this.session.query(CREATE_TABLE_SQL);
-        }
-        this.initialized = true;
-    }
-    escapeString(s) {
-        return s.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
-    }
-    toJson(value) {
-        return JSON.stringify(value ?? {});
-    }
-    parseJson(value) {
-        try {
-            return JSON.parse(value || '{}');
-        }
-        catch {
-            return {};
-        }
-    }
-    nextEventId() {
-        return `evt-${Date.now()}-${++this.eventCounter}`;
-    }
-    // ─────────────────────────────────────────────────────────────────────────────
-    // TrackingBackend Implementation
-    // ─────────────────────────────────────────────────────────────────────────────
-    async track(event) {
-        await this.init();
-        const eventId = this.nextEventId();
-        const timestamp = event.timestamp.toISOString();
-        const data = event.data;
-        // Extract standard fields from event data
-        const experimentId = String(data.experimentId ?? '');
-        const experimentName = String(data.experimentName ?? '');
-        const variantId = String(data.variantId ?? '');
-        const variantName = String(data.variantName ?? '');
-        const runId = String(data.runId ?? '');
-        const success = data.success !== false ? 1 : 0;
-        const durationMs = Number(data.duration ?? data.durationMs ?? 0);
-        const metricName = String(data.metricName ?? '');
-        const metricValue = Number(data.metricValue ?? 0);
-        const errorMessage = data.error instanceof Error ? data.error.message : String(data.errorMessage ?? '');
-        const errorStack = data.error instanceof Error ? (data.error.stack ?? '') : '';
-        // Extract dimensions (for cartesian product tracking)
-        const dimensions = data.dimensions ?? data.config ?? {};
-        const result = data.result ?? {};
-        const metadata = data.metadata ?? {};
-        this.session.query(`
-      INSERT INTO experiments (
-        eventId, eventType, timestamp,
-        experimentId, experimentName,
-        variantId, variantName,
-        dimensions, runId, success, durationMs,
-        result, metricName, metricValue,
-        errorMessage, errorStack, metadata
-      ) VALUES (
-        '${eventId}', '${event.type}', '${timestamp}',
-        '${this.escapeString(experimentId)}', '${this.escapeString(experimentName)}',
-        '${this.escapeString(variantId)}', '${this.escapeString(variantName)}',
-        '${this.escapeString(this.toJson(dimensions))}',
-        '${this.escapeString(runId)}', ${success}, ${durationMs},
-        '${this.escapeString(this.toJson(result))}',
-        '${this.escapeString(metricName)}', ${metricValue},
-        '${this.escapeString(errorMessage)}', '${this.escapeString(errorStack)}',
-        '${this.escapeString(this.toJson(metadata))}'
-      )
-    `);
-    }
-    async flush() {
-        // chdb writes are synchronous, nothing to flush
-    }
-    // ─────────────────────────────────────────────────────────────────────────────
-    // Experiment Result Storage (direct API)
-    // ─────────────────────────────────────────────────────────────────────────────
-    async storeResult(result) {
-        await this.track({
-            type: 'variant.complete',
-            timestamp: result.completedAt,
-            data: {
-                experimentId: result.experimentId,
-                variantId: result.variantId,
-                variantName: result.variantName,
-                runId: result.runId,
-                success: result.success,
-                duration: result.duration,
-                result: result.result,
-                metricValue: result.metricValue,
-                error: result.error,
-                metadata: result.metadata,
-            },
-        });
-    }
-    async storeResults(results) {
-        for (const result of results) {
-            await this.storeResult(result);
-        }
-    }
-    // ─────────────────────────────────────────────────────────────────────────────
-    // Analytics Queries
-    // ─────────────────────────────────────────────────────────────────────────────
-    /**
-     * Get all experiments
-     */
-    async getExperiments() {
-        await this.init();
-        const result = this.session.query(`
-      SELECT
-        experimentId,
-        any(experimentName) AS experimentName,
-        uniq(variantId) AS variantCount,
-        count() AS runCount,
-        min(timestamp) AS firstRun,
-        max(timestamp) AS lastRun
-      FROM experiments
-      WHERE experimentId != '' AND eventType = 'variant.complete'
-      GROUP BY experimentId
-      ORDER BY lastRun DESC
-    `, 'JSONEachRow');
-        if (!result.trim())
-            return [];
-        return result.trim().split('\n')
-            .filter(Boolean)
-            .map(line => JSON.parse(line));
-    }
-    /**
-     * Get variant performance for an experiment
-     */
-    async getVariantStats(experimentId) {
-        await this.init();
-        const result = this.session.query(`
-      SELECT
-        variantId,
-        any(variantName) AS variantName,
-        count() AS runCount,
-        countIf(success = 1) AS successCount,
-        countIf(success = 1) / count() AS successRate,
-        avg(durationMs) AS avgDuration,
-        avg(metricValue) AS avgMetric,
-        min(metricValue) AS minMetric,
-        max(metricValue) AS maxMetric
-      FROM experiments
-      WHERE experimentId = '${this.escapeString(experimentId)}'
-        AND eventType = 'variant.complete'
-      GROUP BY variantId
-      ORDER BY avgMetric DESC
-    `, 'JSONEachRow');
-        if (!result.trim())
-            return [];
-        return result.trim().split('\n')
-            .filter(Boolean)
-            .map(line => {
-            const row = JSON.parse(line);
-            return {
-                ...row,
-                runCount: Number(row.runCount),
-                successCount: Number(row.successCount),
-                successRate: Number(row.successRate),
-                avgDuration: Number(row.avgDuration),
-                avgMetric: Number(row.avgMetric),
-                minMetric: Number(row.minMetric),
-                maxMetric: Number(row.maxMetric),
-            };
-        });
-    }
-    /**
-     * Get the best performing variant for an experiment
-     */
-    async getBestVariant(experimentId, options = {}) {
-        const { metric = 'avgMetric', minimumRuns = 1 } = options;
-        await this.init();
-        const orderBy = metric === 'avgDuration' ? 'ASC' : 'DESC';
-        const metricExpr = metric === 'successRate'
-            ? 'countIf(success = 1) / count()'
-            : metric === 'avgDuration'
-                ? 'avg(durationMs)'
-                : 'avg(metricValue)';
-        const result = this.session.query(`
-      SELECT
-        variantId,
-        any(variantName) AS variantName,
-        ${metricExpr} AS metricValue,
-        count() AS runCount
-      FROM experiments
-      WHERE experimentId = '${this.escapeString(experimentId)}'
-        AND eventType = 'variant.complete'
-      GROUP BY variantId
-      HAVING runCount >= ${minimumRuns}
-      ORDER BY metricValue ${orderBy}
-      LIMIT 1
-    `, 'JSONEachRow');
-        if (!result.trim())
-            return null;
-        const row = JSON.parse(result.trim().split('\n')[0]);
-        return {
-            variantId: row.variantId,
-            variantName: row.variantName,
-            metricValue: Number(row.metricValue),
-            runCount: Number(row.runCount),
-        };
-    }
-    /**
-     * Get cartesian product analysis - performance by dimension values
-     */
-    async getCartesianAnalysis(experimentId, dimension) {
-        await this.init();
-        const result = this.session.query(`
-      SELECT
-        JSONExtractString(dimensions, '${this.escapeString(dimension)}') AS dimensionValue,
-        count() AS runCount,
-        avg(metricValue) AS avgMetric,
-        countIf(success = 1) / count() AS successRate
-      FROM experiments
-      WHERE experimentId = '${this.escapeString(experimentId)}'
-        AND eventType = 'variant.complete'
-        AND dimensionValue != ''
-      GROUP BY dimensionValue
-      ORDER BY avgMetric DESC
-    `, 'JSONEachRow');
-        if (!result.trim())
-            return [];
-        return result.trim().split('\n')
-            .filter(Boolean)
-            .map(line => {
-            const row = JSON.parse(line);
-            return {
-                dimensionValue: row.dimensionValue,
-                runCount: Number(row.runCount),
-                avgMetric: Number(row.avgMetric),
-                successRate: Number(row.successRate),
-            };
-        });
-    }
-    /**
-     * Get multi-dimensional cartesian analysis
-     */
-    async getCartesianGrid(experimentId, dimensions) {
-        await this.init();
-        const dimExtracts = dimensions.map(d => `JSONExtractString(dimensions, '${this.escapeString(d)}') AS dim_${d}`).join(', ');
-        const dimGroupBy = dimensions.map(d => `dim_${d}`).join(', ');
-        const result = this.session.query(`
-      SELECT
-        ${dimExtracts},
-        count() AS runCount,
-        avg(metricValue) AS avgMetric,
-        countIf(success = 1) / count() AS successRate
-      FROM experiments
-      WHERE experimentId = '${this.escapeString(experimentId)}'
-        AND eventType = 'variant.complete'
-      GROUP BY ${dimGroupBy}
-      ORDER BY avgMetric DESC
-    `, 'JSONEachRow');
-        if (!result.trim())
-            return [];
-        return result.trim().split('\n')
-            .filter(Boolean)
-            .map(line => {
-            const row = JSON.parse(line);
-            const dims = {};
-            for (const d of dimensions) {
-                dims[d] = row[`dim_${d}`];
-            }
-            return {
-                dimensions: dims,
-                runCount: Number(row.runCount),
-                avgMetric: Number(row.avgMetric),
-                successRate: Number(row.successRate),
-            };
-        });
-    }
-    /**
-     * Get time series of experiment metrics
-     */
-    async getTimeSeries(experimentId, options = {}) {
-        const { interval = 'day', variantId } = options;
-        await this.init();
-        const dateFunc = interval === 'hour'
-            ? 'toStartOfHour(timestamp)'
-            : interval === 'week'
-                ? 'toStartOfWeek(timestamp)'
-                : 'toStartOfDay(timestamp)';
-        const variantFilter = variantId
-            ? `AND variantId = '${this.escapeString(variantId)}'`
-            : '';
-        const result = this.session.query(`
-      SELECT
-        ${dateFunc} AS period,
-        count() AS runCount,
-        avg(metricValue) AS avgMetric,
-        countIf(success = 1) / count() AS successRate
-      FROM experiments
-      WHERE experimentId = '${this.escapeString(experimentId)}'
-        AND eventType = 'variant.complete'
-        ${variantFilter}
-      GROUP BY period
-      ORDER BY period ASC
-    `, 'JSONEachRow');
-        if (!result.trim())
-            return [];
-        return result.trim().split('\n')
-            .filter(Boolean)
-            .map(line => {
-            const row = JSON.parse(line);
-            return {
-                period: row.period,
-                runCount: Number(row.runCount),
-                avgMetric: Number(row.avgMetric),
-                successRate: Number(row.successRate),
-            };
-        });
-    }
-    /**
-     * Get raw events for an experiment
-     */
-    async getEvents(experimentId, options = {}) {
-        const { eventType, variantId, limit = 100 } = options;
-        await this.init();
-        const filters = [`experimentId = '${this.escapeString(experimentId)}'`];
-        if (eventType)
-            filters.push(`eventType = '${this.escapeString(eventType)}'`);
-        if (variantId)
-            filters.push(`variantId = '${this.escapeString(variantId)}'`);
-        const result = this.session.query(`
-      SELECT
-        eventType,
-        timestamp,
-        experimentId,
-        experimentName,
-        variantId,
-        variantName,
-        dimensions,
-        runId,
-        success,
-        durationMs,
-        result,
-        metricName,
-        metricValue,
-        errorMessage,
-        metadata
-      FROM experiments
-      WHERE ${filters.join(' AND ')}
-      ORDER BY timestamp DESC
-      LIMIT ${limit}
-    `, 'JSONEachRow');
-        if (!result.trim())
-            return [];
-        return result.trim().split('\n')
-            .filter(Boolean)
-            .map(line => {
-            const row = JSON.parse(line);
-            return {
-                type: row.eventType,
-                timestamp: new Date(row.timestamp),
-                data: {
-                    experimentId: row.experimentId,
-                    experimentName: row.experimentName,
-                    variantId: row.variantId,
-                    variantName: row.variantName,
-                    dimensions: this.parseJson(row.dimensions),
-                    runId: row.runId,
-                    success: row.success === 1,
-                    duration: row.durationMs,
-                    result: this.parseJson(row.result),
-                    metricName: row.metricName,
-                    metricValue: row.metricValue,
-                    errorMessage: row.errorMessage,
-                    metadata: this.parseJson(row.metadata),
-                },
-            };
-        });
-    }
-    /**
-     * Raw SQL query access for custom analytics
-     */
-    query(sql, format = 'JSONEachRow') {
-        return this.session.query(sql, format);
-    }
-    /**
-     * Close the storage (cleanup)
-     */
-    close() {
-        // Session cleanup handled by chdb
-    }
-}
-/**
- * Create a chdb storage backend for experiments
- *
- * @example
- * ```ts
- * import { configureTracking } from 'ai-experiments'
- * import { createChdbBackend } from 'ai-experiments/storage'
- *
- * const storage = createChdbBackend({ dataPath: './.my-experiments' })
- *
- * configureTracking({ backend: storage })
- *
- * // Later: analyze results
- * const best = await storage.getBestVariant('my-experiment')
- * console.log(`Best variant: ${best.variantName} (${best.metricValue})`)
- * ```
- */
-export function createChdbBackend(options) {
-    return new ChdbStorage(options);
-}