ai-experiments 2.3.0 → 2.4.0

package/dist/experiment.js

@@ -0,0 +1,292 @@
+/**
+ * Experiment execution and management
+ */
+import { randomUUID } from 'crypto';
+import { track } from './tracking.js';
+/**
+ * Create and run an A/B experiment with multiple variants
+ *
+ * @example
+ * ```ts
+ * import { Experiment } from 'ai-experiments'
+ *
+ * const results = await Experiment({
+ *   id: 'prompt-comparison',
+ *   name: 'Prompt Engineering Test',
+ *   variants: [
+ *     {
+ *       id: 'baseline',
+ *       name: 'Baseline Prompt',
+ *       config: { prompt: 'Summarize this text.' },
+ *     },
+ *     {
+ *       id: 'detailed',
+ *       name: 'Detailed Prompt',
+ *       config: { prompt: 'Provide a comprehensive summary...' },
+ *     },
+ *   ],
+ *   execute: async (config) => {
+ *     return await ai.generate({ prompt: config.prompt })
+ *   },
+ *   metric: (result) => result.quality_score,
+ * })
+ *
+ * console.log('Best variant:', results.bestVariant)
+ * ```
+ */
+export async function Experiment(config, options = {}) {
+    const { parallel = true, maxConcurrency, stopOnError = false, context: contextData, onVariantStart, onVariantComplete, onVariantError, } = options;
+    const experimentStartTime = new Date();
+    // Track experiment start
+    track({
+        type: 'experiment.start',
+        timestamp: experimentStartTime,
+        data: {
+            experimentId: config.id,
+            experimentName: config.name,
+            variantCount: config.variants.length,
+            parallel,
+            ...config.metadata,
+        },
+    });
+    const results = [];
+    // Execute variants
+    if (parallel) {
+        // Parallel execution with optional concurrency limit
+        const executeVariant = async (variant) => {
+            return runVariant(config, variant, contextData, {
+                ...(onVariantStart !== undefined && { onVariantStart }),
+                ...(onVariantComplete !== undefined && { onVariantComplete }),
+                ...(onVariantError !== undefined && { onVariantError }),
+            });
+        };
+        if (maxConcurrency && maxConcurrency > 0) {
+            // Execute with concurrency limit
+            const chunks = chunkArray(config.variants, maxConcurrency);
+            for (const chunk of chunks) {
+                const chunkResults = await Promise.all(chunk.map(executeVariant));
+                results.push(...chunkResults);
+                // Stop on first error if configured
+                if (stopOnError && chunkResults.some((r) => !r.success)) {
+                    break;
+                }
+            }
+        }
+        else {
+            // Execute all in parallel
+            const allResults = await Promise.all(config.variants.map(executeVariant));
+            results.push(...allResults);
+        }
+    }
+    else {
+        // Sequential execution
+        for (const variant of config.variants) {
+            const result = await runVariant(config, variant, contextData, {
+                ...(onVariantStart !== undefined && { onVariantStart }),
+                ...(onVariantComplete !== undefined && { onVariantComplete }),
+                ...(onVariantError !== undefined && { onVariantError }),
+            });
+            results.push(result);
+            // Stop on first error if configured
+            if (stopOnError && !result.success) {
+                break;
+            }
+        }
+    }
+    const experimentEndTime = new Date();
+    const totalDuration = experimentEndTime.getTime() - experimentStartTime.getTime();
+    // Find best variant by metric
+    let bestVariant;
+    const successfulResults = results.filter((r) => r.success && r.metricValue !== undefined);
+    if (successfulResults.length > 0) {
+        const best = successfulResults.reduce((prev, current) => (current.metricValue ?? -Infinity) > (prev.metricValue ?? -Infinity) ? current : prev);
+        bestVariant = {
+            variantId: best.variantId,
+            variantName: best.variantName,
+            metricValue: best.metricValue,
+        };
+    }
+    const summary = {
+        experimentId: config.id,
+        experimentName: config.name,
+        results,
+        ...(bestVariant !== undefined && { bestVariant }),
+        totalDuration,
+        successCount: results.filter((r) => r.success).length,
+        failureCount: results.filter((r) => !r.success).length,
+        startedAt: experimentStartTime,
+        completedAt: experimentEndTime,
+    };
+    // Track experiment completion
+    track({
+        type: 'experiment.complete',
+        timestamp: experimentEndTime,
+        data: {
+            experimentId: config.id,
+            experimentName: config.name,
+            successCount: summary.successCount,
+            failureCount: summary.failureCount,
+            totalDuration,
+            bestVariant: bestVariant?.variantId,
+            ...config.metadata,
+        },
+    });
+    return summary;
+}
+/**
+ * Run a single variant
+ */
+async function runVariant(config, variant, contextData, callbacks) {
+    const runId = randomUUID();
+    const startTime = new Date();
+    const context = {
+        experimentId: config.id,
+        variantId: variant.id,
+        runId,
+        startedAt: startTime,
+        ...(contextData !== undefined && { data: contextData }),
+    };
+    // Track variant start
+    track({
+        type: 'variant.start',
+        timestamp: startTime,
+        data: {
+            experimentId: config.id,
+            variantId: variant.id,
+            variantName: variant.name,
+            runId,
+        },
+    });
+    callbacks.onVariantStart?.(variant.id, variant.name);
+    try {
+        // Execute the variant
+        const result = await config.execute(variant.config, context);
+        const endTime = new Date();
+        const duration = endTime.getTime() - startTime.getTime();
+        // Compute metric if provided
+        let metricValue;
+        if (config.metric) {
+            metricValue = await config.metric(result);
+            track({
+                type: 'metric.computed',
+                timestamp: new Date(),
+                data: {
+                    experimentId: config.id,
+                    variantId: variant.id,
+                    runId,
+                    metricValue,
+                },
+            });
+        }
+        const experimentResult = {
+            experimentId: config.id,
+            variantId: variant.id,
+            variantName: variant.name,
+            runId,
+            result,
+            ...(metricValue !== undefined && { metricValue }),
+            duration,
+            startedAt: startTime,
+            completedAt: endTime,
+            success: true,
+        };
+        // Track variant completion
+        track({
+            type: 'variant.complete',
+            timestamp: endTime,
+            data: {
+                experimentId: config.id,
+                variantId: variant.id,
+                variantName: variant.name,
+                runId,
+                duration,
+                metricValue,
+                success: true,
+            },
+        });
+        callbacks.onVariantComplete?.(experimentResult);
+        return experimentResult;
+    }
+    catch (error) {
+        const endTime = new Date();
+        const duration = endTime.getTime() - startTime.getTime();
+        const err = error instanceof Error ? error : new Error(String(error));
+        // Track variant error
+        track({
+            type: 'variant.error',
+            timestamp: endTime,
+            data: {
+                experimentId: config.id,
+                variantId: variant.id,
+                variantName: variant.name,
+                runId,
+                duration,
+                error: err.message,
+                stack: err.stack,
+            },
+        });
+        callbacks.onVariantError?.(variant.id, err);
+        return {
+            experimentId: config.id,
+            variantId: variant.id,
+            variantName: variant.name,
+            runId,
+            result: undefined,
+            duration,
+            startedAt: startTime,
+            completedAt: endTime,
+            error: err,
+            success: false,
+        };
+    }
+}
+/**
+ * Split array into chunks
+ */
+function chunkArray(array, size) {
+    const chunks = [];
+    for (let i = 0; i < array.length; i += size) {
+        chunks.push(array.slice(i, i + size));
+    }
+    return chunks;
+}
+/**
+ * Helper to create experiment variants from a parameter grid
+ *
+ * @example
+ * ```ts
+ * const variants = createVariantsFromGrid({
+ *   temperature: [0.3, 0.7, 1.0],
+ *   model: ['sonnet', 'opus'],
+ *   maxTokens: [100, 500],
+ * })
+ * // Returns 12 variants (3 * 2 * 2 combinations)
+ * ```
+ */
+export function createVariantsFromGrid(grid) {
+    const keys = Object.keys(grid);
+    const values = keys.map((k) => grid[k]);
+    // Generate all combinations
+    const combinations = cartesianProduct(values);
+    return combinations.map((combo, index) => {
+        const config = Object.fromEntries(keys.map((key, i) => [key, combo[i]]));
+        return {
+            id: `variant-${index}`,
+            name: keys.map((k, i) => `${String(k)}=${combo[i]}`).join(', '),
+            config,
+        };
+    });
+}
+/**
+ * Cartesian product helper
+ */
+function cartesianProduct(arrays) {
+    if (arrays.length === 0)
+        return [[]];
+    if (arrays.length === 1)
+        return arrays[0].map((x) => [x]);
+    const [first, ...rest] = arrays;
+    const restProduct = cartesianProduct(rest);
+    return first.flatMap((x) => restProduct.map((arr) => [x, ...arr]));
+}
+//# sourceMappingURL=experiment.js.map

package/dist/experiment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../src/experiment.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AASnC,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAErC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAA0C,EAC1C,UAAgC,EAAE;IAElC,MAAM,EACJ,QAAQ,GAAG,IAAI,EACf,cAAc,EACd,WAAW,GAAG,KAAK,EACnB,OAAO,EAAE,WAAW,EACpB,cAAc,EACd,iBAAiB,EACjB,cAAc,GACf,GAAG,OAAO,CAAA;IAEX,MAAM,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAA;IAEtC,yBAAyB;IACzB,KAAK,CAAC;QACJ,IAAI,EAAE,kBAAkB;QACxB,SAAS,EAAE,mBAAmB;QAC9B,IAAI,EAAE;YACJ,YAAY,EAAE,MAAM,CAAC,EAAE;YACvB,cAAc,EAAE,MAAM,CAAC,IAAI;YAC3B,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;YACpC,QAAQ;YACR,GAAG,MAAM,CAAC,QAAQ;SACnB;KACF,CAAC,CAAA;IAEF,MAAM,OAAO,GAAgC,EAAE,CAAA;IAE/C,mBAAmB;IACnB,IAAI,QAAQ,EAAE,CAAC;QACb,qDAAqD;QACrD,MAAM,cAAc,GAAG,KAAK,EAC1B,OAAmC,EACC,EAAE;YACtC,OAAO,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;gBAC9C,GAAG,CAAC,cAAc,KAAK,SAAS,IAAI,EAAE,cAAc,EAAE,CAAC;gBACvD,GAAG,CAAC,iBAAiB,KAAK,SAAS,IAAI,EAAE,iBAAiB,EAAE,CAAC;gBAC7D,GAAG,CAAC,cAAc,KAAK,SAAS,IAAI,EAAE,cAAc,EAAE,CAAC;aACxD,CAAC,CAAA;QACJ,CAAC,CAAA;QAED,IAAI,cAAc,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACzC,iCAAiC;YACjC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAA;YAC1D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAA;gBACjE,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAA;gBAE7B,oCAAoC;gBACpC,IAAI,WAAW,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxD,MAAK;gBACP,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,0BAA0B;YAC1B,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAA;YACzE,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;QAC7B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,uBAAuB;QACvB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;gBAC5D,GAAG,CAAC,cAAc,KAAK,SAAS,IAAI,EAAE,cAAc,EAAE,CAAC;gBACvD,GAAG,CAAC,iBAAiB,KAAK,SAAS,IAAI,EAAE,iBAAiB,EAAE,CAAC;gBAC7D,GAAG,CAAC,cAAc,KAAK,SAAS,IAAI,EAAE,cAAc,EAAE,CAAC;aACxD,CAAC,CAAA;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEpB,oCAAoC;YACpC,IAAI,WAAW,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnC,MAAK;YACP,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAA;IACpC,MAAM,aAAa,GAAG,iBAAiB,CAAC,OAAO,EAAE,GAAG,mBAAmB,CAAC,OAAO,EAAE,CAAA;IAEjF,8BAA8B;IAC9B,IAAI,WAAsD,CAAA;IAC1D,MAAM,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAA;IACzF,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,CACtD,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACtF,CAAA;QACD,WAAW,GAAG;YACZ,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAY;SAC/B,CAAA;IACH,CAAC;IAED,MAAM,OAAO,GAA+B;QAC1C,YAAY,EAAE,MAAM,CAAC,EAAE;QACvB,cAAc,EAAE,MAAM,CAAC,IAAI;QAC3B,OAAO;QACP,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,CAAC;QACjD,aAAa;QACb,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM;QACrD,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM;QACtD,SAAS,EAAE,mBAAmB;QAC9B,WAAW,EAAE,iBAAiB;KAC/B,CAAA;IAED,8BAA8B;IAC9B,KAAK,CAAC;QACJ,IAAI,EAAE,qBAAqB;QAC3B,SAAS,EAAE,iBAAiB;QAC5B,IAAI,EAAE;YACJ,YAAY,EAAE,MAAM,CAAC,EAAE;YACvB,cAAc,EAAE,MAAM,CAAC,IAAI;YAC3B,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,aAAa;YACb,WAAW,EAAE,WAAW,EAAE,SAAS;YACnC,GAAG,MAAM,CAAC,QAAQ;SACnB;KACF,CAAC,CAAA;IAEF,OAAO,OAAO,CAAA;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CACvB,MAA0C,EAC1C,OAAmC,EACnC,WAAgD,EAChD,SAIC;IAED,MAAM,KAAK,GAAG,UAAU,EAAE,CAAA;IAC1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAA;IAE5B,MAAM,OAAO,GAAsB;QACjC,YAAY,EAAE,MAAM,CAAC,EAAE;QACvB,SAAS,EAAE,OAAO,CAAC,EAAE;QACrB,KAAK;QACL,SAAS,EAAE,SAAS;QACpB,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;KACxD,CAAA;IAED,sBAAsB;IACtB,KAAK,CAAC;QACJ,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,SAAS;QACpB,IAAI,EAAE;YACJ,YAAY,EAAE,MAAM,CAAC,EAAE;YACvB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,KAAK;SACN;KACF,CAAC,CAAA;IAEF,SAAS,CAAC,cAAc,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IAEpD,IAAI,CAAC;QACH,sBAAsB;QACtB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAC5D,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAA;QAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAA;QAExD,6BAA6B;QAC7B,IAAI,WAA+B,CAAA;QACnC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,WAAW,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAEzC,KAAK,CAAC;gBACJ,IAAI,EAAE,iBAAiB;gBACvB,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,IAAI,EAAE;oBACJ,YAAY,EAAE,MAAM,CAAC,EAAE;oBACvB,SAAS,EAAE,OAAO,CAAC,EAAE;oBACrB,KAAK;oBACL,WAAW;iBACZ;aACF,CAAC,CAAA;QACJ,CAAC;QAED,MAAM,gBAAgB,GAA8B;YAClD,YAAY,EAAE,MAAM,CAAC,EAAE;YACvB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,KAAK;YACL,MAAM;YACN,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,CAAC;YACjD,QAAQ;YACR,SAAS,EAAE,SAAS;YACpB,WAAW,EAAE,OAAO;YACpB,OAAO,EAAE,IAAI;SACd,CAAA;QAED,2BAA2B;QAC3B,KAAK,CAAC;YACJ,IAAI,EAAE,kBAAkB;YACxB,SAAS,EAAE,OAAO;YAClB,IAAI,EAAE;gBACJ,YAAY,EAAE,MAAM,CAAC,EAAE;gBACvB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,KAAK;gBACL,QAAQ;gBACR,WAAW;gBACX,OAAO,EAAE,IAAI;aACd;SACF,CAAC,CAAA;QAEF,SAAS,CAAC,iBAAiB,EAAE,CAAC,gBAAgB,CAAC,CAAA;QAE/C,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAA;QAC1B,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAA;QACxD,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;QAErE,sBAAsB;QACtB,KAAK,CAAC;YACJ,IAAI,EAAE,eAAe;YACrB,SAAS,EAAE,OAAO;YAClB,IAAI,EAAE;gBACJ,YAAY,EAAE,MAAM,CAAC,EAAE;gBACvB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,KAAK;gBACL,QAAQ;gBACR,KAAK,EAAE,GAAG,CAAC,OAAO;gBAClB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB;SACF,CAAC,CAAA;QAEF,SAAS,CAAC,cAAc,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;QAE3C,OAAO;YACL,YAAY,EAAE,MAAM,CAAC,EAAE;YACvB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,KAAK;YACL,MAAM,EAAE,SAA+B;YACvC,QAAQ;YACR,SAAS,EAAE,SAAS;YACpB,WAAW,EAAE,OAAO;YACpB,KAAK,EAAE,GAAG;YACV,OAAO,EAAE,KAAK;SACf,CAAA;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAI,KAAU,EAAE,IAAY;IAC7C,MAAM,MAAM,GAAU,EAAE,CAAA;IACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;IACvC,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,sBAAsB,CACpC,IAAO;IAEP,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAgB,CAAA;IAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEvC,4BAA4B;IAC5B,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAA;IAE7C,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACvC,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAEtE,CAAA;QAED,OAAO;YACL,EAAE,EAAE,WAAW,KAAK,EAAE;YACtB,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAC/D,MAAM;SACP,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAI,MAAa;IACxC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IACpC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAE1D,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,MAAM,CAAA;IAC/B,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAE1C,OAAO,KAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAA;AACrE,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * ai-experiments - AI-powered experimentation primitives for testing and evaluating models
+ *
+ * This package provides tools for A/B testing, parameter exploration, decision making,
+ * and tracking in AI applications.
+ *
+ * @packageDocumentation
+ */
+export * from './types.js';
+export { Experiment, createVariantsFromGrid } from './experiment.js';
+export { cartesian, cartesianFilter, cartesianSample, cartesianCount, cartesianWithLabels, } from './cartesian.js';
+export { decide, decideWeighted, decideEpsilonGreedy, decideThompsonSampling, decideUCB, } from './decide.js';
+export { track, flush, configureTracking, getTrackingConfig, createConsoleBackend, createMemoryBackend, createBatchBackend, createFileBackend, } from './tracking.js';
+export { ClickHouseExperimentStorage, createClickHouseExperimentStorage, bootstrapExperimentsSchema, ChdbStorage, createChdbBackend, } from './clickhouse-storage.js';
+export type { ClickHouseExperimentStorageOptions, ChdbStorageOptions, } from './clickhouse-storage.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,cAAc,YAAY,CAAA;AAG1B,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAA;AAGpE,OAAO,EACL,SAAS,EACT,eAAe,EACf,eAAe,EACf,cAAc,EACd,mBAAmB,GACpB,MAAM,gBAAgB,CAAA;AAGvB,OAAO,EACL,MAAM,EACN,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,GACV,MAAM,aAAa,CAAA;AAGpB,OAAO,EACL,KAAK,EACL,KAAK,EACL,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,eAAe,CAAA;AAGtB,OAAO,EACL,2BAA2B,EAC3B,iCAAiC,EACjC,0BAA0B,EAE1B,WAAW,EACX,iBAAiB,GAClB,MAAM,yBAAyB,CAAA;AAChC,YAAY,EACV,kCAAkC,EAClC,kBAAkB,GACnB,MAAM,yBAAyB,CAAA"}

package/dist/index.js

@@ -0,0 +1,23 @@
+/**
+ * ai-experiments - AI-powered experimentation primitives for testing and evaluating models
+ *
+ * This package provides tools for A/B testing, parameter exploration, decision making,
+ * and tracking in AI applications.
+ *
+ * @packageDocumentation
+ */
+// Export core types
+export * from './types.js';
+// Export experiment functionality
+export { Experiment, createVariantsFromGrid } from './experiment.js';
+// Export cartesian product utilities
+export { cartesian, cartesianFilter, cartesianSample, cartesianCount, cartesianWithLabels, } from './cartesian.js';
+// Export decision making utilities
+export { decide, decideWeighted, decideEpsilonGreedy, decideThompsonSampling, decideUCB, } from './decide.js';
+// Export tracking utilities
+export { track, flush, configureTracking, getTrackingConfig, createConsoleBackend, createMemoryBackend, createBatchBackend, createFileBackend, } from './tracking.js';
+// Export ClickHouse storage backend (canonical adapter via ai-database)
+export { ClickHouseExperimentStorage, createClickHouseExperimentStorage, bootstrapExperimentsSchema, 
+// Backwards-compat aliases
+ChdbStorage, createChdbBackend, } from './clickhouse-storage.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,oBAAoB;AACpB,cAAc,YAAY,CAAA;AAE1B,kCAAkC;AAClC,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,qCAAqC;AACrC,OAAO,EACL,SAAS,EACT,eAAe,EACf,eAAe,EACf,cAAc,EACd,mBAAmB,GACpB,MAAM,gBAAgB,CAAA;AAEvB,mCAAmC;AACnC,OAAO,EACL,MAAM,EACN,cAAc,EACd,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,GACV,MAAM,aAAa,CAAA;AAEpB,4BAA4B;AAC5B,OAAO,EACL,KAAK,EACL,KAAK,EACL,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,eAAe,CAAA;AAEtB,wEAAwE;AACxE,OAAO,EACL,2BAA2B,EAC3B,iCAAiC,EACjC,0BAA0B;AAC1B,2BAA2B;AAC3B,WAAW,EACX,iBAAiB,GAClB,MAAM,yBAAyB,CAAA"}

package/dist/tracking.d.ts

@@ -0,0 +1,159 @@
+/**
+ * Event tracking for experiments
+ */
+import type { TrackingEvent, TrackingBackend, TrackingOptions } from './types.js';
+/**
+ * Configure tracking
+ *
+ * @example
+ * ```ts
+ * import { configureTracking } from 'ai-experiments'
+ *
+ * // Use console backend (default)
+ * configureTracking({
+ *   enabled: true,
+ *   metadata: { projectId: 'my-project' },
+ * })
+ *
+ * // Use custom backend
+ * configureTracking({
+ *   backend: {
+ *     track: async (event) => {
+ *       await fetch('/api/analytics', {
+ *         method: 'POST',
+ *         body: JSON.stringify(event),
+ *       })
+ *     },
+ *   },
+ * })
+ *
+ * // Disable tracking
+ * configureTracking({ enabled: false })
+ * ```
+ */
+export declare function configureTracking(options: TrackingOptions): void;
+/**
+ * Track an event
+ *
+ * @example
+ * ```ts
+ * import { track } from 'ai-experiments'
+ *
+ * track({
+ *   type: 'experiment.start',
+ *   timestamp: new Date(),
+ *   data: {
+ *     experimentId: 'my-experiment',
+ *     variantCount: 3,
+ *   },
+ * })
+ * ```
+ */
+export declare function track(event: TrackingEvent): void;
+/**
+ * Flush pending events
+ *
+ * Call this before the process exits to ensure all events are sent.
+ *
+ * @example
+ * ```ts
+ * import { flush } from 'ai-experiments'
+ *
+ * process.on('SIGINT', async () => {
+ *   await flush()
+ *   process.exit(0)
+ * })
+ * ```
+ */
+export declare function flush(): Promise<void>;
+/**
+ * Create a console-based tracking backend
+ *
+ * Logs events to console.log in a human-readable format.
+ */
+export declare function createConsoleBackend(options?: {
+    /** Whether to include full event data (default: false) */
+    verbose?: boolean;
+}): TrackingBackend;
+/**
+ * Create an in-memory tracking backend that stores events
+ *
+ * Useful for testing or collecting events for batch processing.
+ *
+ * @example
+ * ```ts
+ * import { createMemoryBackend } from 'ai-experiments'
+ *
+ * const backend = createMemoryBackend()
+ * configureTracking({ backend })
+ *
+ * // Run experiments...
+ *
+ * // Get all events
+ * const events = backend.getEvents()
+ * console.log(`Tracked ${events.length} events`)
+ *
+ * // Clear events
+ * backend.clear()
+ * ```
+ */
+export declare function createMemoryBackend(): TrackingBackend & {
+    getEvents: () => TrackingEvent[];
+    clear: () => void;
+};
+/**
+ * Create a batching tracking backend
+ *
+ * Batches events and sends them in groups to reduce network overhead.
+ *
+ * @example
+ * ```ts
+ * import { createBatchBackend } from 'ai-experiments'
+ *
+ * const backend = createBatchBackend({
+ *   batchSize: 10,
+ *   flushInterval: 5000, // 5 seconds
+ *   send: async (events) => {
+ *     await fetch('/api/analytics/batch', {
+ *       method: 'POST',
+ *       body: JSON.stringify({ events }),
+ *     })
+ *   },
+ * })
+ *
+ * configureTracking({ backend })
+ * ```
+ */
+export declare function createBatchBackend(options: {
+    /** Maximum batch size before auto-flush */
+    batchSize: number;
+    /** Interval in ms to auto-flush (default: no auto-flush) */
+    flushInterval?: number;
+    /** Function to send batched events */
+    send: (events: TrackingEvent[]) => Promise<void>;
+}): TrackingBackend;
+/**
+ * Create a file-based tracking backend
+ *
+ * Writes events to a file (JSONL format).
+ *
+ * @example
+ * ```ts
+ * import { createFileBackend } from 'ai-experiments'
+ *
+ * const backend = createFileBackend({
+ *   path: './experiments.jsonl',
+ * })
+ *
+ * configureTracking({ backend })
+ * ```
+ */
+export declare function createFileBackend(options: {
+    /** Path to the file */
+    path: string;
+}): TrackingBackend;
+/**
+ * Get current tracking configuration
+ */
+export declare function getTrackingConfig(): Readonly<Required<TrackingOptions>>;
+//# sourceMappingURL=tracking.d.ts.map

package/dist/tracking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tracking.d.ts","sourceRoot":"","sources":["../src/tracking.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAWjF;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,CAMhE;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAsBhD;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAI3C;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,CAAC,EAAE;IAC7C,0DAA0D;IAC1D,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB,GAAG,eAAe,CAgBlB;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,mBAAmB,IAAI,eAAe,GAAG;IACvD,SAAS,EAAE,MAAM,aAAa,EAAE,CAAA;IAChC,KAAK,EAAE,MAAM,IAAI,CAAA;CAClB,CAYA;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE;IAC1C,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAA;IACjB,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,sCAAsC;IACtC,IAAI,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CACjD,GAAG,eAAe,CAkDlB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,uBAAuB;IACvB,IAAI,EAAE,MAAM,CAAA;CACb,GAAG,eAAe,CAwClB;AAaD;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAEvE"}