ai-experiments 2.0.2 → 2.1.1

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # ai-experiments
 
+## 2.1.1
+
+### Patch Changes
+
+- Updated dependencies [6beb531]
+  - ai-functions@2.1.1
+
+## 2.0.3
+
+### Patch Changes
+
+- Updated dependencies
+  - rpc.do@0.2.0
+  - ai-functions@2.0.3
+
 ## 2.0.2
 
 ### Patch Changes

package/examples.js

@@ -0,0 +1,211 @@
+/**
+ * Usage examples for ai-experiments
+ *
+ * These examples demonstrate the core APIs and patterns.
+ */
+import { Experiment, cartesian, decide, configureTracking, createMemoryBackend, } from './src/index.js';
+// ============================================================================
+// Example 1: Simple A/B Experiment
+// ============================================================================
+async function example1_simpleExperiment() {
+    console.log('\n=== Example 1: Simple A/B Experiment ===\n');
+    const results = await Experiment({
+        id: 'greeting-test',
+        name: 'Greeting Message Test',
+        description: 'Test different greeting messages',
+        variants: [
+            {
+                id: 'formal',
+                name: 'Formal Greeting',
+                config: { message: 'Good day, sir.' },
+            },
+            {
+                id: 'casual',
+                name: 'Casual Greeting',
+                config: { message: 'Hey there!' },
+            },
+            {
+                id: 'friendly',
+                name: 'Friendly Greeting',
+                config: { message: 'Hello friend!' },
+            },
+        ],
+        execute: async (config) => {
+            // Simulate some work
+            await new Promise((resolve) => setTimeout(resolve, 100));
+            return {
+                message: config.message,
+                wordCount: config.message.split(' ').length,
+                charCount: config.message.length,
+            };
+        },
+        metric: (result) => {
+            // Score based on message length (prefer shorter)
+            return 1 / result.charCount;
+        },
+    });
+    console.log('Experiment Results:');
+    console.log('- Best Variant:', results.bestVariant);
+    console.log('- Success Rate:', `${results.successCount}/${results.results.length}`);
+    console.log('- Total Duration:', `${results.totalDuration}ms`);
+    console.log('\nAll Results:');
+    results.results.forEach((r) => {
+        console.log(`  ${r.variantName}: metric=${r.metricValue?.toFixed(4)}, duration=${r.duration}ms`);
+    });
+}
+// ============================================================================
+// Example 2: Parameter Grid with Cartesian Product
+// ============================================================================
+async function example2_parameterGrid() {
+    console.log('\n=== Example 2: Parameter Grid Experiment ===\n');
+    // Generate all combinations of parameters
+    const parameterGrid = cartesian({
+        temperature: [0.3, 0.7, 1.0],
+        maxTokens: [50, 100],
+        topP: [0.9, 1.0],
+    });
+    console.log(`Generated ${parameterGrid.length} parameter combinations:`);
+    parameterGrid.slice(0, 5).forEach((params, i) => {
+        console.log(`  ${i + 1}.`, params);
+    });
+    console.log('  ...');
+    // Create variants from parameter combinations
+    const variants = parameterGrid.map((params, i) => ({
+        id: `variant-${i}`,
+        name: `T=${params.temperature} max=${params.maxTokens} topP=${params.topP}`,
+        config: params,
+    }));
+    const results = await Experiment({
+        id: 'parameter-sweep',
+        name: 'Parameter Sweep Experiment',
+        variants,
+        execute: async (config) => {
+            // Simulate AI generation with different parameters
+            await new Promise((resolve) => setTimeout(resolve, 50));
+            const score = Math.random() * config.temperature + config.maxTokens / 1000;
+            return { score, config };
+        },
+        metric: (result) => result.score,
+    });
+    console.log(`\nBest parameters: ${results.bestVariant?.variantName}`);
+    console.log(`Best score: ${results.bestVariant?.metricValue.toFixed(4)}`);
+}
+// ============================================================================
+// Example 3: Decision Making
+// ============================================================================
+async function example3_decisionMaking() {
+    console.log('\n=== Example 3: Decision Making ===\n');
+    // Simple decision with sync scoring
+    const result1 = await decide({
+        options: ['apple', 'banana', 'orange'],
+        score: (fruit) => {
+            const prices = { apple: 1.5, banana: 0.5, orange: 2.0 };
+            return 1 / prices[fruit]; // Lower price = higher score
+        },
+        context: 'Choosing fruit based on value',
+    });
+    console.log('Decision 1 (best value):');
+    console.log(`  Selected: ${result1.selected}`);
+    console.log(`  Score: ${result1.score.toFixed(4)}`);
+    // Decision with all options returned
+    const result2 = await decide({
+        options: ['fast', 'accurate', 'balanced'],
+        score: async (approach) => {
+            const scores = { fast: 0.7, accurate: 0.85, balanced: 0.9 };
+            return scores[approach];
+        },
+        returnAll: true,
+    });
+    console.log('\nDecision 2 (with all options):');
+    console.log(`  Selected: ${result2.selected}`);
+    result2.allOptions?.forEach((opt) => {
+        console.log(`  - ${opt.option}: ${opt.score}`);
+    });
+}
+// ============================================================================
+// Example 4: Event Tracking
+// ============================================================================
+async function example4_eventTracking() {
+    console.log('\n=== Example 4: Event Tracking ===\n');
+    // Create a memory backend to collect events
+    const backend = createMemoryBackend();
+    configureTracking({ backend });
+    // Run a small experiment
+    await Experiment({
+        id: 'tracked-experiment',
+        name: 'Tracked Experiment',
+        variants: [
+            { id: 'v1', name: 'Variant 1', config: { value: 1 } },
+            { id: 'v2', name: 'Variant 2', config: { value: 2 } },
+        ],
+        execute: async (config) => ({ result: config.value * 2 }),
+        metric: (result) => result.result,
+    });
+    // Get all tracked events
+    const events = backend.getEvents();
+    console.log(`Tracked ${events.length} events:`);
+    events.forEach((event) => {
+        console.log(`  [${event.type}]`, JSON.stringify(event.data).slice(0, 80));
+    });
+    // Reset tracking to default
+    configureTracking({ enabled: true });
+}
+// ============================================================================
+// Example 5: Advanced - Sequential vs Parallel Execution
+// ============================================================================
+async function example5_executionModes() {
+    console.log('\n=== Example 5: Sequential vs Parallel Execution ===\n');
+    const variants = [
+        { id: 'v1', name: 'Variant 1', config: { delay: 100 } },
+        { id: 'v2', name: 'Variant 2', config: { delay: 150 } },
+        { id: 'v3', name: 'Variant 3', config: { delay: 200 } },
+    ];
+    const execute = async (config) => {
+        await new Promise((resolve) => setTimeout(resolve, config.delay));
+        return { completed: true };
+    };
+    // Parallel execution (default)
+    const start1 = Date.now();
+    await Experiment({
+        id: 'parallel-test',
+        name: 'Parallel Execution',
+        variants,
+        execute,
+    });
+    const parallel_duration = Date.now() - start1;
+    // Sequential execution
+    const start2 = Date.now();
+    await Experiment({
+        id: 'sequential-test',
+        name: 'Sequential Execution',
+        variants,
+        execute,
+    }, { parallel: false });
+    const sequential_duration = Date.now() - start2;
+    console.log(`Parallel execution: ${parallel_duration}ms`);
+    console.log(`Sequential execution: ${sequential_duration}ms`);
+    console.log(`Speedup: ${(sequential_duration / parallel_duration).toFixed(2)}x`);
+}
+// ============================================================================
+// Run all examples
+// ============================================================================
+async function main() {
+    // Disable verbose tracking for examples
+    configureTracking({ enabled: false });
+    try {
+        await example1_simpleExperiment();
+        await example2_parameterGrid();
+        await example3_decisionMaking();
+        await example4_eventTracking();
+        await example5_executionModes();
+        console.log('\n✅ All examples completed successfully!\n');
+    }
+    catch (error) {
+        console.error('\n❌ Error running examples:', error);
+        process.exit(1);
+    }
+}
+// Run if this file is executed directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+    main();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-experiments",
-  "version": "2.0.2",
+  "version": "2.1.1",
   "description": "AI-powered experimentation primitives for testing and evaluating models",
   "type": "module",
   "main": "dist/index.js",
@@ -9,6 +9,10 @@
     ".": {
       "import": "./dist/index.js",
       "types": "./dist/index.d.ts"
+    },
+    "./storage": {
+      "import": "./dist/chdb-storage.js",
+      "types": "./dist/chdb-storage.d.ts"
     }
   },
   "scripts": {
@@ -20,12 +24,14 @@
     "clean": "rm -rf dist"
   },
   "dependencies": {
-    "ai-functions": "2.0.2",
-    "rpc.do": "^0.1.0"
+    "ai-functions": "2.1.1",
+    "chdb": "^1.6.0"
   },
   "keywords": [
     "ai",
     "experiments",
+    "clickhouse",
+    "analytics",
     "primitives"
   ],
   "license": "MIT"

package/src/cartesian.js

@@ -0,0 +1,215 @@
+/**
+ * Cartesian product utilities for parameter exploration
+ */
+/**
+ * Generate cartesian product of parameter sets
+ *
+ * Takes an object where each key maps to an array of possible values,
+ * and returns all possible combinations as an array of objects.
+ *
+ * @example
+ * ```ts
+ * import { cartesian } from 'ai-experiments'
+ *
+ * const combinations = cartesian({
+ *   model: ['sonnet', 'opus', 'gpt-4o'],
+ *   temperature: [0.3, 0.7, 1.0],
+ *   maxTokens: [100, 500, 1000],
+ * })
+ *
+ * // Returns 27 combinations (3 * 3 * 3):
+ * // [
+ * //   { model: 'sonnet', temperature: 0.3, maxTokens: 100 },
+ * //   { model: 'sonnet', temperature: 0.3, maxTokens: 500 },
+ * //   { model: 'sonnet', temperature: 0.3, maxTokens: 1000 },
+ * //   { model: 'sonnet', temperature: 0.7, maxTokens: 100 },
+ * //   ...
+ * // ]
+ *
+ * // Use with experiments:
+ * const variants = combinations.map((config, i) => ({
+ *   id: `variant-${i}`,
+ *   name: `${config.model} T=${config.temperature} max=${config.maxTokens}`,
+ *   config,
+ * }))
+ * ```
+ */
+export function cartesian(params) {
+    const keys = Object.keys(params);
+    const values = keys.map((k) => params[k]);
+    // Handle empty input
+    if (keys.length === 0) {
+        return [];
+    }
+    // Handle single parameter
+    if (keys.length === 1) {
+        const key = keys[0];
+        return values[0].map((val) => ({ [key]: val }));
+    }
+    // Generate all combinations using recursive helper
+    const combinations = cartesianProduct(values);
+    // Map back to objects
+    return combinations.map((combo) => {
+        const obj = {};
+        keys.forEach((key, i) => {
+            obj[key] = combo[i];
+        });
+        return obj;
+    });
+}
+/**
+ * Recursive cartesian product implementation
+ */
+function cartesianProduct(arrays) {
+    if (arrays.length === 0)
+        return [[]];
+    const [first, ...rest] = arrays;
+    // Base case: single array
+    if (rest.length === 0) {
+        return first.map((x) => [x]);
+    }
+    // Recursive case
+    const restProduct = cartesianProduct(rest);
+    const result = [];
+    for (const x of first) {
+        for (const restCombo of restProduct) {
+            result.push([x, ...restCombo]);
+        }
+    }
+    return result;
+}
+/**
+ * Generate a grid of parameter combinations with filtering
+ *
+ * Similar to cartesian(), but allows filtering out invalid combinations.
+ *
+ * @example
+ * ```ts
+ * import { cartesianFilter } from 'ai-experiments'
+ *
+ * const combinations = cartesianFilter(
+ *   {
+ *     model: ['sonnet', 'opus'],
+ *     temperature: [0.3, 0.7, 1.0],
+ *     maxTokens: [100, 500],
+ *   },
+ *   // Filter out combinations where opus uses high temperature
+ *   (combo) => !(combo.model === 'opus' && combo.temperature > 0.7)
+ * )
+ * ```
+ */
+export function cartesianFilter(params, filter) {
+    const allCombinations = cartesian(params);
+    return allCombinations.filter(filter);
+}
+/**
+ * Generate a random sample from the cartesian product
+ *
+ * Useful when the full cartesian product is too large to test all combinations.
+ *
+ * @example
+ * ```ts
+ * import { cartesianSample } from 'ai-experiments'
+ *
+ * // Full product would be 1000 combinations (10 * 10 * 10)
+ * // Sample just 20 random combinations
+ * const sample = cartesianSample(
+ *   {
+ *     param1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+ *     param2: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+ *     param3: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+ *   },
+ *   20
+ * )
+ * ```
+ */
+export function cartesianSample(params, sampleSize, options = {}) {
+    const { unique = true } = options;
+    // Generate all combinations first
+    const allCombinations = cartesian(params);
+    // If sample size is larger than available combinations, return all
+    if (sampleSize >= allCombinations.length) {
+        return allCombinations;
+    }
+    // Shuffle and take first n items
+    const shuffled = [...allCombinations];
+    // Simple Fisher-Yates shuffle
+    for (let i = shuffled.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+    }
+    return shuffled.slice(0, sampleSize);
+}
+/**
+ * Count the total number of combinations without generating them
+ *
+ * Useful for checking if cartesian product is feasible before generating.
+ *
+ * @example
+ * ```ts
+ * import { cartesianCount } from 'ai-experiments'
+ *
+ * const count = cartesianCount({
+ *   model: ['sonnet', 'opus', 'gpt-4o'],
+ *   temperature: [0.3, 0.5, 0.7, 0.9],
+ *   maxTokens: [100, 500, 1000, 2000],
+ * })
+ * // Returns 48 (3 * 4 * 4)
+ *
+ * if (count > 100) {
+ *   console.log('Too many combinations, use cartesianSample instead')
+ * }
+ * ```
+ */
+export function cartesianCount(params) {
+    const keys = Object.keys(params);
+    if (keys.length === 0)
+        return 0;
+    return keys.reduce((total, key) => {
+        const arr = params[key];
+        return total * (arr?.length ?? 0);
+    }, 1);
+}
+/**
+ * Generate cartesian product with labels for each dimension
+ *
+ * Returns combinations with additional metadata about which dimension each value came from.
+ *
+ * @example
+ * ```ts
+ * import { cartesianWithLabels } from 'ai-experiments'
+ *
+ * const labeled = cartesianWithLabels({
+ *   model: ['sonnet', 'opus'],
+ *   temperature: [0.3, 0.7],
+ * })
+ * // [
+ * //   { values: { model: 'sonnet', temperature: 0.3 }, labels: { model: 0, temperature: 0 } },
+ * //   { values: { model: 'sonnet', temperature: 0.7 }, labels: { model: 0, temperature: 1 } },
+ * //   { values: { model: 'opus', temperature: 0.3 }, labels: { model: 1, temperature: 0 } },
+ * //   { values: { model: 'opus', temperature: 0.7 }, labels: { model: 1, temperature: 1 } },
+ * // ]
+ * ```
+ */
+export function cartesianWithLabels(params) {
+    const keys = Object.keys(params);
+    const values = keys.map((k) => params[k]);
+    if (keys.length === 0) {
+        return [];
+    }
+    const combinations = cartesianProduct(values);
+    return combinations.map((combo) => {
+        const valuesObj = {};
+        const labelsObj = {};
+        keys.forEach((key, i) => {
+            const value = combo[i];
+            valuesObj[key] = value;
+            const arr = params[key];
+            labelsObj[key] = arr ? arr.indexOf(value) : -1;
+        });
+        return {
+            values: valuesObj,
+            labels: labelsObj,
+        };
+    });
+}