ai-experiments 2.1.1 → 2.3.0

package/dist/types.d.ts

@@ -1,198 +0,0 @@
-/**
- * Core types for AI experimentation
- */
-/**
- * A variant within an experiment
- */
-export interface ExperimentVariant<TConfig = unknown> {
-    /** Unique identifier for the variant */
-    id: string;
-    /** Human-readable name */
-    name: string;
-    /** Variant configuration */
-    config: TConfig;
-    /** Weight for weighted random selection (default: 1) */
-    weight?: number;
-    /** Optional description */
-    description?: string;
-}
-/**
- * Configuration for an experiment
- */
-export interface ExperimentConfig<TConfig = unknown, TResult = unknown> {
-    /** Unique experiment identifier */
-    id: string;
-    /** Human-readable name */
-    name: string;
-    /** Experiment description */
-    description?: string;
-    /** List of variants to test */
-    variants: ExperimentVariant<TConfig>[];
-    /** Function to execute for each variant */
-    execute: (config: TConfig, context?: ExperimentContext) => Promise<TResult> | TResult;
-    /** Optional success metric function */
-    metric?: (result: TResult) => number | Promise<number>;
-    /** Metadata for the experiment */
-    metadata?: Record<string, unknown>;
-}
-/**
- * Context passed to experiment execution
- */
-export interface ExperimentContext {
-    /** Experiment ID */
-    experimentId: string;
-    /** Variant ID */
-    variantId: string;
-    /** Run ID (unique per execution) */
-    runId: string;
-    /** Timestamp when execution started */
-    startedAt: Date;
-    /** Additional context data */
-    data?: Record<string, unknown>;
-}
-/**
- * Result of executing an experiment variant
- */
-export interface ExperimentResult<TResult = unknown> {
-    /** Experiment ID */
-    experimentId: string;
-    /** Variant ID */
-    variantId: string;
-    /** Variant name */
-    variantName: string;
-    /** Run ID */
-    runId: string;
-    /** Execution result */
-    result: TResult;
-    /** Computed metric value (if metric function provided) */
-    metricValue?: number;
-    /** Execution duration in milliseconds */
-    duration: number;
-    /** Timestamp when execution started */
-    startedAt: Date;
-    /** Timestamp when execution completed */
-    completedAt: Date;
-    /** Error if execution failed */
-    error?: Error;
-    /** Success flag */
-    success: boolean;
-    /** Additional metadata */
-    metadata?: Record<string, unknown>;
-}
-/**
- * Summary of experiment results across all variants
- */
-export interface ExperimentSummary<TResult = unknown> {
-    /** Experiment ID */
-    experimentId: string;
-    /** Experiment name */
-    experimentName: string;
-    /** Results for each variant */
-    results: ExperimentResult<TResult>[];
-    /** Best performing variant (by metric) */
-    bestVariant?: {
-        variantId: string;
-        variantName: string;
-        metricValue: number;
-    };
-    /** Total execution duration */
-    totalDuration: number;
-    /** Number of successful runs */
-    successCount: number;
-    /** Number of failed runs */
-    failureCount: number;
-    /** Timestamp when experiment started */
-    startedAt: Date;
-    /** Timestamp when experiment completed */
-    completedAt: Date;
-}
-/**
- * Options for running an experiment
- */
-export interface RunExperimentOptions {
-    /** Run variants in parallel (default: true) */
-    parallel?: boolean;
-    /** Maximum concurrent executions (default: unlimited) */
-    maxConcurrency?: number;
-    /** Stop on first error (default: false) */
-    stopOnError?: boolean;
-    /** Custom context data */
-    context?: Record<string, unknown>;
-    /** Event callbacks */
-    onVariantStart?: (variantId: string, variantName: string) => void;
-    onVariantComplete?: (result: ExperimentResult) => void;
-    onVariantError?: (variantId: string, error: Error) => void;
-}
-/**
- * Parameters for cartesian product generation
- */
-export type CartesianParams = Record<string, unknown[]>;
-/**
- * Result of cartesian product - array of parameter combinations
- */
-export type CartesianResult<T extends CartesianParams> = Array<{
-    [K in keyof T]: T[K][number];
-}>;
-/**
- * Decision options
- */
-export interface DecideOptions<T> {
-    /** Options to choose from */
-    options: T[];
-    /** Scoring function for each option */
-    score: (option: T) => number | Promise<number>;
-    /** Context or prompt for decision making */
-    context?: string;
-    /** Whether to return all options sorted by score (default: false) */
-    returnAll?: boolean;
-}
-/**
- * Result of a decision
- */
-export interface DecisionResult<T> {
-    /** The selected option */
-    selected: T;
-    /** Score of the selected option */
-    score: number;
-    /** All options with their scores (if returnAll was true) */
-    allOptions?: Array<{
-        option: T;
-        score: number;
-    }>;
-}
-/**
- * Tracking event types
- */
-export type TrackingEventType = 'experiment.start' | 'experiment.complete' | 'variant.start' | 'variant.complete' | 'variant.error' | 'metric.computed' | 'decision.made';
-/**
- * Tracking event
- */
-export interface TrackingEvent {
-    /** Event type */
-    type: TrackingEventType;
-    /** Timestamp */
-    timestamp: Date;
-    /** Event data */
-    data: Record<string, unknown>;
-}
-/**
- * Tracking backend interface
- */
-export interface TrackingBackend {
-    /** Track an event */
-    track(event: TrackingEvent): void | Promise<void>;
-    /** Flush pending events */
-    flush?(): void | Promise<void>;
-}
-/**
- * Options for tracking configuration
- */
-export interface TrackingOptions {
-    /** Custom tracking backend */
-    backend?: TrackingBackend;
-    /** Whether tracking is enabled (default: true) */
-    enabled?: boolean;
-    /** Additional metadata to include with all events */
-    metadata?: Record<string, unknown>;
-}
-//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,OAAO,GAAG,OAAO;IAClD,wCAAwC;IACxC,EAAE,EAAE,MAAM,CAAA;IACV,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,4BAA4B;IAC5B,MAAM,EAAE,OAAO,CAAA;IACf,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,2BAA2B;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO;IACpE,mCAAmC;IACnC,EAAE,EAAE,MAAM,CAAA;IACV,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,6BAA6B;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,+BAA+B;IAC/B,QAAQ,EAAE,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAA;IACtC,2CAA2C;IAC3C,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,iBAAiB,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;IACrF,uCAAuC;IACvC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACtD,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAA;IACb,uCAAuC;IACvC,SAAS,EAAE,IAAI,CAAA;IACf,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB,CAAC,OAAO,GAAG,OAAO;IACjD,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,mBAAmB;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa;IACb,KAAK,EAAE,MAAM,CAAA;IACb,uBAAuB;IACvB,MAAM,EAAE,OAAO,CAAA;IACf,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,yCAAyC;IACzC,QAAQ,EAAE,MAAM,CAAA;IAChB,uCAAuC;IACvC,SAAS,EAAE,IAAI,CAAA;IACf,yCAAyC;IACzC,WAAW,EAAE,IAAI,CAAA;IACjB,gCAAgC;IAChC,KAAK,CAAC,EAAE,KAAK,CAAA;IACb,mBAAmB;IACnB,OAAO,EAAE,OAAO,CAAA;IAChB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,OAAO,GAAG,OAAO;IAClD,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,+BAA+B;IAC/B,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAA;IACpC,0CAA0C;IAC1C,WAAW,CAAC,EAAE;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAA;IACD,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAA;IACrB,gCAAgC;IAChC,YAAY,EAAE,MAAM,CAAA;IACpB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,wCAAwC;IACxC,SAAS,EAAE,IAAI,CAAA;IACf,0CAA0C;IAC1C,WAAW,EAAE,IAAI,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,yDAAyD;IACzD,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACjC,sBAAsB;IACtB,cAAc,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAA;IACjE,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,IAAI,CAAA;IACtD,cAAc,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;CAC3D;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;AAEvD;;GAEG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,SAAS,eAAe,IAAI,KAAK,CAAC;KAC5D,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;CAC7B,CAAC,CAAA;AAEF;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,6BAA6B;IAC7B,OAAO,EAAE,CAAC,EAAE,CAAA;IACZ,uCAAuC;IACvC,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAC9C,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,qEAAqE;IACrE,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,0BAA0B;IAC1B,QAAQ,EAAE,CAAC,CAAA;IACX,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAA;IACb,4DAA4D;IAC5D,UAAU,CAAC,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,CAAC,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GACzB,kBAAkB,GAClB,qBAAqB,GACrB,eAAe,GACf,kBAAkB,GAClB,eAAe,GACf,iBAAiB,GACjB,eAAe,CAAA;AAEnB;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,iBAAiB;IACjB,IAAI,EAAE,iBAAiB,CAAA;IACvB,gBAAgB;IAChB,SAAS,EAAE,IAAI,CAAA;IACf,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qBAAqB;IACrB,KAAK,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjD,2BAA2B;IAC3B,KAAK,CAAC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,OAAO,CAAC,EAAE,eAAe,CAAA;IACzB,kDAAkD;IAClD,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC"}

package/dist/types.js

@@ -1,5 +0,0 @@
-/**
- * Core types for AI experimentation
- */
-export {};
-//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/src/cartesian.js

@@ -1,215 +0,0 @@
-/**
- * Cartesian product utilities for parameter exploration
- */
-/**
- * Generate cartesian product of parameter sets
- *
- * Takes an object where each key maps to an array of possible values,
- * and returns all possible combinations as an array of objects.
- *
- * @example
- * ```ts
- * import { cartesian } from 'ai-experiments'
- *
- * const combinations = cartesian({
- *   model: ['sonnet', 'opus', 'gpt-4o'],
- *   temperature: [0.3, 0.7, 1.0],
- *   maxTokens: [100, 500, 1000],
- * })
- *
- * // Returns 27 combinations (3 * 3 * 3):
- * // [
- * //   { model: 'sonnet', temperature: 0.3, maxTokens: 100 },
- * //   { model: 'sonnet', temperature: 0.3, maxTokens: 500 },
- * //   { model: 'sonnet', temperature: 0.3, maxTokens: 1000 },
- * //   { model: 'sonnet', temperature: 0.7, maxTokens: 100 },
- * //   ...
- * // ]
- *
- * // Use with experiments:
- * const variants = combinations.map((config, i) => ({
- *   id: `variant-${i}`,
- *   name: `${config.model} T=${config.temperature} max=${config.maxTokens}`,
- *   config,
- * }))
- * ```
- */
-export function cartesian(params) {
-    const keys = Object.keys(params);
-    const values = keys.map((k) => params[k]);
-    // Handle empty input
-    if (keys.length === 0) {
-        return [];
-    }
-    // Handle single parameter
-    if (keys.length === 1) {
-        const key = keys[0];
-        return values[0].map((val) => ({ [key]: val }));
-    }
-    // Generate all combinations using recursive helper
-    const combinations = cartesianProduct(values);
-    // Map back to objects
-    return combinations.map((combo) => {
-        const obj = {};
-        keys.forEach((key, i) => {
-            obj[key] = combo[i];
-        });
-        return obj;
-    });
-}
-/**
- * Recursive cartesian product implementation
- */
-function cartesianProduct(arrays) {
-    if (arrays.length === 0)
-        return [[]];
-    const [first, ...rest] = arrays;
-    // Base case: single array
-    if (rest.length === 0) {
-        return first.map((x) => [x]);
-    }
-    // Recursive case
-    const restProduct = cartesianProduct(rest);
-    const result = [];
-    for (const x of first) {
-        for (const restCombo of restProduct) {
-            result.push([x, ...restCombo]);
-        }
-    }
-    return result;
-}
-/**
- * Generate a grid of parameter combinations with filtering
- *
- * Similar to cartesian(), but allows filtering out invalid combinations.
- *
- * @example
- * ```ts
- * import { cartesianFilter } from 'ai-experiments'
- *
- * const combinations = cartesianFilter(
- *   {
- *     model: ['sonnet', 'opus'],
- *     temperature: [0.3, 0.7, 1.0],
- *     maxTokens: [100, 500],
- *   },
- *   // Filter out combinations where opus uses high temperature
- *   (combo) => !(combo.model === 'opus' && combo.temperature > 0.7)
- * )
- * ```
- */
-export function cartesianFilter(params, filter) {
-    const allCombinations = cartesian(params);
-    return allCombinations.filter(filter);
-}
-/**
- * Generate a random sample from the cartesian product
- *
- * Useful when the full cartesian product is too large to test all combinations.
- *
- * @example
- * ```ts
- * import { cartesianSample } from 'ai-experiments'
- *
- * // Full product would be 1000 combinations (10 * 10 * 10)
- * // Sample just 20 random combinations
- * const sample = cartesianSample(
- *   {
- *     param1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
- *     param2: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
- *     param3: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
- *   },
- *   20
- * )
- * ```
- */
-export function cartesianSample(params, sampleSize, options = {}) {
-    const { unique = true } = options;
-    // Generate all combinations first
-    const allCombinations = cartesian(params);
-    // If sample size is larger than available combinations, return all
-    if (sampleSize >= allCombinations.length) {
-        return allCombinations;
-    }
-    // Shuffle and take first n items
-    const shuffled = [...allCombinations];
-    // Simple Fisher-Yates shuffle
-    for (let i = shuffled.length - 1; i > 0; i--) {
-        const j = Math.floor(Math.random() * (i + 1));
-        [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
-    }
-    return shuffled.slice(0, sampleSize);
-}
-/**
- * Count the total number of combinations without generating them
- *
- * Useful for checking if cartesian product is feasible before generating.
- *
- * @example
- * ```ts
- * import { cartesianCount } from 'ai-experiments'
- *
- * const count = cartesianCount({
- *   model: ['sonnet', 'opus', 'gpt-4o'],
- *   temperature: [0.3, 0.5, 0.7, 0.9],
- *   maxTokens: [100, 500, 1000, 2000],
- * })
- * // Returns 48 (3 * 4 * 4)
- *
- * if (count > 100) {
- *   console.log('Too many combinations, use cartesianSample instead')
- * }
- * ```
- */
-export function cartesianCount(params) {
-    const keys = Object.keys(params);
-    if (keys.length === 0)
-        return 0;
-    return keys.reduce((total, key) => {
-        const arr = params[key];
-        return total * (arr?.length ?? 0);
-    }, 1);
-}
-/**
- * Generate cartesian product with labels for each dimension
- *
- * Returns combinations with additional metadata about which dimension each value came from.
- *
- * @example
- * ```ts
- * import { cartesianWithLabels } from 'ai-experiments'
- *
- * const labeled = cartesianWithLabels({
- *   model: ['sonnet', 'opus'],
- *   temperature: [0.3, 0.7],
- * })
- * // [
- * //   { values: { model: 'sonnet', temperature: 0.3 }, labels: { model: 0, temperature: 0 } },
- * //   { values: { model: 'sonnet', temperature: 0.7 }, labels: { model: 0, temperature: 1 } },
- * //   { values: { model: 'opus', temperature: 0.3 }, labels: { model: 1, temperature: 0 } },
- * //   { values: { model: 'opus', temperature: 0.7 }, labels: { model: 1, temperature: 1 } },
- * // ]
- * ```
- */
-export function cartesianWithLabels(params) {
-    const keys = Object.keys(params);
-    const values = keys.map((k) => params[k]);
-    if (keys.length === 0) {
-        return [];
-    }
-    const combinations = cartesianProduct(values);
-    return combinations.map((combo) => {
-        const valuesObj = {};
-        const labelsObj = {};
-        keys.forEach((key, i) => {
-            const value = combo[i];
-            valuesObj[key] = value;
-            const arr = params[key];
-            labelsObj[key] = arr ? arr.indexOf(value) : -1;
-        });
-        return {
-            values: valuesObj,
-            labels: labelsObj,
-        };
-    });
-}