ai-experiments 2.1.3 → 2.4.0

package/.turbo/turbo-build.log

@@ -1,4 +1,4 @@
 
-> ai-experiments@2.1.3 build /Users/nathanclevenger/projects/primitives.org.ai/packages/ai-experiments
+> ai-experiments@2.3.0 build /Users/nathanclevenger/projects/primitives.org.ai/packages/ai-experiments
 > tsc
 

package/CHANGELOG.md

@@ -1,5 +1,22 @@
 # ai-experiments
 
+## 2.4.0
+
+### Patch Changes
+
+- Updated dependencies [4d58f5f]
+  - ai-functions@2.4.0
+  - ai-database@2.4.0
+
+## 2.3.0
+
+### Patch Changes
+
+- Updated dependencies [9e2779a]
+- Updated dependencies [b7c7c57]
+  - ai-functions@2.3.0
+  - ai-database@2.3.0
+
 ## 2.1.3
 
 ### Patch Changes

package/README.md

@@ -1,6 +1,18 @@
 # ai-experiments
 
-AI-powered experimentation primitives for testing and evaluating models.
+![Stability: Experimental](https://img.shields.io/badge/stability-experimental-red)
+
+Runtime A/B testing for AI variants in production. Traffic splitting, multi-armed bandits, parameter sweeps, and ClickHouse-backed outcome tracking via the canonical `ai-database` adapter.
+
+## Lifecycle
+
+**Runtime (production).** `ai-experiments` is for splitting live traffic across model/prompt/parameter variants and measuring outcomes against real users. Reach for it when you need to learn which variant wins under production conditions — not in a test suite.
+
+## Not for
+
+- **Unit testing.** If you want to assert that a specific input produces a specific output during development or CI, use [`ai-tests`](../ai-tests) plus `vitest`. `ai-experiments` exists to *measure* outcomes across users, not to *assert* them against fixtures.
+- **Sandboxing untrusted code.** That's [`ai-evaluate`](../ai-evaluate)'s job.
+- **Offline-only experiments.** While `Experiment()` works offline for parameter sweeps, the package's value comes from production tracking — if you're never deploying the variants, a plain `for` loop is simpler.
 
 ## Overview
 

package/dist/clickhouse-storage.d.ts

@@ -0,0 +1,227 @@
+/**
+ * ClickHouse storage backend for AI experiments.
+ *
+ * Migrated (per [ADR-0003](../../../docs/adr/0003-storage-strategy-pg-clickhouse-default.md))
+ * from the standalone embedded `chdb` adapter to the canonical
+ * `ClickHouseProvider` from `ai-database`. Experiment runs are recorded as
+ * SVO Actions on the canonical `actions` table:
+ *
+ * - `verb` — event type (e.g. `variant.complete`)
+ * - `subject` — `experimentId`
+ * - `object` — `variantId`
+ * - `roles` — `{ runId }` (other named roles can be carried here)
+ * - `data` — JSON payload (`experimentName`, `variantName`, `dimensions`,
+ *    `runId`, `success`, `durationMs`, `result`, `metricName`,
+ *    `metricValue`, `errorMessage`, `errorStack`, `metadata`)
+ * - `status` — `'completed'` for successful runs, `'failed'` otherwise
+ *
+ * Writes go through {@link ClickHouseProvider.recordAction} (and
+ * {@link ClickHouseProvider.commitBatch} when bulk-storing). Aggregations
+ * use {@link ClickHouseProvider.analyticsQuery} against the `actions` table
+ * with `JSONExtract*` against the `data` column. This is the **option (c)
+ * hybrid** approach from the migration plan: SVO surface for writes, raw
+ * analytical SQL for the experiment-specific aggregations the SVO surface
+ * doesn't expose directly (cartesian grids, time-series rollups,
+ * best-variant selection by metric).
+ *
+ * Deployment shift: the previous `chdb` adapter used embedded ClickHouse
+ * (file-system path). This adapter is HTTP-based — callers wire up a
+ * ClickHouse server (self-hosted or ClickHouse Cloud) via
+ * {@link createClickHouseHttpFetcher}. Tests use a fake fetcher.
+ *
+ * @packageDocumentation
+ */
+import { type ClickHouseHttpFetcher, type ClickHouseProvider } from 'ai-database';
+import type { TrackingBackend, TrackingEvent, ExperimentResult } from './types.js';
+/** Options for {@link ClickHouseExperimentStorage}. */
+export interface ClickHouseExperimentStorageOptions {
+    /** HTTP fetcher (typically constructed via `createClickHouseHttpFetcher`). */
+    fetcher: ClickHouseHttpFetcher;
+    /**
+     * ClickHouse database. Must already exist (or call
+     * {@link bootstrapExperimentsSchema}).
+     *
+     * @default 'aidb'
+     */
+    database?: string;
+    /**
+     * Namespace partition for experiment events.
+     *
+     * @default 'experiments'
+     */
+    namespace?: string;
+    /**
+     * Whether to bootstrap the canonical schema (`things`/`actions`/`verbs`)
+     * before the first write.
+     *
+     * @default false
+     */
+    autoBootstrap?: boolean;
+}
+/**
+ * Backwards-compat alias for the prior `ChdbStorageOptions`. The
+ * `dataPath` and `autoInit` fields are retained as no-ops to ease the
+ * upgrade — they are ignored in favor of HTTP fetcher options.
+ */
+export interface ChdbStorageOptions extends Partial<ClickHouseExperimentStorageOptions> {
+    /** @deprecated Embedded chdb is no longer used; supply `fetcher`. */
+    dataPath?: string;
+    /** @deprecated Replaced by `autoBootstrap`. */
+    autoInit?: boolean;
+}
+/**
+ * ClickHouse-backed experiment storage built on the canonical
+ * {@link ClickHouseProvider} adapter. Implements
+ * {@link TrackingBackend}.
+ *
+ * @example
+ * ```ts
+ * import { configureTracking } from 'ai-experiments'
+ * import { createClickHouseExperimentStorage } from 'ai-experiments/storage'
+ * import { createClickHouseHttpFetcher } from 'ai-database'
+ *
+ * const fetcher = createClickHouseHttpFetcher({
+ *   url: process.env.CLICKHOUSE_URL!,
+ *   username: process.env.CLICKHOUSE_USER,
+ *   password: process.env.CLICKHOUSE_PASSWORD,
+ *   database: 'aidb',
+ * })
+ * const storage = createClickHouseExperimentStorage({ fetcher })
+ *
+ * configureTracking({ backend: storage })
+ *
+ * // Later: analyse results
+ * const best = await storage.getBestVariant('my-experiment')
+ * ```
+ */
+export declare class ClickHouseExperimentStorage implements TrackingBackend {
+    private readonly provider;
+    private readonly fetcher;
+    private readonly database;
+    private readonly namespace;
+    private readonly autoBootstrap;
+    private bootstrapped;
+    private readonly nextEventId;
+    constructor(options: ClickHouseExperimentStorageOptions);
+    /** Underlying canonical CH provider (escape hatch for advanced callers). */
+    getProvider(): ClickHouseProvider;
+    private ensureBootstrap;
+    track(event: TrackingEvent): Promise<void>;
+    flush(): Promise<void>;
+    storeResult(result: ExperimentResult): Promise<void>;
+    /**
+     * Bulk-store results via {@link ClickHouseProvider.commitBatch}. One
+     * HTTP POST per call carries the entire batch as `JSONEachRow`.
+     */
+    storeResults(results: ExperimentResult[]): Promise<void>;
+    /** Quote a SQL string literal — same shape as the canonical adapter. */
+    private quote;
+    /**
+     * Get all experiments seen in the actions table (variant.complete events).
+     */
+    getExperiments(): Promise<Array<{
+        experimentId: string;
+        experimentName: string;
+        variantCount: number;
+        runCount: number;
+        firstRun: string;
+        lastRun: string;
+    }>>;
+    getVariantStats(experimentId: string): Promise<Array<{
+        variantId: string;
+        variantName: string;
+        runCount: number;
+        successCount: number;
+        successRate: number;
+        avgDuration: number;
+        avgMetric: number;
+        minMetric: number;
+        maxMetric: number;
+    }>>;
+    getBestVariant(experimentId: string, options?: {
+        metric?: 'avgMetric' | 'successRate' | 'avgDuration';
+        minimumRuns?: number;
+    }): Promise<{
+        variantId: string;
+        variantName: string;
+        metricValue: number;
+        runCount: number;
+    } | null>;
+    getCartesianAnalysis(experimentId: string, dimension: string): Promise<Array<{
+        dimensionValue: string;
+        runCount: number;
+        avgMetric: number;
+        successRate: number;
+    }>>;
+    getCartesianGrid(experimentId: string, dimensions: string[]): Promise<Array<{
+        dimensions: Record<string, string>;
+        runCount: number;
+        avgMetric: number;
+        successRate: number;
+    }>>;
+    getTimeSeries(experimentId: string, options?: {
+        interval?: 'hour' | 'day' | 'week';
+        variantId?: string;
+    }): Promise<Array<{
+        period: string;
+        runCount: number;
+        avgMetric: number;
+        successRate: number;
+    }>>;
+    getEvents(experimentId: string, options?: {
+        eventType?: string;
+        variantId?: string;
+        limit?: number;
+    }): Promise<TrackingEvent[]>;
+    /**
+     * Raw analytical SQL pass-through. Routes to
+     * {@link ClickHouseProvider.analyticsQuery}.
+     */
+    query(sql: string): Promise<Array<Record<string, unknown>>>;
+    /** No-op (kept for backwards compat with the chdb-era API). */
+    close(): void;
+}
+/**
+ * Bootstrap the canonical CH schema (`things`, `actions`, `verbs`).
+ * Convenience wrapper around the canonical
+ * {@link bootstrapClickHouseSchema}.
+ */
+export declare function bootstrapExperimentsSchema(fetcher: ClickHouseHttpFetcher, options?: {
+    database?: string;
+}): Promise<void>;
+/**
+ * Create a ClickHouse-backed experiment storage backend.
+ *
+ * @example
+ * ```ts
+ * import { configureTracking } from 'ai-experiments'
+ * import { createClickHouseExperimentStorage } from 'ai-experiments/storage'
+ * import { createClickHouseHttpFetcher } from 'ai-database'
+ *
+ * const fetcher = createClickHouseHttpFetcher({
+ *   url: process.env.CLICKHOUSE_URL!,
+ *   username: process.env.CLICKHOUSE_USER,
+ *   password: process.env.CLICKHOUSE_PASSWORD,
+ *   database: 'aidb',
+ * })
+ * const storage = createClickHouseExperimentStorage({ fetcher })
+ *
+ * configureTracking({ backend: storage })
+ *
+ * const best = await storage.getBestVariant('my-experiment')
+ * ```
+ */
+export declare function createClickHouseExperimentStorage(options: ClickHouseExperimentStorageOptions): ClickHouseExperimentStorage;
+/**
+ * @deprecated Renamed to {@link ClickHouseExperimentStorage}. The class
+ * now wraps the canonical `ClickHouseProvider` from `ai-database` instead
+ * of embedded chdb. The old `dataPath` constructor option is ignored —
+ * supply a `fetcher` instead.
+ */
+export declare const ChdbStorage: typeof ClickHouseExperimentStorage;
+export type ChdbStorage = ClickHouseExperimentStorage;
+/**
+ * @deprecated Renamed to {@link createClickHouseExperimentStorage}.
+ */
+export declare function createChdbBackend(options: ClickHouseExperimentStorageOptions): ClickHouseExperimentStorage;
+//# sourceMappingURL=clickhouse-storage.d.ts.map

package/dist/clickhouse-storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clickhouse-storage.d.ts","sourceRoot":"","sources":["../src/clickhouse-storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EAGxB,MAAM,aAAa,CAAA;AACpB,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAElF,uDAAuD;AACvD,MAAM,WAAW,kCAAkC;IACjD,8EAA8E;IAC9E,OAAO,EAAE,qBAAqB,CAAA;IAC9B;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAA;CACxB;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAmB,SAAQ,OAAO,CAAC,kCAAkC,CAAC;IACrF,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAqFD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,2BAA4B,YAAW,eAAe;IACjE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuB;gBAEvC,OAAO,EAAE,kCAAkC;IAYvD,4EAA4E;IAC5E,WAAW,IAAI,kBAAkB;YAInB,eAAe;IAYvB,KAAK,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAc1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQtB,WAAW,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmB1D;;;OAGG;IACG,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgC9D,wEAAwE;IACxE,OAAO,CAAC,KAAK;IAIb;;OAEG;IACG,cAAc,IAAI,OAAO,CAC7B,KAAK,CAAC;QACJ,YAAY,EAAE,MAAM,CAAA;QACpB,cAAc,EAAE,MAAM,CAAA;QACtB,YAAY,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,OAAO,EAAE,MAAM,CAAA;KAChB,CAAC,CACH;IA2BK,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAClD,KAAK,CAAC;QACJ,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,MAAM,CAAA;QAChB,YAAY,EAAE,MAAM,CAAA;QACpB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,SAAS,EAAE,MAAM,CAAA;QACjB,SAAS,EAAE,MAAM,CAAA;QACjB,SAAS,EAAE,MAAM,CAAA;KAClB,CAAC,CACH;IAkCK,cAAc,CAClB,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE;QACP,MAAM,CAAC,EAAE,WAAW,GAAG,aAAa,GAAG,aAAa,CAAA;QACpD,WAAW,CAAC,EAAE,MAAM,CAAA;KAChB,GACL,OAAO,CAAC;QACT,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,MAAM,CAAA;KACjB,GAAG,IAAI,CAAC;IAqCH,oBAAoB,CACxB,YAAY,EAAE,MAAM,EACpB,SAAS,EAAE,MAAM,GAChB,OAAO,CACR,KAAK,CAAC;QACJ,cAAc,EAAE,MAAM,CAAA;QACtB,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAC,CACH;IA0BK,gBAAgB,CACpB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,EAAE,GACnB,OAAO,CACR,KAAK,CAAC;QACJ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAClC,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAC,CACH;IAqCK,aAAa,CACjB,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE;QACP,QAAQ,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAAA;QAClC,SAAS,CAAC,EAAE,MAAM,CAAA;KACd,GACL,OAAO,CACR,KAAK,CAAC;QACJ,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAC,CACH;IAkCK,SAAS,CACb,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE;QACP,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,KAAK,CAAC,EAAE,MAAM,CAAA;KACV,GACL,OAAO,CAAC,aAAa,EAAE,CAAC;IA0C3B;;;OAGG;IACG,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAIjE,+DAA+D;IAC/D,KAAK,IAAI,IAAI;CAGd;AAED;;;;GAIG;AACH,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,qBAAqB,EAC9B,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAClC,OAAO,CAAC,IAAI,CAAC,CAKf;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,iCAAiC,CAC/C,OAAO,EAAE,kCAAkC,GAC1C,2BAA2B,CAE7B;AAMD;;;;;GAKG;AACH,eAAO,MAAM,WAAW,oCAA8B,CAAA;AACtD,MAAM,MAAM,WAAW,GAAG,2BAA2B,CAAA;AAErD;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,kCAAkC,GAC1C,2BAA2B,CAE7B"}