@amodalai/core 0.1.0

package/dist/src/eval/eval-runner.js

@@ -0,0 +1,92 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import { judgeAllAssertions } from './eval-judge.js';
+import { computeEvalCost, aggregateRunCost } from './eval-cost.js';
+/**
+ * Run the eval suite from a loaded repo.
+ * Yields progress events and returns the full suite result.
+ */
+export async function* runEvalSuite(repo, options) {
+    const startTime = Date.now();
+    let evals = repo.evals;
+    if (options.filter) {
+        const pattern = options.filter.toLowerCase();
+        evals = evals.filter((e) => e.name.toLowerCase().includes(pattern) || e.title.toLowerCase().includes(pattern));
+    }
+    const results = [];
+    const perCaseCosts = [];
+    for (let i = 0; i < evals.length; i++) {
+        const ev = evals[i];
+        yield { type: 'eval_start', evalName: ev.name, current: i + 1, total: evals.length };
+        const result = await runSingleEval(ev, options);
+        results.push(result);
+        if (result.cost) {
+            perCaseCosts.push(result.cost);
+        }
+        yield { type: 'eval_complete', evalName: ev.name, passed: result.passed, current: i + 1, total: evals.length };
+    }
+    const totalCost = perCaseCosts.length > 0 ? aggregateRunCost(perCaseCosts) : undefined;
+    const suiteResult = {
+        results,
+        totalPassed: results.filter((r) => r.passed).length,
+        totalFailed: results.filter((r) => !r.passed).length,
+        totalSkipped: 0,
+        totalDurationMs: Date.now() - startTime,
+        totalCost,
+        model: options.model,
+        gitSha: options.gitSha,
+        timestamp: new Date().toISOString(),
+    };
+    yield { type: 'suite_complete' };
+    return suiteResult;
+}
+async function runSingleEval(ev, options) {
+    const start = Date.now();
+    try {
+        const { response, toolCalls, usage } = await options.queryProvider.query(ev.query, ev.setup.tenant);
+        // Build enriched response that includes tool call info for the judge.
+        // The judge only sees text, so we append a structured summary of
+        // tool calls so assertions about tool usage can be evaluated.
+        let enrichedResponse = response;
+        if (toolCalls.length > 0) {
+            const toolSummary = toolCalls
+                .map((tc) => `- ${tc.name}(${JSON.stringify(tc.parameters)})`)
+                .join('\n');
+            enrichedResponse += `\n\n## Tool Calls Made\n${toolSummary}`;
+        }
+        const assertions = await judgeAllAssertions(enrichedResponse, ev.assertions, options.judgeProvider);
+        const passed = assertions.every((a) => a.passed);
+        const cost = usage && options.model
+            ? computeEvalCost(usage.inputTokens, usage.outputTokens, options.model.model)
+            : undefined;
+        return {
+            eval: ev,
+            response,
+            toolCalls,
+            assertions,
+            passed,
+            durationMs: Date.now() - start,
+            cost,
+        };
+    }
+    catch (err) {
+        return {
+            eval: ev,
+            response: '',
+            toolCalls: [],
+            assertions: ev.assertions.map((a) => ({
+                text: a.text,
+                negated: a.negated,
+                passed: false,
+                reason: `Eval execution error: ${err instanceof Error ? err.message : String(err)}`,
+            })),
+            passed: false,
+            durationMs: Date.now() - start,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+//# sourceMappingURL=eval-runner.js.map

package/dist/src/eval/eval-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.js","sourceRoot":"","sources":["../../../src/eval/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAC,kBAAkB,EAAC,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAC,eAAe,EAAE,gBAAgB,EAAC,MAAM,gBAAgB,CAAC;AAqBjE;;;GAGG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CACjC,IAAgB,EAChB,OAA0B;IAE1B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAEvB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC7C,KAAK,GAAG,KAAK,CAAC,MAAM,CAClB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CACzF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,YAAY,GAAmB,EAAE,CAAC;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,EAAC,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAC,CAAC;QAEnF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,EAAC,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAC,CAAC;IAC/G,CAAC;IAED,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEvF,MAAM,WAAW,GAAoB;QACnC,OAAO;QACP,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM;QACnD,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM;QACpD,YAAY,EAAE,CAAC;QACf,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QACvC,SAAS;QACT,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC;IAEF,MAAM,EAAC,IAAI,EAAE,gBAAgB,EAAC,CAAC;IAC/B,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,EAAc,EACd,OAA0B;IAE1B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,IAAI,CAAC;QACH,MAAM,EAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAC,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,KAAK,CACpE,EAAE,CAAC,KAAK,EACR,EAAE,CAAC,KAAK,CAAC,MAAM,CAChB,CAAC;QAEF,sEAAsE;QACtE,iEAAiE;QACjE,8DAA8D;QAC9D,IAAI,gBAAgB,GAAG,QAAQ,CAAC;QAChC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,WAAW,GAAG,SAAS;iBAC1B,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;iBAC7D,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,gBAAgB,IAAI,2BAA2B,WAAW,EAAE,CAAC;QAC/D,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,kBAAkB,CACzC,gBAAgB,EAChB,EAAE,CAAC,UAAU,EACb,OAAO,CAAC,aAAa,CACtB,CAAC;QAEF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,IAAI,GAAG,KAAK,IAAI,OAAO,CAAC,KAAK;YACjC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;YAC7E,CAAC,CAAC,SAAS,CAAC;QAEd,OAAO;YACL,IAAI,EAAE,EAAE;YACR,QAAQ;YACR,SAAS;YACT,UAAU;YACV,MAAM;YACN,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC9B,IAAI;SACL,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,EAAE;YACR,QAAQ,EAAE,EAAE;YACZ,SAAS,EAAE,EAAE;YACb,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACpC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,yBAAyB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;aACpF,CAAC,CAAC;YACH,MAAM,EAAE,KAAK;YACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC9B,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/src/eval/eval-session-provider.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { EvalQueryProvider } from './eval-runner.js';
+import type { ModelConfig } from '../repo/config-schema.js';
+import type { LLMToolDefinition } from '../providers/runtime/runtime-provider-types.js';
+/**
+ * Options for creating a SessionEvalQueryProvider.
+ */
+export interface SessionEvalProviderOptions {
+    modelConfig: ModelConfig;
+    systemPrompt?: string;
+    tools?: LLMToolDefinition[];
+    maxTokens?: number;
+}
+/**
+ * An EvalQueryProvider that uses the runtime provider infrastructure.
+ * Creates an ephemeral LLM call for each query, capturing token usage.
+ */
+export declare class SessionEvalQueryProvider implements EvalQueryProvider {
+    private readonly provider;
+    private readonly systemPrompt;
+    private readonly tools;
+    private readonly model;
+    private readonly maxTokens;
+    constructor(options: SessionEvalProviderOptions);
+    query(message: string, _tenantId?: string): Promise<{
+        response: string;
+        toolCalls: Array<{
+            name: string;
+            parameters: Record<string, unknown>;
+        }>;
+        usage?: {
+            inputTokens: number;
+            outputTokens: number;
+        };
+    }>;
+}

package/dist/src/eval/eval-session-provider.js

@@ -0,0 +1,46 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import { createRuntimeProvider } from '../providers/runtime/provider-factory.js';
+/**
+ * An EvalQueryProvider that uses the runtime provider infrastructure.
+ * Creates an ephemeral LLM call for each query, capturing token usage.
+ */
+export class SessionEvalQueryProvider {
+    provider;
+    systemPrompt;
+    tools;
+    model;
+    maxTokens;
+    constructor(options) {
+        this.provider = createRuntimeProvider(options.modelConfig);
+        this.systemPrompt = options.systemPrompt ?? 'You are a helpful assistant.';
+        this.tools = options.tools ?? [];
+        this.model = options.modelConfig.model;
+        this.maxTokens = options.maxTokens ?? 4096;
+    }
+    async query(message, _tenantId) {
+        const result = await this.provider.chat({
+            model: this.model,
+            systemPrompt: this.systemPrompt,
+            messages: [{ role: 'user', content: message }],
+            tools: this.tools,
+            maxTokens: this.maxTokens,
+        });
+        const responseText = result.content
+            .filter((b) => b.type === 'text')
+            .map((b) => b.text)
+            .join('');
+        const toolCalls = result.content
+            .filter((b) => b.type === 'tool_use')
+            .map((b) => ({ name: b.name, parameters: b.input }));
+        return {
+            response: responseText,
+            toolCalls,
+            usage: result.usage,
+        };
+    }
+}
+//# sourceMappingURL=eval-session-provider.js.map

package/dist/src/eval/eval-session-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-session-provider.js","sourceRoot":"","sources":["../../../src/eval/eval-session-provider.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAC,qBAAqB,EAAC,MAAM,0CAA0C,CAAC;AAY/E;;;GAGG;AACH,MAAM,OAAO,wBAAwB;IAClB,QAAQ,CAAkB;IAC1B,YAAY,CAAS;IACrB,KAAK,CAAsB;IAC3B,KAAK,CAAS;IACd,SAAS,CAAS;IAEnC,YAAY,OAAmC;QAC7C,IAAI,CAAC,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,8BAA8B,CAAC;QAC3E,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,KAAK,CACT,OAAe,EACf,SAAkB;QAMlB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACtC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,QAAQ,EAAE,CAAC,EAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAC,CAAC;YAC5C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO;aAChC,MAAM,CAAC,CAAC,CAAC,EAAqC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;aACnE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,IAAI,CAAC,EAAE,CAAC,CAAC;QAEZ,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO;aAC7B,MAAM,CAAC,CAAC,CAAC,EAAqF,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;aACvH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAC,CAAC,CAAC,CAAC;QAErD,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,SAAS;YACT,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC;IACJ,CAAC;CACF"}

package/dist/src/eval/eval-types.d.ts

@@ -0,0 +1,146 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { LoadedEval } from '../repo/repo-types.js';
+/**
+ * Result of judging a single assertion.
+ */
+export interface AssertionResult {
+    text: string;
+    negated: boolean;
+    passed: boolean;
+    reason: string;
+}
+/**
+ * Token and cost tracking for a single eval case or aggregated run.
+ */
+export interface EvalCostInfo {
+    inputTokens: number;
+    outputTokens: number;
+    totalTokens: number;
+    estimatedCostMicros: number;
+}
+/**
+ * Model identity for an eval run.
+ */
+export interface EvalModelInfo {
+    provider: string;
+    model: string;
+}
+/**
+ * Result of running a single eval case.
+ */
+export interface EvalResult {
+    eval: LoadedEval;
+    response: string;
+    toolCalls: Array<{
+        name: string;
+        parameters: Record<string, unknown>;
+    }>;
+    assertions: AssertionResult[];
+    passed: boolean;
+    durationMs: number;
+    cost?: EvalCostInfo;
+    error?: string;
+}
+/**
+ * Result of running an entire eval suite.
+ */
+export interface EvalSuiteResult {
+    results: EvalResult[];
+    totalPassed: number;
+    totalFailed: number;
+    totalSkipped: number;
+    totalDurationMs: number;
+    totalCost?: EvalCostInfo;
+    model?: EvalModelInfo;
+    gitSha?: string;
+    timestamp: string;
+}
+/**
+ * A stored baseline for comparison.
+ */
+export interface EvalBaseline {
+    id: string;
+    orgId: string;
+    gitSha: string;
+    isProduction: boolean;
+    result: EvalSuiteResult;
+    createdAt: string;
+}
+/**
+ * Status of a single eval comparison.
+ */
+export type EvalDiffStatus = 'unchanged' | 'improved' | 'regressed' | 'new' | 'removed';
+/**
+ * Diff between current and baseline eval results.
+ */
+export interface EvalDiff {
+    evalName: string;
+    status: EvalDiffStatus;
+    currentPassed: boolean | null;
+    baselinePassed: boolean | null;
+    assertionChanges: Array<{
+        text: string;
+        currentPassed: boolean | null;
+        baselinePassed: boolean | null;
+        status: EvalDiffStatus;
+    }>;
+}
+/**
+ * Progress event emitted during eval execution.
+ */
+export interface EvalProgress {
+    type: 'eval_start' | 'eval_complete' | 'suite_complete';
+    evalName?: string;
+    passed?: boolean;
+    current?: number;
+    total?: number;
+}
+/**
+ * A complete eval run with model and cost tracking.
+ */
+export interface EvalRunRecord {
+    id: string;
+    orgId: string;
+    appId?: string;
+    model: EvalModelInfo;
+    suite: EvalSuiteResult;
+    perCaseCosts: EvalCostInfo[];
+    totalCost: EvalCostInfo;
+    gitSha?: string;
+    label?: string;
+    triggeredBy: 'manual' | 'ci' | 'automation';
+    createdAt: string;
+}
+/**
+ * Comparison of two eval runs.
+ */
+export interface EvalRunComparison {
+    runA: EvalRunRecord;
+    runB: EvalRunRecord;
+    diff: EvalDiff[];
+    costDelta: {
+        totalMicros: number;
+        perCase: number[];
+    };
+    qualityDelta: {
+        passRateDelta: number;
+        durationDeltaMs: number;
+    };
+}
+/**
+ * Aggregated trend point for time-series visualization.
+ */
+export interface EvalTrendPoint {
+    runId: string;
+    label?: string;
+    gitSha?: string;
+    model: EvalModelInfo;
+    passRate: number;
+    totalCostMicros: number;
+    avgDurationMs: number;
+    timestamp: string;
+}

package/dist/src/eval/eval-types.js

@@ -0,0 +1,7 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+export {};
+//# sourceMappingURL=eval-types.js.map

package/dist/src/eval/eval-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-types.js","sourceRoot":"","sources":["../../../src/eval/eval-types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}

package/dist/src/eval/experiment-runner.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { AmodalRepo } from '../repo/repo-types.js';
+import type { ExperimentConfig, ExperimentEvalResult, ExperimentAssignment, ExperimentDeployment } from './experiment-types.js';
+import type { EvalRunnerOptions } from './eval-runner.js';
+/**
+ * Run an experiment: eval suite against both control and variant configs.
+ */
+export declare function runExperiment(repo: AmodalRepo, config: ExperimentConfig, evalOptions: Omit<EvalRunnerOptions, 'gitSha'>): Promise<ExperimentEvalResult>;
+/**
+ * Assign a session to control or variant based on traffic split.
+ */
+export declare function assignExperiment(deployment: ExperimentDeployment): ExperimentAssignment;

package/dist/src/eval/experiment-runner.js

@@ -0,0 +1,73 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import { runEvalSuite } from './eval-runner.js';
+/**
+ * Run an experiment: eval suite against both control and variant configs.
+ */
+export async function runExperiment(repo, config, evalOptions) {
+    // Run control
+    const controlGen = runEvalSuite(repo, evalOptions);
+    let controlResult;
+    while (true) {
+        const next = await controlGen.next();
+        if (next.done) {
+            controlResult = next.value;
+            break;
+        }
+    }
+    // Run variant (in a real implementation, this would apply config changes)
+    const variantGen = runEvalSuite(repo, evalOptions);
+    let variantResult;
+    while (true) {
+        const next = await variantGen.next();
+        if (next.done) {
+            variantResult = next.value;
+            break;
+        }
+    }
+    const controlPassRate = controlResult.results.length > 0
+        ? controlResult.totalPassed / controlResult.results.length
+        : 0;
+    const variantPassRate = variantResult.results.length > 0
+        ? variantResult.totalPassed / variantResult.results.length
+        : 0;
+    const controlAvgDuration = controlResult.results.length > 0
+        ? controlResult.results.reduce((sum, r) => sum + r.durationMs, 0) / controlResult.results.length
+        : 0;
+    const variantAvgDuration = variantResult.results.length > 0
+        ? variantResult.results.reduce((sum, r) => sum + r.durationMs, 0) / variantResult.results.length
+        : 0;
+    let winner = 'tie';
+    if (variantPassRate > controlPassRate) {
+        winner = 'variant';
+    }
+    else if (controlPassRate > variantPassRate) {
+        winner = 'control';
+    }
+    return {
+        experimentName: config.name,
+        control: controlResult,
+        variant: variantResult,
+        comparison: {
+            controlPassRate,
+            variantPassRate,
+            controlAvgDuration,
+            variantAvgDuration,
+            winner,
+        },
+    };
+}
+/**
+ * Assign a session to control or variant based on traffic split.
+ */
+export function assignExperiment(deployment) {
+    const roll = Math.random() * 100;
+    return {
+        experimentId: deployment.id,
+        variant: roll < deployment.trafficPercent ? 'variant' : 'control',
+    };
+}
+//# sourceMappingURL=experiment-runner.js.map

package/dist/src/eval/experiment-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-runner.js","sourceRoot":"","sources":["../../../src/eval/experiment-runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAE9C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAgB,EAChB,MAAwB,EACxB,WAA8C;IAE9C,cAAc;IACd,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACnD,IAAI,aAAa,CAAC;IAClB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3B,MAAM;QACR,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACnD,IAAI,aAAa,CAAC;IAClB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC;YAC3B,MAAM;QACR,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QACtD,CAAC,CAAC,aAAa,CAAC,WAAW,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM;QAC1D,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,eAAe,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QACtD,CAAC,CAAC,aAAa,CAAC,WAAW,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM;QAC1D,CAAC,CAAC,CAAC,CAAC;IAEN,MAAM,kBAAkB,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QACzD,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM;QAChG,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,kBAAkB,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QACzD,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM;QAChG,CAAC,CAAC,CAAC,CAAC;IAEN,IAAI,MAAM,GAAkC,KAAK,CAAC;IAClD,IAAI,eAAe,GAAG,eAAe,EAAE,CAAC;QACtC,MAAM,GAAG,SAAS,CAAC;IACrB,CAAC;SAAM,IAAI,eAAe,GAAG,eAAe,EAAE,CAAC;QAC7C,MAAM,GAAG,SAAS,CAAC;IACrB,CAAC;IAED,OAAO;QACL,cAAc,EAAE,MAAM,CAAC,IAAI;QAC3B,OAAO,EAAE,aAAa;QACtB,OAAO,EAAE,aAAa;QACtB,UAAU,EAAE;YACV,eAAe;YACf,eAAe;YACf,kBAAkB;YAClB,kBAAkB;YAClB,MAAM;SACP;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,UAAgC;IAEhC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;IACjC,OAAO;QACL,YAAY,EAAE,UAAU,CAAC,EAAE;QAC3B,OAAO,EAAE,IAAI,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;KAClE,CAAC;AACJ,CAAC"}

package/dist/src/eval/experiment-types.d.ts

@@ -0,0 +1,56 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { EvalSuiteResult } from './eval-types.js';
+/**
+ * A model config change to apply for the variant.
+ */
+export interface ExperimentChange {
+    path: string;
+    value: unknown;
+}
+/**
+ * Configuration for an A/B experiment.
+ */
+export interface ExperimentConfig {
+    name: string;
+    description?: string;
+    controlConfig: Record<string, unknown>;
+    variantConfig: Record<string, unknown>;
+    changes: ExperimentChange[];
+    trafficPercent: number;
+}
+/**
+ * Result of running an eval-based experiment.
+ */
+export interface ExperimentEvalResult {
+    experimentName: string;
+    control: EvalSuiteResult;
+    variant: EvalSuiteResult;
+    comparison: {
+        controlPassRate: number;
+        variantPassRate: number;
+        controlAvgDuration: number;
+        variantAvgDuration: number;
+        winner: 'control' | 'variant' | 'tie';
+    };
+}
+/**
+ * A deployed experiment with traffic splitting.
+ */
+export interface ExperimentDeployment {
+    id: string;
+    name: string;
+    controlConfig: Record<string, unknown>;
+    variantConfig: Record<string, unknown>;
+    trafficPercent: number;
+}
+/**
+ * Assignment for a session in an active experiment.
+ */
+export interface ExperimentAssignment {
+    experimentId: string;
+    variant: 'control' | 'variant';
+}

package/dist/src/eval/experiment-types.js

@@ -0,0 +1,7 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+export {};
+//# sourceMappingURL=experiment-types.js.map

package/dist/src/eval/experiment-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-types.js","sourceRoot":"","sources":["../../../src/eval/experiment-types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}

package/dist/src/eval/index.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+export * from './eval-types.js';
+export { judgeAssertion, judgeAllAssertions } from './eval-judge.js';
+export type { JudgeProvider } from './eval-judge.js';
+export { runEvalSuite } from './eval-runner.js';
+export type { EvalQueryProvider, EvalRunnerOptions } from './eval-runner.js';
+export { diffEvalResults } from './eval-diff.js';
+export { formatEvalTable, formatDiffTable, formatComparisonTable, formatEvalMarkdown } from './eval-formatter.js';
+export { PlatformEvalClient } from './platform-eval-client.js';
+export type { EvalRunSummary, PlatformEvalComparison } from './platform-eval-client.js';
+export * from './experiment-types.js';
+export { runExperiment, assignExperiment } from './experiment-runner.js';
+export { MODEL_PRICING, getModelPricing, computeEvalCost, aggregateRunCost, formatCostMicros } from './eval-cost.js';
+export { buildEvalRun, compareRuns, buildTrendPoints } from './eval-run-builder.js';
+export { SessionEvalQueryProvider } from './eval-session-provider.js';
+export type { SessionEvalProviderOptions } from './eval-session-provider.js';
+export { runMultiModelEval } from './multi-model-runner.js';
+export type { MultiModelProgress, MultiModelEvalOptions } from './multi-model-runner.js';

package/dist/src/eval/index.js

@@ -0,0 +1,18 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+export * from './eval-types.js';
+export { judgeAssertion, judgeAllAssertions } from './eval-judge.js';
+export { runEvalSuite } from './eval-runner.js';
+export { diffEvalResults } from './eval-diff.js';
+export { formatEvalTable, formatDiffTable, formatComparisonTable, formatEvalMarkdown } from './eval-formatter.js';
+export { PlatformEvalClient } from './platform-eval-client.js';
+export * from './experiment-types.js';
+export { runExperiment, assignExperiment } from './experiment-runner.js';
+export { MODEL_PRICING, getModelPricing, computeEvalCost, aggregateRunCost, formatCostMicros } from './eval-cost.js';
+export { buildEvalRun, compareRuns, buildTrendPoints } from './eval-run-builder.js';
+export { SessionEvalQueryProvider } from './eval-session-provider.js';
+export { runMultiModelEval } from './multi-model-runner.js';
+//# sourceMappingURL=index.js.map

package/dist/src/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,iBAAiB,CAAC;AAChC,OAAO,EAAC,cAAc,EAAE,kBAAkB,EAAC,MAAM,iBAAiB,CAAC;AAEnE,OAAO,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAC,eAAe,EAAC,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAC,eAAe,EAAE,eAAe,EAAE,qBAAqB,EAAE,kBAAkB,EAAC,MAAM,qBAAqB,CAAC;AAChH,OAAO,EAAC,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAE7D,cAAc,uBAAuB,CAAC;AACtC,OAAO,EAAC,aAAa,EAAE,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAC,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,EAAC,MAAM,gBAAgB,CAAC;AACnH,OAAO,EAAC,YAAY,EAAE,WAAW,EAAE,gBAAgB,EAAC,MAAM,uBAAuB,CAAC;AAClF,OAAO,EAAC,wBAAwB,EAAC,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC"}

package/dist/src/eval/multi-model-runner.d.ts

@@ -0,0 +1,42 @@
+/**
+ * @license
+ * Copyright 2026 Amodal Labs, Inc.
+ * SPDX-License-Identifier: MIT
+ */
+import type { AmodalRepo } from '../repo/repo-types.js';
+import type { ModelConfig } from '../repo/config-schema.js';
+import type { EvalRunRecord, EvalModelInfo } from './eval-types.js';
+import type { JudgeProvider } from './eval-judge.js';
+import type { LLMToolDefinition } from '../providers/runtime/runtime-provider-types.js';
+/**
+ * Progress event for multi-model eval runs.
+ */
+export interface MultiModelProgress {
+    type: 'model_start' | 'model_complete' | 'all_complete';
+    model?: EvalModelInfo;
+    currentModel?: number;
+    totalModels?: number;
+    passRate?: number;
+    costMicros?: number;
+}
+/**
+ * Options for multi-model eval runs.
+ */
+export interface MultiModelEvalOptions {
+    models: ModelConfig[];
+    judgeProvider: JudgeProvider;
+    orgId: string;
+    appId?: string;
+    gitSha?: string;
+    label?: string;
+    triggeredBy?: 'manual' | 'ci' | 'automation';
+    filter?: string;
+    systemPrompt?: string;
+    tools?: LLMToolDefinition[];
+    maxTokens?: number;
+}
+/**
+ * Run the same eval suite against multiple models.
+ * Yields progress events and returns an EvalRunRecord per model.
+ */
+export declare function runMultiModelEval(repo: AmodalRepo, options: MultiModelEvalOptions): AsyncGenerator<MultiModelProgress, EvalRunRecord[]>;