@agentlensai/server 0.6.0 → 0.7.0

package/dist/lib/benchmark/engine.js

@@ -0,0 +1,159 @@
+/**
+ * BenchmarkEngine (Story 3.4)
+ *
+ * Orchestrator that combines MetricAggregator and StatisticalComparator
+ * to produce full benchmark results with human-readable summaries.
+ */
+import { MetricAggregator } from './metric-aggregator.js';
+import { StatisticalComparator } from './statistical.js';
+// ─── Constants ─────────────────────────────────────────────
+const MIN_SESSIONS_FOR_RELIABLE_RESULTS = 30;
+// ─── Metric display names ──────────────────────────────────
+const METRIC_DISPLAY_NAMES = {
+    health_score: 'health score',
+    error_rate: 'error rate',
+    avg_cost: 'cost',
+    avg_latency: 'latency',
+    tool_success_rate: 'tool success rate',
+    completion_rate: 'completion rate',
+    avg_tokens: 'tokens',
+    avg_duration: 'duration',
+};
+// ─── BenchmarkEngine ───────────────────────────────────────
+export class BenchmarkEngine {
+    aggregator = new MetricAggregator();
+    comparator = new StatisticalComparator();
+    /**
+     * Compute full benchmark results: aggregation + statistical comparison.
+     *
+     * For completed benchmarks, results are cached via store.saveResults().
+     * For running benchmarks, results are computed on-the-fly (not cached).
+     */
+    async computeResults(benchmark, eventStore, benchmarkStore) {
+        // For completed benchmarks, try to return cached results first
+        if (benchmark.status === 'completed' && benchmarkStore) {
+            const cached = benchmarkStore.getResults(benchmark.tenantId, benchmark.id);
+            if (cached)
+                return cached;
+        }
+        const variants = benchmark.variants;
+        const metrics = benchmark.metrics;
+        // Step 1: Aggregate metrics for each variant
+        const variantMetricsList = [];
+        const variantData = [];
+        for (const variant of variants) {
+            const aggregated = await this.aggregator.aggregate(eventStore, variant, metrics, benchmark.timeRange);
+            variantMetricsList.push({
+                variantId: variant.id,
+                variantName: variant.name,
+                sessionCount: aggregated.sessionCount,
+                metrics: aggregated.metrics,
+            });
+            variantData.push({
+                id: variant.id,
+                name: variant.name,
+                metrics: aggregated.metrics,
+            });
+        }
+        // Step 2: Pairwise comparisons for each metric
+        const comparisons = [];
+        for (let i = 0; i < variantData.length; i++) {
+            for (let j = i + 1; j < variantData.length; j++) {
+                const vA = variantData[i];
+                const vB = variantData[j];
+                for (const metric of metrics) {
+                    try {
+                        const comparison = this.comparator.compare(vA, vB, metric);
+                        comparisons.push(comparison);
+                    }
+                    catch {
+                        // Skip metrics where data is missing for a variant
+                    }
+                }
+            }
+        }
+        // Step 3: Generate summary
+        const summary = this.formatSummary(comparisons, variantMetricsList);
+        const results = {
+            benchmarkId: benchmark.id,
+            tenantId: benchmark.tenantId,
+            variants: variantMetricsList,
+            comparisons,
+            summary,
+            computedAt: new Date().toISOString(),
+        };
+        // Cache results for completed benchmarks
+        if (benchmark.status === 'completed' && benchmarkStore) {
+            benchmarkStore.saveResults(benchmark.tenantId, benchmark.id, results);
+        }
+        return results;
+    }
+    /**
+     * Generate a human-readable summary of benchmark comparisons.
+     */
+    formatSummary(comparisons, variants) {
+        const parts = [];
+        // Warning: insufficient data
+        const lowDataVariants = variants.filter((v) => v.sessionCount < MIN_SESSIONS_FOR_RELIABLE_RESULTS);
+        if (lowDataVariants.length > 0) {
+            const names = lowDataVariants
+                .map((v) => `${v.variantName}: ${v.sessionCount} sessions`)
+                .join(', ');
+            parts.push(`⚠️ Insufficient data (${names}). Results may be unreliable.`);
+        }
+        // Group significant comparisons by winner
+        const significant = comparisons.filter((c) => c.significant);
+        if (significant.length === 0) {
+            parts.push('No significant differences found between variants.' +
+                (variants.length > 0
+                    ? ` Current sample sizes: ${variants.map((v) => `${v.variantName}: ${v.sessionCount}`).join(', ')}.`
+                    : ''));
+            return parts.join('\n');
+        }
+        // Build wins map: winnerId → list of comparisons
+        const winsByVariant = new Map();
+        for (const comp of significant) {
+            if (!comp.winner)
+                continue;
+            const wins = winsByVariant.get(comp.winner) ?? [];
+            wins.push(comp);
+            winsByVariant.set(comp.winner, wins);
+        }
+        // Find variant names
+        const variantNameMap = new Map();
+        for (const v of variants) {
+            variantNameMap.set(v.variantId, v.variantName);
+        }
+        // Also look in comparisons for variant names
+        for (const c of comparisons) {
+            variantNameMap.set(c.variantA.id, c.variantA.name);
+            variantNameMap.set(c.variantB.id, c.variantB.name);
+        }
+        // Generate per-winner summaries
+        for (const [winnerId, wins] of winsByVariant) {
+            const winnerName = variantNameMap.get(winnerId) ?? winnerId;
+            // Group by loser
+            const loserIds = new Set(wins.map((c) => c.winner === c.variantA.id ? c.variantB.id : c.variantA.id));
+            for (const loserId of loserIds) {
+                const loserName = variantNameMap.get(loserId) ?? loserId;
+                const relevantWins = wins.filter((c) => (c.winner === c.variantA.id && c.variantB.id === loserId) ||
+                    (c.winner === c.variantB.id && c.variantA.id === loserId));
+                const metricTexts = relevantWins.map((comp) => {
+                    const metricName = METRIC_DISPLAY_NAMES[comp.metric] ?? comp.metric;
+                    const pctSign = comp.percentDiff >= 0 ? '+' : '';
+                    const pctStr = `${pctSign}${comp.percentDiff.toFixed(0)}%`;
+                    return `${metricName} (${pctStr}, ${comp.confidence})`;
+                });
+                parts.push(`${winnerName} outperforms ${loserName} on ${metricTexts.join(' and ')}.`);
+            }
+        }
+        // Note metrics with no significant differences
+        const notSignificant = comparisons.filter((c) => !c.significant);
+        if (notSignificant.length > 0) {
+            const nsMetrics = [...new Set(notSignificant.map((c) => METRIC_DISPLAY_NAMES[c.metric] ?? c.metric))];
+            parts.push(`No significant difference on ${nsMetrics.join(', ')}.`);
+        }
+        return parts.join('\n');
+    }
+}
+//# sourceMappingURL=engine.js.map

package/dist/lib/benchmark/engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../../src/lib/benchmark/engine.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAUH,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,8DAA8D;AAE9D,MAAM,iCAAiC,GAAG,EAAE,CAAC;AAE7C,8DAA8D;AAE9D,MAAM,oBAAoB,GAAoC;IAC5D,YAAY,EAAE,cAAc;IAC5B,UAAU,EAAE,YAAY;IACxB,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,SAAS;IACtB,iBAAiB,EAAE,mBAAmB;IACtC,eAAe,EAAE,iBAAiB;IAClC,UAAU,EAAE,QAAQ;IACpB,YAAY,EAAE,UAAU;CACzB,CAAC;AAEF,8DAA8D;AAE9D,MAAM,OAAO,eAAe;IACT,UAAU,GAAG,IAAI,gBAAgB,EAAE,CAAC;IACpC,UAAU,GAAG,IAAI,qBAAqB,EAAE,CAAC;IAE1D;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAClB,SAAgC,EAChC,UAAuB,EACvB,cAA+B;QAE/B,+DAA+D;QAC/D,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW,IAAI,cAAc,EAAE,CAAC;YACvD,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;YAC3E,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAC;QAC5B,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;QACpC,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;QAElC,6CAA6C;QAC7C,MAAM,kBAAkB,GAAqB,EAAE,CAAC;QAChD,MAAM,WAAW,GAIZ,EAAE,CAAC;QAER,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAChD,UAAU,EACV,OAAO,EACP,OAAO,EACP,SAAS,CAAC,SAAS,CACpB,CAAC;YAEF,kBAAkB,CAAC,IAAI,CAAC;gBACtB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,WAAW,EAAE,OAAO,CAAC,IAAI;gBACzB,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,OAAO,EAAE,UAAU,CAAC,OAAO;aAC5B,CAAC,CAAC;YAEH,WAAW,CAAC,IAAI,CAAC;gBACf,EAAE,EAAE,OAAO,CAAC,EAAE;gBACd,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,UAAU,CAAC,OAAO;aAC5B,CAAC,CAAC;QACL,CAAC;QAED,+CAA+C;QAC/C,MAAM,WAAW,GAAuB,EAAE,CAAC;QAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,EAAE,GAAG,WAAW,CAAC,CAAC,CAAE,CAAC;gBAC3B,MAAM,EAAE,GAAG,WAAW,CAAC,CAAC,CAAE,CAAC;gBAE3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,CAAC;wBACH,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;wBAC3D,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;oBAC/B,CAAC;oBAAC,MAAM,CAAC;wBACP,mDAAmD;oBACrD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,2BAA2B;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QAEpE,MAAM,OAAO,GAAqB;YAChC,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,QAAQ,EAAE,SAAS,CAAC,QAAQ;YAC5B,QAAQ,EAAE,kBAAkB;YAC5B,WAAW;YACX,OAAO;YACP,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACrC,CAAC;QAEF,yCAAyC;QACzC,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW,IAAI,cAAc,EAAE,CAAC;YACvD,cAAc,CAAC,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACxE,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,aAAa,CACX,WAA+B,EAC/B,QAA0B;QAE1B,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,6BAA6B;QAC7B,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,iCAAiC,CAC1D,CAAC;QACF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,eAAe;iBAC1B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,YAAY,WAAW,CAAC;iBAC1D,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,yBAAyB,KAAK,+BAA+B,CAAC,CAAC;QAC5E,CAAC;QAED,0CAA0C;QAC1C,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAE7D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CACR,oDAAoD;gBAClD,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;oBAClB,CAAC,CAAC,0BAA0B,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;oBACpG,CAAC,CAAC,EAAE,CAAC,CACV,CAAC;YACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,iDAAiD;QACjD,MAAM,aAAa,GAAG,IAAI,GAAG,EAA8B,CAAC;QAC5D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,MAAM;gBAAE,SAAS;YAC3B,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAClD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChB,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACvC,CAAC;QAED,qBAAqB;QACrB,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;QACjD,CAAC;QACD,6CAA6C;QAC7C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;YAC5B,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACnD,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QAED,gCAAgC;QAChC,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,aAAa,EAAE,CAAC;YAC7C,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC;YAE5D,iBAAiB;YACjB,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACb,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAC3D,CACF,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC;gBACzD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAC9B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,OAAO,CAAC;oBACzD,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,OAAO,CAAC,CAC5D,CAAC;gBACF,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC5C,MAAM,UAAU,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC;oBACpE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjD,MAAM,MAAM,GAAG,GAAG,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;oBAC3D,OAAO,GAAG,UAAU,KAAK,MAAM,KAAK,IAAI,CAAC,UAAU,GAAG,CAAC;gBACzD,CAAC,CAAC,CAAC;gBAEH,KAAK,CAAC,IAAI,CACR,GAAG,UAAU,gBAAgB,SAAS,OAAO,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAC1E,CAAC;YACJ,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACjE,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACtG,KAAK,CAAC,IAAI,CAAC,gCAAgC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtE,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF"}

package/dist/lib/benchmark/metric-aggregator.d.ts

@@ -0,0 +1,38 @@
+/**
+ * MetricAggregator (Story 3.2)
+ *
+ * Computes descriptive statistics for each benchmark metric across a variant's sessions.
+ * Provides clean inputs for the StatisticalComparator.
+ */
+import type { BenchmarkMetric, BenchmarkVariant, IEventStore, MetricStats, Session } from '@agentlensai/core';
+export declare class MetricAggregator {
+    /**
+     * Aggregate metrics for a variant by querying sessions matching the variant's tag.
+     *
+     * @param store - Event store to query sessions from
+     * @param variant - The benchmark variant (identifies sessions via tag)
+     * @param metrics - Which metrics to compute
+     * @param timeRange - Optional time range filter
+     * @returns Record of metric → MetricStats
+     */
+    aggregate(store: IEventStore, variant: BenchmarkVariant, metrics: BenchmarkMetric[], timeRange?: {
+        from: string;
+        to: string;
+    }): Promise<{
+        sessionCount: number;
+        metrics: Record<BenchmarkMetric, MetricStats>;
+    }>;
+    /**
+     * Extract numeric values for a metric from a list of sessions.
+     */
+    extractMetricValues(sessions: Session[], metric: BenchmarkMetric): number[];
+    /**
+     * Extract a single metric value from a session.
+     */
+    private extractSingleValue;
+    /**
+     * Compute descriptive statistics from an array of numeric values.
+     */
+    computeStats(values: number[]): MetricStats;
+}
+//# sourceMappingURL=metric-aggregator.d.ts.map

package/dist/lib/benchmark/metric-aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metric-aggregator.d.ts","sourceRoot":"","sources":["../../../src/lib/benchmark/metric-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,OAAO,EACR,MAAM,mBAAmB,CAAC;AAI3B,qBAAa,gBAAgB;IAC3B;;;;;;;;OAQG;IACG,SAAS,CACb,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,gBAAgB,EACzB,OAAO,EAAE,eAAe,EAAE,EAC1B,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,GACvC,OAAO,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,eAAe,EAAE,WAAW,CAAC,CAAA;KAAE,CAAC;IAiCnF;;OAEG;IACH,mBAAmB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE;IAa3E;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAsD1B;;OAEG;IACH,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,WAAW;CA6C5C"}

package/dist/lib/benchmark/metric-aggregator.js

@@ -0,0 +1,159 @@
+/**
+ * MetricAggregator (Story 3.2)
+ *
+ * Computes descriptive statistics for each benchmark metric across a variant's sessions.
+ * Provides clean inputs for the StatisticalComparator.
+ */
+// ─── Public Interface ──────────────────────────────────────
+export class MetricAggregator {
+    /**
+     * Aggregate metrics for a variant by querying sessions matching the variant's tag.
+     *
+     * @param store - Event store to query sessions from
+     * @param variant - The benchmark variant (identifies sessions via tag)
+     * @param metrics - Which metrics to compute
+     * @param timeRange - Optional time range filter
+     * @returns Record of metric → MetricStats
+     */
+    async aggregate(store, variant, metrics, timeRange) {
+        // Paginate through all sessions (store caps at 500 per query)
+        const sessions = [];
+        const pageSize = 500;
+        let offset = 0;
+        while (true) {
+            const page = await store.querySessions({
+                tenantId: variant.tenantId,
+                agentId: variant.agentId,
+                tags: [variant.tag],
+                from: timeRange?.from,
+                to: timeRange?.to,
+                limit: pageSize,
+                offset,
+            });
+            sessions.push(...page.sessions);
+            if (page.sessions.length < pageSize)
+                break;
+            offset += pageSize;
+        }
+        const result = {};
+        for (const metric of metrics) {
+            const values = this.extractMetricValues(sessions, metric);
+            result[metric] = this.computeStats(values);
+        }
+        return {
+            sessionCount: sessions.length,
+            metrics: result,
+        };
+    }
+    /**
+     * Extract numeric values for a metric from a list of sessions.
+     */
+    extractMetricValues(sessions, metric) {
+        const values = [];
+        for (const session of sessions) {
+            const value = this.extractSingleValue(session, metric);
+            if (value !== null && !isNaN(value)) {
+                values.push(value);
+            }
+        }
+        return values;
+    }
+    /**
+     * Extract a single metric value from a session.
+     */
+    extractSingleValue(session, metric) {
+        switch (metric) {
+            case 'avg_cost':
+                return session.totalCostUsd;
+            case 'error_rate':
+                // Error rate = errorCount / eventCount
+                if (session.eventCount === 0)
+                    return null;
+                return session.errorCount / session.eventCount;
+            case 'completion_rate':
+                // Binary: 1 if completed, 0 otherwise
+                return session.status === 'completed' ? 1 : 0;
+            case 'tool_success_rate':
+                // errorCount includes ALL errors (LLM, system, etc.), not just tool errors.
+                // Use conservative proxy: cap tool errors at toolCallCount to avoid negative rates.
+                if (session.toolCallCount === 0)
+                    return null;
+                const toolErrors = Math.min(session.errorCount, session.toolCallCount);
+                return Math.max(0, (session.toolCallCount - toolErrors) / session.toolCallCount);
+            case 'avg_tokens':
+                return session.totalInputTokens + session.totalOutputTokens;
+            case 'avg_duration': {
+                // Duration in milliseconds
+                if (!session.endedAt || !session.startedAt)
+                    return null;
+                const start = new Date(session.startedAt).getTime();
+                const end = new Date(session.endedAt).getTime();
+                const durationMs = end - start;
+                return durationMs > 0 ? durationMs : null;
+            }
+            case 'avg_latency':
+                // avg_latency requires event-level query; we approximate from session-level data
+                // latency ≈ duration / llmCallCount (average time per LLM call)
+                if (!session.endedAt || !session.startedAt || session.llmCallCount === 0)
+                    return null;
+                const startMs = new Date(session.startedAt).getTime();
+                const endMs = new Date(session.endedAt).getTime();
+                const totalDuration = endMs - startMs;
+                return totalDuration > 0 ? totalDuration / session.llmCallCount : null;
+            case 'health_score':
+                // health_score is a composite score that would need the HealthComputer.
+                // At the session level, we compute a simplified score:
+                // Based on error rate, completion, and tool success
+                // This is a placeholder that returns null if not computed elsewhere.
+                return null;
+            default:
+                return null;
+        }
+    }
+    /**
+     * Compute descriptive statistics from an array of numeric values.
+     */
+    computeStats(values) {
+        if (values.length === 0) {
+            return {
+                mean: 0,
+                median: 0,
+                stddev: 0,
+                min: 0,
+                max: 0,
+                count: 0,
+            };
+        }
+        const n = values.length;
+        const sorted = [...values].sort((a, b) => a - b);
+        const sum = values.reduce((acc, v) => acc + v, 0);
+        const mean = sum / n;
+        // Median
+        let median;
+        if (n % 2 === 0) {
+            median = (sorted[n / 2 - 1] + sorted[n / 2]) / 2;
+        }
+        else {
+            median = sorted[Math.floor(n / 2)];
+        }
+        // Sample standard deviation (Bessel's correction)
+        let variance;
+        if (n < 2) {
+            variance = 0;
+        }
+        else {
+            const sumSquaredDiffs = values.reduce((acc, v) => acc + (v - mean) ** 2, 0);
+            variance = sumSquaredDiffs / (n - 1);
+        }
+        const stddev = Math.sqrt(variance);
+        return {
+            mean,
+            median,
+            stddev,
+            min: sorted[0],
+            max: sorted[n - 1],
+            count: n,
+        };
+    }
+}
+//# sourceMappingURL=metric-aggregator.js.map

package/dist/lib/benchmark/metric-aggregator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metric-aggregator.js","sourceRoot":"","sources":["../../../src/lib/benchmark/metric-aggregator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAUH,8DAA8D;AAE9D,MAAM,OAAO,gBAAgB;IAC3B;;;;;;;;OAQG;IACH,KAAK,CAAC,SAAS,CACb,KAAkB,EAClB,OAAyB,EACzB,OAA0B,EAC1B,SAAwC;QAExC,8DAA8D;QAC9D,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,GAAG,CAAC;QACrB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,aAAa,CAAC;gBACrC,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;gBACnB,IAAI,EAAE,SAAS,EAAE,IAAI;gBACrB,EAAE,EAAE,SAAS,EAAE,EAAE;gBACjB,KAAK,EAAE,QAAQ;gBACf,MAAM;aACP,CAAC,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;YAChC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,QAAQ;gBAAE,MAAM;YAC3C,MAAM,IAAI,QAAQ,CAAC;QACrB,CAAC;QAED,MAAM,MAAM,GAAG,EAA0C,CAAC;QAE1D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO;YACL,YAAY,EAAE,QAAQ,CAAC,MAAM;YAC7B,OAAO,EAAE,MAAM;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,QAAmB,EAAE,MAAuB;QAC9D,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACvD,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,OAAgB,EAAE,MAAuB;QAClE,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,UAAU;gBACb,OAAO,OAAO,CAAC,YAAY,CAAC;YAE9B,KAAK,YAAY;gBACf,uCAAuC;gBACvC,IAAI,OAAO,CAAC,UAAU,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC;gBAC1C,OAAO,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;YAEjD,KAAK,iBAAiB;gBACpB,sCAAsC;gBACtC,OAAO,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEhD,KAAK,mBAAmB;gBACtB,4EAA4E;gBAC5E,oFAAoF;gBACpF,IAAI,OAAO,CAAC,aAAa,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC;gBAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;gBACvE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;YAEnF,KAAK,YAAY;gBACf,OAAO,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC;YAE9D,KAAK,cAAc,CAAC,CAAC,CAAC;gBACpB,2BAA2B;gBAC3B,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS;oBAAE,OAAO,IAAI,CAAC;gBACxD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChD,MAAM,UAAU,GAAG,GAAG,GAAG,KAAK,CAAC;gBAC/B,OAAO,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YAC5C,CAAC;YAED,KAAK,aAAa;gBAChB,iFAAiF;gBACjF,gEAAgE;gBAChE,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,KAAK,CAAC;oBAAE,OAAO,IAAI,CAAC;gBACtF,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;gBACtD,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;gBAClD,MAAM,aAAa,GAAG,KAAK,GAAG,OAAO,CAAC;gBACtC,OAAO,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;YAEzE,KAAK,cAAc;gBACjB,wEAAwE;gBACxE,uDAAuD;gBACvD,oDAAoD;gBACpD,qEAAqE;gBACrE,OAAO,IAAI,CAAC;YAEd;gBACE,OAAO,IAAI,CAAC;QAChB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,MAAgB;QAC3B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,CAAC;gBACT,MAAM,EAAE,CAAC;gBACT,GAAG,EAAE,CAAC;gBACN,GAAG,EAAE,CAAC;gBACN,KAAK,EAAE,CAAC;aACT,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAEjD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;QAErB,SAAS;QACT,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAChB,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;QACtC,CAAC;QAED,kDAAkD;QAClD,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACV,QAAQ,GAAG,CAAC,CAAC;QACf,CAAC;aAAM,CAAC;YACN,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5E,QAAQ,GAAG,eAAe,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACvC,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI;YACJ,MAAM;YACN,MAAM;YACN,GAAG,EAAE,MAAM,CAAC,CAAC,CAAE;YACf,GAAG,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE;YACnB,KAAK,EAAE,CAAC;SACT,CAAC;IACJ,CAAC;CACF"}

package/dist/lib/benchmark/statistical.d.ts

@@ -0,0 +1,51 @@
+/**
+ * StatisticalComparator (Story 3.3)
+ *
+ * Hypothesis testing for benchmark variant comparisons.
+ * Implements Welch's t-test and chi-squared test with numerical p-value computation.
+ */
+import type { BenchmarkMetric, MetricComparison, MetricStats } from '@agentlensai/core';
+export interface TTestResult {
+    tStatistic: number;
+    degreesOfFreedom: number;
+    pValue: number;
+    confidenceInterval: {
+        lower: number;
+        upper: number;
+    };
+    cohenD: number;
+}
+export interface ChiSquaredResult {
+    chiSquared: number;
+    pValue: number;
+    phi: number;
+}
+export declare class StatisticalComparator {
+    /**
+     * Welch's t-test for comparing two sets of continuous metric statistics.
+     */
+    welchTTest(statsA: MetricStats, statsB: MetricStats): TTestResult;
+    /**
+     * Chi-squared test with Yates' continuity correction for comparing two proportions.
+     */
+    chiSquaredTest(successesA: number, totalA: number, successesB: number, totalB: number): ChiSquaredResult;
+    /**
+     * Compare two variants on a single metric. Selects appropriate test.
+     */
+    compare(variantA: {
+        id: string;
+        name: string;
+        metrics: Record<string, MetricStats>;
+    }, variantB: {
+        id: string;
+        name: string;
+        metrics: Record<string, MetricStats>;
+    }, metric: BenchmarkMetric): MetricComparison;
+    private compareContinuous;
+    private compareProportions;
+    /**
+     * Confidence star rating based on p-value.
+     */
+    confidenceStars(pValue: number): '★★★' | '★★' | '★' | '—';
+}
+//# sourceMappingURL=statistical.d.ts.map

package/dist/lib/benchmark/statistical.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../../src/lib/benchmark/statistical.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,eAAe,EACf,gBAAgB,EAChB,WAAW,EAEZ,MAAM,mBAAmB,CAAC;AAiC3B,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,MAAM,EAAE,MAAM,CAAC;CAChB;AAID,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;CACb;AAgKD,qBAAa,qBAAqB;IAChC;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,GAAG,WAAW;IAmEjE;;OAEG;IACH,cAAc,CACZ,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,gBAAgB;IA8CnB;;OAEG;IACH,OAAO,CACL,QAAQ,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;KAAE,EAC5E,QAAQ,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;KAAE,EAC5E,MAAM,EAAE,eAAe,GACtB,gBAAgB;IAiBnB,OAAO,CAAC,iBAAiB;IAwCzB,OAAO,CAAC,kBAAkB;IAwD1B;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,GAAG,IAAI,GAAG,GAAG,GAAG,GAAG;CAM1D"}