@cogitator-ai/evals 0.1.0

package/README.md

@@ -0,0 +1,456 @@
+# @cogitator-ai/evals
+
+Evaluation framework for Cogitator AI agents. Run eval suites, compare models with A/B tests, enforce quality thresholds, and track regressions — all with built-in statistical significance testing.
+
+## Installation
+
+```bash
+pnpm add @cogitator-ai/evals
+
+# Optional dependencies
+pnpm add papaparse  # CSV dataset loading
+```
+
+## Features
+
+- **EvalSuite** — Run datasets against agents or plain functions with configurable concurrency, timeouts, and retries
+- **4 Deterministic Metrics** — exactMatch, contains, regex, jsonSchema (Zod)
+- **5 LLM-as-Judge Metrics** — faithfulness, relevance, coherence, helpfulness, custom llmMetric
+- **3 Statistical Metrics** — latency, cost, tokenUsage with full percentile breakdowns
+- **Custom Metrics** — `metric()` factory for anything domain-specific
+- **Assertions** — threshold, noRegression, custom assertion with auto-detection of lower-is-better metrics
+- **A/B Testing** — EvalComparison with paired t-test and McNemar's test for statistical significance
+- **4 Reporters** — console (colored table), JSON, CSV, CI (exit code on failure)
+- **Builder API** — Fluent `EvalBuilder` for composable eval pipelines
+- **Baseline Workflow** — Save baselines, compare against them, catch regressions in CI
+- **Zod Validation** — Type-safe configuration with runtime checks
+
+---
+
+## Quick Start
+
+```typescript
+import { EvalSuite, Dataset, exactMatch, contains, threshold, latency } from '@cogitator-ai/evals';
+
+const dataset = Dataset.from([
+  { input: 'What is 2+2?', expected: '4' },
+  { input: 'Capital of France?', expected: 'Paris' },
+  { input: 'Largest planet?', expected: 'Jupiter' },
+]);
+
+const suite = new EvalSuite({
+  dataset,
+  target: {
+    fn: async (input) => {
+      // replace with your agent or LLM call
+      return `The answer is ${input}`;
+    },
+  },
+  metrics: [exactMatch(), contains()],
+  statisticalMetrics: [latency()],
+  assertions: [threshold('exactMatch', 0.8)],
+  concurrency: 5,
+  timeout: 30_000,
+});
+
+const result = await suite.run();
+
+result.report('console');
+result.saveBaseline('./baseline.json');
+```
+
+---
+
+## Datasets
+
+Datasets are immutable collections of eval cases. Each case has an `input`, optional `expected`, optional `context`, and optional `metadata`.
+
+### From inline data
+
+```typescript
+import { Dataset } from '@cogitator-ai/evals';
+
+const dataset = Dataset.from([
+  { input: 'Translate hello to French', expected: 'Bonjour' },
+  { input: 'Summarize this article', context: { article: '...' } },
+]);
+```
+
+### From JSONL
+
+```typescript
+const dataset = await Dataset.fromJsonl('./evals/qa.jsonl');
+```
+
+Each line must be a JSON object with at least an `input` field:
+
+```jsonl
+{"input": "What is TypeScript?", "expected": "A typed superset of JavaScript"}
+{"input": "What is Zod?", "expected": "A TypeScript-first schema validation library"}
+```
+
+### From CSV
+
+Requires `papaparse` as an optional dependency.
+
+```typescript
+const dataset = await Dataset.fromCsv('./evals/qa.csv');
+```
+
+CSV must have an `input` column. Optional columns: `expected`, `metadata.*`, `context.*`.
+
+### Transformations
+
+```typescript
+const filtered = dataset.filter((c) => c.expected !== undefined);
+const sampled = dataset.sample(50);
+const shuffled = dataset.shuffle();
+```
+
+All transformations return new `Dataset` instances — the original is never mutated.
+
+---
+
+## Metrics
+
+### Deterministic
+
+Binary (0 or 1) metrics that compare output against expected values.
+
+| Metric       | Description                                | Requires `expected` |
+| ------------ | ------------------------------------------ | ------------------- |
+| `exactMatch` | Exact string match (case optional)         | Yes                 |
+| `contains`   | Output contains expected substring         | Yes                 |
+| `regex`      | Output matches a regex pattern             | No                  |
+| `jsonSchema` | Output is valid JSON matching a Zod schema | No                  |
+
+```typescript
+import { exactMatch, contains, regex, jsonSchema } from '@cogitator-ai/evals';
+import { z } from 'zod';
+
+const metrics = [
+  exactMatch({ caseSensitive: true }),
+  contains(),
+  regex(/\d{4}-\d{2}-\d{2}/),
+  jsonSchema(z.object({ answer: z.string(), confidence: z.number() })),
+];
+```
+
+### LLM-as-Judge
+
+Metrics scored by an LLM judge (0.0 to 1.0). Require a `judge` config on the suite.
+
+| Metric         | Evaluates                               |
+| -------------- | --------------------------------------- |
+| `faithfulness` | Factual accuracy relative to input      |
+| `relevance`    | How on-topic the response is            |
+| `coherence`    | Logical structure and readability       |
+| `helpfulness`  | Practical usefulness to the user        |
+| `llmMetric`    | Custom prompt — you define the criteria |
+
+```typescript
+import { faithfulness, relevance, llmMetric } from '@cogitator-ai/evals';
+
+const suite = new EvalSuite({
+  dataset,
+  target: { fn: myFunction },
+  metrics: [
+    faithfulness(),
+    relevance(),
+    llmMetric({
+      name: 'technicalAccuracy',
+      prompt: 'Rate how technically accurate the response is for a software engineering audience.',
+    }),
+  ],
+  judge: { model: 'gpt-4o', temperature: 0 },
+});
+```
+
+### Statistical
+
+Aggregate metrics computed across all results. These report percentile breakdowns (p50, p95, p99) rather than per-case scores.
+
+```typescript
+import { latency, cost, tokenUsage } from '@cogitator-ai/evals';
+
+const suite = new EvalSuite({
+  dataset,
+  target: { agent, cogitator },
+  metrics: [exactMatch()],
+  statisticalMetrics: [latency(), cost(), tokenUsage()],
+});
+```
+
+### Custom
+
+Build domain-specific metrics with the `metric()` factory.
+
+```typescript
+import { metric } from '@cogitator-ai/evals';
+
+const wordCount = metric({
+  name: 'wordCount',
+  evaluate: ({ output }) => {
+    const count = output.split(/\s+/).length;
+    return { score: Math.min(count / 100, 1), details: `${count} words` };
+  },
+});
+
+const suite = new EvalSuite({
+  dataset,
+  target: { fn: myFunction },
+  metrics: [wordCount],
+});
+```
+
+Scores are automatically clamped to [0, 1].
+
+---
+
+## Assertions
+
+Assertions check aggregated metrics after a suite run and produce pass/fail results.
+
+### threshold
+
+Enforces a minimum (or maximum for latency/cost) value on a metric's mean.
+
+```typescript
+import { threshold } from '@cogitator-ai/evals';
+
+const assertions = [
+  threshold('exactMatch', 0.9),
+  threshold('latency', 5000),
+  threshold('relevance', 0.7),
+];
+```
+
+Latency and cost metrics are automatically detected as lower-is-better.
+
+### noRegression
+
+Compares current results against a saved baseline file.
+
+```typescript
+import { noRegression } from '@cogitator-ai/evals';
+
+const assertions = [noRegression('./baseline.json', { tolerance: 0.05 })];
+```
+
+### Custom assertion
+
+```typescript
+import { assertion } from '@cogitator-ai/evals';
+
+const assertions = [
+  assertion({
+    name: 'totalCostBudget',
+    check: (_aggregated, stats) => stats.cost < 1.0,
+    message: 'Total eval cost exceeded $1.00 budget',
+  }),
+];
+```
+
+---
+
+## A/B Testing
+
+`EvalComparison` runs two targets on the same dataset and determines a winner using statistical significance tests (paired t-test for continuous metrics, McNemar's test for binary metrics).
+
+```typescript
+import { EvalComparison, Dataset, exactMatch, contains } from '@cogitator-ai/evals';
+
+const dataset = Dataset.from([
+  { input: 'What is 2+2?', expected: '4' },
+  { input: 'Capital of Japan?', expected: 'Tokyo' },
+  { input: 'Boiling point of water?', expected: '100°C' },
+]);
+
+const comparison = new EvalComparison({
+  dataset,
+  targets: {
+    baseline: { fn: async (input) => baselineModel(input) },
+    challenger: { fn: async (input) => challengerModel(input) },
+  },
+  metrics: [exactMatch(), contains()],
+  concurrency: 5,
+});
+
+const result = await comparison.run();
+
+console.log(`Winner: ${result.summary.winner}`);
+for (const [name, mc] of Object.entries(result.summary.metrics)) {
+  console.log(
+    `  ${name}: baseline=${mc.baseline.toFixed(3)} challenger=${mc.challenger.toFixed(3)} p=${mc.pValue.toFixed(4)} ${mc.significant ? '*' : ''}`
+  );
+}
+```
+
+Access full suite results via `result.baseline` and `result.challenger`.
+
+---
+
+## Reporters
+
+Call `result.report()` after a suite run to output results.
+
+| Reporter  | Output                                          |
+| --------- | ----------------------------------------------- |
+| `console` | Colored table with metrics, assertions, summary |
+| `json`    | Writes `eval-report.json` (configurable path)   |
+| `csv`     | Writes `eval-report.csv` (configurable path)    |
+| `ci`      | Compact output, `process.exit(1)` on failure    |
+
+```typescript
+const result = await suite.run();
+
+result.report('console');
+result.report('json', { path: './reports/eval.json' });
+result.report(['console', 'json', 'csv']);
+result.report('ci');
+```
+
+---
+
+## Builder API
+
+`EvalBuilder` provides a fluent interface for constructing eval suites.
+
+```typescript
+import {
+  EvalBuilder,
+  Dataset,
+  exactMatch,
+  contains,
+  faithfulness,
+  latency,
+  threshold,
+  noRegression,
+} from '@cogitator-ai/evals';
+
+const suite = new EvalBuilder()
+  .withDataset(await Dataset.fromJsonl('./evals/qa.jsonl'))
+  .withTarget({ fn: async (input) => myModel(input) })
+  .withMetrics([exactMatch(), contains(), faithfulness()])
+  .withStatisticalMetrics([latency()])
+  .withJudge({ model: 'gpt-4o', temperature: 0 })
+  .withAssertions([threshold('exactMatch', 0.85), noRegression('./baseline.json')])
+  .withConcurrency(10)
+  .withTimeout(60_000)
+  .withRetries(2)
+  .onProgress(({ completed, total }) => {
+    console.log(`${completed}/${total}`);
+  })
+  .build();
+
+const result = await suite.run();
+result.report('console');
+```
+
+---
+
+## Baseline Workflow
+
+Save a baseline after a successful run, then use `noRegression` to guard against regressions in CI.
+
+```typescript
+const result = await suite.run();
+
+result.saveBaseline('./baseline.json');
+```
+
+The baseline file is a simple JSON map of metric names to mean scores:
+
+```json
+{
+  "exactMatch": 0.92,
+  "contains": 0.97,
+  "latency": 1234
+}
+```
+
+In subsequent runs, use `noRegression` to compare:
+
+```typescript
+const suite = new EvalSuite({
+  dataset,
+  target: { fn: myFunction },
+  metrics: [exactMatch(), contains()],
+  assertions: [noRegression('./baseline.json', { tolerance: 0.05 })],
+});
+
+const result = await suite.run();
+result.report('ci');
+```
+
+---
+
+## API Reference
+
+### Core
+
+| Export           | Description                                                         |
+| ---------------- | ------------------------------------------------------------------- |
+| `EvalSuite`      | Main evaluation runner                                              |
+| `EvalComparison` | A/B testing runner with statistical significance                    |
+| `EvalBuilder`    | Fluent builder for EvalSuite                                        |
+| `Dataset`        | Immutable dataset with from/fromJsonl/fromCsv/filter/sample/shuffle |
+| `loadJsonl`      | Low-level JSONL file loader                                         |
+| `loadCsv`        | Low-level CSV file loader                                           |
+
+### Metrics
+
+| Export         | Type          | Description                         |
+| -------------- | ------------- | ----------------------------------- |
+| `exactMatch`   | Deterministic | Exact string match                  |
+| `contains`     | Deterministic | Substring match                     |
+| `regex`        | Deterministic | Regex pattern match                 |
+| `jsonSchema`   | Deterministic | Zod schema validation               |
+| `faithfulness` | LLM Judge     | Factual accuracy                    |
+| `relevance`    | LLM Judge     | Topical relevance                   |
+| `coherence`    | LLM Judge     | Logical structure                   |
+| `helpfulness`  | LLM Judge     | Practical usefulness                |
+| `llmMetric`    | LLM Judge     | Custom judge prompt                 |
+| `latency`      | Statistical   | Response time percentiles           |
+| `cost`         | Statistical   | Token cost aggregation              |
+| `tokenUsage`   | Statistical   | Input/output token counts           |
+| `metric`       | Custom        | Factory for domain-specific metrics |
+
+### Assertions
+
+| Export         | Description                                    |
+| -------------- | ---------------------------------------------- |
+| `threshold`    | Enforce min/max on metric mean                 |
+| `noRegression` | Compare against saved baseline                 |
+| `assertion`    | Custom assertion with arbitrary check function |
+
+### Reporters
+
+| Export   | Description                            |
+| -------- | -------------------------------------- |
+| `report` | Dispatch to one or more reporter types |
+
+### Statistics
+
+| Export         | Description                                               |
+| -------------- | --------------------------------------------------------- |
+| `pairedTTest`  | Paired t-test for continuous metric comparison            |
+| `mcnemarsTest` | McNemar's test for binary metric comparison               |
+| `mean`         | Arithmetic mean                                           |
+| `median`       | Median value                                              |
+| `stdDev`       | Sample standard deviation                                 |
+| `percentile`   | Arbitrary percentile                                      |
+| `aggregate`    | Full stats: mean, median, min, max, stdDev, p50, p95, p99 |
+
+### Agent Tools
+
+| Export              | Description                          |
+| ------------------- | ------------------------------------ |
+| `createRunEvalTool` | Creates a `run_eval` tool for agents |
+| `evalTools`         | Returns all eval tools as an array   |
+
+---
+
+## License
+
+MIT

package/dist/assertions/custom.d.ts

@@ -0,0 +1,11 @@
+import type { AssertionFn, AggregatedMetric } from './index';
+export declare function assertion(opts: {
+    name: string;
+    check: (aggregated: Record<string, AggregatedMetric>, stats: {
+        total: number;
+        duration: number;
+        cost: number;
+    }) => boolean;
+    message?: string;
+}): AssertionFn;
+//# sourceMappingURL=custom.d.ts.map

package/dist/assertions/custom.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/assertions/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE7D,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,CACL,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAC5C,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,KACrD,OAAO,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,WAAW,CAYd"}

package/dist/assertions/custom.js

@@ -0,0 +1,13 @@
+export function assertion(opts) {
+    return (aggregated, stats) => {
+        const passed = opts.check(aggregated, stats);
+        return {
+            name: opts.name,
+            passed,
+            message: passed
+                ? `Custom assertion '${opts.name}' passed`
+                : (opts.message ?? `Custom assertion '${opts.name}' failed`),
+        };
+    };
+}
+//# sourceMappingURL=custom.js.map

package/dist/assertions/custom.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/assertions/custom.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,SAAS,CAAC,IAOzB;IACC,OAAO,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAE7C,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM;YACN,OAAO,EAAE,MAAM;gBACb,CAAC,CAAC,qBAAqB,IAAI,CAAC,IAAI,UAAU;gBAC1C,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,qBAAqB,IAAI,CAAC,IAAI,UAAU,CAAC;SAC/D,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/assertions/index.d.ts

@@ -0,0 +1,27 @@
+export interface AggregatedMetric {
+    name: string;
+    mean: number;
+    median: number;
+    min: number;
+    max: number;
+    stdDev: number;
+    p50: number;
+    p95: number;
+    p99: number;
+}
+export interface AssertionResult {
+    name: string;
+    passed: boolean;
+    message: string;
+    actual?: number;
+    expected?: number;
+}
+export type AssertionFn = (aggregated: Record<string, AggregatedMetric>, stats: {
+    total: number;
+    duration: number;
+    cost: number;
+}) => AssertionResult;
+export { threshold } from './threshold';
+export { noRegression } from './regression';
+export { assertion } from './custom';
+//# sourceMappingURL=index.d.ts.map

package/dist/assertions/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,WAAW,GAAG,CACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAC5C,KAAK,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,KACrD,eAAe,CAAC;AAErB,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}

package/dist/assertions/index.js

@@ -0,0 +1,4 @@
+export { threshold } from './threshold';
+export { noRegression } from './regression';
+export { assertion } from './custom';
+//# sourceMappingURL=index.js.map

package/dist/assertions/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/assertions/index.ts"],"names":[],"mappings":"AAyBA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC"}

package/dist/assertions/regression.d.ts

@@ -0,0 +1,5 @@
+import type { AssertionFn } from './index';
+export declare function noRegression(baselinePath: string, opts?: {
+    tolerance?: number;
+}): AssertionFn;
+//# sourceMappingURL=regression.d.ts.map

package/dist/assertions/regression.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"regression.d.ts","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAM3C,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAsD7F"}

package/dist/assertions/regression.js

@@ -0,0 +1,58 @@
+import * as fs from 'node:fs';
+function isLowerBetter(name) {
+    return name.startsWith('latency') || name.startsWith('cost');
+}
+export function noRegression(baselinePath, opts) {
+    return (aggregated, _stats) => {
+        let baseline;
+        try {
+            const raw = fs.readFileSync(baselinePath, 'utf-8');
+            baseline = JSON.parse(raw);
+        }
+        catch {
+            return {
+                name: 'noRegression',
+                passed: false,
+                message: `Failed to read baseline file: ${baselinePath}`,
+            };
+        }
+        const tolerance = opts?.tolerance ?? 0.05;
+        for (const [metric, baselineValue] of Object.entries(baseline)) {
+            const agg = aggregated[metric];
+            if (!agg)
+                continue;
+            const actual = agg.mean;
+            const lowerBetter = isLowerBetter(metric);
+            if (lowerBetter) {
+                const limit = baselineValue * (1 + tolerance);
+                if (actual > limit) {
+                    return {
+                        name: 'noRegression',
+                        passed: false,
+                        message: `Regression in '${metric}': ${actual} > ${limit} (baseline ${baselineValue}, tolerance ${tolerance * 100}%)`,
+                        actual,
+                        expected: limit,
+                    };
+                }
+            }
+            else {
+                const limit = baselineValue * (1 - tolerance);
+                if (actual < limit) {
+                    return {
+                        name: 'noRegression',
+                        passed: false,
+                        message: `Regression in '${metric}': ${actual} < ${limit} (baseline ${baselineValue}, tolerance ${tolerance * 100}%)`,
+                        actual,
+                        expected: limit,
+                    };
+                }
+            }
+        }
+        return {
+            name: 'noRegression',
+            passed: true,
+            message: 'All metrics within tolerance of baseline',
+        };
+    };
+}
+//# sourceMappingURL=regression.js.map

package/dist/assertions/regression.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"regression.js","sourceRoot":"","sources":["../../src/assertions/regression.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAG9B,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,YAAoB,EAAE,IAA6B;IAC9E,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE;QAC5B,IAAI,QAAgC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,iCAAiC,YAAY,EAAE;aACzD,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC;QAE1C,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/B,IAAI,CAAC,GAAG;gBAAE,SAAS;YAEnB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC;YACxB,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;YAE1C,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,KAAK,GAAG,aAAa,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC9C,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;oBACnB,OAAO;wBACL,IAAI,EAAE,cAAc;wBACpB,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,kBAAkB,MAAM,MAAM,MAAM,MAAM,KAAK,cAAc,aAAa,eAAe,SAAS,GAAG,GAAG,IAAI;wBACrH,MAAM;wBACN,QAAQ,EAAE,KAAK;qBAChB,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAG,aAAa,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC9C,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;oBACnB,OAAO;wBACL,IAAI,EAAE,cAAc;wBACpB,MAAM,EAAE,KAAK;wBACb,OAAO,EAAE,kBAAkB,MAAM,MAAM,MAAM,MAAM,KAAK,cAAc,aAAa,eAAe,SAAS,GAAG,GAAG,IAAI;wBACrH,MAAM;wBACN,QAAQ,EAAE,KAAK;qBAChB,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO;YACL,IAAI,EAAE,cAAc;YACpB,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,0CAA0C;SACpD,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/assertions/threshold.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionFn } from './index';
+export declare function threshold(metricName: string, value: number): AssertionFn;
+//# sourceMappingURL=threshold.d.ts.map

package/dist/assertions/threshold.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"threshold.d.ts","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAoB,MAAM,SAAS,CAAC;AA6B7D,wBAAgB,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CA0BxE"}

package/dist/assertions/threshold.js

@@ -0,0 +1,45 @@
+function isLowerBetter(name) {
+    const base = name.split('.')[0];
+    return (base.startsWith('latency') ||
+        base.startsWith('cost') ||
+        base.endsWith('Duration') ||
+        base.endsWith('Latency'));
+}
+function resolve(aggregated, path) {
+    const parts = path.split('.');
+    const metric = aggregated[parts[0]];
+    if (!metric)
+        return { value: 0, found: false };
+    if (parts.length === 1)
+        return { value: metric.mean, found: true };
+    const field = parts[1];
+    const val = metric[field];
+    if (typeof val !== 'number')
+        return { value: 0, found: false };
+    return { value: val, found: true };
+}
+export function threshold(metricName, value) {
+    return (aggregated, _stats) => {
+        const { value: actual, found } = resolve(aggregated, metricName);
+        if (!found) {
+            return {
+                name: `threshold(${metricName})`,
+                passed: false,
+                message: `Metric '${metricName}' not found in aggregated results`,
+            };
+        }
+        const lowerBetter = isLowerBetter(metricName);
+        const passed = lowerBetter ? actual <= value : actual >= value;
+        const direction = lowerBetter ? '<=' : '>=';
+        return {
+            name: `threshold(${metricName})`,
+            passed,
+            message: passed
+                ? `${metricName} = ${actual} ${direction} ${value}`
+                : `${metricName} = ${actual}, expected ${direction} ${value}`,
+            actual,
+            expected: value,
+        };
+    };
+}
+//# sourceMappingURL=threshold.js.map

package/dist/assertions/threshold.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"threshold.js","sourceRoot":"","sources":["../../src/assertions/threshold.ts"],"names":[],"mappings":"AAEA,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAChC,OAAO,CACL,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;QACzB,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CACzB,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CACd,UAA4C,EAC5C,IAAY;IAEZ,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAE/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAEnE,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAA2B,CAAC;IACjD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAE/D,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,UAAkB,EAAE,KAAa;IACzD,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE;QAC5B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAEjE,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO;gBACL,IAAI,EAAE,aAAa,UAAU,GAAG;gBAChC,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,WAAW,UAAU,mCAAmC;aAClE,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;QAC/D,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QAE5C,OAAO;YACL,IAAI,EAAE,aAAa,UAAU,GAAG;YAChC,MAAM;YACN,OAAO,EAAE,MAAM;gBACb,CAAC,CAAC,GAAG,UAAU,MAAM,MAAM,IAAI,SAAS,IAAI,KAAK,EAAE;gBACnD,CAAC,CAAC,GAAG,UAAU,MAAM,MAAM,cAAc,SAAS,IAAI,KAAK,EAAE;YAC/D,MAAM;YACN,QAAQ,EAAE,KAAK;SAChB,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}

package/dist/datasets/csv-loader.d.ts

@@ -0,0 +1,3 @@
+import type { EvalCase } from '../schema';
+export declare function loadCsv(path: string): Promise<EvalCase[]>;
+//# sourceMappingURL=csv-loader.d.ts.map

package/dist/datasets/csv-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/datasets/csv-loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CA6C/D"}