@cogitator-ai/evals 0.8.0 → 0.10.0

package/dist/eval-suite.d.ts

@@ -1,10 +1,15 @@
 import { Dataset } from './datasets/index.js';
 import type { EvalCase } from './schema.js';
 import type { JudgeConfigInput } from './schema.js';
-import type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types.js';
+import type { MetricFn, Passage, ScoredCaseResult, StatisticalMetricFn } from './metrics/types.js';
 import type { LLMMetricFn } from './metrics/llm-judge.js';
 import type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
 import type { ReporterType, ReporterOptions } from './reporters/index.js';
+/** A turn of a conversation: what the user said and what the target answered */
+export interface EvalTurn {
+    input: string;
+    output: string;
+}
 /** What a function target gets besides the input */
 export interface EvalTargetContext {
     /**
@@ -17,11 +22,27 @@ export interface EvalTargetContext {
     case: EvalCase;
     /** 0 for the first attempt, then 1, 2 and so on for retries */
     attempt: number;
+    /** 0 for the case's `input`, then 1, 2 and so on for each of its further `turns` */
+    turn: number;
+    /** The turns of the case before this one, with the answers the target gave */
+    history: readonly EvalTurn[];
+}
+/** A tool result as the `retrieval` of a target reads it */
+export interface RetrievalToolResult {
+    name: string;
+    result: unknown;
 }
 export interface EvalTarget {
     agent?: unknown;
     cogitator?: unknown;
+    /** Called once per turn of a case: with `input`, then with each of its further `turns` */
     fn?: (input: string, context: EvalTargetContext) => Promise<string>;
+    /**
+     * Reads the passages a tool result of an agent target surfaced, into the case's `retrieved`,
+     * undefined for a result that holds none. Default: `ragSearchPassages`, which reads the
+     * results of the `rag_search` tool of `@cogitator-ai/rag`
+     */
+    retrieval?: (toolResult: RetrievalToolResult) => Passage[] | undefined;
 }
 export interface EvalProgress {
     completed: number;
@@ -38,7 +59,19 @@ export interface EvalSuiteOptions {
     concurrency?: number;
     timeout?: number;
     retries?: number;
-    /** Called after each case. A callback that throws is reported as a process warning */
+    /**
+     * Runs of each case, a positive integer, 1 by default. Each trial is scored on its own, the
+     * case result is the first trial that answered (the first when all errored) with every trial in
+     * `trials`, and the aggregate of a metric
+     * takes the mean of a case's scored trials as the case's score. A case counts as errored in
+     * `stats.errors` when every trial errored, and `latency()`, `cost()` and `tokenUsage()` see
+     * every trial. Use `passAtK()` to score how often a case passes
+     */
+    trials?: number;
+    /**
+     * Called after each case, once all its trials ran. A callback that throws is reported as a
+     * process warning
+     */
     onProgress?: (progress: EvalProgress) => void;
 }
 export interface EvalRunOptions {
@@ -53,9 +86,11 @@ export interface EvalRunOptions {
 /** The run-wide figures of a finished suite run, every field filled in */
 export type EvalSuiteStats = Required<EvalStats>;
 export interface EvalSuiteResult {
-    results: Array<EvalCaseResult & {
-        scores: MetricScore[];
-    }>;
+    /**
+     * One result per case, in dataset order. With several trials it is the first trial that
+     * answered, or the first when every trial errored, with every trial in its `trials`
+     */
+    results: ScoredCaseResult[];
     aggregated: Record<string, AggregatedMetric>;
     assertions: AssertionResult[];
     stats: EvalSuiteStats;
@@ -73,11 +108,16 @@ export declare class EvalSuite {
     private readonly concurrency;
     private readonly timeout;
     private readonly retries;
+    private readonly trials;
     private readonly onProgress?;
     constructor(opts: EvalSuiteOptions);
     private validateTarget;
+    /** A `passAtK()` metric needs the metric it reads and at least `k` trials */
+    private validatePassAtK;
     /** Two metrics with one name would be aggregated, compared and reported as one */
     private validateMetricNames;
+    /** The cases a run with `maxCases` goes through, in order */
+    cases(maxCases?: number): readonly EvalCase[];
     run(options?: EvalRunOptions): Promise<EvalSuiteResult>;
     private reportProgress;
     private runCase;
@@ -89,14 +129,21 @@ export declare class EvalSuite {
      */
     private executeCaseAttempt;
     private executeFnTarget;
+    /**
+     * Runs the agent on each turn of the case. A case with further `turns` runs them all on one
+     * thread of its own, so the Cogitator's memory carries the conversation from turn to turn
+     */
     private executeAgentTarget;
     private evaluateCaseMetrics;
     /**
-     * A metric's score for one case. A metric that throws, or whose score is not a finite number,
-     * scores 0 with the reason in `error`, so one bad metric neither drops the case nor poisons
-     * the aggregate with NaN.
+     * A metric's score for one case. Only cases with an output are scored: a case whose every
+     * attempt failed is left unscored, so its empty output neither costs a judge call nor counts as
+     * a score of 0 (`stats.errors` and the `maxErrorRate` assertion catch those). A metric that
+     * throws, or whose score is not a finite number, scores 0 with the reason in `error`, so one
+     * bad metric neither drops the case nor poisons the aggregate with NaN.
      */
     private scoreMetric;
+    /** One score per case and metric: the mean over the case's trials the metric scored */
     private aggregateScores;
 }
 //# sourceMappingURL=eval-suite.d.ts.map

package/dist/eval-suite.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAElG,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9F,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,EAAE,WAAW,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACrE;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sFAAsF;IACtF,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,wFAAwF;IACxF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AASD,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AA+DD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IA0ClC,OAAO,CAAC,cAAc;IAuBtB,kFAAkF;IAClF,OAAO,CAAC,mBAAmB;IAcrB,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IAsGjE,OAAO,CAAC,cAAc;YAcR,OAAO;YASP,WAAW;IA8BzB;;;;OAIG;YACW,kBAAkB;YAoClB,eAAe;YASf,kBAAkB;YAoClB,mBAAmB;IAKjC;;;;OAIG;YACW,WAAW;IAuBzB,OAAO,CAAC,eAAe;CAuBxB"}
+{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EACV,QAAQ,EAMR,OAAO,EACP,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,iBAAiB,CAAC;AAIzB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9F,OAAO,KAAK,EAAgB,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE/E,gFAAgF;AAChF,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,EAAE,WAAW,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,oFAAoF;IACpF,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,OAAO,EAAE,SAAS,QAAQ,EAAE,CAAC;CAC9B;AAED,4DAA4D;AAC5D,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,0FAA0F;IAC1F,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACpE;;;;OAIG;IACH,SAAS,CAAC,EAAE,CAAC,UAAU,EAAE,mBAAmB,KAAK,OAAO,EAAE,GAAG,SAAS,CAAC;CACxE;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,wFAAwF;IACxF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAyDD,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAmFD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IA6ClC,OAAO,CAAC,cAAc;IA2BtB,6EAA6E;IAC7E,OAAO,CAAC,eAAe;IAkBvB,kFAAkF;IAClF,OAAO,CAAC,mBAAmB;IAc3B,6DAA6D;IAC7D,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,QAAQ,EAAE;IAIvC,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IAyGjE,OAAO,CAAC,cAAc;YAcR,OAAO;YASP,WAAW;IAiCzB;;;;OAIG;YACW,kBAAkB;YAsClB,eAAe;IAmC7B;;;OAGG;YACW,kBAAkB;YA4ElB,mBAAmB;IAKjC;;;;;;OAMG;YACW,WAAW;IAuBzB,uFAAuF;IACvF,OAAO,CAAC,eAAe;CA4BxB"}

package/dist/eval-suite.js

@@ -1,9 +1,30 @@
+import { randomUUID } from 'node:crypto';
 import { writeFileSync } from 'node:fs';
 import { EvalSuiteConfigSchema, JudgeConfigSchema } from './schema.js';
 import { nonFiniteScoreError } from './metrics/types.js';
+import { isPassAtKMetric } from './metrics/statistical.js';
+import { ragSearchPassages } from './metrics/retrieval.js';
 import { bindJudgeContext, judgeContextFor } from './metrics/llm-judge.js';
-import { aggregate } from './stats/index.js';
+import { aggregate, mean } from './stats/index.js';
 import { report } from './reporters/index.js';
+/**
+ * A turn of a multi-turn case failed: the error of the turn, with the turns run so far (the
+ * failed one last) and what they spent
+ */
+class TurnFailure extends Error {
+    turns;
+    usage;
+    constructor(cause, turns, usage) {
+        super(errorMessage(cause), { cause });
+        this.turns = turns;
+        this.usage = usage;
+        this.name = 'TurnFailure';
+    }
+}
+/** What the user says to the target, turn by turn: the case's `input`, then its `turns` */
+function turnInputs(evalCase) {
+    return [evalCase.input, ...(evalCase.turns ?? [])];
+}
 export function isLLMMetric(m) {
     return 'requiresJudge' in m && m.requiresJudge === true;
 }
@@ -67,6 +88,25 @@ async function waitAtMost(promise, ms) {
         clearTimeout(timer);
     }
 }
+/**
+ * The aggregate of a metric no case was scored on, every case having errored: its figures are
+ * NaN, so a threshold, a regression check or a baseline over it fails instead of passing on
+ * nothing
+ */
+function unscored(name) {
+    return {
+        name,
+        mean: Number.NaN,
+        median: Number.NaN,
+        min: Number.NaN,
+        max: Number.NaN,
+        stdDev: Number.NaN,
+        p50: Number.NaN,
+        p95: Number.NaN,
+        p99: Number.NaN,
+        count: 0,
+    };
+}
 export class EvalSuite {
     dataset;
     target;
@@ -76,12 +116,14 @@ export class EvalSuite {
     concurrency;
     timeout;
     retries;
+    trials;
     onProgress;
     constructor(opts) {
         const config = EvalSuiteConfigSchema.parse({
             concurrency: opts.concurrency,
             timeout: opts.timeout,
             retries: opts.retries,
+            trials: opts.trials,
         });
         this.dataset = opts.dataset;
         this.target = opts.target;
@@ -90,10 +132,12 @@ export class EvalSuite {
         this.concurrency = config.concurrency;
         this.timeout = config.timeout;
         this.retries = config.retries;
+        this.trials = config.trials;
         this.onProgress = opts.onProgress;
         this.validateTarget();
         const rawMetrics = opts.metrics ?? [];
         this.validateMetricNames(rawMetrics);
+        this.validatePassAtK(rawMetrics);
         const hasLLMMetrics = rawMetrics.some(isLLMMetric);
         if (hasLLMMetrics && !opts.judge) {
             throw new Error('LLM metrics require a judge config');
@@ -123,6 +167,24 @@ export class EvalSuite {
         if (hasCogitator && !hasAgent) {
             throw new Error('Cogitator target requires agent instance');
         }
+        if (hasFn && this.target.retrieval !== undefined) {
+            throw new Error('Target retrieval reads tool results, so it needs an agent target, not fn');
+        }
+    }
+    /** A `passAtK()` metric needs the metric it reads and at least `k` trials */
+    validatePassAtK(metrics) {
+        const names = new Set(metrics.map((m) => m.metricName));
+        for (const statMetric of this.statisticalMetrics) {
+            if (!isPassAtKMetric(statMetric))
+                continue;
+            const { metric, k } = statMetric.passAtK;
+            if (!names.has(metric)) {
+                throw new Error(`${statMetric.metricName} reads the metric '${metric}', which the suite does not have: add it to metrics`);
+            }
+            if (k > this.trials) {
+                throw new Error(`${statMetric.metricName} needs at least ${k} trials per case, the suite runs ${this.trials}: set trials to ${k} or more`);
+            }
+        }
     }
     /** Two metrics with one name would be aggregated, compared and reported as one */
     validateMetricNames(metrics) {
@@ -137,6 +199,10 @@ export class EvalSuite {
             seen.add(name);
         }
     }
+    /** The cases a run with `maxCases` goes through, in order */
+    cases(maxCases) {
+        return this.dataset.cases.slice(0, maxCases);
+    }
     async run(options = {}) {
         const { maxCases, signal } = options;
         if (maxCases !== undefined && (!Number.isInteger(maxCases) || maxCases < 1)) {
@@ -144,37 +210,45 @@ export class EvalSuite {
         }
         signal?.throwIfAborted();
         const suiteStart = Date.now();
-        const cases = this.dataset.cases.slice(0, maxCases);
+        const cases = this.cases(maxCases);
         const total = cases.length;
-        const orderedResults = new Array(total);
-        let nextIdx = 0;
+        const trials = this.trials;
+        const trialResults = cases.map(() => new Array(trials));
+        const trialsLeft = cases.map(() => trials);
+        const runs = total * trials;
+        let nextRun = 0;
         let completed = 0;
         const worker = async () => {
-            while (nextIdx < total && !signal?.aborted) {
-                const i = nextIdx++;
+            while (nextRun < runs && !signal?.aborted) {
+                const run = nextRun++;
+                const i = Math.floor(run / trials);
                 const evalCase = cases[i];
-                orderedResults[i] = await this.runCase(evalCase, signal);
+                trialResults[i][run % trials] = await this.runCase(evalCase, signal);
+                if (--trialsLeft[i] > 0)
+                    continue;
                 completed++;
                 this.reportProgress({ completed, total, currentCase: evalCase });
             }
         };
-        await Promise.all(Array.from({ length: Math.min(this.concurrency, total) }, worker));
+        await Promise.all(Array.from({ length: Math.min(this.concurrency, runs) }, worker));
         signal?.throwIfAborted();
+        const orderedResults = trialResults.map((caseTrials) => trials === 1 ? caseTrials[0] : { ...leadTrial(caseTrials), trials: caseTrials });
+        const allTrials = orderedResults.flatMap(trialsOf);
         const aggregated = this.aggregateScores(orderedResults);
         for (const statMetric of this.statisticalMetrics) {
-            const score = statMetric(orderedResults);
+            const score = statMetric(isPassAtKMetric(statMetric) ? orderedResults : allTrials);
             aggregated[score.name] = {
                 name: score.name,
                 ...aggregate(score.values ?? [score.score]),
                 ...(score.metadata && { metadata: score.metadata }),
             };
         }
-        const targetCost = orderedResults.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0);
-        const judgeCost = orderedResults.reduce((sum, r) => sum + r.scores.reduce((s, score) => s + (score.usage?.cost ?? 0), 0), 0);
+        const targetCost = allTrials.reduce((sum, r) => sum + (r.usage?.cost ?? 0), 0);
+        const judgeCost = allTrials.reduce((sum, r) => sum + r.scores.reduce((s, score) => s + (score.usage?.cost ?? 0), 0), 0);
         const stats = {
             total,
             errors: orderedResults.filter((r) => r.error !== undefined).length,
-            metricErrors: orderedResults.reduce((n, r) => n + r.scores.filter((s) => s.error !== undefined).length, 0),
+            metricErrors: allTrials.reduce((n, r) => n + r.scores.filter((s) => s.error !== undefined).length, 0),
             duration: Date.now() - suiteStart,
             cost: targetCost + judgeCost,
             targetCost,
@@ -188,13 +262,7 @@ export class EvalSuite {
             stats,
             report: (type, options) => {
                 report({
-                    results: orderedResults.map((r) => ({
-                        case: { input: r.case.input, expected: r.case.expected },
-                        output: r.output,
-                        duration: r.duration,
-                        ...(r.error !== undefined && { error: r.error }),
-                        scores: r.scores,
-                    })),
+                    results: orderedResults.map(reportedCase),
                     aggregated,
                     assertions: assertionResults,
                     stats,
@@ -234,13 +302,14 @@ export class EvalSuite {
     }
     async runCase(evalCase, signal) {
         const caseResult = await this.executeCase(evalCase, signal);
-        if (signal?.aborted)
+        if (signal?.aborted || caseResult.error !== undefined)
             return { ...caseResult, scores: [] };
         return { ...caseResult, scores: await this.evaluateCaseMetrics(caseResult) };
     }
     async executeCase(evalCase, signal) {
         const start = Date.now();
         let lastError;
+        let lastTurns;
         let usage;
         let attempts = 0;
         for (let attempt = 0; attempt <= this.retries && !signal?.aborted; attempt++) {
@@ -252,6 +321,7 @@ export class EvalSuite {
             }
             usage = addUsage(usage, outcome.usage);
             lastError = outcome.error;
+            lastTurns = outcome.turns;
         }
         return {
             case: evalCase,
@@ -260,6 +330,7 @@ export class EvalSuite {
             error: errorMessage(signal?.aborted ? signal.reason : lastError),
             attempts,
             ...(usage && { usage }),
+            ...(lastTurns && { turns: lastTurns }),
         };
     }
     /**
@@ -280,10 +351,12 @@ export class EvalSuite {
         try {
             const first = await Promise.race([settled, abort.aborted]);
             if (first !== 'aborted' && (first.ok || !signal.aborted))
-                return first;
+                return failureDetails(first);
             const late = await waitAtMost(settled, this.timeout);
-            const usage = late?.ok ? late.result.usage : undefined;
-            return { ok: false, error: signal.reason, ...(usage && { usage }) };
+            const lateOutcome = late && failureDetails(late);
+            const usage = lateOutcome?.ok ? lateOutcome.result.usage : lateOutcome?.usage;
+            const turns = lateOutcome && !lateOutcome.ok ? lateOutcome.turns : undefined;
+            return { ok: false, error: signal.reason, ...(usage && { usage }), ...(turns && { turns }) };
         }
         finally {
             clearTimeout(timer);
@@ -292,31 +365,105 @@ export class EvalSuite {
     }
     async executeFnTarget(evalCase, context) {
         const start = Date.now();
-        const output = await this.target.fn(evalCase.input, context);
-        return { case: evalCase, output, duration: Date.now() - start };
+        const inputs = turnInputs(evalCase);
+        const multiTurn = inputs.length > 1;
+        const turns = [];
+        for (const [turn, input] of inputs.entries()) {
+            if (turn > 0 && context.signal.aborted) {
+                throw new TurnFailure(context.signal.reason, [...turns], undefined);
+            }
+            const history = turns.map(({ input, output }) => ({ input, output }));
+            try {
+                const output = await this.target.fn(input, { ...context, turn, history });
+                turns.push({ input, output });
+            }
+            catch (error) {
+                if (!multiTurn)
+                    throw error;
+                throw new TurnFailure(error, [...turns, { input, output: '', error: errorMessage(error) }], undefined);
+            }
+        }
+        return {
+            case: evalCase,
+            output: turns[turns.length - 1].output,
+            duration: Date.now() - start,
+            ...(multiTurn && { turns }),
+        };
     }
+    /**
+     * Runs the agent on each turn of the case. A case with further `turns` runs them all on one
+     * thread of its own, so the Cogitator's memory carries the conversation from turn to turn
+     */
     async executeAgentTarget(evalCase, signal) {
         const start = Date.now();
         const cogitator = this.target.cogitator;
-        const runResult = await cogitator.run(this.target.agent, {
-            input: evalCase.input,
-            context: evalCase.context,
-            signal,
-        });
-        const duration = Date.now() - start;
-        const result = {
+        const retrieval = this.target.retrieval ?? ragSearchPassages;
+        const inputs = turnInputs(evalCase);
+        const threadId = inputs.length > 1 ? `eval-${randomUUID()}` : undefined;
+        const turns = [];
+        const toolCalls = [];
+        const toolResults = [];
+        const retrieved = [];
+        let reportsToolCalls = false;
+        let usage;
+        const flags = {};
+        for (const input of inputs) {
+            if (turns.length > 0 && signal.aborted) {
+                throw new TurnFailure(signal.reason, [...turns], usage);
+            }
+            const turnResults = [];
+            let runResult;
+            try {
+                runResult = await cogitator.run(this.target.agent, {
+                    input,
+                    context: evalCase.context,
+                    signal,
+                    ...(threadId !== undefined && { threadId }),
+                    onToolResult: (toolResult) => turnResults.push(toolResultOf(toolResult)),
+                });
+            }
+            catch (error) {
+                if (threadId === undefined)
+                    throw error;
+                throw new TurnFailure(error, [...turns, { input, output: '', error: errorMessage(error) }], usage);
+            }
+            const output = typeof runResult.output === 'string' ? runResult.output : '';
+            const turnCalls = Array.isArray(runResult.toolCalls)
+                ? runResult.toolCalls
+                : undefined;
+            if (turnCalls)
+                reportsToolCalls = true;
+            toolCalls.push(...(turnCalls ?? []));
+            toolResults.push(...turnResults);
+            for (const toolResult of turnResults) {
+                if (toolResult.error !== undefined)
+                    continue;
+                retrieved.push(...(retrieval({ name: toolResult.name, result: toolResult.result }) ?? []));
+            }
+            usage = addUsage(usage, targetUsageOf(runResult.usage));
+            if (runResult.truncated === true)
+                flags.truncated = true;
+            if (runResult.iterationLimitReached === true)
+                flags.iterationLimitReached = true;
+            if (runResult.status === 'paused')
+                flags.paused = true;
+            turns.push({
+                input,
+                output,
+                ...(turnCalls && turnCalls.length > 0 && { toolCalls: turnCalls }),
+            });
+        }
+        return {
             case: evalCase,
-            output: runResult.output ?? '',
-            duration,
+            output: turns[turns.length - 1].output,
+            duration: Date.now() - start,
+            ...(usage && { usage }),
+            ...(reportsToolCalls && { toolCalls }),
+            ...(toolResults.length > 0 && { toolResults }),
+            ...(retrieved.length > 0 && { retrieved }),
+            ...(threadId !== undefined && { turns }),
+            ...flags,
         };
-        const usage = targetUsageOf(runResult.usage);
-        if (usage) {
-            result.usage = usage;
-        }
-        if (runResult.toolCalls) {
-            result.toolCalls = runResult.toolCalls;
-        }
-        return result;
     }
     async evaluateCaseMetrics(result) {
         if (this.boundMetrics.length === 0)
@@ -324,9 +471,11 @@ export class EvalSuite {
         return Promise.all(this.boundMetrics.map((m) => this.scoreMetric(m, result)));
     }
     /**
-     * A metric's score for one case. A metric that throws, or whose score is not a finite number,
-     * scores 0 with the reason in `error`, so one bad metric neither drops the case nor poisons
-     * the aggregate with NaN.
+     * A metric's score for one case. Only cases with an output are scored: a case whose every
+     * attempt failed is left unscored, so its empty output neither costs a judge call nor counts as
+     * a score of 0 (`stats.errors` and the `maxErrorRate` assertion catch those). A metric that
+     * throws, or whose score is not a finite number, scores 0 with the reason in `error`, so one
+     * bad metric neither drops the case nor poisons the aggregate with NaN.
      */
     async scoreMetric(metric, result) {
         try {
@@ -352,23 +501,75 @@ export class EvalSuite {
             return { name: metric.metricName, score: 0, details: `metric error: ${error}`, error };
         }
     }
+    /** One score per case and metric: the mean over the case's trials the metric scored */
     aggregateScores(results) {
         const scoresByMetric = new Map();
+        const push = (map, name, value) => {
+            const values = map.get(name);
+            if (values)
+                values.push(value);
+            else
+                map.set(name, [value]);
+        };
         for (const r of results) {
-            for (const s of r.scores) {
-                let arr = scoresByMetric.get(s.name);
-                if (!arr) {
-                    arr = [];
-                    scoresByMetric.set(s.name, arr);
-                }
-                arr.push(s.score);
+            const caseScores = new Map();
+            for (const trial of trialsOf(r)) {
+                for (const s of trial.scores)
+                    push(caseScores, s.name, s.score);
             }
+            for (const [name, values] of caseScores)
+                push(scoresByMetric, name, mean(values));
         }
         const aggregated = {};
         for (const [name, values] of scoresByMetric) {
-            aggregated[name] = { name, ...aggregate(values) };
+            aggregated[name] = { name, ...aggregate(values), count: values.length };
+        }
+        if (results.length > 0 && results.flatMap(trialsOf).every((r) => r.error !== undefined)) {
+            for (const metric of this.boundMetrics) {
+                aggregated[metric.metricName] ??= unscored(metric.metricName);
+            }
         }
         return aggregated;
     }
 }
+function trialsOf(result) {
+    return result.trials ?? [result];
+}
+function toolResultOf(toolResult) {
+    return {
+        callId: toolResult.callId,
+        name: toolResult.name,
+        ...(toolResult.result !== undefined && { result: toolResult.result }),
+        ...(toolResult.error !== undefined && { error: toolResult.error }),
+    };
+}
+/** The turns and usage a failed attempt leaves, when a turn of a multi-turn case failed */
+function failureDetails(settled) {
+    if (settled.ok || !(settled.error instanceof TurnFailure))
+        return settled;
+    const { turns, usage } = settled.error;
+    return { ok: false, error: settled.error, turns, ...(usage && { usage }) };
+}
+/** A case result as the reporters get it, the agent's data and the trials kept */
+/** The trial a case is reported by: the first that answered, the first when every trial errored */
+function leadTrial(trials) {
+    return trials.find((trial) => trial.error === undefined) ?? trials[0];
+}
+function reportedCase(r) {
+    return {
+        case: { input: r.case.input, expected: r.case.expected },
+        output: r.output,
+        duration: r.duration,
+        ...(r.error !== undefined && { error: r.error }),
+        ...(r.truncated && { truncated: true }),
+        ...(r.iterationLimitReached && { iterationLimitReached: true }),
+        ...(r.paused && { paused: true }),
+        ...(r.toolCalls && { toolCalls: r.toolCalls }),
+        ...(r.toolResults && { toolResults: r.toolResults }),
+        ...(r.retrieved && { retrieved: r.retrieved }),
+        ...(r.turns && { turns: r.turns }),
+        scores: r.scores,
+        ...(r.trials && { trials: r.trials.map(reportedCase) }),
+    };
+}
 //# sourceMappingURL=eval-suite.js.map