@cogitator-ai/evals 0.9.0 → 0.11.0

package/dist/metrics/statistical.js

@@ -75,4 +75,65 @@ export function tokenUsage() {
         };
     });
 }
+/** Whether a statistical metric is a `passAtK()` one */
+export function isPassAtKMetric(metric) {
+    return 'passAtK' in metric && typeof metric.passAtK === 'object';
+}
+/**
+ * The chance that at least one of `k` trials drawn from `n`, `c` of which passed, passes:
+ * 1 - C(n - c, k) / C(n, k), computed as a product so large counts do not overflow
+ */
+export function passAtKEstimate(n, c, k) {
+    if (n - c < k)
+        return 1;
+    let allFail = 1;
+    for (let i = n - c + 1; i <= n; i++)
+        allFail *= 1 - k / i;
+    return 1 - allFail;
+}
+/**
+ * pass@k of a per-case metric over the suite's trials: for each case, the unbiased estimate of
+ * the chance that at least one of `k` trials scores `threshold` or more, averaged over the cases
+ * the metric scored at least one trial of. A trial that errored counts as a failure. Named
+ * `passAtK(<metric>,<k>)`. The suite must run at least `k` trials and have the metric.
+ */
+export function passAtK(metricName, opts) {
+    const { k, threshold = 1 } = opts;
+    if (!Number.isInteger(k) || k < 1) {
+        throw new Error(`passAtK k must be a positive integer, got ${k}`);
+    }
+    if (!Number.isFinite(threshold)) {
+        throw new Error(`passAtK threshold must be a finite number, got ${threshold}`);
+    }
+    const name = `passAtK(${metricName},${k})`;
+    const fn = createStatisticalFn(name, (results) => {
+        const values = [];
+        for (const result of results) {
+            const trials = result.trials ?? [result];
+            if (trials.length < k) {
+                throw new Error(`${name} needs at least ${k} trials per case, a case has ${trials.length}: run the suite with trials: ${k} or more`);
+            }
+            const scores = trials.map((trial) => trial.error === undefined ? trial.scores?.find((s) => s.name === metricName) : undefined);
+            if (scores.every((s) => s === undefined))
+                continue;
+            const passed = scores.filter((s) => s !== undefined && s.score >= threshold).length;
+            values.push(passAtKEstimate(trials.length, passed, k));
+        }
+        const metadata = { metric: metricName, k, threshold, cases: values.length };
+        if (values.length === 0) {
+            return {
+                name,
+                score: Number.NaN,
+                details: `no case has a trial scored on ${metricName}`,
+                metadata,
+            };
+        }
+        return { name, score: mean(values), values, metadata };
+    });
+    Object.defineProperty(fn, 'passAtK', {
+        value: Object.freeze({ metric: metricName, k, threshold }),
+        writable: false,
+    });
+    return fn;
+}
 //# sourceMappingURL=statistical.js.map

package/dist/metrics/statistical.js.map

@@ -1 +1 @@
-{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAAmD;IAEnD,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,GAAG,CAAC,MAAgB;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAE/B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC;gBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,GAAG,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;YACnB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAAmD;IAEnD,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,GAAG,CAAC,MAAgB;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAE/B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC;gBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,GAAG,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;YACnB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAcD,wDAAwD;AACxD,MAAM,UAAU,eAAe,CAAC,MAA2B;IACzD,OAAO,SAAS,IAAI,MAAM,IAAI,OAAQ,MAA0B,CAAC,OAAO,KAAK,QAAQ,CAAC;AACxF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS;IAC7D,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,OAAO,CAAC;AACrB,CAAC;AAID;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,UAAkB,EAAE,IAAoB;IAC9D,MAAM,EAAE,CAAC,EAAE,SAAS,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC;IAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,kDAAkD,SAAS,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,UAAU,IAAI,CAAC,GAAG,CAAC;IAE3C,MAAM,EAAE,GAAG,mBAAmB,CAAC,IAAI,EAAE,CAAC,OAAyB,EAAE,EAAE;QACjE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAA2B,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,mBAAmB,CAAC,gCAAgC,MAAM,CAAC,MAAM,gCAAgC,CAAC,UAAU,CACpH,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAClC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CACzF,CAAC;YACF,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC;gBAAE,SAAS;YACnD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,MAAM,CAAC;YACpF,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,MAAM,CAAC,GAAG;gBACjB,OAAO,EAAE,iCAAiC,UAAU,EAAE;gBACtD,QAAQ;aACT,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE;QACnC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC;QAC1D,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;IACH,OAAO,EAAqB,CAAC;AAC/B,CAAC"}

package/dist/metrics/tool-calls.d.ts

@@ -0,0 +1,43 @@
+import type { ExpectedToolCall } from '../schema.js';
+import type { EvalToolCall, MetricFn, MetricOptions } from './types.js';
+/** How the arguments of an expected tool call are compared with the actual call's */
+export type ToolArgumentsMatch = 'ignore' | 'subset' | 'exact';
+/** Which figure of the tool call match `toolCalls()` scores */
+export type ToolCallsScore = 'f1' | 'recall' | 'precision';
+/** How strictly `trajectory()` compares the order of the tool calls */
+export type TrajectoryOrder = 'exact' | 'in-order' | 'any-order';
+export interface ToolCallsOptions extends MetricOptions {
+    /**
+     * `'subset'` (default): the expected arguments must all be in the call with equal values,
+     * others may be there too. `'exact'`: the arguments must be equal. `'ignore'`: only the name
+     * counts. An expected call without `arguments` matches on its name alone
+     */
+    arguments?: ToolArgumentsMatch;
+    /** `'f1'` (default), `'recall'` (expected calls made) or `'precision'` (calls that were expected) */
+    score?: ToolCallsScore;
+}
+export interface TrajectoryOptions extends MetricOptions {
+    /**
+     * `'exact'` (default): 1 when the tool names are the expected ones in the same order, else 0.
+     * `'in-order'`: the longest common subsequence over the longer of the two sequences.
+     * `'any-order'`: the names both share, as multisets, over the longer of the two
+     */
+    order?: TrajectoryOrder;
+}
+/** Whether an actual call is the expected one, its arguments compared as `mode` says */
+export declare function toolCallMatches(expected: ExpectedToolCall, actual: EvalToolCall, mode?: ToolArgumentsMatch): boolean;
+/**
+ * Compares the tool calls of the run, over every turn, with the case's `expectedToolCalls` as
+ * multisets: each expected call pairs with at most one actual call. Scores the F1 of the match
+ * by default, or its recall or precision. A case expecting no calls scores 1 when the agent made
+ * none and 0 otherwise, and so does a case expecting calls when the agent made none.
+ */
+export declare function toolCalls(opts?: ToolCallsOptions): MetricFn;
+/** How well a sequence of tool names follows the expected one, 0 to 1 */
+export declare function trajectoryScore(expected: readonly string[], actual: readonly string[], order: TrajectoryOrder): number;
+/**
+ * Compares the sequence of tools the agent called, over every turn, with the names of the
+ * case's `expectedToolCalls`. Arguments are not looked at: pair it with `toolCalls()` for them.
+ */
+export declare function trajectory(opts?: TrajectoryOptions): MetricFn;
+//# sourceMappingURL=tool-calls.d.ts.map

package/dist/metrics/tool-calls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-calls.d.ts","sourceRoot":"","sources":["../../src/metrics/tool-calls.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AASrE,qFAAqF;AACrF,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/D,+DAA+D;AAC/D,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE3D,uEAAuE;AACvE,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,UAAU,GAAG,WAAW,CAAC;AAEjE,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACrD;;;;OAIG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B,qGAAqG;IACrG,KAAK,CAAC,EAAE,cAAc,CAAC;CACxB;AAED,MAAM,WAAW,iBAAkB,SAAQ,aAAa;IACtD;;;;OAIG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAwBD,wFAAwF;AACxF,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,YAAY,EACpB,IAAI,GAAE,kBAA6B,GAClC,OAAO,CAMT;AAYD;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,gBAAgB,GAAG,QAAQ,CAqC3D;AAED,yEAAyE;AACzE,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,EAAE,eAAe,GACrB,MAAM,CAWR;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,iBAAiB,GAAG,QAAQ,CAsB7D"}

package/dist/metrics/tool-calls.js

@@ -0,0 +1,106 @@
+import { deepEqual, isSubset, longestCommonSubsequence, maxMatching, multisetOverlap, } from './matching.js';
+const ARGUMENT_MODES = ['ignore', 'subset', 'exact'];
+const SCORE_MODES = ['f1', 'recall', 'precision'];
+const ORDERS = ['exact', 'in-order', 'any-order'];
+function checkOption(metric, option, value, allowed) {
+    if (!allowed.includes(value)) {
+        throw new Error(`${metric} ${option} must be one of ${allowed.map((v) => `'${v}'`).join(', ')}, got ${String(value)}`);
+    }
+}
+function createMetricFn(name, fn) {
+    fn.metricName = name;
+    return fn;
+}
+/** Whether an actual call is the expected one, its arguments compared as `mode` says */
+export function toolCallMatches(expected, actual, mode = 'subset') {
+    if (expected.name !== actual.name)
+        return false;
+    if (mode === 'ignore' || expected.arguments === undefined)
+        return true;
+    return mode === 'exact'
+        ? deepEqual(expected.arguments, actual.arguments)
+        : isSubset(expected.arguments, actual.arguments);
+}
+function describeCall(call) {
+    return call.arguments === undefined
+        ? call.name
+        : `${call.name}(${JSON.stringify(call.arguments)})`;
+}
+function ratio(part, whole) {
+    return whole === 0 ? 0 : part / whole;
+}
+/**
+ * Compares the tool calls of the run, over every turn, with the case's `expectedToolCalls` as
+ * multisets: each expected call pairs with at most one actual call. Scores the F1 of the match
+ * by default, or its recall or precision. A case expecting no calls scores 1 when the agent made
+ * none and 0 otherwise, and so does a case expecting calls when the agent made none.
+ */
+export function toolCalls(opts) {
+    const mode = opts?.arguments ?? 'subset';
+    const scoring = opts?.score ?? 'f1';
+    const name = opts?.name ?? 'toolCalls';
+    checkOption('toolCalls', 'arguments', mode, ARGUMENT_MODES);
+    checkOption('toolCalls', 'score', scoring, SCORE_MODES);
+    return createMetricFn(name, (async (result) => {
+        const expected = result.case.expectedToolCalls;
+        if (expected === undefined) {
+            return { name, score: 0, details: 'no expected tool calls provided' };
+        }
+        const actual = result.toolCalls ?? [];
+        const pairs = maxMatching(expected, actual, (e, a) => toolCallMatches(e, a, mode));
+        const precision = actual.length === 0 ? (expected.length === 0 ? 1 : 0) : ratio(pairs.size, actual.length);
+        const recall = expected.length === 0 ? (actual.length === 0 ? 1 : 0) : ratio(pairs.size, expected.length);
+        const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
+        const score = scoring === 'precision' ? precision : scoring === 'recall' ? recall : f1;
+        const matchedActual = new Set(pairs.values());
+        const missing = expected.filter((_, i) => !pairs.has(i)).map(describeCall);
+        const unexpected = actual.filter((_, j) => !matchedActual.has(j)).map(describeCall);
+        const problems = [
+            ...(missing.length > 0 ? [`missing ${missing.join(', ')}`] : []),
+            ...(unexpected.length > 0 ? [`unexpected ${unexpected.join(', ')}`] : []),
+        ];
+        return {
+            name,
+            score,
+            details: problems.length > 0 ? problems.join(' | ') : undefined,
+            metadata: { precision, recall, f1, matched: pairs.size },
+        };
+    }));
+}
+/** How well a sequence of tool names follows the expected one, 0 to 1 */
+export function trajectoryScore(expected, actual, order) {
+    if (order === 'exact') {
+        return expected.length === actual.length && expected.every((n, i) => n === actual[i]) ? 1 : 0;
+    }
+    const longest = Math.max(expected.length, actual.length);
+    if (longest === 0)
+        return 1;
+    const shared = order === 'in-order'
+        ? longestCommonSubsequence(expected, actual)
+        : multisetOverlap(expected, actual);
+    return shared / longest;
+}
+/**
+ * Compares the sequence of tools the agent called, over every turn, with the names of the
+ * case's `expectedToolCalls`. Arguments are not looked at: pair it with `toolCalls()` for them.
+ */
+export function trajectory(opts) {
+    const order = opts?.order ?? 'exact';
+    const name = opts?.name ?? 'trajectory';
+    checkOption('trajectory', 'order', order, ORDERS);
+    return createMetricFn(name, (async (result) => {
+        const expectedCalls = result.case.expectedToolCalls;
+        if (expectedCalls === undefined) {
+            return { name, score: 0, details: 'no expected tool calls provided' };
+        }
+        const expected = expectedCalls.map((c) => c.name);
+        const actual = (result.toolCalls ?? []).map((c) => c.name);
+        const score = trajectoryScore(expected, actual, order);
+        return {
+            name,
+            score,
+            details: score < 1 ? `expected [${expected.join(', ')}], got [${actual.join(', ')}]` : undefined,
+        };
+    }));
+}
+//# sourceMappingURL=tool-calls.js.map

package/dist/metrics/tool-calls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-calls.js","sourceRoot":"","sources":["../../src/metrics/tool-calls.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,SAAS,EACT,QAAQ,EACR,wBAAwB,EACxB,WAAW,EACX,eAAe,GAChB,MAAM,YAAY,CAAC;AA+BpB,MAAM,cAAc,GAAkC,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACpF,MAAM,WAAW,GAA8B,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;AAC7E,MAAM,MAAM,GAA+B,CAAC,OAAO,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AAE9E,SAAS,WAAW,CAClB,MAAc,EACd,MAAc,EACd,KAAQ,EACR,OAAqB;IAErB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,IAAI,MAAM,mBAAmB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,eAAe,CAC7B,QAA0B,EAC1B,MAAoB,EACpB,OAA2B,QAAQ;IAEnC,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAChD,IAAI,IAAI,KAAK,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACvE,OAAO,IAAI,KAAK,OAAO;QACrB,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QACjD,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,YAAY,CAAC,IAA2D;IAC/E,OAAO,IAAI,CAAC,SAAS,KAAK,SAAS;QACjC,CAAC,CAAC,IAAI,CAAC,IAAI;QACX,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;AACxD,CAAC;AAED,SAAS,KAAK,CAAC,IAAY,EAAE,KAAa;IACxC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC;AACxC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,IAAuB;IAC/C,MAAM,IAAI,GAAG,IAAI,EAAE,SAAS,IAAI,QAAQ,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,WAAW,CAAC;IACvC,WAAW,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;IAC5D,WAAW,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;IAExD,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC;QAC/C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QACnF,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3F,MAAM,MAAM,GACV,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC7F,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,OAAO,KAAK,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAEvF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC3E,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpF,MAAM,QAAQ,GAAG;YACf,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1E,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;YAC/D,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE;SACzD,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAC7B,QAA2B,EAC3B,MAAyB,EACzB,KAAsB;IAEtB,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,OAAO,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC5B,MAAM,MAAM,GACV,KAAK,KAAK,UAAU;QAClB,CAAC,CAAC,wBAAwB,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC5C,CAAC,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,MAAM,GAAG,OAAO,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,IAAwB;IACjD,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,OAAO,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IACxC,WAAW,CAAC,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAElD,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACpD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC;QACxE,CAAC;QAED,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAEvD,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;SAC1F,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}

package/dist/metrics/types.d.ts

@@ -19,6 +19,34 @@ export interface MetricScore {
     /** What computing the score cost, for metrics that call a model such as the LLM judge */
     usage?: MetricUsage;
 }
+/** A tool call the agent made */
+export interface EvalToolCall {
+    id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+}
+/** What a tool call returned, as the run's `onToolResult` reported it */
+export interface EvalToolResult {
+    callId: string;
+    name: string;
+    result?: unknown;
+    error?: string;
+}
+/** A passage the agent's retrieval surfaced, such as one result of a `rag_search` call */
+export interface Passage {
+    id?: string;
+    content: string;
+    score?: number;
+    source?: string;
+}
+/** One user turn of a multi-turn case and the answer to it */
+export interface EvalTurnResult {
+    input: string;
+    output: string;
+    toolCalls?: readonly EvalToolCall[];
+    /** Set on the turn that failed, the last one the case ran */
+    error?: string;
+}
 export interface EvalCaseResult {
     case: EvalCase;
     output: string;
@@ -38,11 +66,21 @@ export interface EvalCaseResult {
         cost: number;
         duration: number;
     };
-    toolCalls?: readonly {
-        id: string;
-        name: string;
-        arguments: Record<string, unknown>;
-    }[];
+    /** The tool calls of the run, over every turn */
+    toolCalls?: readonly EvalToolCall[];
+    /** What the tool calls returned, over every turn, raw */
+    toolResults?: readonly EvalToolResult[];
+    /** The passages the tool results surfaced, over every turn, in the order they came */
+    retrieved?: readonly Passage[];
+    /** Each turn of a case with `turns`: `input` first, then the further turns */
+    turns?: readonly EvalTurnResult[];
+    /**
+     * Every run of the case when the suite runs several `trials`, each with its own scores. The
+     * case result itself is the first of them
+     */
+    trials?: ReadonlyArray<EvalCaseResult & {
+        scores: MetricScore[];
+    }>;
     /** The agent's answer stopped at its output token limit */
     truncated?: boolean;
     /** Tool calls used up the agent's `maxIterations` before it answered */
@@ -63,6 +101,10 @@ export interface StatisticalScore extends MetricScore {
 export type StatisticalMetricFn = ((results: EvalCaseResult[]) => StatisticalScore) & {
     metricName: string;
 };
+/** A case result with the scores of its metrics, as a suite run returns it */
+export type ScoredCaseResult = EvalCaseResult & {
+    scores: MetricScore[];
+};
 /** Options every built-in per-case metric factory takes */
 export interface MetricOptions {
     /**

package/dist/metrics/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,CAAC,EAAE,SAAS;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,EAAE,CAAC;IACJ,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,0FAA0F;AAC1F,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,8DAA8D;AAC9D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,yDAAyD;IACzD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,sFAAsF;IACtF,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,8EAA8E;IAC9E,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IACnE,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,8EAA8E;AAC9E,MAAM,MAAM,gBAAgB,GAAG,cAAc,GAAG;IAAE,MAAM,EAAE,WAAW,EAAE,CAAA;CAAE,CAAC;AAE1E,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}

package/dist/metrics/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAiFA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AA4HA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}

package/dist/reporters/console.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"console.d.ts","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAgB/C,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAiD3D"}
+{"version":3,"file":"console.d.ts","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiB/C,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAoD3D"}

package/dist/reporters/console.js

@@ -1,3 +1,4 @@
+import { trialsPerCase } from './format.js';
 const GREEN = '\x1b[32m';
 const RED = '\x1b[31m';
 const BOLD = '\x1b[1m';
@@ -12,7 +13,8 @@ function fmt(n) {
 export function consoleReport(result) {
     const metrics = Object.values(result.aggregated);
     if (metrics.length > 0) {
-        const cols = { metric: 14, mean: 10, median: 10, p95: 10, min: 10, max: 10, cases: 8 };
+        const metricWidth = Math.max(14, ...metrics.map((m) => m.name.length + 2));
+        const cols = { metric: metricWidth, mean: 10, median: 10, p95: 10, min: 10, max: 10, cases: 8 };
         const showCount = metrics.some((m) => m.count !== undefined);
         const header = pad('Metric', cols.metric) +
             pad('Mean', cols.mean) +
@@ -46,6 +48,8 @@ export function consoleReport(result) {
     const failed = result.assertions.filter((a) => !a.passed).length;
     const errors = result.stats.errors ?? 0;
     const errorPart = errors > 0 ? ` | ${RED}${errors} errored${RESET}` : '';
-    console.log(`\n${BOLD}Summary${RESET}: ${result.stats.total} cases${errorPart} | ${result.stats.duration}ms | $${result.stats.cost} | ${GREEN}${passed} passed${RESET} ${failed > 0 ? `${RED}${failed} failed${RESET}` : ''}`);
+    const trials = trialsPerCase(result);
+    const trialPart = trials > 1 ? ` x ${trials} trials` : '';
+    console.log(`\n${BOLD}Summary${RESET}: ${result.stats.total} cases${trialPart}${errorPart} | ${result.stats.duration}ms | $${result.stats.cost} | ${GREEN}${passed} passed${RESET} ${failed > 0 ? `${RED}${failed} failed${RESET}` : ''}`);
 }
 //# sourceMappingURL=console.js.map

package/dist/reporters/console.js.map

@@ -1 +1 @@
-{"version":3,"file":"console.js","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,GAAW,EAAE,GAAW;IACnC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACvF,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QAC7D,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;YACtB,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAE9C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GACP,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,aAAa,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,MAAM,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEzE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,SAAS,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,MAAM,UAAU,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClN,CAAC;AACJ,CAAC"}
+{"version":3,"file":"console.js","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,GAAW,EAAE,GAAW;IACnC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAChG,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QAC7D,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;YACtB,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAE9C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GACP,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,aAAa,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,MAAM,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACzE,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1D,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,SAAS,GAAG,SAAS,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,MAAM,UAAU,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9N,CAAC;AACJ,CAAC"}

package/dist/reporters/format.d.ts

@@ -10,4 +10,10 @@ export declare function caseFlags(result: Partial<Record<CaseFlag, boolean>>): C
 export declare function truncateInput(input: string, max?: number): string;
 /** The label of the case at `index` (zero based) in a report or test name */
 export declare function caseLabel(index: number, input: string): string;
+/** How many trials each case ran, 1 for a suite without several trials */
+export declare function trialsPerCase(result: {
+    results: ReadonlyArray<{
+        trials?: readonly unknown[];
+    }>;
+}): number;
 //# sourceMappingURL=format.d.ts.map

package/dist/reporters/format.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,eAAO,MAAM,UAAU,2DAA4D,CAAC;AACpF,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,CAEhF;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAA6B,GAAG,MAAM,CAQvF;AAED,6EAA6E;AAC7E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAG9D"}
+{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,eAAO,MAAM,UAAU,2DAA4D,CAAC;AACpF,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,CAEhF;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAA6B,GAAG,MAAM,CAQvF;AAED,6EAA6E;AAC7E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAG9D;AAED,0EAA0E;AAC1E,wBAAgB,aAAa,CAAC,MAAM,EAAE;IACpC,OAAO,EAAE,aAAa,CAAC;QAAE,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,CAAA;KAAE,CAAC,CAAC;CACzD,GAAG,MAAM,CAET"}

package/dist/reporters/format.js

@@ -24,4 +24,8 @@ export function caseLabel(index, input) {
     const text = truncateInput(input);
     return `[${index + 1}] ${text === '' ? '(empty input)' : text}`;
 }
+/** How many trials each case ran, 1 for a suite without several trials */
+export function trialsPerCase(result) {
+    return result.results.reduce((most, r) => Math.max(most, r.trials?.length ?? 1), 1);
+}
 //# sourceMappingURL=format.js.map

package/dist/reporters/format.js.map

@@ -1 +1 @@
-{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,uBAAuB,EAAE,QAAQ,CAAU,CAAC;AAGpF,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,MAA0C;IAClE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,MAAc,oBAAoB;IAC7E,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxB,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,KAAK;SACZ,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;SACjB,IAAI,CAAC,EAAE,CAAC;SACR,OAAO,EAAE,KAAK,CAAC;AACpB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO,IAAI,KAAK,GAAG,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAClE,CAAC"}
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,uBAAuB,EAAE,QAAQ,CAAU,CAAC;AAGpF,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,MAA0C;IAClE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,MAAc,oBAAoB;IAC7E,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxB,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,KAAK;SACZ,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;SACjB,IAAI,CAAC,EAAE,CAAC;SACR,OAAO,EAAE,KAAK,CAAC;AACpB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO,IAAI,KAAK,GAAG,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAClE,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,aAAa,CAAC,MAE7B;IACC,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtF,CAAC"}

package/dist/reporters/github.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiB/C,iGAAiG;AACjG,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,CAoBnE;AAmBD,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA0C7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAK1D"}
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiB/C,iGAAiG;AACjG,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,CAoBnE;AAmBD,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA2C7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAK1D"}

package/dist/reporters/github.js

@@ -1,5 +1,5 @@
 import { appendFileSync } from 'node:fs';
-import { caseFlags, caseLabel } from './format.js';
+import { caseFlags, caseLabel, trialsPerCase } from './format.js';
 /** Escapes the message of a workflow command */
 function escapeData(value) {
     return value.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
@@ -52,8 +52,9 @@ export function githubSummary(result) {
     const passed = result.assertions.filter((a) => a.passed).length;
     const failed = result.assertions.length - passed;
     const flagged = result.results.filter((r) => caseFlags(r).length > 0).length;
+    const trials = trialsPerCase(result);
     const totals = [
-        `**${stats.total}** cases`,
+        `**${stats.total}** cases${trials > 1 ? ` x **${trials}** trials` : ''}`,
         `**${stats.errors ?? 0}** errored`,
         ...(stats.metricErrors ? [`**${stats.metricErrors}** metric errors`] : []),
         ...(flagged > 0 ? [`**${flagged}** truncated, paused or at the iteration limit`] : []),

package/dist/reporters/github.js.map

@@ -1 +1 @@
-{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAEhD,gDAAgD;AAChD,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAChF,CAAC;AAED,4EAA4E;AAC5E,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,OAAO,CAAC,IAAyB,EAAE,KAAa,EAAE,OAAe;IACxE,OAAO,KAAK,IAAI,UAAU,cAAc,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,eAAe,KAAK,CAAC,IAAI,YAAY,KAAK,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,KAAK;SACT,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAE7E,MAAM,MAAM,GAAG;QACb,KAAK,KAAK,CAAC,KAAK,UAAU;QAC1B,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY;QAClC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,gDAAgD,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,GAAG,KAAK,CAAC,QAAQ,KAAK;QACtB,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM;QACxB,iBAAiB,MAAM,gBAAgB,MAAM,WAAW;KACzD,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,iBAAiB,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAE9D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,aAAa,EACb,EAAE,EACF,uCAAuC,EACvC,qCAAqC,CACtC,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,EAAE,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAuB;IAClD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEhE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACpD,IAAI,WAAW;QAAE,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC"}
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,gDAAgD;AAChD,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAChF,CAAC;AAED,4EAA4E;AAC5E,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,OAAO,CAAC,IAAyB,EAAE,KAAa,EAAE,OAAe;IACxE,OAAO,KAAK,IAAI,UAAU,cAAc,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,eAAe,KAAK,CAAC,IAAI,YAAY,KAAK,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,KAAK;SACT,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAE7E,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG;QACb,KAAK,KAAK,CAAC,KAAK,WAAW,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAM,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;QACxE,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY;QAClC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,gDAAgD,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,GAAG,KAAK,CAAC,QAAQ,KAAK;QACtB,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM;QACxB,iBAAiB,MAAM,gBAAgB,MAAM,WAAW;KACzD,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,iBAAiB,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAE9D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,aAAa,EACb,EAAE,EACF,uCAAuC,EACvC,qCAAqC,CACtC,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,EAAE,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAuB;IAClD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEhE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACpD,IAAI,WAAW;QAAE,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC"}

package/dist/reporters/index.d.ts

@@ -1,4 +1,5 @@
 import type { EvalStats } from '../assertions/index.js';
+import type { EvalToolCall, EvalToolResult, EvalTurnResult, Passage } from '../metrics/types.js';
 export interface AggregatedMetric {
     name: string;
     mean: number;
@@ -21,29 +22,41 @@ export interface AssertionResult {
     actual?: number;
     expected?: number;
 }
-export interface EvalSuiteResult {
-    results: Array<{
-        case: {
-            input: string;
-            expected?: string;
-        };
-        output: string;
-        duration: number;
-        /** Why the case has no output: every attempt failed or timed out */
+/** A case result as the reporters get it */
+export interface ReportedCase {
+    case: {
+        input: string;
+        expected?: string;
+    };
+    output: string;
+    duration: number;
+    /** Why the case has no output: every attempt failed or timed out */
+    error?: string;
+    /** The agent's answer stopped at its output token limit */
+    truncated?: boolean;
+    /** Tool calls used up the agent's `maxIterations` before it answered */
+    iterationLimitReached?: boolean;
+    /** The run paused for tool approvals, so `output` holds no answer */
+    paused?: boolean;
+    /** The tool calls of the run, over every turn */
+    toolCalls?: readonly EvalToolCall[];
+    /** What the tool calls returned, over every turn */
+    toolResults?: readonly EvalToolResult[];
+    /** The passages the run retrieved */
+    retrieved?: readonly Passage[];
+    /** Each turn of a multi-turn case */
+    turns?: readonly EvalTurnResult[];
+    scores: Array<{
+        name: string;
+        score: number;
+        details?: string;
         error?: string;
-        /** The agent's answer stopped at its output token limit */
-        truncated?: boolean;
-        /** Tool calls used up the agent's `maxIterations` before it answered */
-        iterationLimitReached?: boolean;
-        /** The run paused for tool approvals, so `output` holds no answer */
-        paused?: boolean;
-        scores: Array<{
-            name: string;
-            score: number;
-            details?: string;
-            error?: string;
-        }>;
     }>;
+    /** Every run of the case when the suite ran several trials, the first being the case itself */
+    trials?: readonly ReportedCase[];
+}
+export interface EvalSuiteResult {
+    results: ReportedCase[];
     aggregated: Record<string, AggregatedMetric>;
     assertions: AssertionResult[];
     stats: EvalStats;

package/dist/reporters/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,oEAAoE;QACpE,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,2DAA2D;QAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,wEAAwE;QACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAChC,qEAAqE;QACrE,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClF,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,QAAQ,CAAC;AAClF,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAG9F;AAOD;;;;GAIG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CA0BN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE9F,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,oDAAoD;IACpD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,qCAAqC;IACrC,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,qCAAqC;IACrC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjF,+FAA+F;IAC/F,MAAM,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,QAAQ,CAAC;AAClF,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAG9F;AAOD;;;;GAIG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CA0BN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/reporters/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAwDxC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAiC;IACpF,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;IACtE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,OAAO;gBACV,WAAW,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,QAAQ;gBACX,YAAY,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAsExC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAiC;IACpF,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;IACtE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,OAAO;gBACV,WAAW,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,QAAQ;gBACX,YAAY,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/schema.d.ts

@@ -1,14 +1,26 @@
 import { z } from 'zod';
+/** A tool call a case expects: the tool's name and, optionally, arguments to compare */
+export declare const ExpectedToolCallSchema: z.ZodObject<{
+    name: z.ZodString;
+    arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strip>;
 export declare const EvalCaseSchema: z.ZodObject<{
     input: z.ZodString;
     expected: z.ZodOptional<z.ZodString>;
     context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>>;
+    expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 export declare const EvalSuiteConfigSchema: z.ZodObject<{
     concurrency: z.ZodDefault<z.ZodNumber>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
+    trials: z.ZodDefault<z.ZodNumber>;
 }, z.core.$strip>;
 /**
  * The part of a Cogitator runtime the judge needs: running an agent. The judge's `usage`, when
@@ -47,7 +59,9 @@ export declare const EvalComparisonConfigSchema: z.ZodObject<{
     concurrency: z.ZodDefault<z.ZodNumber>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
+    trials: z.ZodDefault<z.ZodNumber>;
 }, z.core.$strip>;
+export type ExpectedToolCall = z.output<typeof ExpectedToolCallSchema>;
 export type EvalCase = z.output<typeof EvalCaseSchema>;
 export type EvalCaseInput = z.input<typeof EvalCaseSchema>;
 export type EvalSuiteConfig = z.output<typeof EvalSuiteConfigSchema>;

package/dist/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAmBD,eAAO,MAAM,iBAAiB;;;;;;;iBAgB5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,eAAO,MAAM,sBAAsB;;;iBAGjC,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;iBAWzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;iBAKhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAmBD,eAAO,MAAM,iBAAiB;;;;;;;iBAgB5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;iBAKrC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACvE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}