@cogitator-ai/evals 0.8.0 → 0.10.0

package/dist/metrics/statistical.js

@@ -75,4 +75,65 @@ export function tokenUsage() {
         };
     });
 }
+/** Whether a statistical metric is a `passAtK()` one */
+export function isPassAtKMetric(metric) {
+    return 'passAtK' in metric && typeof metric.passAtK === 'object';
+}
+/**
+ * The chance that at least one of `k` trials drawn from `n`, `c` of which passed, passes:
+ * 1 - C(n - c, k) / C(n, k), computed as a product so large counts do not overflow
+ */
+export function passAtKEstimate(n, c, k) {
+    if (n - c < k)
+        return 1;
+    let allFail = 1;
+    for (let i = n - c + 1; i <= n; i++)
+        allFail *= 1 - k / i;
+    return 1 - allFail;
+}
+/**
+ * pass@k of a per-case metric over the suite's trials: for each case, the unbiased estimate of
+ * the chance that at least one of `k` trials scores `threshold` or more, averaged over the cases
+ * the metric scored at least one trial of. A trial that errored counts as a failure. Named
+ * `passAtK(<metric>,<k>)`. The suite must run at least `k` trials and have the metric.
+ */
+export function passAtK(metricName, opts) {
+    const { k, threshold = 1 } = opts;
+    if (!Number.isInteger(k) || k < 1) {
+        throw new Error(`passAtK k must be a positive integer, got ${k}`);
+    }
+    if (!Number.isFinite(threshold)) {
+        throw new Error(`passAtK threshold must be a finite number, got ${threshold}`);
+    }
+    const name = `passAtK(${metricName},${k})`;
+    const fn = createStatisticalFn(name, (results) => {
+        const values = [];
+        for (const result of results) {
+            const trials = result.trials ?? [result];
+            if (trials.length < k) {
+                throw new Error(`${name} needs at least ${k} trials per case, a case has ${trials.length}: run the suite with trials: ${k} or more`);
+            }
+            const scores = trials.map((trial) => trial.error === undefined ? trial.scores?.find((s) => s.name === metricName) : undefined);
+            if (scores.every((s) => s === undefined))
+                continue;
+            const passed = scores.filter((s) => s !== undefined && s.score >= threshold).length;
+            values.push(passAtKEstimate(trials.length, passed, k));
+        }
+        const metadata = { metric: metricName, k, threshold, cases: values.length };
+        if (values.length === 0) {
+            return {
+                name,
+                score: Number.NaN,
+                details: `no case has a trial scored on ${metricName}`,
+                metadata,
+            };
+        }
+        return { name, score: mean(values), values, metadata };
+    });
+    Object.defineProperty(fn, 'passAtK', {
+        value: Object.freeze({ metric: metricName, k, threshold }),
+        writable: false,
+    });
+    return fn;
+}
 //# sourceMappingURL=statistical.js.map

package/dist/metrics/statistical.js.map

@@ -1 +1 @@
-{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAAmD;IAEnD,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,GAAG,CAAC,MAAgB;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAE/B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC;gBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,GAAG,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;YACnB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"statistical.js","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAE3C,SAAS,mBAAmB,CAC1B,IAAY,EACZ,EAAmD;IAEnD,MAAM,MAAM,GAAG,EAAyB,CAAC;IACzC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,GAAG,CAAC,MAAgB;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC;IAC3C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,OAAO;IACrB,OAAO,mBAAmB,CAAC,SAAS,EAAE,CAAC,OAAyB,EAAE,EAAE;QAClE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAEnC,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,IAAI;IAClB,OAAO,mBAAmB,CAAC,MAAM,EAAE,CAAC,OAAyB,EAAE,EAAE;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAE/B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE;gBACR,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC;gBACjB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,EAAE,KAAK,CAAC,GAAG;aACf;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,UAAU;IACxB,OAAO,mBAAmB,CAAC,YAAY,EAAE,CAAC,OAAyB,EAAE,EAAE;QACrE,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QACjE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,WAAW,GAAG,CAAC,CAAC,KAAM,CAAC,YAAY,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;QACpC,MAAM,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;YACnB,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW;gBACX,WAAW,EAAE,UAAU,GAAG,WAAW;gBACrC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC;gBAC5B,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;aAC/B;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAcD,wDAAwD;AACxD,MAAM,UAAU,eAAe,CAAC,MAA2B;IACzD,OAAO,SAAS,IAAI,MAAM,IAAI,OAAQ,MAA0B,CAAC,OAAO,KAAK,QAAQ,CAAC;AACxF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS;IAC7D,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACxB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,OAAO,CAAC;AACrB,CAAC;AAID;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,UAAkB,EAAE,IAAoB;IAC9D,MAAM,EAAE,CAAC,EAAE,SAAS,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC;IAClC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,kDAAkD,SAAS,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,UAAU,IAAI,CAAC,GAAG,CAAC;IAE3C,MAAM,EAAE,GAAG,mBAAmB,CAAC,IAAI,EAAE,CAAC,OAAyB,EAAE,EAAE;QACjE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAA2B,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,mBAAmB,CAAC,gCAAgC,MAAM,CAAC,MAAM,gCAAgC,CAAC,UAAU,CACpH,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAClC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CACzF,CAAC;YACF,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC;gBAAE,SAAS;YACnD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,MAAM,CAAC;YACpF,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,QAAQ,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,MAAM,CAAC,GAAG;gBACjB,OAAO,EAAE,iCAAiC,UAAU,EAAE;gBACtD,QAAQ;aACT,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE;QACnC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC;QAC1D,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;IACH,OAAO,EAAqB,CAAC;AAC/B,CAAC"}

package/dist/metrics/tool-calls.d.ts

@@ -0,0 +1,43 @@
+import type { ExpectedToolCall } from '../schema.js';
+import type { EvalToolCall, MetricFn, MetricOptions } from './types.js';
+/** How the arguments of an expected tool call are compared with the actual call's */
+export type ToolArgumentsMatch = 'ignore' | 'subset' | 'exact';
+/** Which figure of the tool call match `toolCalls()` scores */
+export type ToolCallsScore = 'f1' | 'recall' | 'precision';
+/** How strictly `trajectory()` compares the order of the tool calls */
+export type TrajectoryOrder = 'exact' | 'in-order' | 'any-order';
+export interface ToolCallsOptions extends MetricOptions {
+    /**
+     * `'subset'` (default): the expected arguments must all be in the call with equal values,
+     * others may be there too. `'exact'`: the arguments must be equal. `'ignore'`: only the name
+     * counts. An expected call without `arguments` matches on its name alone
+     */
+    arguments?: ToolArgumentsMatch;
+    /** `'f1'` (default), `'recall'` (expected calls made) or `'precision'` (calls that were expected) */
+    score?: ToolCallsScore;
+}
+export interface TrajectoryOptions extends MetricOptions {
+    /**
+     * `'exact'` (default): 1 when the tool names are the expected ones in the same order, else 0.
+     * `'in-order'`: the longest common subsequence over the longer of the two sequences.
+     * `'any-order'`: the names both share, as multisets, over the longer of the two
+     */
+    order?: TrajectoryOrder;
+}
+/** Whether an actual call is the expected one, its arguments compared as `mode` says */
+export declare function toolCallMatches(expected: ExpectedToolCall, actual: EvalToolCall, mode?: ToolArgumentsMatch): boolean;
+/**
+ * Compares the tool calls of the run, over every turn, with the case's `expectedToolCalls` as
+ * multisets: each expected call pairs with at most one actual call. Scores the F1 of the match
+ * by default, or its recall or precision. A case expecting no calls scores 1 when the agent made
+ * none and 0 otherwise, and so does a case expecting calls when the agent made none.
+ */
+export declare function toolCalls(opts?: ToolCallsOptions): MetricFn;
+/** How well a sequence of tool names follows the expected one, 0 to 1 */
+export declare function trajectoryScore(expected: readonly string[], actual: readonly string[], order: TrajectoryOrder): number;
+/**
+ * Compares the sequence of tools the agent called, over every turn, with the names of the
+ * case's `expectedToolCalls`. Arguments are not looked at: pair it with `toolCalls()` for them.
+ */
+export declare function trajectory(opts?: TrajectoryOptions): MetricFn;
+//# sourceMappingURL=tool-calls.d.ts.map

package/dist/metrics/tool-calls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-calls.d.ts","sourceRoot":"","sources":["../../src/metrics/tool-calls.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AASrE,qFAAqF;AACrF,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE/D,+DAA+D;AAC/D,MAAM,MAAM,cAAc,GAAG,IAAI,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE3D,uEAAuE;AACvE,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,UAAU,GAAG,WAAW,CAAC;AAEjE,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACrD;;;;OAIG;IACH,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B,qGAAqG;IACrG,KAAK,CAAC,EAAE,cAAc,CAAC;CACxB;AAED,MAAM,WAAW,iBAAkB,SAAQ,aAAa;IACtD;;;;OAIG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAwBD,wFAAwF;AACxF,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,YAAY,EACpB,IAAI,GAAE,kBAA6B,GAClC,OAAO,CAMT;AAYD;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,gBAAgB,GAAG,QAAQ,CAqC3D;AAED,yEAAyE;AACzE,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,KAAK,EAAE,eAAe,GACrB,MAAM,CAWR;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,iBAAiB,GAAG,QAAQ,CAsB7D"}

package/dist/metrics/tool-calls.js

@@ -0,0 +1,106 @@
+import { deepEqual, isSubset, longestCommonSubsequence, maxMatching, multisetOverlap, } from './matching.js';
+const ARGUMENT_MODES = ['ignore', 'subset', 'exact'];
+const SCORE_MODES = ['f1', 'recall', 'precision'];
+const ORDERS = ['exact', 'in-order', 'any-order'];
+function checkOption(metric, option, value, allowed) {
+    if (!allowed.includes(value)) {
+        throw new Error(`${metric} ${option} must be one of ${allowed.map((v) => `'${v}'`).join(', ')}, got ${String(value)}`);
+    }
+}
+function createMetricFn(name, fn) {
+    fn.metricName = name;
+    return fn;
+}
+/** Whether an actual call is the expected one, its arguments compared as `mode` says */
+export function toolCallMatches(expected, actual, mode = 'subset') {
+    if (expected.name !== actual.name)
+        return false;
+    if (mode === 'ignore' || expected.arguments === undefined)
+        return true;
+    return mode === 'exact'
+        ? deepEqual(expected.arguments, actual.arguments)
+        : isSubset(expected.arguments, actual.arguments);
+}
+function describeCall(call) {
+    return call.arguments === undefined
+        ? call.name
+        : `${call.name}(${JSON.stringify(call.arguments)})`;
+}
+function ratio(part, whole) {
+    return whole === 0 ? 0 : part / whole;
+}
+/**
+ * Compares the tool calls of the run, over every turn, with the case's `expectedToolCalls` as
+ * multisets: each expected call pairs with at most one actual call. Scores the F1 of the match
+ * by default, or its recall or precision. A case expecting no calls scores 1 when the agent made
+ * none and 0 otherwise, and so does a case expecting calls when the agent made none.
+ */
+export function toolCalls(opts) {
+    const mode = opts?.arguments ?? 'subset';
+    const scoring = opts?.score ?? 'f1';
+    const name = opts?.name ?? 'toolCalls';
+    checkOption('toolCalls', 'arguments', mode, ARGUMENT_MODES);
+    checkOption('toolCalls', 'score', scoring, SCORE_MODES);
+    return createMetricFn(name, (async (result) => {
+        const expected = result.case.expectedToolCalls;
+        if (expected === undefined) {
+            return { name, score: 0, details: 'no expected tool calls provided' };
+        }
+        const actual = result.toolCalls ?? [];
+        const pairs = maxMatching(expected, actual, (e, a) => toolCallMatches(e, a, mode));
+        const precision = actual.length === 0 ? (expected.length === 0 ? 1 : 0) : ratio(pairs.size, actual.length);
+        const recall = expected.length === 0 ? (actual.length === 0 ? 1 : 0) : ratio(pairs.size, expected.length);
+        const f1 = precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
+        const score = scoring === 'precision' ? precision : scoring === 'recall' ? recall : f1;
+        const matchedActual = new Set(pairs.values());
+        const missing = expected.filter((_, i) => !pairs.has(i)).map(describeCall);
+        const unexpected = actual.filter((_, j) => !matchedActual.has(j)).map(describeCall);
+        const problems = [
+            ...(missing.length > 0 ? [`missing ${missing.join(', ')}`] : []),
+            ...(unexpected.length > 0 ? [`unexpected ${unexpected.join(', ')}`] : []),
+        ];
+        return {
+            name,
+            score,
+            details: problems.length > 0 ? problems.join(' | ') : undefined,
+            metadata: { precision, recall, f1, matched: pairs.size },
+        };
+    }));
+}
+/** How well a sequence of tool names follows the expected one, 0 to 1 */
+export function trajectoryScore(expected, actual, order) {
+    if (order === 'exact') {
+        return expected.length === actual.length && expected.every((n, i) => n === actual[i]) ? 1 : 0;
+    }
+    const longest = Math.max(expected.length, actual.length);
+    if (longest === 0)
+        return 1;
+    const shared = order === 'in-order'
+        ? longestCommonSubsequence(expected, actual)
+        : multisetOverlap(expected, actual);
+    return shared / longest;
+}
+/**
+ * Compares the sequence of tools the agent called, over every turn, with the names of the
+ * case's `expectedToolCalls`. Arguments are not looked at: pair it with `toolCalls()` for them.
+ */
+export function trajectory(opts) {
+    const order = opts?.order ?? 'exact';
+    const name = opts?.name ?? 'trajectory';
+    checkOption('trajectory', 'order', order, ORDERS);
+    return createMetricFn(name, (async (result) => {
+        const expectedCalls = result.case.expectedToolCalls;
+        if (expectedCalls === undefined) {
+            return { name, score: 0, details: 'no expected tool calls provided' };
+        }
+        const expected = expectedCalls.map((c) => c.name);
+        const actual = (result.toolCalls ?? []).map((c) => c.name);
+        const score = trajectoryScore(expected, actual, order);
+        return {
+            name,
+            score,
+            details: score < 1 ? `expected [${expected.join(', ')}], got [${actual.join(', ')}]` : undefined,
+        };
+    }));
+}
+//# sourceMappingURL=tool-calls.js.map

package/dist/metrics/tool-calls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-calls.js","sourceRoot":"","sources":["../../src/metrics/tool-calls.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,SAAS,EACT,QAAQ,EACR,wBAAwB,EACxB,WAAW,EACX,eAAe,GAChB,MAAM,YAAY,CAAC;AA+BpB,MAAM,cAAc,GAAkC,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AACpF,MAAM,WAAW,GAA8B,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;AAC7E,MAAM,MAAM,GAA+B,CAAC,OAAO,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AAE9E,SAAS,WAAW,CAClB,MAAc,EACd,MAAc,EACd,KAAQ,EACR,OAAqB;IAErB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,GAAG,MAAM,IAAI,MAAM,mBAAmB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,eAAe,CAC7B,QAA0B,EAC1B,MAAoB,EACpB,OAA2B,QAAQ;IAEnC,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAChD,IAAI,IAAI,KAAK,QAAQ,IAAI,QAAQ,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACvE,OAAO,IAAI,KAAK,OAAO;QACrB,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;QACjD,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,YAAY,CAAC,IAA2D;IAC/E,OAAO,IAAI,CAAC,SAAS,KAAK,SAAS;QACjC,CAAC,CAAC,IAAI,CAAC,IAAI;QACX,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;AACxD,CAAC;AAED,SAAS,KAAK,CAAC,IAAY,EAAE,KAAa;IACxC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC;AACxC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,IAAuB;IAC/C,MAAM,IAAI,GAAG,IAAI,EAAE,SAAS,IAAI,QAAQ,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,WAAW,CAAC;IACvC,WAAW,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;IAC5D,WAAW,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC;IAExD,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC;QAC/C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;QACnF,MAAM,SAAS,GACb,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3F,MAAM,MAAM,GACV,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC7F,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,OAAO,KAAK,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAEvF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC3E,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpF,MAAM,QAAQ,GAAG;YACf,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1E,CAAC;QAEF,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;YAC/D,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE;SACzD,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,eAAe,CAC7B,QAA2B,EAC3B,MAAyB,EACzB,KAAsB;IAEtB,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,OAAO,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC5B,MAAM,MAAM,GACV,KAAK,KAAK,UAAU;QAClB,CAAC,CAAC,wBAAwB,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC5C,CAAC,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,MAAM,GAAG,OAAO,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,IAAwB;IACjD,MAAM,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,OAAO,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IACxC,WAAW,CAAC,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAElD,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC;QACpD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC;QACxE,CAAC;QAED,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAEvD,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS;SAC1F,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}

package/dist/metrics/types.d.ts

@@ -19,6 +19,34 @@ export interface MetricScore {
     /** What computing the score cost, for metrics that call a model such as the LLM judge */
     usage?: MetricUsage;
 }
+/** A tool call the agent made */
+export interface EvalToolCall {
+    id: string;
+    name: string;
+    arguments: Record<string, unknown>;
+}
+/** What a tool call returned, as the run's `onToolResult` reported it */
+export interface EvalToolResult {
+    callId: string;
+    name: string;
+    result?: unknown;
+    error?: string;
+}
+/** A passage the agent's retrieval surfaced, such as one result of a `rag_search` call */
+export interface Passage {
+    id?: string;
+    content: string;
+    score?: number;
+    source?: string;
+}
+/** One user turn of a multi-turn case and the answer to it */
+export interface EvalTurnResult {
+    input: string;
+    output: string;
+    toolCalls?: readonly EvalToolCall[];
+    /** Set on the turn that failed, the last one the case ran */
+    error?: string;
+}
 export interface EvalCaseResult {
     case: EvalCase;
     output: string;
@@ -38,11 +66,27 @@ export interface EvalCaseResult {
         cost: number;
         duration: number;
     };
-    toolCalls?: readonly {
-        id: string;
-        name: string;
-        arguments: Record<string, unknown>;
-    }[];
+    /** The tool calls of the run, over every turn */
+    toolCalls?: readonly EvalToolCall[];
+    /** What the tool calls returned, over every turn, raw */
+    toolResults?: readonly EvalToolResult[];
+    /** The passages the tool results surfaced, over every turn, in the order they came */
+    retrieved?: readonly Passage[];
+    /** Each turn of a case with `turns`: `input` first, then the further turns */
+    turns?: readonly EvalTurnResult[];
+    /**
+     * Every run of the case when the suite runs several `trials`, each with its own scores. The
+     * case result itself is the first of them
+     */
+    trials?: ReadonlyArray<EvalCaseResult & {
+        scores: MetricScore[];
+    }>;
+    /** The agent's answer stopped at its output token limit */
+    truncated?: boolean;
+    /** Tool calls used up the agent's `maxIterations` before it answered */
+    iterationLimitReached?: boolean;
+    /** The run paused for tool approvals, so `output` holds no answer */
+    paused?: boolean;
 }
 export type MetricFn = ((result: EvalCaseResult) => Promise<MetricScore>) & {
     metricName: string;
@@ -57,6 +101,10 @@ export interface StatisticalScore extends MetricScore {
 export type StatisticalMetricFn = ((results: EvalCaseResult[]) => StatisticalScore) & {
     metricName: string;
 };
+/** A case result with the scores of its metrics, as a suite run returns it */
+export type ScoredCaseResult = EvalCaseResult & {
+    scores: MetricScore[];
+};
 /** Options every built-in per-case metric factory takes */
 export interface MetricOptions {
     /**

package/dist/metrics/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,CAAC,EAAE,SAAS;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,EAAE,CAAC;CACL;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,0FAA0F;AAC1F,MAAM,WAAW,OAAO;IACtB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,8DAA8D;AAC9D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,6DAA6D;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,yDAAyD;IACzD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,sFAAsF;IACtF,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,8EAA8E;IAC9E,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC;;;OAGG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IACnE,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,8EAA8E;AAC9E,MAAM,MAAM,gBAAgB,GAAG,cAAc,GAAG;IAAE,MAAM,EAAE,WAAW,EAAE,CAAA;CAAE,CAAC;AAE1E,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}

package/dist/metrics/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AA2EA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AA4HA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}

package/dist/reporters/ci.d.ts

@@ -1,3 +1,7 @@
 import type { EvalSuiteResult } from './index.js';
+/**
+ * Prints a short pass or fail summary and sets `process.exitCode` to 1 when an assertion
+ * failed. It does not end the process, so output flushes and later hooks still run.
+ */
 export declare function ciReport(result: EvalSuiteResult): void;
 //# sourceMappingURL=ci.d.ts.map

package/dist/reporters/ci.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAM/C,wBAAgB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAuBtD"}
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAM/C;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAuBtD"}

package/dist/reporters/ci.js

@@ -1,6 +1,10 @@
 const GREEN = '\x1b[32m';
 const RED = '\x1b[31m';
 const RESET = '\x1b[0m';
+/**
+ * Prints a short pass or fail summary and sets `process.exitCode` to 1 when an assertion
+ * failed. It does not end the process, so output flushes and later hooks still run.
+ */
 export function ciReport(result) {
     const passed = result.assertions.filter((a) => a.passed).length;
     const failed = result.assertions.filter((a) => !a.passed).length;
@@ -16,7 +20,7 @@ export function ciReport(result) {
     }
     console.log(`Result: ${passed} passed, ${failed} failed`);
     if (failed > 0) {
-        process.exit(1);
+        process.exitCode = 1;
     }
 }
 //# sourceMappingURL=ci.js.map

package/dist/reporters/ci.js.map

@@ -1 +1 @@
-{"version":3,"file":"ci.js","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAExC,OAAO,CAAC,GAAG,CACT,SAAS,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CACpI,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,YAAY,MAAM,SAAS,CAAC,CAAC;IAE1D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
+{"version":3,"file":"ci.js","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAExC,OAAO,CAAC,GAAG,CACT,SAAS,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CACpI,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,YAAY,MAAM,SAAS,CAAC,CAAC;IAE1D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC"}

package/dist/reporters/console.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"console.d.ts","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAgB/C,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CA8C3D"}
+{"version":3,"file":"console.d.ts","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiB/C,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAoD3D"}

package/dist/reporters/console.js

@@ -1,3 +1,4 @@
+import { trialsPerCase } from './format.js';
 const GREEN = '\x1b[32m';
 const RED = '\x1b[31m';
 const BOLD = '\x1b[1m';
@@ -12,13 +13,16 @@ function fmt(n) {
 export function consoleReport(result) {
     const metrics = Object.values(result.aggregated);
     if (metrics.length > 0) {
-        const cols = { metric: 14, mean: 10, median: 10, p95: 10, min: 10, max: 10 };
+        const metricWidth = Math.max(14, ...metrics.map((m) => m.name.length + 2));
+        const cols = { metric: metricWidth, mean: 10, median: 10, p95: 10, min: 10, max: 10, cases: 8 };
+        const showCount = metrics.some((m) => m.count !== undefined);
         const header = pad('Metric', cols.metric) +
             pad('Mean', cols.mean) +
             pad('Median', cols.median) +
             pad('P95', cols.p95) +
             pad('Min', cols.min) +
-            pad('Max', cols.max);
+            pad('Max', cols.max) +
+            (showCount ? pad('Cases', cols.cases) : '');
         console.log(`\n${BOLD}${header}${RESET}`);
         console.log(DIM + '─'.repeat(header.length) + RESET);
         for (const m of metrics) {
@@ -27,7 +31,8 @@ export function consoleReport(result) {
                 pad(fmt(m.median), cols.median) +
                 pad(fmt(m.p95), cols.p95) +
                 pad(fmt(m.min), cols.min) +
-                pad(fmt(m.max), cols.max);
+                pad(fmt(m.max), cols.max) +
+                (showCount ? pad(m.count === undefined ? '-' : String(m.count), cols.cases) : '');
             console.log(row);
         }
     }
@@ -43,6 +48,8 @@ export function consoleReport(result) {
     const failed = result.assertions.filter((a) => !a.passed).length;
     const errors = result.stats.errors ?? 0;
     const errorPart = errors > 0 ? ` | ${RED}${errors} errored${RESET}` : '';
-    console.log(`\n${BOLD}Summary${RESET}: ${result.stats.total} cases${errorPart} | ${result.stats.duration}ms | $${result.stats.cost} | ${GREEN}${passed} passed${RESET} ${failed > 0 ? `${RED}${failed} failed${RESET}` : ''}`);
+    const trials = trialsPerCase(result);
+    const trialPart = trials > 1 ? ` x ${trials} trials` : '';
+    console.log(`\n${BOLD}Summary${RESET}: ${result.stats.total} cases${trialPart}${errorPart} | ${result.stats.duration}ms | $${result.stats.cost} | ${GREEN}${passed} passed${RESET} ${failed > 0 ? `${RED}${failed} failed${RESET}` : ''}`);
 }
 //# sourceMappingURL=console.js.map

package/dist/reporters/console.js.map

@@ -1 +1 @@
-{"version":3,"file":"console.js","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,GAAW,EAAE,GAAW;IACnC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAC7E,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;YACtB,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAEvB,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GACP,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,aAAa,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,MAAM,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEzE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,SAAS,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,MAAM,UAAU,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClN,CAAC;AACJ,CAAC"}
+{"version":3,"file":"console.js","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,GAAW,EAAE,GAAW;IACnC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAChG,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QAC7D,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;YACtB,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAE9C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GACP,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,aAAa,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,MAAM,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACzE,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1D,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,SAAS,GAAG,SAAS,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,MAAM,UAAU,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9N,CAAC;AACJ,CAAC"}

package/dist/reporters/csv.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAS/C,wBAAgB,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAyBlF"}
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAU/C,wBAAgB,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CA2BlF"}

package/dist/reporters/csv.js

@@ -1,6 +1,7 @@
 import { writeFileSync } from 'node:fs';
+import { CASE_FLAGS } from './format.js';
 function escapeField(value) {
-    if (value.includes(',') || value.includes('"') || value.includes('\n')) {
+    if (/[",\r\n]/.test(value)) {
         return `"${value.replace(/"/g, '""')}"`;
     }
     return value;
@@ -13,7 +14,8 @@ export function csvReport(result, options) {
         }
     }
     const metrics = [...metricNames];
-    const headers = ['input', 'expected', 'output', 'duration', ...metrics];
+    const flags = CASE_FLAGS.filter((flag) => result.results.some((r) => r[flag] === true));
+    const headers = ['input', 'expected', 'output', 'duration', ...flags, ...metrics];
     const lines = [headers.join(',')];
     for (const r of result.results) {
         const scoreMap = new Map(r.scores.map((s) => [s.name, s.score]));
@@ -22,6 +24,7 @@ export function csvReport(result, options) {
             escapeField(r.case.expected ?? ''),
             escapeField(r.output),
             String(r.duration),
+            ...flags.map((flag) => (r[flag] === true ? 'true' : '')),
             ...metrics.map((m) => String(scoreMap.get(m) ?? '')),
         ];
         lines.push(row.join(','));

package/dist/reporters/csv.js.map

@@ -1 +1 @@
-{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACvE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAuB,EAAE,OAAyB;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;IAEjC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,CAAC;IACxE,MAAM,KAAK,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG;YACV,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAClC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;YACrB,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SACrD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}
+{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAuB,EAAE,OAAyB;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAExF,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC,CAAC;IAClF,MAAM,KAAK,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG;YACV,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAClC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;YACrB,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClB,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxD,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SACrD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}

package/dist/reporters/format.d.ts

@@ -0,0 +1,19 @@
+/** The run flags a case result can carry, in the order reports list them */
+export declare const CASE_FLAGS: readonly ["truncated", "iterationLimitReached", "paused"];
+export type CaseFlag = (typeof CASE_FLAGS)[number];
+/** The flags set on a case result, in `CASE_FLAGS` order */
+export declare function caseFlags(result: Partial<Record<CaseFlag, boolean>>): CaseFlag[];
+/**
+ * A one-line label for a case input: whitespace collapsed and cut to `max` characters with an
+ * ellipsis, so test names and annotation titles stay readable
+ */
+export declare function truncateInput(input: string, max?: number): string;
+/** The label of the case at `index` (zero based) in a report or test name */
+export declare function caseLabel(index: number, input: string): string;
+/** How many trials each case ran, 1 for a suite without several trials */
+export declare function trialsPerCase(result: {
+    results: ReadonlyArray<{
+        trials?: readonly unknown[];
+    }>;
+}): number;
+//# sourceMappingURL=format.d.ts.map

package/dist/reporters/format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,eAAO,MAAM,UAAU,2DAA4D,CAAC;AACpF,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,CAEhF;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAA6B,GAAG,MAAM,CAQvF;AAED,6EAA6E;AAC7E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAG9D;AAED,0EAA0E;AAC1E,wBAAgB,aAAa,CAAC,MAAM,EAAE;IACpC,OAAO,EAAE,aAAa,CAAC;QAAE,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,CAAA;KAAE,CAAC,CAAC;CACzD,GAAG,MAAM,CAET"}

package/dist/reporters/format.js

@@ -0,0 +1,31 @@
+/** The run flags a case result can carry, in the order reports list them */
+export const CASE_FLAGS = ['truncated', 'iterationLimitReached', 'paused'];
+/** The flags set on a case result, in `CASE_FLAGS` order */
+export function caseFlags(result) {
+    return CASE_FLAGS.filter((flag) => result[flag] === true);
+}
+const DEFAULT_LABEL_LENGTH = 60;
+/**
+ * A one-line label for a case input: whitespace collapsed and cut to `max` characters with an
+ * ellipsis, so test names and annotation titles stay readable
+ */
+export function truncateInput(input, max = DEFAULT_LABEL_LENGTH) {
+    const line = input.replace(/\s+/g, ' ').trim();
+    const chars = [...line];
+    if (chars.length <= max)
+        return line;
+    return `${chars
+        .slice(0, max - 3)
+        .join('')
+        .trimEnd()}...`;
+}
+/** The label of the case at `index` (zero based) in a report or test name */
+export function caseLabel(index, input) {
+    const text = truncateInput(input);
+    return `[${index + 1}] ${text === '' ? '(empty input)' : text}`;
+}
+/** How many trials each case ran, 1 for a suite without several trials */
+export function trialsPerCase(result) {
+    return result.results.reduce((most, r) => Math.max(most, r.trials?.length ?? 1), 1);
+}
+//# sourceMappingURL=format.js.map

package/dist/reporters/format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,uBAAuB,EAAE,QAAQ,CAAU,CAAC;AAGpF,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,MAA0C;IAClE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,MAAc,oBAAoB;IAC7E,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxB,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,KAAK;SACZ,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;SACjB,IAAI,CAAC,EAAE,CAAC;SACR,OAAO,EAAE,KAAK,CAAC;AACpB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO,IAAI,KAAK,GAAG,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAClE,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,aAAa,CAAC,MAE7B;IACC,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACtF,CAAC"}

package/dist/reporters/github.d.ts

@@ -0,0 +1,11 @@
+import type { EvalSuiteResult } from './index.js';
+/** The workflow commands that annotate the run: failures as errors, metric errors as warnings */
+export declare function githubAnnotations(result: EvalSuiteResult): string[];
+/** The Markdown summary of the run for `$GITHUB_STEP_SUMMARY` */
+export declare function githubSummary(result: EvalSuiteResult): string;
+/**
+ * Prints GitHub Actions annotations to stdout, and appends a Markdown summary to the file in
+ * `GITHUB_STEP_SUMMARY` when the variable is set
+ */
+export declare function githubReport(result: EvalSuiteResult): void;
+//# sourceMappingURL=github.d.ts.map

package/dist/reporters/github.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiB/C,iGAAiG;AACjG,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,CAoBnE;AAmBD,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA2C7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAK1D"}