@cogitator-ai/evals 0.7.0 → 0.9.0

package/dist/metrics/llm-judge.js

@@ -81,6 +81,8 @@ function usageOf(value) {
  * asking for a `{ score, reasoning }` JSON answer.
  */
 export function judgeContextFor(cogitator, judgeConfig) {
+    const cache = judgeConfig.cache === true ? memoryCache() : judgeConfig.cache || undefined;
+    const slots = judgeConfig.concurrency === undefined ? undefined : limiter(judgeConfig.concurrency);
     let judgeAgent;
     const agent = () => (judgeAgent ??= import('@cogitator-ai/core').then(({ Agent }) => new Agent({
         name: 'eval-judge',
@@ -93,18 +95,104 @@ export function judgeContextFor(cogitator, judgeConfig) {
     }), (error) => {
         throw new Error(`The LLM judge needs @cogitator-ai/core: ${error instanceof Error ? error.message : String(error)}`);
     }));
+    const judge = async (input) => {
+        const result = await cogitator.run(await agent(), { input, useMemory: false });
+        const usage = usageOf(result.usage);
+        return {
+            output: result.structured !== undefined ? JSON.stringify(result.structured) : result.output,
+            ...(usage && { usage }),
+        };
+    };
+    const limited = (input) => (slots ? slots(() => judge(input)) : judge(input));
+    const inFlight = new Map();
+    const lookup = async (input, key) => {
+        const hit = await cacheRead(cache, key);
+        if (hit !== undefined && parseJudgeOutput(hit))
+            return { output: hit };
+        const answered = await limited(input);
+        if (parseJudgeOutput(answered.output))
+            await cacheWrite(cache, key, answered.output);
+        return answered;
+    };
+    const cached = async (input) => {
+        const key = cacheKey(judgeConfig, input);
+        const running = inFlight.get(key);
+        if (running)
+            return { output: (await running).output };
+        const verdict = lookup(input, key).finally(() => inFlight.delete(key));
+        inFlight.set(key, verdict);
+        return verdict;
+    };
     return {
         judgeConfig,
-        cogitator: {
-            run: async ({ input }) => {
-                const result = await cogitator.run(await agent(), { input, useMemory: false });
-                const usage = usageOf(result.usage);
-                return {
-                    output: result.structured !== undefined ? JSON.stringify(result.structured) : result.output,
-                    ...(usage && { usage }),
-                };
-            },
-        },
+        cogitator: { run: ({ input }) => (cache ? cached(input) : limited(input)) },
+    };
+}
+/** A cached verdict, none on a miss or when the cache cannot be read: the judge then runs */
+async function cacheRead(cache, key) {
+    try {
+        const hit = await cache.get(key);
+        return typeof hit === 'string' ? hit : undefined;
+    }
+    catch (error) {
+        warnCache('read', error);
+        return undefined;
+    }
+}
+/** Keeps a verdict, warning instead of failing the score when the cache cannot be written */
+async function cacheWrite(cache, key, verdict) {
+    try {
+        await cache.set(key, verdict);
+    }
+    catch (error) {
+        warnCache('write', error);
+    }
+}
+function warnCache(action, error) {
+    process.emitWarning(`The judge cache could not ${action} a verdict: ${error instanceof Error ? error.message : String(error)}`, { code: 'COGITATOR_EVALS_JUDGE_CACHE' });
+}
+/** Bumped whenever how the judge is asked changes, so kept verdicts of an older judge go unused */
+const CACHE_KEY_VERSION = 1;
+/**
+ * The key of a verdict: what the judge was asked (its instructions and the whole prompt), by
+ * which model and with which settings
+ */
+function cacheKey(judgeConfig, input) {
+    return JSON.stringify([
+        CACHE_KEY_VERSION,
+        JUDGE_INSTRUCTIONS,
+        judgeConfig.model,
+        judgeConfig.temperature,
+        judgeConfig.maxTokens ?? null,
+        input,
+    ]);
+}
+function memoryCache() {
+    const verdicts = new Map();
+    return {
+        get: (key) => verdicts.get(key),
+        set: (key, verdict) => void verdicts.set(key, verdict),
+    };
+}
+/** Runs at most `limit` tasks at once, the rest waiting in order: a task that ends hands its slot on */
+function limiter(limit) {
+    let running = 0;
+    const waiting = [];
+    return async (task) => {
+        if (running >= limit)
+            await new Promise((resolve) => waiting.push(resolve));
+        else
+            running++;
+        try {
+            return await task();
+        }
+        finally {
+            const next = waiting.shift();
+            if (next)
+                next();
+            else
+                running--;
+        }
     };
 }
 function createJudgeMetric(name, systemPrompt) {

package/dist/metrics/llm-judge.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAiBxB,MAAM,YAAY,GAAG,6BAA6B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAClC,MAAM,cAAc,GAClB,yFAAyF,CAAC;AAE5F,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,MAAM,WAAW,GAAG,gCAAgC,CAAC;AAOrD,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,sGAAsG;AACtG,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,6FAA6F;AAC7F,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,IAAkC,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;QAAE,IAAI,GAAG,KAAK,CAAC;IAC/D,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,KAAK,MAAM,SAAS,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAChD,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,KAAgC,CAAC;IAC1F,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,OAAO;QACL,WAAW;QACX,SAAS,EAAE;YACT,GAAG,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;gBACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpC,OAAO;oBACL,MAAM,EACJ,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;oBACrF,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;iBACxB,CAAC;YACJ,CAAC;SACF;KACF,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,OAAgC;IACnD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3B,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,GAAG,GAAG,KAAK,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAClF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;IACpC,MAAM,YAAY,GAChB,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1F,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,YAAY,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AACzG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,WAAW,EAAE,CAAC;YAEnD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI;oBACJ,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,WAAW;oBACpB,KAAK,EAAE,WAAW;oBAClB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;iBACxB,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;gBACpE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAgB,OAAO,EAAE;gBAClC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAoB;IAC/C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,cAAc,EAC5B,qaAAqa,CACta,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAoB;IAC9C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,aAAa,EAC3B,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAiBxB,MAAM,YAAY,GAAG,6BAA6B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAClC,MAAM,cAAc,GAClB,yFAAyF,CAAC;AAE5F,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,MAAM,WAAW,GAAG,gCAAgC,CAAC;AAOrD,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,sGAAsG;AACtG,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,6FAA6F;AAC7F,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,IAAkC,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;QAAE,IAAI,GAAG,KAAK,CAAC;IAC/D,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,KAAK,MAAM,SAAS,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAChD,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,KAAgC,CAAC;IAC1F,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,IAAI,SAAS,CAAC;IAC1F,MAAM,KAAK,GACT,WAAW,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;IACvF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,MAAM,KAAK,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;YAC3F,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4D,CAAC;IAErF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE;QAClD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,KAAM,EAAE,GAAG,CAAC,CAAC;QACzC,IAAI,GAAG,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,UAAU,CAAC,KAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtF,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,OAAO;YAAE,OAAO,EAAE,MAAM,EAAE,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,OAAO;QACL,WAAW;QACX,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE;KAC5E,CAAC;AACJ,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,SAAS,CAAC,KAAiB,EAAE,GAAW;IACrD,IAAI,CAAC;QACH,MAAM,GAAG,GAAY,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,UAAU,CAAC,KAAiB,EAAE,GAAW,EAAE,OAAe;IACvE,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,MAAwB,EAAE,KAAc;IACzD,OAAO,CAAC,WAAW,CACjB,6BAA6B,MAAM,eAAe,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC1G,EAAE,IAAI,EAAE,6BAA6B,EAAE,CACxC,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,SAAS,QAAQ,CAAC,WAAwB,EAAE,KAAa;IACvD,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,iBAAiB;QACjB,kBAAkB;QAClB,WAAW,CAAC,KAAK;QACjB,WAAW,CAAC,WAAW;QACvB,WAAW,CAAC,SAAS,IAAI,IAAI;QAC7B,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW;IAClB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,OAAO;QACL,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC;KACvD,CAAC;AACJ,CAAC;AAED,wGAAwG;AACxG,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,OAAO,KAAK,EAAK,IAAsB,EAAc,EAAE;QACrD,IAAI,OAAO,IAAI,KAAK;YAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;;YAC7E,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;;gBACZ,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,OAAgC;IACnD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3B,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,GAAG,GAAG,KAAK,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAClF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;IACpC,MAAM,YAAY,GAChB,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1F,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,YAAY,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AACzG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,WAAW,EAAE,CAAC;YAEnD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI;oBACJ,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,WAAW;oBACpB,KAAK,EAAE,WAAW;oBAClB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;iBACxB,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;gBACpE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAgB,OAAO,EAAE;gBAClC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAoB;IAC/C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,cAAc,EAC5B,qaAAqa,CACta,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAoB;IAC9C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,aAAa,EAC3B,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}

package/dist/metrics/types.d.ts

@@ -43,6 +43,12 @@ export interface EvalCaseResult {
         name: string;
         arguments: Record<string, unknown>;
     }[];
+    /** The agent's answer stopped at its output token limit */
+    truncated?: boolean;
+    /** Tool calls used up the agent's `maxIterations` before it answered */
+    iterationLimitReached?: boolean;
+    /** The run paused for tool approvals, so `output` holds no answer */
+    paused?: boolean;
 }
 export type MetricFn = ((result: EvalCaseResult) => Promise<MetricScore>) & {
     metricName: string;

package/dist/metrics/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,CAAC,EAAE,SAAS;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,EAAE,CAAC;CACL;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,QAAQ,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,SAAS,CAAC,EAAE,SAAS;QACnB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,EAAE,CAAC;IACJ,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC,GAAG;IAC1E,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,gBAAgB,CAAC,GAAG;IACpF,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,yEAAyE;AACzE,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAGtE"}

package/dist/metrics/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AA2EA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/metrics/types.ts"],"names":[],"mappings":"AAiFA,yEAAyE;AACzE,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1E,OAAO,SAAS,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;AACzD,CAAC"}

package/dist/reporters/ci.d.ts

@@ -1,3 +1,7 @@
 import type { EvalSuiteResult } from './index.js';
+/**
+ * Prints a short pass or fail summary and sets `process.exitCode` to 1 when an assertion
+ * failed. It does not end the process, so output flushes and later hooks still run.
+ */
 export declare function ciReport(result: EvalSuiteResult): void;
 //# sourceMappingURL=ci.d.ts.map

package/dist/reporters/ci.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAM/C,wBAAgB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAuBtD"}
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAM/C;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAuBtD"}

package/dist/reporters/ci.js

@@ -1,6 +1,10 @@
 const GREEN = '\x1b[32m';
 const RED = '\x1b[31m';
 const RESET = '\x1b[0m';
+/**
+ * Prints a short pass or fail summary and sets `process.exitCode` to 1 when an assertion
+ * failed. It does not end the process, so output flushes and later hooks still run.
+ */
 export function ciReport(result) {
     const passed = result.assertions.filter((a) => a.passed).length;
     const failed = result.assertions.filter((a) => !a.passed).length;
@@ -16,7 +20,7 @@ export function ciReport(result) {
     }
     console.log(`Result: ${passed} passed, ${failed} failed`);
     if (failed > 0) {
-        process.exit(1);
+        process.exitCode = 1;
     }
 }
 //# sourceMappingURL=ci.js.map

package/dist/reporters/ci.js.map

@@ -1 +1 @@
-{"version":3,"file":"ci.js","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAExC,OAAO,CAAC,GAAG,CACT,SAAS,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CACpI,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,YAAY,MAAM,SAAS,CAAC,CAAC;IAE1D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
+{"version":3,"file":"ci.js","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAExC,OAAO,CAAC,GAAG,CACT,SAAS,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CACpI,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,YAAY,MAAM,SAAS,CAAC,CAAC;IAE1D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC"}

package/dist/reporters/console.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"console.d.ts","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAgB/C,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CA8C3D"}
+{"version":3,"file":"console.d.ts","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAgB/C,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAiD3D"}

package/dist/reporters/console.js

@@ -12,13 +12,15 @@ function fmt(n) {
 export function consoleReport(result) {
     const metrics = Object.values(result.aggregated);
     if (metrics.length > 0) {
-        const cols = { metric: 14, mean: 10, median: 10, p95: 10, min: 10, max: 10 };
+        const cols = { metric: 14, mean: 10, median: 10, p95: 10, min: 10, max: 10, cases: 8 };
+        const showCount = metrics.some((m) => m.count !== undefined);
         const header = pad('Metric', cols.metric) +
             pad('Mean', cols.mean) +
             pad('Median', cols.median) +
             pad('P95', cols.p95) +
             pad('Min', cols.min) +
-            pad('Max', cols.max);
+            pad('Max', cols.max) +
+            (showCount ? pad('Cases', cols.cases) : '');
         console.log(`\n${BOLD}${header}${RESET}`);
         console.log(DIM + '─'.repeat(header.length) + RESET);
         for (const m of metrics) {
@@ -27,7 +29,8 @@ export function consoleReport(result) {
                 pad(fmt(m.median), cols.median) +
                 pad(fmt(m.p95), cols.p95) +
                 pad(fmt(m.min), cols.min) +
-                pad(fmt(m.max), cols.max);
+                pad(fmt(m.max), cols.max) +
+                (showCount ? pad(m.count === undefined ? '-' : String(m.count), cols.cases) : '');
             console.log(row);
         }
     }

package/dist/reporters/console.js.map

@@ -1 +1 @@
-{"version":3,"file":"console.js","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,GAAW,EAAE,GAAW;IACnC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAC7E,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;YACtB,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAEvB,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GACP,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,aAAa,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,MAAM,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEzE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,SAAS,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,MAAM,UAAU,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClN,CAAC;AACJ,CAAC"}
+{"version":3,"file":"console.js","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,GAAW,EAAE,GAAW;IACnC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACvF,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QAC7D,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;YACtB,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAE9C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GACP,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,aAAa,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,MAAM,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEzE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,SAAS,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,MAAM,UAAU,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClN,CAAC;AACJ,CAAC"}

package/dist/reporters/csv.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAS/C,wBAAgB,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAyBlF"}
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAU/C,wBAAgB,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CA2BlF"}

package/dist/reporters/csv.js

@@ -1,6 +1,7 @@
 import { writeFileSync } from 'node:fs';
+import { CASE_FLAGS } from './format.js';
 function escapeField(value) {
-    if (value.includes(',') || value.includes('"') || value.includes('\n')) {
+    if (/[",\r\n]/.test(value)) {
         return `"${value.replace(/"/g, '""')}"`;
     }
     return value;
@@ -13,7 +14,8 @@ export function csvReport(result, options) {
         }
     }
     const metrics = [...metricNames];
-    const headers = ['input', 'expected', 'output', 'duration', ...metrics];
+    const flags = CASE_FLAGS.filter((flag) => result.results.some((r) => r[flag] === true));
+    const headers = ['input', 'expected', 'output', 'duration', ...flags, ...metrics];
     const lines = [headers.join(',')];
     for (const r of result.results) {
         const scoreMap = new Map(r.scores.map((s) => [s.name, s.score]));
@@ -22,6 +24,7 @@ export function csvReport(result, options) {
             escapeField(r.case.expected ?? ''),
             escapeField(r.output),
             String(r.duration),
+            ...flags.map((flag) => (r[flag] === true ? 'true' : '')),
             ...metrics.map((m) => String(scoreMap.get(m) ?? '')),
         ];
         lines.push(row.join(','));

package/dist/reporters/csv.js.map

@@ -1 +1 @@
-{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACvE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAuB,EAAE,OAAyB;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;IAEjC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,CAAC;IACxE,MAAM,KAAK,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG;YACV,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAClC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;YACrB,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SACrD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}
+{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAuB,EAAE,OAAyB;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAExF,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC,CAAC;IAClF,MAAM,KAAK,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG;YACV,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAClC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;YACrB,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClB,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxD,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SACrD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}

package/dist/reporters/format.d.ts

@@ -0,0 +1,13 @@
+/** The run flags a case result can carry, in the order reports list them */
+export declare const CASE_FLAGS: readonly ["truncated", "iterationLimitReached", "paused"];
+export type CaseFlag = (typeof CASE_FLAGS)[number];
+/** The flags set on a case result, in `CASE_FLAGS` order */
+export declare function caseFlags(result: Partial<Record<CaseFlag, boolean>>): CaseFlag[];
+/**
+ * A one-line label for a case input: whitespace collapsed and cut to `max` characters with an
+ * ellipsis, so test names and annotation titles stay readable
+ */
+export declare function truncateInput(input: string, max?: number): string;
+/** The label of the case at `index` (zero based) in a report or test name */
+export declare function caseLabel(index: number, input: string): string;
+//# sourceMappingURL=format.d.ts.map

package/dist/reporters/format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,eAAO,MAAM,UAAU,2DAA4D,CAAC;AACpF,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,CAEhF;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAA6B,GAAG,MAAM,CAQvF;AAED,6EAA6E;AAC7E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAG9D"}

package/dist/reporters/format.js

@@ -0,0 +1,27 @@
+/** The run flags a case result can carry, in the order reports list them */
+export const CASE_FLAGS = ['truncated', 'iterationLimitReached', 'paused'];
+/** The flags set on a case result, in `CASE_FLAGS` order */
+export function caseFlags(result) {
+    return CASE_FLAGS.filter((flag) => result[flag] === true);
+}
+const DEFAULT_LABEL_LENGTH = 60;
+/**
+ * A one-line label for a case input: whitespace collapsed and cut to `max` characters with an
+ * ellipsis, so test names and annotation titles stay readable
+ */
+export function truncateInput(input, max = DEFAULT_LABEL_LENGTH) {
+    const line = input.replace(/\s+/g, ' ').trim();
+    const chars = [...line];
+    if (chars.length <= max)
+        return line;
+    return `${chars
+        .slice(0, max - 3)
+        .join('')
+        .trimEnd()}...`;
+}
+/** The label of the case at `index` (zero based) in a report or test name */
+export function caseLabel(index, input) {
+    const text = truncateInput(input);
+    return `[${index + 1}] ${text === '' ? '(empty input)' : text}`;
+}
+//# sourceMappingURL=format.js.map

package/dist/reporters/format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,uBAAuB,EAAE,QAAQ,CAAU,CAAC;AAGpF,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,MAA0C;IAClE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,MAAc,oBAAoB;IAC7E,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxB,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,KAAK;SACZ,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;SACjB,IAAI,CAAC,EAAE,CAAC;SACR,OAAO,EAAE,KAAK,CAAC;AACpB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO,IAAI,KAAK,GAAG,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAClE,CAAC"}

package/dist/reporters/github.d.ts

@@ -0,0 +1,11 @@
+import type { EvalSuiteResult } from './index.js';
+/** The workflow commands that annotate the run: failures as errors, metric errors as warnings */
+export declare function githubAnnotations(result: EvalSuiteResult): string[];
+/** The Markdown summary of the run for `$GITHUB_STEP_SUMMARY` */
+export declare function githubSummary(result: EvalSuiteResult): string;
+/**
+ * Prints GitHub Actions annotations to stdout, and appends a Markdown summary to the file in
+ * `GITHUB_STEP_SUMMARY` when the variable is set
+ */
+export declare function githubReport(result: EvalSuiteResult): void;
+//# sourceMappingURL=github.d.ts.map

package/dist/reporters/github.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiB/C,iGAAiG;AACjG,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,CAoBnE;AAmBD,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA0C7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAK1D"}

package/dist/reporters/github.js

@@ -0,0 +1,94 @@
+import { appendFileSync } from 'node:fs';
+import { caseFlags, caseLabel } from './format.js';
+/** Escapes the message of a workflow command */
+function escapeData(value) {
+    return value.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
+}
+/** Escapes a workflow command property, where `:` and `,` are separators */
+function escapeProperty(value) {
+    return escapeData(value).replace(/:/g, '%3A').replace(/,/g, '%2C');
+}
+function command(kind, title, message) {
+    return `::${kind} title=${escapeProperty(title)}::${escapeData(message)}`;
+}
+/** The workflow commands that annotate the run: failures as errors, metric errors as warnings */
+export function githubAnnotations(result) {
+    const lines = [];
+    result.results.forEach((r, i) => {
+        const label = caseLabel(i, r.case.input);
+        if (r.error !== undefined) {
+            lines.push(command('error', `Eval case ${label}`, r.error));
+        }
+        for (const score of r.scores) {
+            if (score.error !== undefined) {
+                lines.push(command('warning', `Eval metric ${score.name} on case ${label}`, score.error));
+            }
+        }
+    });
+    for (const a of result.assertions) {
+        if (!a.passed)
+            lines.push(command('error', `Eval assertion ${a.name}`, a.message));
+    }
+    return lines;
+}
+/**
+ * A table cell of the step summary: any line break (a lone `\r` too) becomes `<br>` so the row
+ * holds, and `<` and `&` are escaped so no tag or comment in an answer hides the rest
+ */
+function cell(value) {
+    return value
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/\\/g, '\\\\')
+        .replace(/\|/g, '\\|')
+        .replace(/\r\n|\r|\n/g, '<br>');
+}
+function num(n) {
+    return Number.isInteger(n) ? String(n) : n.toFixed(4);
+}
+/** The Markdown summary of the run for `$GITHUB_STEP_SUMMARY` */
+export function githubSummary(result) {
+    const { stats } = result;
+    const passed = result.assertions.filter((a) => a.passed).length;
+    const failed = result.assertions.length - passed;
+    const flagged = result.results.filter((r) => caseFlags(r).length > 0).length;
+    const totals = [
+        `**${stats.total}** cases`,
+        `**${stats.errors ?? 0}** errored`,
+        ...(stats.metricErrors ? [`**${stats.metricErrors}** metric errors`] : []),
+        ...(flagged > 0 ? [`**${flagged}** truncated, paused or at the iteration limit`] : []),
+        `${stats.duration} ms`,
+        `${num(stats.cost)} USD`,
+        `assertions: **${passed}** passed, **${failed}** failed`,
+    ];
+    const lines = ['## Eval results', '', totals.join(' | '), ''];
+    const metrics = Object.values(result.aggregated);
+    if (metrics.length > 0) {
+        lines.push('### Metrics', '', '| Metric | Mean | P50 | P95 | Cases |', '| --- | ---: | ---: | ---: | ---: |');
+        for (const m of metrics) {
+            const count = m.count === undefined ? '-' : String(m.count);
+            lines.push(`| ${cell(m.name)} | ${num(m.mean)} | ${num(m.p50)} | ${num(m.p95)} | ${count} |`);
+        }
+        lines.push('');
+    }
+    if (result.assertions.length > 0) {
+        lines.push('### Assertions', '', '| Result | Assertion | Details |', '| --- | --- | --- |');
+        for (const a of result.assertions) {
+            lines.push(`| ${a.passed ? 'pass' : '**fail**'} | ${cell(a.name)} | ${cell(a.message)} |`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n') + '\n';
+}
+/**
+ * Prints GitHub Actions annotations to stdout, and appends a Markdown summary to the file in
+ * `GITHUB_STEP_SUMMARY` when the variable is set
+ */
+export function githubReport(result) {
+    for (const line of githubAnnotations(result))
+        console.log(line);
+    const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+    if (summaryPath)
+        appendFileSync(summaryPath, githubSummary(result), 'utf-8');
+}
+//# sourceMappingURL=github.js.map

package/dist/reporters/github.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAEhD,gDAAgD;AAChD,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAChF,CAAC;AAED,4EAA4E;AAC5E,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,OAAO,CAAC,IAAyB,EAAE,KAAa,EAAE,OAAe;IACxE,OAAO,KAAK,IAAI,UAAU,cAAc,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,eAAe,KAAK,CAAC,IAAI,YAAY,KAAK,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,KAAK;SACT,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAE7E,MAAM,MAAM,GAAG;QACb,KAAK,KAAK,CAAC,KAAK,UAAU;QAC1B,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY;QAClC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,gDAAgD,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,GAAG,KAAK,CAAC,QAAQ,KAAK;QACtB,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM;QACxB,iBAAiB,MAAM,gBAAgB,MAAM,WAAW;KACzD,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,iBAAiB,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAE9D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,aAAa,EACb,EAAE,EACF,uCAAuC,EACvC,qCAAqC,CACtC,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,EAAE,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAuB;IAClD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEhE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACpD,IAAI,WAAW;QAAE,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC"}

package/dist/reporters/index.d.ts

@@ -9,6 +9,8 @@ export interface AggregatedMetric {
     p50: number;
     p95: number;
     p99: number;
+    /** Cases the metric was scored on: errored cases are not scored */
+    count?: number;
     /** Extra figures a statistical metric reports, such as the total cost */
     metadata?: Record<string, unknown>;
 }
@@ -29,6 +31,12 @@ export interface EvalSuiteResult {
         duration: number;
         /** Why the case has no output: every attempt failed or timed out */
         error?: string;
+        /** The agent's answer stopped at its output token limit */
+        truncated?: boolean;
+        /** Tool calls used up the agent's `maxIterations` before it answered */
+        iterationLimitReached?: boolean;
+        /** The run paused for tool approvals, so `output` holds no answer */
+        paused?: boolean;
         scores: Array<{
             name: string;
             score: number;
@@ -40,23 +48,26 @@ export interface EvalSuiteResult {
     assertions: AssertionResult[];
     stats: EvalStats;
 }
-export type ReporterType = 'console' | 'json' | 'csv' | 'ci';
+export type ReporterType = 'console' | 'json' | 'csv' | 'ci' | 'junit' | 'github';
 export type ReporterOptions = {
     /**
      * Base path of file reports. Each file reporter writes it with its own
-     * extension, so `./reports/eval` gives `eval.json` and `eval.csv`.
+     * extension, so `./reports/eval` gives `eval.json`, `eval.csv` and `eval.xml`.
      */
     path?: string;
 };
 /** The file a reporter writes: the base path with the reporter's extension. */
-export declare function reportPath(base: string | undefined, extension: 'json' | 'csv'): string;
+export declare function reportPath(base: string | undefined, extension: 'json' | 'csv' | 'xml'): string;
 /**
- * Runs the reporters in the order given, except `ci`, which runs last because
- * it ends the process when an assertion failed.
+ * Runs the reporters in the order given, except `ci`, which runs last so its verdict is the
+ * final line of the output. `ci` sets `process.exitCode` to 1 when an assertion failed instead
+ * of ending the process, so output still flushes and later hooks still run.
  */
 export declare function report(result: EvalSuiteResult, type: ReporterType | ReporterType[], options?: ReporterOptions): void;
 export { consoleReport } from './console.js';
 export { jsonReport } from './json.js';
 export { csvReport } from './csv.js';
 export { ciReport } from './ci.js';
+export { junitReport } from './junit.js';
+export { githubReport } from './github.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/reporters/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,oEAAoE;QACpE,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClF,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAGtF;AAOD;;;GAGG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAoBN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,oEAAoE;QACpE,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,2DAA2D;QAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,wEAAwE;QACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAChC,qEAAqE;QACrE,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClF,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,QAAQ,CAAC;AAClF,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAG9F;AAOD;;;;GAIG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CA0BN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/reporters/index.js

@@ -4,10 +4,12 @@ import { consoleReport } from './console.js';
 import { jsonReport } from './json.js';
 import { csvReport } from './csv.js';
 import { ciReport } from './ci.js';
+import { junitReport } from './junit.js';
+import { githubReport } from './github.js';
 const DEFAULT_BASE = 'eval-report';
 /** The file a reporter writes: the base path with the reporter's extension. */
 export function reportPath(base, extension) {
-    const stem = (base ?? DEFAULT_BASE).replace(/\.(json|csv)$/i, '');
+    const stem = (base ?? DEFAULT_BASE).replace(/\.(json|csv|xml)$/i, '');
     return `${stem}.${extension}`;
 }
 function prepare(path) {
@@ -15,8 +17,9 @@ function prepare(path) {
     return path;
 }
 /**
- * Runs the reporters in the order given, except `ci`, which runs last because
- * it ends the process when an assertion failed.
+ * Runs the reporters in the order given, except `ci`, which runs last so its verdict is the
+ * final line of the output. `ci` sets `process.exitCode` to 1 when an assertion failed instead
+ * of ending the process, so output still flushes and later hooks still run.
  */
 export function report(result, type, options) {
     const types = Array.isArray(type) ? type : [type];
@@ -32,6 +35,12 @@ export function report(result, type, options) {
             case 'csv':
                 csvReport(result, { path: prepare(reportPath(options?.path, 'csv')) });
                 break;
+            case 'junit':
+                junitReport(result, { path: prepare(reportPath(options?.path, 'xml')) });
+                break;
+            case 'github':
+                githubReport(result);
+                break;
             case 'ci':
                 ciReport(result);
                 break;
@@ -42,4 +51,6 @@ export { consoleReport } from './console.js';
 export { jsonReport } from './json.js';
 export { csvReport } from './csv.js';
 export { ciReport } from './ci.js';
+export { junitReport } from './junit.js';
+export { githubReport } from './github.js';
 //# sourceMappingURL=index.js.map