@cogitator-ai/evals 0.7.0 → 0.9.0

package/dist/datasets/issues.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"issues.js","sourceRoot":"","sources":["../../src/datasets/issues.ts"],"names":[],"mappings":"AAEA,4EAA4E;AAC5E,MAAM,UAAU,cAAc,CAAC,KAAiB;IAC9C,OAAO,KAAK,CAAC,MAAM;SAChB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;SAC9F,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}

package/dist/datasets/jsonl-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-loader.d.ts","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAyBjE"}
+{"version":3,"file":"jsonl-loader.d.ts","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CA6BjE"}

package/dist/datasets/jsonl-loader.js

@@ -1,6 +1,7 @@
 import { createReadStream } from 'node:fs';
 import { createInterface } from 'node:readline';
 import { EvalCaseSchema } from '../schema.js';
+import { describeIssues } from './issues.js';
 export async function loadJsonl(path) {
     const cases = [];
     let lineNumber = 0;
@@ -20,7 +21,11 @@ export async function loadJsonl(path) {
         catch {
             throw new Error(`Invalid JSON at line ${lineNumber}: ${trimmed}`);
         }
-        cases.push(EvalCaseSchema.parse(parsed));
+        const checked = EvalCaseSchema.safeParse(parsed);
+        if (!checked.success) {
+            throw new Error(`Invalid case at line ${lineNumber}: ${describeIssues(checked.error)}`);
+        }
+        cases.push(checked.data);
     }
     return cases;
 }

package/dist/datasets/jsonl-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-loader.js","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAG3C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,EAAE,GAAG,eAAe,CAAC;QACzB,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QACpD,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,UAAU,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,KAAK,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"file":"jsonl-loader.js","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAG1C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,EAAE,GAAG,eAAe,CAAC;QACzB,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QACpD,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,UAAU,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,KAAK,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,KAAK,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/eval-comparison.d.ts

@@ -27,6 +27,8 @@ export interface EvalComparisonRunOptions {
 export interface MetricComparison {
     baseline: number;
     challenger: number;
+    /** Cases compared: those at least one side scored, an errored side counting as a 0 */
+    n: number;
     pValue: number;
     significant: boolean;
     winner: 'baseline' | 'challenger' | 'tie';

package/dist/eval-comparison.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAe,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAKrC,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QACP,QAAQ,EAAE,UAAU,CAAC;QACrB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACvF;AAED,MAAM,WAAW,wBAAwB;IACvC,kEAAkE;IAClE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;QAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,eAAe,CAAC;CAC7B;AAkED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAwB;gBAEjC,IAAI,EAAE,qBAAqB;IASjC,GAAG,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAoE7E"}
+{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAe,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAKrC,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QACP,QAAQ,EAAE,UAAU,CAAC;QACrB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACvF;AAED,MAAM,WAAW,wBAAwB;IACvC,kEAAkE;IAClE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,CAAC,EAAE,MAAM,CAAC;IACV,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;QAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,eAAe,CAAC;CAC7B;AAmED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAwB;gBAEjC,IAAI,EAAE,qBAAqB;IASjC,GAAG,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAuE7E"}

package/dist/eval-comparison.js

@@ -14,6 +14,7 @@ function compareMetric(baselineScores, challengerScores) {
         return {
             baseline: baselineMean,
             challenger: challengerMean,
+            n,
             pValue: 1,
             significant: false,
             winner: 'tie',
@@ -44,7 +45,7 @@ function compareMetric(baselineScores, challengerScores) {
     if (significant) {
         winner = challengerMean > baselineMean ? 'challenger' : 'baseline';
     }
-    return { baseline: baselineMean, challenger: challengerMean, pValue, significant, winner };
+    return { baseline: baselineMean, challenger: challengerMean, n, pValue, significant, winner };
 }
 function determineOverallWinner(metrics) {
     let baselineWins = 0;
@@ -101,15 +102,15 @@ export class EvalComparison {
             challengerSuite.run({ signal: options.signal }),
         ]);
         const metricNames = new Set();
-        for (const r of baselineResult.results) {
-            for (const s of r.scores) {
-                metricNames.add(s.name);
+        for (const result of [baselineResult, challengerResult]) {
+            for (const r of result.results) {
+                for (const s of r.scores)
+                    metricNames.add(s.name);
             }
         }
         const metricComparisons = {};
         for (const name of metricNames) {
-            const baselineScores = extractScores(baselineResult, name);
-            const challengerScores = extractScores(challengerResult, name);
+            const [baselineScores, challengerScores] = pairedScores(baselineResult, challengerResult, name);
             metricComparisons[name] = compareMetric(baselineScores, challengerScores);
         }
         return {
@@ -122,10 +123,23 @@ export class EvalComparison {
         };
     }
 }
-function extractScores(result, metricName) {
-    return result.results.map((r) => {
-        const score = r.scores.find((s) => s.name === metricName);
-        return score?.score ?? 0;
-    });
+/**
+ * The scores of a metric for each case, side by side. A case one side errored on counts as a 0
+ * for that side, since a target that fails loses the case, and a case both sides errored on is
+ * left out, telling nothing about either
+ */
+function pairedScores(baseline, challenger, metricName) {
+    const scoreOf = (result, at) => result.results[at]?.scores.find((s) => s.name === metricName)?.score;
+    const baselineScores = [];
+    const challengerScores = [];
+    for (let at = 0; at < baseline.results.length; at++) {
+        const b = scoreOf(baseline, at);
+        const c = scoreOf(challenger, at);
+        if (b === undefined && c === undefined)
+            continue;
+        baselineScores.push(b ?? 0);
+        challengerScores.push(c ?? 0);
+    }
+    return [baselineScores, challengerScores];
 }
 //# sourceMappingURL=eval-comparison.js.map

package/dist/eval-comparison.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAuC3C,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,cAAwB,EAAE,gBAA0B;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;IAEhC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,cAAc;YAC1B,MAAM,EAAE,CAAC;YACT,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,WAAoB,CAAC;IAEzB,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;YAChF,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;QACtE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC7D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,MAAM,GAAsC,KAAK,CAAC;IACtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,GAAG,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AAC7F,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAyC;IAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAE7C,YAAY,IAA2B;QACrC,0BAA0B,CAAC,KAAK,CAAC;YAC/B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;SACR,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YACvC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,iBAAiB,GAAqC,EAAE,CAAC;QAE/D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,cAAc,GAAG,aAAa,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;YAC3D,MAAM,gBAAgB,GAAG,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;YAC/D,iBAAiB,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;aAC3B;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;CACF;AAED,SAAS,aAAa,CAAC,MAAuB,EAAE,UAAkB;IAChE,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QACvE,OAAO,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAyC3C,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,cAAwB,EAAE,gBAA0B;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;IAEhC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,cAAc;YAC1B,CAAC;YACD,MAAM,EAAE,CAAC;YACT,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,WAAoB,CAAC;IAEzB,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;YAChF,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;QACtE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC7D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,MAAM,GAAsC,KAAK,CAAC;IACtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,GAAG,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AAChG,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAyC;IAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAE7C,YAAY,IAA2B;QACrC,0BAA0B,CAAC,KAAK,CAAC;YAC/B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;SACR,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM;oBAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;QAED,MAAM,iBAAiB,GAAqC,EAAE,CAAC;QAE/D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,YAAY,CACrD,cAAc,EACd,gBAAgB,EAChB,IAAI,CACL,CAAC;YACF,iBAAiB,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;aAC3B;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;CACF;AAED;;;;GAIG;AACH,SAAS,YAAY,CACnB,QAAyB,EACzB,UAA2B,EAC3B,UAAkB;IAElB,MAAM,OAAO,GAAG,CAAC,MAAuB,EAAE,EAAU,EAAE,EAAE,CACtD,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,EAAE,KAAK,CAAC;IACpF,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5B,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC"}

package/dist/eval-suite.d.ts

@@ -78,6 +78,8 @@ export declare class EvalSuite {
     private validateTarget;
     /** Two metrics with one name would be aggregated, compared and reported as one */
     private validateMetricNames;
+    /** The cases a run with `maxCases` goes through, in order */
+    cases(maxCases?: number): readonly EvalCase[];
     run(options?: EvalRunOptions): Promise<EvalSuiteResult>;
     private reportProgress;
     private runCase;
@@ -92,9 +94,11 @@ export declare class EvalSuite {
     private executeAgentTarget;
     private evaluateCaseMetrics;
     /**
-     * A metric's score for one case. A metric that throws, or whose score is not a finite number,
-     * scores 0 with the reason in `error`, so one bad metric neither drops the case nor poisons
-     * the aggregate with NaN.
+     * A metric's score for one case. Only cases with an output are scored: a case whose every
+     * attempt failed is left unscored, so its empty output neither costs a judge call nor counts as
+     * a score of 0 (`stats.errors` and the `maxErrorRate` assertion catch those). A metric that
+     * throws, or whose score is not a finite number, scores 0 with the reason in `error`, so one
+     * bad metric neither drops the case nor poisons the aggregate with NaN.
      */
     private scoreMetric;
     private aggregateScores;

package/dist/eval-suite.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAElG,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9F,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,EAAE,WAAW,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACrE;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sFAAsF;IACtF,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,wFAAwF;IACxF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AASD,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AA+DD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IA0ClC,OAAO,CAAC,cAAc;IAuBtB,kFAAkF;IAClF,OAAO,CAAC,mBAAmB;IAcrB,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IAsGjE,OAAO,CAAC,cAAc;YAcR,OAAO;YASP,WAAW;IA8BzB;;;;OAIG;YACW,kBAAkB;YAoClB,eAAe;YASf,kBAAkB;YAoClB,mBAAmB;IAKjC;;;;OAIG;YACW,WAAW;IAuBzB,OAAO,CAAC,eAAe;CAuBxB"}
+{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAkB,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAElG,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAG9F,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,MAAM,EAAE,WAAW,CAAC;IACpB,yBAAyB;IACzB,IAAI,EAAE,QAAQ,CAAC;IACf,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACrE;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sFAAsF;IACtF,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,0EAA0E;AAC1E,MAAM,MAAM,cAAc,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,cAAc,CAAC;IACtB,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,wFAAwF;IACxF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AASD,wBAAgB,WAAW,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,IAAI,WAAW,CAEzD;AAmFD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IA0ClC,OAAO,CAAC,cAAc;IAuBtB,kFAAkF;IAClF,OAAO,CAAC,mBAAmB;IAc3B,6DAA6D;IAC7D,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,QAAQ,EAAE;IAIvC,GAAG,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IAyGjE,OAAO,CAAC,cAAc;YAcR,OAAO;YASP,WAAW;IA8BzB;;;;OAIG;YACW,kBAAkB;YAoClB,eAAe;YASf,kBAAkB;YAuClB,mBAAmB;IAKjC;;;;;;OAMG;YACW,WAAW;IAuBzB,OAAO,CAAC,eAAe;CA4BxB"}

package/dist/eval-suite.js

@@ -67,6 +67,25 @@ async function waitAtMost(promise, ms) {
         clearTimeout(timer);
     }
 }
+/**
+ * The aggregate of a metric no case was scored on, every case having errored: its figures are
+ * NaN, so a threshold, a regression check or a baseline over it fails instead of passing on
+ * nothing
+ */
+function unscored(name) {
+    return {
+        name,
+        mean: Number.NaN,
+        median: Number.NaN,
+        min: Number.NaN,
+        max: Number.NaN,
+        stdDev: Number.NaN,
+        p50: Number.NaN,
+        p95: Number.NaN,
+        p99: Number.NaN,
+        count: 0,
+    };
+}
 export class EvalSuite {
     dataset;
     target;
@@ -137,6 +156,10 @@ export class EvalSuite {
             seen.add(name);
         }
     }
+    /** The cases a run with `maxCases` goes through, in order */
+    cases(maxCases) {
+        return this.dataset.cases.slice(0, maxCases);
+    }
     async run(options = {}) {
         const { maxCases, signal } = options;
         if (maxCases !== undefined && (!Number.isInteger(maxCases) || maxCases < 1)) {
@@ -144,7 +167,7 @@ export class EvalSuite {
         }
         signal?.throwIfAborted();
         const suiteStart = Date.now();
-        const cases = this.dataset.cases.slice(0, maxCases);
+        const cases = this.cases(maxCases);
         const total = cases.length;
         const orderedResults = new Array(total);
         let nextIdx = 0;
@@ -193,6 +216,9 @@ export class EvalSuite {
                         output: r.output,
                         duration: r.duration,
                         ...(r.error !== undefined && { error: r.error }),
+                        ...(r.truncated && { truncated: true }),
+                        ...(r.iterationLimitReached && { iterationLimitReached: true }),
+                        ...(r.paused && { paused: true }),
                         scores: r.scores,
                     })),
                     aggregated,
@@ -234,7 +260,7 @@ export class EvalSuite {
     }
     async runCase(evalCase, signal) {
         const caseResult = await this.executeCase(evalCase, signal);
-        if (signal?.aborted)
+        if (signal?.aborted || caseResult.error !== undefined)
             return { ...caseResult, scores: [] };
         return { ...caseResult, scores: await this.evaluateCaseMetrics(caseResult) };
     }
@@ -316,6 +342,12 @@ export class EvalSuite {
         if (runResult.toolCalls) {
             result.toolCalls = runResult.toolCalls;
         }
+        if (runResult.truncated === true)
+            result.truncated = true;
+        if (runResult.iterationLimitReached === true)
+            result.iterationLimitReached = true;
+        if (runResult.status === 'paused')
+            result.paused = true;
         return result;
     }
     async evaluateCaseMetrics(result) {
@@ -324,9 +356,11 @@ export class EvalSuite {
         return Promise.all(this.boundMetrics.map((m) => this.scoreMetric(m, result)));
     }
     /**
-     * A metric's score for one case. A metric that throws, or whose score is not a finite number,
-     * scores 0 with the reason in `error`, so one bad metric neither drops the case nor poisons
-     * the aggregate with NaN.
+     * A metric's score for one case. Only cases with an output are scored: a case whose every
+     * attempt failed is left unscored, so its empty output neither costs a judge call nor counts as
+     * a score of 0 (`stats.errors` and the `maxErrorRate` assertion catch those). A metric that
+     * throws, or whose score is not a finite number, scores 0 with the reason in `error`, so one
+     * bad metric neither drops the case nor poisons the aggregate with NaN.
      */
     async scoreMetric(metric, result) {
         try {
@@ -366,7 +400,12 @@ export class EvalSuite {
         }
         const aggregated = {};
         for (const [name, values] of scoresByMetric) {
-            aggregated[name] = { name, ...aggregate(values) };
+            aggregated[name] = { name, ...aggregate(values), count: values.length };
+        }
+        if (results.length > 0 && results.every((r) => r.error !== undefined)) {
+            for (const metric of this.boundMetrics) {
+                aggregated[metric.metricName] ??= unscored(metric.metricName);
+            }
         }
         return aggregated;
     }

package/dist/eval-suite.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAGpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEtD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAyErC,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,IAA6B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;QACpD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;KACzC,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,MAAmB;IACtC,IAAI,MAAM,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,SAAS,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,KAAK,UAAU,UAAU,CAAI,OAAmB,EAAE,EAAU;IAC1D,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACrC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,kFAAkF;IAC1E,mBAAmB,CAAC,OAAmB;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,wEAAwE,CACvG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAG3B,MAAM,cAAc,GAAG,IAAI,KAAK,CAAe,KAAK,CAAC,CAAC;QACtD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,OAAO,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC3C,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;gBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,cAAc,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACzD,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACrF,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpF,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CACrC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAChF,CAAC,CACF,CAAC;QACF,MAAM,KAAK,GAAmB;YAC5B,KAAK;YACL,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAClE,YAAY,EAAE,cAAc,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAClE,CAAC,CACF;YACD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;YACjC,IAAI,EAAE,UAAU,GAAG,SAAS;YAC5B,UAAU;YACV,SAAS;SACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;wBAChD,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;;wBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,mEAAmE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;gBACJ,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,QAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAC9B,OAAO,CAAC,WAAW,CAAC,+BAA+B,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE;YACxE,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;QACL,IAAI,CAAC;YACH,MAAM,QAAQ,GAAY,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,QAAQ,YAAY,OAAO;gBAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC1D,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,IAAI,KAA8B,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,QAAQ;YACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,OAAe,EACf,WAAoC;QAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,WAAW;YACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,KAAK,IAAI,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,EAAE;YACZ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CACrD,CAAC,MAAM,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAC3C,CAAC,KAAc,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO,KAAK,CAAC;YAEvE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YACvD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACtE,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,OAA0B;QAE1B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC9D,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,MAAmB;QAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,MAAM;SACP,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACvB,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,MAAgB,EAAE,MAAsB;QAChE,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;YACzF,CAAC;YACD,MAAM,MAAM,GAAG,KAAoB,CAAC;YACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC;YAC5B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU;gBACtC,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,OAAO,EAAE;gBACnC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
+{"version":3,"file":"eval-suite.js","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAGpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEtD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAExE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAyErC,MAAM,UAAU,WAAW,CAAC,CAAW;IACrC,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;QACxC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAA8B,EAAE,IAA6B;IAC7E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,YAAY,EAAE,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY;QACpD,WAAW,EAAE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW;QACjD,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;KACzC,CAAC;AACJ,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,MAAmB;IACtC,IAAI,MAAM,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,OAAO,CAAC,SAAS,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7B,CAAC;AAED,8FAA8F;AAC9F,KAAK,UAAU,UAAU,CAAI,OAAmB,EAAE,EAAU;IAC1D,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO;QACL,IAAI;QACJ,IAAI,EAAE,MAAM,CAAC,GAAG;QAChB,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,MAAM,EAAE,MAAM,CAAC,GAAG;QAClB,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,SAAS;IACH,OAAO,CAAU;IACjB,MAAM,CAAa;IACnB,YAAY,CAAa;IACzB,kBAAkB,CAAwB;IAC1C,YAAY,CAAgB;IAC5B,WAAW,CAAS;IACpB,OAAO,CAAS;IAChB,OAAO,CAAS;IAChB,UAAU,CAAoC;IAE/D,YAAY,IAAsB;QAChC,MAAM,MAAM,GAAG,qBAAqB,CAAC,KAAK,CAAC;YACzC,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAElC,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACrC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEnD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACjF,MAAM,cAAc,GAClB,WAAW,EAAE,SAAS,IAAK,IAAI,CAAC,MAAM,CAAC,SAAwC,CAAC;QAClF,IAAI,aAAa,IAAI,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,6FAA6F,CAC9F,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GACT,WAAW,IAAI,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEtF,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,EAAE,KAAK,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,KAAK,KAAK,SAAS,CAAC;QACrC,MAAM,YAAY,GAAG,SAAS,KAAK,SAAS,CAAC;QAE7C,IAAI,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,kFAAkF;IAC1E,mBAAmB,CAAC,OAAmB;QAC7C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,0BAA0B,IAAI,wEAAwE,CACvG,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,KAAK,CAAC,QAAiB;QACrB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAA0B,EAAE;QACpC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAG3B,MAAM,cAAc,GAAG,IAAI,KAAK,CAAe,KAAK,CAAC,CAAC;QACtD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,OAAO,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC3C,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;gBACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,cAAc,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACzD,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,cAAc,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACrF,MAAM,EAAE,cAAc,EAAE,CAAC;QAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC;QAExD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,cAAc,CAAC,CAAC;YACzC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;gBACvB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC3C,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;aACpD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpF,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CACrC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAChF,CAAC,CACF,CAAC;QACF,MAAM,KAAK,GAAmB;YAC5B,KAAK;YACL,MAAM,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;YAClE,YAAY,EAAE,cAAc,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,EAClE,CAAC,CACF;YACD,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;YACjC,IAAI,EAAE,UAAU,GAAG,SAAS;YAC5B,UAAU;YACV,SAAS;SACV,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9E,MAAM,WAAW,GAAoB;YACnC,OAAO,EAAE,cAAc;YACvB,UAAU;YACV,UAAU,EAAE,gBAAgB;YAC5B,KAAK;YACL,MAAM,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;gBACxB,MAAM,CACJ;oBACE,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE;wBACxD,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;wBACpB,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;wBAChD,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;wBACvC,GAAG,CAAC,CAAC,CAAC,qBAAqB,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC;wBAC/D,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;wBACjC,MAAM,EAAE,CAAC,CAAC,MAAM;qBACjB,CAAC,CAAC;oBACH,UAAU;oBACV,UAAU,EAAE,gBAAgB;oBAC5B,KAAK;iBACN,EACD,IAAI,EACJ,OAAO,CACR,CAAC;YACJ,CAAC;YACD,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE;gBACrB,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;gBAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC;;wBACpD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CACb,mEAAmE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;gBACJ,CAAC;gBACD,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,QAAsB;QAC3C,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC7B,MAAM,IAAI,GAAG,CAAC,KAAc,EAAE,EAAE,CAC9B,OAAO,CAAC,WAAW,CAAC,+BAA+B,YAAY,CAAC,KAAK,CAAC,EAAE,EAAE;YACxE,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;QACL,IAAI,CAAC;YACH,MAAM,QAAQ,GAAY,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpD,IAAI,QAAQ,YAAY,OAAO;gBAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,MAAM,EAAE,OAAO,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;YAAE,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAC5F,OAAO,EAAE,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,QAAkB,EAClB,MAA+B;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,SAAkB,CAAC;QACvB,IAAI,KAA8B,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;YAC7E,QAAQ,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YACzE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACzE,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YAC5B,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChE,QAAQ;YACR,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,OAAe,EACf,WAAoC;QAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,WAAW;YACxB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QACtB,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAC;QAEF,MAAM,OAAO,GAAqB,CAAC,KAAK,IAAI,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,EAAE;YACZ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YACrE,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CACrD,CAAC,MAAM,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAC3C,CAAC,KAAc,EAAW,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;gBAAE,OAAO,KAAK,CAAC;YAEvE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;YACvD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACtE,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAkB,EAClB,OAA0B;QAE1B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC9D,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;IAClE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAkB,EAClB,MAAmB;QAEnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAK7B,CAAC;QACF,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE;YACvD,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,MAAM;SACP,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QAEpC,MAAM,MAAM,GAAmB;YAC7B,IAAI,EAAE,QAAQ;YACd,MAAM,EAAG,SAAS,CAAC,MAAiB,IAAI,EAAE;YAC1C,QAAQ;SACT,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QACvB,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;YACxB,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAwC,CAAC;QACxE,CAAC;QACD,IAAI,SAAS,CAAC,SAAS,KAAK,IAAI;YAAE,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;QAC1D,IAAI,SAAS,CAAC,qBAAqB,KAAK,IAAI;YAAE,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAClF,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ;YAAE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC;QAExD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAsB;QACtD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,WAAW,CAAC,MAAgB,EAAE,MAAsB;QAChE,IAAI,CAAC;YACH,MAAM,KAAK,GAAY,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC;gBACpE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;YACzF,CAAC;YACD,MAAM,MAAM,GAAG,KAAoB,CAAC;YACpC,MAAM,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,OAAO;gBAAE,OAAO,MAAM,CAAC;YAC5B,OAAO;gBACL,GAAG,MAAM;gBACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,UAAU;gBACtC,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAiB,OAAO,EAAE;gBACnC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,iBAAiB,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC;QACzF,CAAC;IACH,CAAC;IAEO,eAAe,CACrB,OAA0D;QAE1D,MAAM,cAAc,GAAG,IAAI,GAAG,EAAoB,CAAC;QAEnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACrC,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,GAAG,GAAG,EAAE,CAAC;oBACT,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAClC,CAAC;gBACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAqC,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;YAC5C,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1E,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;YACtE,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}

package/dist/index.d.ts

@@ -7,7 +7,7 @@ export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
-export { exactMatch, contains, similarity, regex, jsonSchema } from './metrics/deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
 export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge.js';
 export { latency, cost, tokenUsage } from './metrics/statistical.js';
@@ -17,6 +17,7 @@ export type { CustomMetricConfig } from './metrics/custom.js';
 export { threshold } from './assertions/index.js';
 export { noRegression } from './assertions/index.js';
 export { assertion } from './assertions/index.js';
+export { maxErrorRate } from './assertions/index.js';
 export type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
 export { report } from './reporters/index.js';
 export type { ReporterType, ReporterOptions } from './reporters/index.js';
@@ -25,7 +26,7 @@ export type { TTestResult } from './stats/t-test.js';
 export { mcnemarsTest } from './stats/mcnemar.js';
 export type { McNemarResult } from './stats/mcnemar.js';
 export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
-export type { EvalCase, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, EvalComparisonConfig, } from './schema.js';
+export type { EvalCase, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, JudgeCache, JudgeCogitator, EvalComparisonConfig, } from './schema.js';
 export { createRunEvalTool, evalTools } from './tools.js';
 export type { EvalTool } from './tools.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAC9F,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -7,13 +7,14 @@ export { EvalBuilder } from './eval-builder.js';
 export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
-export { exactMatch, contains, similarity, regex, jsonSchema } from './metrics/deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
 export { latency, cost, tokenUsage } from './metrics/statistical.js';
 export { metric } from './metrics/custom.js';
 export { threshold } from './assertions/index.js';
 export { noRegression } from './assertions/index.js';
 export { assertion } from './assertions/index.js';
+export { maxErrorRate } from './assertions/index.js';
 export { report } from './reporters/index.js';
 export { pairedTTest } from './stats/t-test.js';
 export { mcnemarsTest } from './stats/mcnemar.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAUtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAC9F,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAY1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAUlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAUtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAY1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAYlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}

package/dist/metrics/deterministic.d.ts

@@ -6,6 +6,10 @@ interface MatchOptions extends MetricOptions {
 export declare function exactMatch(opts?: MatchOptions): MetricFn;
 export declare function similarity(opts?: MatchOptions): MetricFn;
 export declare function contains(opts?: MatchOptions): MetricFn;
+/** Scores the fraction of keywords found in the output. */
+export declare function containsAll(keywords: string[], opts?: MatchOptions): MetricFn;
+/** Scores 1 when the output contains at least one keyword, and 0 otherwise. */
+export declare function containsAny(keywords: string[], opts?: MatchOptions): MetricFn;
 /**
  * Scores 1 when the output matches `pattern`. The `g` and `y` flags are dropped: they make
  * `RegExp.test` resume from the previous match, so identical outputs would alternate between

package/dist/metrics/deterministic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvD,UAAU,YAAa,SAAQ,aAAa;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAOD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBxD;AAkCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkCxD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBtD;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa9E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CA0B1E"}
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC;AACnC,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEvD,UAAU,YAAa,SAAQ,aAAa;IAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAuCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBxD;AAkCD,wBAAgB,UAAU,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkCxD;AAED,wBAAgB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAoBtD;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAkB7E;AAED,+EAA+E;AAC/E,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,YAAY,GAAG,QAAQ,CAiB7E;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa9E;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CA0B1E"}

package/dist/metrics/deterministic.js

@@ -2,6 +2,24 @@ function createMetricFn(name, fn) {
     fn.metricName = name;
     return fn;
 }
+function prepareKeywordTargets(metricName, keywords, caseSensitive) {
+    if (keywords.length === 0) {
+        throw new Error(`${metricName} requires at least one keyword`);
+    }
+    const targets = [];
+    const seen = new Set();
+    for (const keyword of keywords) {
+        if (keyword.trim().length === 0) {
+            throw new Error(`${metricName} requires non-empty keywords`);
+        }
+        const search = caseSensitive ? keyword : keyword.toLowerCase();
+        if (!seen.has(search)) {
+            seen.add(search);
+            targets.push({ original: keyword, search });
+        }
+    }
+    return targets;
+}
 export function exactMatch(opts) {
     const caseSensitive = opts?.caseSensitive ?? false;
     const name = opts?.name ?? 'exactMatch';
@@ -93,6 +111,40 @@ export function contains(opts) {
         };
     }));
 }
+/** Scores the fraction of keywords found in the output. */
+export function containsAll(keywords, opts) {
+    const caseSensitive = opts?.caseSensitive ?? false;
+    const name = opts?.name ?? 'containsAll';
+    const targets = prepareKeywordTargets('containsAll', keywords, caseSensitive);
+    return createMetricFn(name, (async (result) => {
+        const output = caseSensitive ? result.output : result.output.toLowerCase();
+        const missing = targets.filter((keyword) => !output.includes(keyword.search));
+        return {
+            name,
+            score: (targets.length - missing.length) / targets.length,
+            details: missing.length > 0
+                ? `missing keywords: ${missing.map((keyword) => JSON.stringify(keyword.original)).join(', ')}`
+                : undefined,
+        };
+    }));
+}
+/** Scores 1 when the output contains at least one keyword, and 0 otherwise. */
+export function containsAny(keywords, opts) {
+    const caseSensitive = opts?.caseSensitive ?? false;
+    const name = opts?.name ?? 'containsAny';
+    const targets = prepareKeywordTargets('containsAny', keywords, caseSensitive);
+    return createMetricFn(name, (async (result) => {
+        const output = caseSensitive ? result.output : result.output.toLowerCase();
+        const found = targets.some((keyword) => output.includes(keyword.search));
+        return {
+            name,
+            score: found ? 1 : 0,
+            details: found
+                ? undefined
+                : `output does not contain any of: ${targets.map((keyword) => JSON.stringify(keyword.original)).join(', ')}`,
+        };
+    }));
+}
 /**
  * Scores 1 when the output matches `pattern`. The `g` and `y` flags are dropped: they make
  * `RegExp.test` resume from the previous match, so identical outputs would alternate between

package/dist/metrics/deterministic.js.map

@@ -1 +1 @@
-{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhG,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAqB,EAAE,EAAqB;IACvE,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtB,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACZ,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,WAAW;YACxB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY;YAC7B,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe;aACnC,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE/B,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAEzD,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC;QAEpC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,gBAAgB,QAAQ,GAAG,CAAC,CAAC,CAAC,SAAS;SAC1F,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAmB;IAC1C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC;IAEtC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,QAAQ,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,OAAwB,EAAE,IAAoB;IAClE,MAAM,MAAM,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3E,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,OAAO,CAAC;IAEnC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,MAAM,EAAE;SACvE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe,EAAE,IAAoB;IAC9D,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,6BAA6B,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;SACjE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}
+{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../src/metrics/deterministic.ts"],"names":[],"mappings":"AAOA,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAOD,SAAS,qBAAqB,CAC5B,UAAyC,EACzC,QAAkB,EAClB,aAAsB;IAEtB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,gCAAgC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8BAA8B,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACjB,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhG,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,EAAqB,EAAE,EAAqB;IACvE,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;IACpB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtB,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACZ,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,WAAW;YACxB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY;YAC7B,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe;aACnC,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,IAAI,GAAG,IAAI,CAAC;QACZ,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAmB;IAC5C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;QAE/B,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAEzD,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC1D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,GAAG,MAAM,CAAC;QAEpC,OAAO;YACL,IAAI;YACJ,KAAK;YACL,OAAO,EACL,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,MAAM,WAAW,MAAM,gBAAgB,QAAQ,GAAG,CAAC,CAAC,CAAC,SAAS;SAC1F,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAmB;IAC1C,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC;IAEtC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;QACtC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;QACnE,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEtC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,QAAQ,GAAG;SACrE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9E,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM;YACzD,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,qBAAqB,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC9F,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,IAAmB;IACjE,MAAM,aAAa,GAAG,IAAI,EAAE,aAAa,IAAI,KAAK,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,aAAa,CAAC;IACzC,MAAM,OAAO,GAAG,qBAAqB,CAAC,aAAa,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE9E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAEzE,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK;gBACZ,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,mCAAmC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;SAC/G,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,OAAwB,EAAE,IAAoB;IAClE,MAAM,MAAM,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3E,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,OAAO,CAAC;IAEnC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpB,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,iCAAiC,MAAM,EAAE;SACvE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAAe,EAAE,IAAoB;IAC9D,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,YAAY,CAAC;IAExC,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE;aACjD,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC5B,CAAC;QAED,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,6BAA6B,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;SACjE,CAAC;IACJ,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC"}

package/dist/metrics/index.d.ts

@@ -1,5 +1,5 @@
 export type { MetricScore, MetricUsage, MetricOptions, EvalCaseResult, MetricFn, StatisticalMetricFn, StatisticalScore, } from './types.js';
-export { exactMatch, contains, similarity, regex, jsonSchema } from './deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './deterministic.js';
 export { metric } from './custom.js';
 export type { CustomMetricConfig } from './custom.js';
 export { latency, cost, tokenUsage } from './statistical.js';

package/dist/metrics/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACtF,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}

package/dist/metrics/index.js

@@ -1,4 +1,4 @@
-export { exactMatch, contains, similarity, regex, jsonSchema } from './deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './deterministic.js';
 export { metric } from './custom.js';
 export { latency, cost, tokenUsage } from './statistical.js';
 export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge.js';

package/dist/metrics/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACtF,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}

package/dist/metrics/llm-judge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,KAAK,EAAE,QAAQ,EAA+B,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEjG,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QACT,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,WAAW,CAAA;SAAE,CAAC,CAAC;KACpF,CAAC;IACF,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAsFD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAmCjG;AAoCD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CA6CrF;AAED,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK9D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,WAAW,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK7D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}
+{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAc,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,KAAK,EAAE,QAAQ,EAA+B,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEjG,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QACT,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,WAAW,CAAA;SAAE,CAAC,CAAC;KACpF,CAAC;IACF,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAsFD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAyDjG;AA0GD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CA6CrF;AAED,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK9D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,WAAW,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK7D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}