oh-my-knowledge 0.18.0 → 0.20.0

package/dist/src/grading/index.js

@@ -2,13 +2,26 @@
  * Mixed grading: deterministic assertions + LLM judge + multi-dimensional scoring.
  */
 import { ASYNC_ASSERTION_TYPES, ratioToScore, runAssertions, runAsyncAssertions } from './assertions.js';
-import { buildTraceSummary, llmJudge } from './judge.js';
+import { buildTraceSummary, llmJudgeEnsemble, llmJudgeRepeat } from './judge.js';
 import { computeLayeredScores } from './layered-scores.js';
 /**
  * Grade a model output against a sample's criteria.
  */
-export async function grade({ output, sample, executor, judgeModel, allowLlmJudge = true, execMetrics = {}, samplesDir = '.' }) {
+export async function grade({ output, sample, executor, judgeModel, allowLlmJudge = true, execMetrics = {}, samplesDir = '.', judgeRepeat = 1, judgeModels, judgeExecutors, lengthDebias = true }) {
+    const useEnsemble = !!(judgeModels && judgeModels.length >= 2);
     const results = { compositeScore: 0 };
+    // Helper: build a config-keyed function for ensemble pathway. Each judge config
+    // names an executor by string; we resolve via judgeExecutors map (set by pipeline).
+    const executorByName = (name) => {
+        if (judgeExecutors && judgeExecutors[name])
+            return judgeExecutors[name];
+        // Fallback: pipeline only set up the default judgeExecutor and didn't pre-create
+        // others. Tests / single-judge callers hit this path. Use the passed-in executor
+        // for the base case; ensemble with mismatched executor names should fail loudly.
+        if (!judgeExecutors)
+            return executor;
+        throw new Error(`No executor registered for "${name}"; pipeline must populate judgeExecutors`);
+    };
     // 1. Deterministic assertions (pure, no LLM)
     const allAssertions = sample.assertions || [];
     const syncAssertions = allAssertions.filter((a) => !ASYNC_ASSERTION_TYPES.has(a.type));
@@ -54,14 +67,10 @@ export async function grade({ output, sample, executor, judgeModel, allowLlmJudg
         // Multi-dimensional scoring
         results.dimensions = {};
         for (const [dim, rubric] of Object.entries(sample.dimensions)) {
-            results.dimensions[dim] = await llmJudge({
-                output,
-                rubric,
-                prompt: sample.prompt,
-                executor,
-                model: judgeModel,
-                traceSummary,
-            });
+            const dimOptions = { output, rubric, prompt: sample.prompt, executor, model: judgeModel, traceSummary, lengthDebias };
+            results.dimensions[dim] = useEnsemble
+                ? await llmJudgeEnsemble(dimOptions, judgeModels, executorByName, judgeRepeat)
+                : await llmJudgeRepeat(dimOptions, judgeRepeat);
         }
         const dimValues = Object.values(results.dimensions);
         const dimScores = dimValues.map((d) => d.score).filter((s) => s > 0);
@@ -75,16 +84,25 @@ export async function grade({ output, sample, executor, judgeModel, allowLlmJudg
     }
     else if (allowLlmJudge && sample.rubric) {
         // Single rubric scoring
-        const judge = await llmJudge({
-            output,
-            rubric: sample.rubric,
-            prompt: sample.prompt,
-            executor,
-            model: judgeModel,
-            traceSummary,
-        });
+        const rubricOptions = { output, rubric: sample.rubric, prompt: sample.prompt, executor, model: judgeModel, traceSummary, lengthDebias };
+        const judge = useEnsemble
+            ? await llmJudgeEnsemble(rubricOptions, judgeModels, executorByName, judgeRepeat)
+            : await llmJudgeRepeat(rubricOptions, judgeRepeat);
         results.llmScore = judge.score;
         results.llmReason = judge.reason;
+        if (judge.reasoning)
+            results.llmReasoning = judge.reasoning;
+        if (judge.scoreSamples && judge.scoreSamples.length > 1) {
+            results.llmScoreSamples = judge.scoreSamples;
+            results.llmScoreStddev = judge.scoreStddev;
+            if (judge.judgeFailureCount && judge.judgeFailureCount > 0) {
+                results.llmScoreFailures = judge.judgeFailureCount;
+            }
+        }
+        if (judge.ensemble) {
+            results.llmEnsemble = judge.ensemble;
+            results.llmAgreement = judge.agreement;
+        }
         results.judgeCostUSD = (results.judgeCostUSD || 0) + (judge.judgeCostUSD || 0);
     }
     // 3. Layered scores + composite

package/dist/src/grading/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/grading/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACzG,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAY3D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,GAAG,IAAI,EAAE,WAAW,GAAG,EAAE,EAAE,UAAU,GAAG,GAAG,EAAgB;IAC1I,MAAM,OAAO,GAQT,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IAE1B,6CAA6C;IAC7C,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;IAC9C,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACzD,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7D,IAAI,aAAa;YAAE,OAAO,IAAI,CAAC;QAC/B,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,UAAU,GAAG,aAAa,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,wDAAwD;IACxD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACnD,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,eAAe,EAAE;YACrE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU;SACzC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,wCAAwC;YACxC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YACzD,OAAO,CAAC,UAAU,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,CAAC;YAC/C,OAAO,CAAC,UAAU,CAAC,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC;YACjD,sCAAsC;YACtC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;YAC9C,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACjE,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC1F,MAAM,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,OAAO,CAAC,UAAU,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,UAAU,GAAG,YAAY,CAAC;QACpC,CAAC;QACD,wCAAwC;QACxC,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC;QACjF,CAAC;IACH,CAAC;IAED,iBAAiB;IACjB,wDAAwD;IACxD,MAAM,YAAY,GAAG,iBAAiB,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC;IAEjF,IAAI,aAAa,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpF,4BAA4B;QAC5B,OAAO,CAAC,UAAU,GAAG,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9D,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,MAAM,QAAQ,CAAC;gBACvC,MAAM;gBACN,MAAM;gBACN,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ;gBACR,KAAK,EAAE,UAAU;gBACjB,YAAY;aACb,CAAC,CAAC;QACL,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,CAAC;QACD,uFAAuF;QACvF,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzE,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC;IAChF,CAAC;SAAM,IAAI,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1C,wBAAwB;QACxB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ;YACR,KAAK,EAAE,UAAU;YACjB,YAAY;SACb,CAAC,CAAC;QACH,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QAC/B,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;QACjC,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,gCAAgC;IAChC,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACxE,OAAO,CAAC,cAAc,GAAG,cAAc,CAAC;IACxC,OAAO,CAAC,aAAa,GAAG,aAAa,CAAC;IAEtC,OAAO,OAAO,CAAC;AACjB,CAAC;AACD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/grading/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACzG,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjF,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAqC3D;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,GAAG,IAAI,EAAE,WAAW,GAAG,EAAE,EAAE,UAAU,GAAG,GAAG,EAAE,WAAW,GAAG,CAAC,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,GAAG,IAAI,EAAgB;IAC7M,MAAM,WAAW,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAgB,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IAEnD,gFAAgF;IAChF,oFAAoF;IACpF,MAAM,cAAc,GAAG,CAAC,IAAY,EAAc,EAAE;QAClD,IAAI,cAAc,IAAI,cAAc,CAAC,IAAI,CAAC;YAAE,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;QACxE,iFAAiF;QACjF,iFAAiF;QACjF,iFAAiF;QACjF,IAAI,CAAC,cAAc;YAAE,OAAO,QAAQ,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,0CAA0C,CAAC,CAAC;IACjG,CAAC,CAAC;IAEF,6CAA6C;IAC7C,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC;IAC9C,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE;QACzD,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7D,IAAI,aAAa;YAAE,OAAO,IAAI,CAAC;QAC/B,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,UAAU,GAAG,aAAa,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,GAAG,WAAW,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,wDAAwD;IACxD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACnD,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAAC,MAAM,EAAE,eAAe,EAAE;YACrE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU;SACzC,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,wCAAwC;YACxC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;YACzD,OAAO,CAAC,UAAU,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,CAAC;YAC/C,OAAO,CAAC,UAAU,CAAC,MAAM,IAAI,YAAY,CAAC,MAAM,CAAC;YACjD,sCAAsC;YACtC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;YAC9C,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACjE,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC1F,MAAM,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,OAAO,CAAC,UAAU,CAAC,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,UAAU,GAAG,YAAY,CAAC;QACpC,CAAC;QACD,wCAAwC;QACxC,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC;QACjF,CAAC;IACH,CAAC;IAED,iBAAiB;IACjB,wDAAwD;IACxD,MAAM,YAAY,GAAG,iBAAiB,CAAC,WAAW,CAAC,KAAK,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC;IAEjF,IAAI,aAAa,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpF,4BAA4B;QAC5B,OAAO,CAAC,UAAU,GAAG,EAAE,CAAC;QACxB,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9D,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;YACtH,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,WAAW;gBACnC,CAAC,CAAC,MAAM,gBAAgB,CAAC,UAAU,EAAE,WAAY,EAAE,cAAc,EAAE,WAAW,CAAC;gBAC/E,CAAC,CAAC,MAAM,cAAc,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAClG,CAAC;QACD,uFAAuF;QACvF,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzE,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC;IAChF,CAAC;SAAM,IAAI,aAAa,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1C,wBAAwB;QACxB,MAAM,aAAa,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;QACxI,MAAM,KAAK,GAAG,WAAW;YACvB,CAAC,CAAC,MAAM,gBAAgB,CAAC,aAAa,EAAE,WAAY,EAAE,cAAc,EAAE,WAAW,CAAC;YAClF,CAAC,CAAC,MAAM,cAAc,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;QACrD,OAAO,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QAC/B,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;QACjC,IAAI,KAAK,CAAC,SAAS;YAAE,OAAO,CAAC,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC;QAC5D,IAAI,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxD,OAAO,CAAC,eAAe,GAAG,KAAK,CAAC,YAAY,CAAC;YAC7C,OAAO,CAAC,cAAc,GAAG,KAAK,CAAC,WAAW,CAAC;YAC3C,IAAI,KAAK,CAAC,iBAAiB,IAAI,KAAK,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;gBAC3D,OAAO,CAAC,gBAAgB,GAAG,KAAK,CAAC,iBAAiB,CAAC;YACrD,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC;YACrC,OAAO,CAAC,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC;QACzC,CAAC;QACD,OAAO,CAAC,YAAY,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,gCAAgC;IAChC,MAAM,EAAE,cAAc,EAAE,aAAa,EAAE,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;IACxE,OAAO,CAAC,cAAc,GAAG,cAAc,CAAC;IACxC,OAAO,CAAC,aAAa,GAAG,aAAa,CAAC;IAEtC,OAAO,OAAO,CAAC;AACjB,CAAC;AACD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}

package/dist/src/grading/judge.d.ts

@@ -1,4 +1,13 @@
-import type { DimensionResult, ExecutorFn, ToolCallInfo, TurnInfo } from '../types.js';
+import type { DimensionResult, ExecutorFn, JudgeAgreement, JudgeConfig, ToolCallInfo, TurnInfo } from '../types.js';
+export declare function buildJudgePrompt(prompt: string, rubric: string, output: string, traceSummary: string | null, lengthDebias?: boolean): string;
+/**
+ * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
+ * downstream readers can detect "the judge prompt changed between these two reports".
+ *
+ * `lengthDebias` defaults to true (v0.21+ default). Pass false when running under
+ * `--no-debias-length` so the hash matches historical v2-cot reports.
+ */
+export declare function getJudgePromptHash(lengthDebias?: boolean): string;
 interface LlmJudgeOptions {
     output: string;
     rubric: string;
@@ -6,8 +15,62 @@ interface LlmJudgeOptions {
     executor: ExecutorFn;
     model: string;
     traceSummary?: string | null;
+    /**
+     * When true (default since v0.21), the judge prompt includes an explicit
+     * "length is not a quality signal" instruction. Pass false to fall back to
+     * the legacy v2-cot prompt — only useful for reproducing pre-v0.21 reports
+     * or running A/B comparisons inside `omk bench debias-validate length`.
+     */
+    lengthDebias?: boolean;
 }
 export declare function buildTraceSummary(turns?: TurnInfo[], toolCalls?: ToolCallInfo[]): string | null;
-export declare function llmJudge({ output, rubric, prompt, executor, model, traceSummary }: LlmJudgeOptions): Promise<DimensionResult>;
+export declare function llmJudge({ output, rubric, prompt, executor, model, traceSummary, lengthDebias }: LlmJudgeOptions): Promise<DimensionResult>;
+/**
+ * Judge a single (output, rubric) pair N times and aggregate. Returns mean score,
+ * stddev across runs, raw score samples, and the first-call reasoning (we keep one
+ * reasoning sample, not N — only the score distribution matters for stability).
+ *
+ * Calls run with bounded concurrency (JUDGE_REPEAT_CONCURRENCY) to amortize latency
+ * without burning rate limit. N=3 finishes in ~1 round-trip; N=10 in ~4. Cost is
+ * summed across all N calls. When N <= 1 this is equivalent to llmJudge() with
+ * scoreSamples = [score], scoreStddev = 0. The repeat value is clamped to >= 1
+ * here as well as at the CLI layer — library callers shouldn't see surprises.
+ *
+ * Failures: any call returning score=0 (non-JSON / executor error / parse error) is
+ * counted in `judgeFailureCount`. stddev is computed only over successful calls. If
+ * stddev is 0 but judgeFailureCount > 0, that's NOT "judge agreed perfectly" — it
+ * means most calls failed and one happened to succeed. Always inspect both fields.
+ */
+export declare function llmJudgeRepeat(options: LlmJudgeOptions, repeat: number): Promise<DimensionResult>;
+/** Format a JudgeConfig as "executor:model" identifier for reports / logs. */
+export declare function judgeId(config: JudgeConfig): string;
+/**
+ * Compute pairwise inter-judge agreement metrics across an ensemble. For N judges
+ * we get N*(N-1)/2 pairs; metrics are pairwise-averaged.
+ *
+ * Each judge contributes ONE score per sample (its mean if judge-repeat > 1). We
+ * then compute Pearson and mean-abs-diff over the N-judge × M-sample score matrix.
+ *
+ * NOTE: Within a single (sample × dimension) call, each judge gives ONE aggregated
+ * score. So `pairwise` here means "two judges' scores on this one sample". With a
+ * single sample point Pearson is undefined (need ≥ 2). For per-sample agreement we
+ * fall back to mean-abs-diff alone; Pearson kicks in at the report-aggregate level
+ * (across many samples).
+ */
+export declare function computeJudgeAgreement(judgeScores: number[][]): JudgeAgreement;
+/**
+ * Judge a single (output, rubric) pair with N judge models in parallel. Each judge
+ * may use a different executor (e.g. claude:opus + openai:gpt-4o + gemini:pro). Each
+ * judge can also be repeated `judgeRepeat` times — final per-judge score is its mean.
+ *
+ * Returns: aggregate DimensionResult (score = mean across judges; this is the "consensus"
+ * score), per-judge breakdown in `ensemble`, and agreement metrics in `agreement`.
+ *
+ * The aggregate score is the mean of per-judge scores. This is a defensible default
+ * but not the only choice — one could argue median (robust to outlier judges) or
+ * majority vote (if scores are categorical). Mean is what most papers report; we
+ * provide the raw ensemble so downstream can recompute.
+ */
+export declare function llmJudgeEnsemble(options: LlmJudgeOptions, judges: JudgeConfig[], executorByName: (name: string) => ExecutorFn, judgeRepeat?: number): Promise<DimensionResult>;
 export {};
 //# sourceMappingURL=judge.d.ts.map

package/dist/src/grading/judge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../src/grading/judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAOvF,UAAU,eAAe;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAMD,wBAAgB,iBAAiB,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,EAAE,SAAS,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,GAAG,IAAI,CA6C/F;AAED,wBAAsB,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAiDnI"}
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../src/grading/judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAuB,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAkCzI,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,GAAG,IAAI,EAC3B,YAAY,UAAO,GAClB,MAAM,CA8BR;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,YAAY,UAAO,GAAG,MAAM,CAO9D;AAED,UAAU,eAAe;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAMD,wBAAgB,iBAAiB,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,EAAE,SAAS,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,GAAG,IAAI,CA6C/F;AAED,wBAAsB,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CA6BjJ;AA0BD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,eAAe,CAAC,CAmC1B;AAMD,8EAA8E;AAC9E,wBAAgB,OAAO,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CAEnD;AAkCD;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,GAAG,cAAc,CA8B7E;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,WAAW,EAAE,EACrB,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,UAAU,EAC5C,WAAW,SAAI,GACd,OAAO,CAAC,eAAe,CAAC,CAwD1B"}

package/dist/src/grading/judge.js

@@ -1,3 +1,72 @@
+import { createHash } from 'node:crypto';
+/**
+ * Judge prompt template version.
+ *
+ *  - 'v2-cot'         — legacy; CoT scoring without explicit length-debias instruction.
+ *                       Kept for `--no-debias-length` so users can reproduce historical
+ *                       reports byte-for-byte.
+ *  - 'v3-cot-length'  — adds a paragraph telling the judge that length is not a quality
+ *                       signal. Default since v0.21 Phase 3a (research consistently shows
+ *                       LLM judges over-weight verbosity; explicit instruction mitigates).
+ *
+ * Bump when the prompt's intent or structure changes meaningfully — reports tagged
+ * with the same hash are score-comparable; mismatched hashes mean "we changed how we
+ * ask the judge to think" and should not be compared blind.
+ */
+const JUDGE_PROMPT_VERSION_DEBIAS_OFF = 'v2-cot';
+const JUDGE_PROMPT_VERSION_DEBIAS_ON = 'v3-cot-length';
+const JUDGE_SYSTEM_PROMPT = '你是一个严格的 AI 输出质量评审员。先逐条对照评分标准做推理，再给最终分数。只返回 JSON，不要其他内容。';
+const LENGTH_DEBIAS_INSTRUCTION = [
+    '## 重要：长度不是质量信号',
+    '评分时聚焦内容实质与正确性。回答的篇幅、行文丰富度、结构复杂度本身不是质量指标 ——',
+    '简洁正确的回答不应因短而扣分；冗长但偏题或重复的回答不应因长而加分。',
+    '研究显示 LLM 评委容易隐性偏向更长的回答，请在打分前先警觉这一点。',
+].join('\n');
+export function buildJudgePrompt(prompt, rubric, output, traceSummary, lengthDebias = true) {
+    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
+    const traceSection = traceSummary
+        ? ['', '## Agent 执行过程', traceSummary, '', '请同时考虑执行过程的合理性（工具选择、步骤效率、错误恢复）。']
+        : [];
+    const debiasSection = lengthDebias ? ['', LENGTH_DEBIAS_INSTRUCTION] : [];
+    return [
+        `请对以下 AI 输出进行质量评分（template ${version}）。`,
+        '',
+        '## 原始任务',
+        prompt,
+        '',
+        '## 评分标准',
+        rubric,
+        '',
+        '## AI 输出',
+        output,
+        ...traceSection,
+        ...debiasSection,
+        '',
+        '## 评分流程',
+        '1. 逐条对照评分标准，先做推理（reasoning）：列出 AI 输出哪些点对应哪条标准，哪些缺失，哪些有歧义。',
+        '2. 基于推理给出最终分数（1-5 的整数）和简短理由。',
+        '',
+        '请返回 JSON（不要包含 markdown 代码块标记）：',
+        '{"reasoning": "<对照标准的逐条推理>", "score": <1-5的整数>, "reason": "<最终结论的简短理由>"}',
+        '',
+        '评分标准：1=完全不达标, 2=部分涉及, 3=基本达标, 4=较好, 5=优秀',
+    ].join('\n');
+}
+/**
+ * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
+ * downstream readers can detect "the judge prompt changed between these two reports".
+ *
+ * `lengthDebias` defaults to true (v0.21+ default). Pass false when running under
+ * `--no-debias-length` so the hash matches historical v2-cot reports.
+ */
+export function getJudgePromptHash(lengthDebias = true) {
+    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
+    // Hash the template-shaping function source + the version tag together. We hash a
+    // deterministic stringified form of the template (with placeholder inputs) so any
+    // structural edit shows up.
+    const sample = buildJudgePrompt('<P>', '<R>', '<O>', '<T>', lengthDebias);
+    return createHash('sha256').update(version + '\n' + sample).digest('hex').slice(0, 12);
+}
 function getErrorMessage(err) {
     return err instanceof Error ? err.message : String(err);
 }
@@ -47,31 +116,11 @@ export function buildTraceSummary(turns, toolCalls) {
     }
     return lines.join('\n');
 }
-export async function llmJudge({ output, rubric, prompt, executor, model, traceSummary }) {
-    const traceSection = traceSummary
-        ? ['', '## Agent 执行过程', traceSummary, '', '请同时考虑执行过程的合理性（工具选择、步骤效率、错误恢复）。']
-        : [];
-    const judgePrompt = [
-        '请对以下 AI 输出进行质量评分。',
-        '',
-        '## 原始任务',
-        prompt,
-        '',
-        '## 评分标准',
-        rubric,
-        '',
-        '## AI 输出',
-        output,
-        ...traceSection,
-        '',
-        '请返回 JSON（不要包含 markdown 代码块标记）：',
-        '{"score": <1-5的整数>, "reason": "<简短理由>"}',
-        '',
-        '评分标准：1=完全不达标, 2=部分涉及, 3=基本达标, 4=较好, 5=优秀',
-    ].join('\n');
+export async function llmJudge({ output, rubric, prompt, executor, model, traceSummary, lengthDebias }) {
+    const judgePrompt = buildJudgePrompt(prompt, rubric, output, traceSummary || null, lengthDebias ?? true);
     const result = await executor({
         model,
-        system: '你是一个严格的 AI 输出质量评审员。只返回 JSON，不要其他内容。',
+        system: JUDGE_SYSTEM_PROMPT,
         prompt: judgePrompt,
     });
     if (!result.ok)
@@ -87,6 +136,7 @@ export async function llmJudge({ output, rubric, prompt, executor, model, traceS
         return {
             score: Number(parsed.score) || 0,
             reason: String(parsed.reason || ''),
+            reasoning: parsed.reasoning ? String(parsed.reasoning) : undefined,
             judgeCostUSD: result.costUSD,
         };
     }
@@ -95,4 +145,211 @@ export async function llmJudge({ output, rubric, prompt, executor, model, traceS
         return { score: 0, reason: 'failed to parse judge response', judgeCostUSD: result.costUSD };
     }
 }
+/**
+ * Max concurrent judge calls within a single sample × dimension. Each judge call
+ * is one API request; running N=3 sequentially means 3× latency, but going wide
+ * open risks hitting per-account RPM limits. 3 is a conservative middle ground —
+ * for N=3 (the common case) it's fully parallel; for N=10 it's 4 batches.
+ */
+const JUDGE_REPEAT_CONCURRENCY = 3;
+async function runInChunks(items, chunkSize, fn) {
+    const results = new Array(items);
+    for (let start = 0; start < items; start += chunkSize) {
+        const end = Math.min(start + chunkSize, items);
+        const batch = await Promise.all(Array.from({ length: end - start }, (_, k) => fn(start + k)));
+        for (let k = 0; k < batch.length; k++)
+            results[start + k] = batch[k];
+    }
+    return results;
+}
+/**
+ * Judge a single (output, rubric) pair N times and aggregate. Returns mean score,
+ * stddev across runs, raw score samples, and the first-call reasoning (we keep one
+ * reasoning sample, not N — only the score distribution matters for stability).
+ *
+ * Calls run with bounded concurrency (JUDGE_REPEAT_CONCURRENCY) to amortize latency
+ * without burning rate limit. N=3 finishes in ~1 round-trip; N=10 in ~4. Cost is
+ * summed across all N calls. When N <= 1 this is equivalent to llmJudge() with
+ * scoreSamples = [score], scoreStddev = 0. The repeat value is clamped to >= 1
+ * here as well as at the CLI layer — library callers shouldn't see surprises.
+ *
+ * Failures: any call returning score=0 (non-JSON / executor error / parse error) is
+ * counted in `judgeFailureCount`. stddev is computed only over successful calls. If
+ * stddev is 0 but judgeFailureCount > 0, that's NOT "judge agreed perfectly" — it
+ * means most calls failed and one happened to succeed. Always inspect both fields.
+ */
+export async function llmJudgeRepeat(options, repeat) {
+    const n = Math.max(1, Math.floor(repeat) || 1);
+    if (n === 1) {
+        const single = await llmJudge(options);
+        const failed = single.score <= 0 ? 1 : 0;
+        return { ...single, scoreSamples: [single.score], scoreStddev: 0, judgeFailureCount: failed };
+    }
+    // Run N judge calls with bounded concurrency. Result array preserves input order
+    // (call 0 → results[0]) so "first call reasoning" is well-defined regardless of
+    // which physical call returned first.
+    const calls = await runInChunks(n, JUDGE_REPEAT_CONCURRENCY, () => llmJudge(options));
+    const samples = calls.map((c) => c.score);
+    const totalCost = calls.reduce((sum, c) => sum + (c.judgeCostUSD || 0), 0);
+    const firstReasoning = calls[0]?.reasoning;
+    const firstReason = calls.find((c) => c.reason)?.reason || '';
+    const validSamples = samples.filter((s) => s > 0);
+    const failures = samples.length - validSamples.length;
+    const mean = validSamples.length > 0 ? validSamples.reduce((a, b) => a + b, 0) / validSamples.length : 0;
+    const variance = validSamples.length > 1
+        ? validSamples.reduce((s, x) => s + (x - mean) ** 2, 0) / (validSamples.length - 1)
+        : 0;
+    const stddev = Math.sqrt(variance);
+    return {
+        score: Number(mean.toFixed(2)),
+        reason: firstReason,
+        reasoning: firstReasoning,
+        judgeCostUSD: totalCost,
+        scoreSamples: samples,
+        scoreStddev: Number(stddev.toFixed(3)),
+        judgeFailureCount: failures,
+    };
+}
+// ===========================================================================
+// Multi-judge ensemble — cross-model agreement
+// ===========================================================================
+/** Format a JudgeConfig as "executor:model" identifier for reports / logs. */
+export function judgeId(config) {
+    return `${config.executor}:${config.model}`;
+}
+/**
+ * Pearson correlation between two number arrays of equal length. Returns null when
+ * either array has zero variance (constant scores) — Pearson is undefined in that
+ * case (division by zero), and reporting 0 would be misleading.
+ */
+function pearson(a, b) {
+    if (a.length !== b.length || a.length < 2)
+        return null;
+    const n = a.length;
+    const meanA = a.reduce((s, x) => s + x, 0) / n;
+    const meanB = b.reduce((s, x) => s + x, 0) / n;
+    let num = 0;
+    let denomA = 0;
+    let denomB = 0;
+    for (let i = 0; i < n; i++) {
+        const da = a[i] - meanA;
+        const db = b[i] - meanB;
+        num += da * db;
+        denomA += da * da;
+        denomB += db * db;
+    }
+    if (denomA === 0 || denomB === 0)
+        return null;
+    return num / Math.sqrt(denomA * denomB);
+}
+/** Mean absolute difference between two equal-length number arrays. */
+function meanAbsDiff(a, b) {
+    if (a.length !== b.length || a.length === 0)
+        return 0;
+    let sum = 0;
+    for (let i = 0; i < a.length; i++)
+        sum += Math.abs(a[i] - b[i]);
+    return sum / a.length;
+}
+/**
+ * Compute pairwise inter-judge agreement metrics across an ensemble. For N judges
+ * we get N*(N-1)/2 pairs; metrics are pairwise-averaged.
+ *
+ * Each judge contributes ONE score per sample (its mean if judge-repeat > 1). We
+ * then compute Pearson and mean-abs-diff over the N-judge × M-sample score matrix.
+ *
+ * NOTE: Within a single (sample × dimension) call, each judge gives ONE aggregated
+ * score. So `pairwise` here means "two judges' scores on this one sample". With a
+ * single sample point Pearson is undefined (need ≥ 2). For per-sample agreement we
+ * fall back to mean-abs-diff alone; Pearson kicks in at the report-aggregate level
+ * (across many samples).
+ */
+export function computeJudgeAgreement(judgeScores) {
+    // judgeScores[i][j] = score from judge i on sample j. All rows same length.
+    const n = judgeScores.length;
+    if (n < 2)
+        return { meanAbsDiff: 0, pairCount: 0 };
+    let madSum = 0;
+    let pearsonSum = 0;
+    let pearsonCount = 0;
+    let pairs = 0;
+    for (let i = 0; i < n; i++) {
+        for (let j = i + 1; j < n; j++) {
+            pairs++;
+            madSum += meanAbsDiff(judgeScores[i], judgeScores[j]);
+            const p = pearson(judgeScores[i], judgeScores[j]);
+            if (p !== null) {
+                pearsonSum += p;
+                pearsonCount++;
+            }
+        }
+    }
+    const result = {
+        meanAbsDiff: Number((madSum / pairs).toFixed(3)),
+        pairCount: pairs,
+    };
+    if (pearsonCount > 0) {
+        result.pearson = Number((pearsonSum / pearsonCount).toFixed(3));
+    }
+    return result;
+}
+/**
+ * Judge a single (output, rubric) pair with N judge models in parallel. Each judge
+ * may use a different executor (e.g. claude:opus + openai:gpt-4o + gemini:pro). Each
+ * judge can also be repeated `judgeRepeat` times — final per-judge score is its mean.
+ *
+ * Returns: aggregate DimensionResult (score = mean across judges; this is the "consensus"
+ * score), per-judge breakdown in `ensemble`, and agreement metrics in `agreement`.
+ *
+ * The aggregate score is the mean of per-judge scores. This is a defensible default
+ * but not the only choice — one could argue median (robust to outlier judges) or
+ * majority vote (if scores are categorical). Mean is what most papers report; we
+ * provide the raw ensemble so downstream can recompute.
+ */
+export async function llmJudgeEnsemble(options, judges, executorByName, judgeRepeat = 1) {
+    if (judges.length === 0) {
+        throw new Error('llmJudgeEnsemble called with empty judges array');
+    }
+    if (judges.length === 1) {
+        // Degenerate case — fall through to non-ensemble path.
+        return llmJudgeRepeat({ ...options, executor: executorByName(judges[0].executor), model: judges[0].model }, judgeRepeat);
+    }
+    // Run all judges in parallel — they're independent. Each judge internally handles
+    // its judge-repeat sequence.
+    const perJudge = await Promise.all(judges.map(async (jc) => {
+        const r = await llmJudgeRepeat({ ...options, executor: executorByName(jc.executor), model: jc.model }, judgeRepeat);
+        const entry = {
+            judge: judgeId(jc),
+            score: r.score,
+            scoreStddev: r.scoreStddev,
+            scoreSamples: r.scoreSamples,
+            judgeFailureCount: r.judgeFailureCount,
+            reasoning: r.reasoning,
+            costUSD: r.judgeCostUSD,
+        };
+        return { entry, raw: r };
+    }));
+    const ensemble = perJudge.map((p) => p.entry);
+    const totalCost = perJudge.reduce((s, p) => s + (p.raw.judgeCostUSD || 0), 0);
+    // Aggregate score = mean of per-judge means (consensus).
+    const validScores = ensemble.map((e) => e.score).filter((s) => s > 0);
+    const consensusScore = validScores.length > 0
+        ? Number((validScores.reduce((a, b) => a + b, 0) / validScores.length).toFixed(2))
+        : 0;
+    // Per-judge agreement: only one sample point here (this single output), so Pearson
+    // is undefined and we report mean-abs-diff. Aggregate-level Pearson (across all
+    // samples in the run) is computed by the report-level aggregator, not here.
+    const judgeScoreMatrix = ensemble.map((e) => [e.score]);
+    const agreement = computeJudgeAgreement(judgeScoreMatrix);
+    // Pick the "spokesperson" reasoning from the first judge that produced one.
+    const spokesperson = ensemble.find((e) => e.reasoning);
+    return {
+        score: consensusScore,
+        reason: `consensus across ${judges.length} judges`,
+        reasoning: spokesperson?.reasoning,
+        judgeCostUSD: totalCost,
+        ensemble,
+        agreement,
+    };
+}
 //# sourceMappingURL=judge.js.map

package/dist/src/grading/judge.js.map

@@ -1 +1 @@
-{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../../src/grading/judge.ts"],"names":[],"mappings":"AAgBA,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAkB,EAAE,SAA0B;IAC9E,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1F,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,OAAO,SAAS,CAAC,MAAM,OAAO,CAAC,CAAC;QAC3C,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,GAAG,YAAY,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,QAAQ,YAAY,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,IAAI,YAAY,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,YAAY,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7E,MAAM,IAAI,GAA2B,EAAE,CAAC;QACxC,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC/E,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;QAChF,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QACtE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,MAAM,gBAAgB,cAAc,WAAW,SAAS,GAAG,CAAC,CAAC;QACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACnB,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChF,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;gBAClD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1F,CAAC;iBAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,MAAM,GAAG,QAAQ,IAAI,CAAC,CAAC;IACnF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAmB;IACvG,MAAM,YAAY,GAAG,YAAY;QAC/B,CAAC,CAAC,CAAC,EAAE,EAAE,eAAe,EAAE,YAAY,EAAE,EAAE,EAAE,gCAAgC,CAAC;QAC3E,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,WAAW,GAAG;QAClB,mBAAmB;QACnB,EAAE;QACF,SAAS;QACT,MAAM;QACN,EAAE;QACF,SAAS;QACT,MAAM;QACN,EAAE;QACF,UAAU;QACV,MAAM;QACN,GAAG,YAAY;QACf,EAAE;QACF,gCAAgC;QAChC,yCAAyC;QACzC,EAAE;QACF,0CAA0C;KAC3C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;QAC5B,KAAK;QACL,MAAM,EAAE,qCAAqC;QAC7C,MAAM,EAAE,WAAW;KACpB,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,MAAM,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IAE1G,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACnF,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,yBAAyB,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;QACvF,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;QACzD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YAChC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;YACnC,YAAY,EAAE,MAAM,CAAC,OAAO;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,QAAiB,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpF,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gCAAgC,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IAC9F,CAAC;AACH,CAAC"}
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../../src/grading/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AASzC;;;;;;;;;;;;;GAaG;AACH,MAAM,+BAA+B,GAAG,QAAQ,CAAC;AACjD,MAAM,8BAA8B,GAAG,eAAe,CAAC;AAEvD,MAAM,mBAAmB,GAAG,yDAAyD,CAAC;AAEtF,MAAM,yBAAyB,GAAG;IAChC,gBAAgB;IAChB,4CAA4C;IAC5C,oCAAoC;IACpC,qCAAqC;CACtC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,UAAU,gBAAgB,CAC9B,MAAc,EACd,MAAc,EACd,MAAc,EACd,YAA2B,EAC3B,YAAY,GAAG,IAAI;IAEnB,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,+BAA+B,CAAC;IAChG,MAAM,YAAY,GAAG,YAAY;QAC/B,CAAC,CAAC,CAAC,EAAE,EAAE,eAAe,EAAE,YAAY,EAAE,EAAE,EAAE,gCAAgC,CAAC;QAC3E,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,aAAa,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,yBAAyB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1E,OAAO;QACL,6BAA6B,OAAO,IAAI;QACxC,EAAE;QACF,SAAS;QACT,MAAM;QACN,EAAE;QACF,SAAS;QACT,MAAM;QACN,EAAE;QACF,UAAU;QACV,MAAM;QACN,GAAG,YAAY;QACf,GAAG,aAAa;QAChB,EAAE;QACF,SAAS;QACT,2DAA2D;QAC3D,8BAA8B;QAC9B,EAAE;QACF,gCAAgC;QAChC,0EAA0E;QAC1E,EAAE;QACF,0CAA0C;KAC3C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,YAAY,GAAG,IAAI;IACpD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,+BAA+B,CAAC;IAChG,kFAAkF;IAClF,kFAAkF;IAClF,4BAA4B;IAC5B,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;IAC1E,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzF,CAAC;AAkBD,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAkB,EAAE,SAA0B;IAC9E,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1F,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,OAAO,SAAS,CAAC,MAAM,OAAO,CAAC,CAAC;QAC3C,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,GAAG,YAAY,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,QAAQ,YAAY,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,IAAI,YAAY,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,YAAY,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7E,MAAM,IAAI,GAA2B,EAAE,CAAC;QACxC,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC/E,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;QAChF,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QACtE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,MAAM,gBAAgB,cAAc,WAAW,SAAS,GAAG,CAAC,CAAC;QACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACnB,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChF,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;gBAClD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1F,CAAC;iBAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC7B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,MAAM,GAAG,QAAQ,IAAI,CAAC,CAAC;IACnF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,EAAmB;IACrH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,IAAI,IAAI,EAAE,YAAY,IAAI,IAAI,CAAC,CAAC;IAEzG,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;QAC5B,KAAK;QACL,MAAM,EAAE,mBAAmB;QAC3B,MAAM,EAAE,WAAW;KACpB,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,MAAM,CAAC,KAAK,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IAE1G,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACnF,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,yBAAyB,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;QACvF,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAkB,CAAC;QACzD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YAChC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;YACnC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;YAClE,YAAY,EAAE,MAAM,CAAC,OAAO;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,QAAiB,EAAE,CAAC;QAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpF,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gCAAgC,EAAE,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IAC9F,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAEnC,KAAK,UAAU,WAAW,CACxB,KAAa,EACb,SAAiB,EACjB,EAA6B;IAE7B,MAAM,OAAO,GAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;IACtC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,GAAG,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAC7D,CAAC;QACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAwB,EACxB,MAAc;IAEd,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,OAAO,EAAE,GAAG,MAAM,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAChG,CAAC;IAED,iFAAiF;IACjF,gFAAgF;IAChF,sCAAsC;IACtC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,CAAC,EAAE,wBAAwB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAEtF,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3E,MAAM,cAAc,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;IAC3C,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;IAE9D,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IACtD,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC;QACtC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAEnC,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,EAAE,WAAW;QACnB,SAAS,EAAE,cAAc;QACzB,YAAY,EAAE,SAAS;QACvB,YAAY,EAAE,OAAO;QACrB,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtC,iBAAiB,EAAE,QAAQ;KAC5B,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,+CAA+C;AAC/C,8EAA8E;AAE9E,8EAA8E;AAC9E,MAAM,UAAU,OAAO,CAAC,MAAmB;IACzC,OAAO,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;AAC9C,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,CAAW,EAAE,CAAW;IACvC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACvD,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACnB,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/C,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;QACxB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;QACxB,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,MAAM,IAAI,EAAE,GAAG,EAAE,CAAC;QAClB,MAAM,IAAI,EAAE,GAAG,EAAE,CAAC;IACpB,CAAC;IACD,IAAI,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9C,OAAO,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,uEAAuE;AACvE,SAAS,WAAW,CAAC,CAAW,EAAE,CAAW;IAC3C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;AACxB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,qBAAqB,CAAC,WAAuB;IAC3D,4EAA4E;IAC5E,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC;IAC7B,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAEnD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,KAAK,EAAE,CAAC;YACR,MAAM,IAAI,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YACtD,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;gBACf,UAAU,IAAI,CAAC,CAAC;gBAChB,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAmB;QAC7B,WAAW,EAAE,MAAM,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAChD,SAAS,EAAE,KAAK;KACjB,CAAC;IACF,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,UAAU,GAAG,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAwB,EACxB,MAAqB,EACrB,cAA4C,EAC5C,WAAW,GAAG,CAAC;IAEf,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,uDAAuD;QACvD,OAAO,cAAc,CAAC,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,WAAW,CAAC,CAAC;IAC3H,CAAC;IAED,kFAAkF;IAClF,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;QACtB,MAAM,CAAC,GAAG,MAAM,cAAc,CAC5B,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,cAAc,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,EACtE,WAAW,CACZ,CAAC;QACF,MAAM,KAAK,GAAwB;YACjC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;YAClB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;YACtC,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,YAAY;SACxB,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IAC3B,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE9E,yDAAyD;IACzD,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACtE,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC;QAC3C,CAAC,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC,CAAC;IAEN,mFAAmF;IACnF,gFAAgF;IAChF,4EAA4E;IAC5E,MAAM,gBAAgB,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,qBAAqB,CAAC,gBAAgB,CAAC,CAAC;IAE1D,4EAA4E;IAC5E,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAEvD,OAAO;QACL,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,oBAAoB,MAAM,CAAC,MAAM,SAAS;QAClD,SAAS,EAAE,YAAY,EAAE,SAAS;QAClC,YAAY,EAAE,SAAS;QACvB,QAAQ;QACR,SAAS;KACV,CAAC;AACJ,CAAC"}

package/dist/src/inputs/eval-config.js

@@ -103,6 +103,24 @@ function validateEvalConfig(parsed, configPath) {
     assertBoolOpt('noCache');
     assertBoolOpt('blind');
     assertStringOpt('mcpConfig');
+    // v0.22 — budget validation. Top-level `budget: { totalUSD?, perSampleUSD?, perSampleMs? }`.
+    let budget;
+    if (obj.budget !== undefined) {
+        if (typeof obj.budget !== 'object' || obj.budget === null || Array.isArray(obj.budget)) {
+            throw new Error(`${configPath}: budget 必须是对象`);
+        }
+        const b = obj.budget;
+        for (const k of ['totalUSD', 'perSampleUSD', 'perSampleMs']) {
+            if (b[k] !== undefined && (typeof b[k] !== 'number' || b[k] < 0)) {
+                throw new Error(`${configPath}: budget.${k} 必须是非负数字`);
+            }
+        }
+        budget = {
+            totalUSD: b.totalUSD,
+            perSampleUSD: b.perSampleUSD,
+            perSampleMs: b.perSampleMs,
+        };
+    }
     return {
         samples: obj.samples,
         executor: obj.executor,
@@ -115,6 +133,7 @@ function validateEvalConfig(parsed, configPath) {
         blind: obj.blind,
         mcpConfig: obj.mcpConfig,
         variants,
+        budget,
     };
 }
 function resolveConfigPaths(config, configDir) {

package/dist/src/inputs/eval-config.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-config.js","sourceRoot":"","sources":["../../../src/inputs/eval-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAQ9C,MAAM,WAAW,GAA8B,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;AAExE;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,UAAkB;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,MAAM,GAAY,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAA6B;IACjE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;KACpD,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAe,EAAE,UAAkB;IAC7D,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,WAAW,CAAC,CAAC;IAC5C,CAAC;IACD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAE9C,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,sBAAsB,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,wBAAwB,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAK,GAAG,CAAC,QAAsB,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7D,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,SAAS,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,CAAC,GAAG,GAA8B,CAAC;QACzC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,iBAAiB,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAsB,CAAC,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,cAAc,CAAC,yCAAyC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAC/F,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,qBAAqB,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,cAAc,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,WAAW,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjB,UAAU,GAAG,IAAI,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAsB;YAC9B,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,GAAG,EAAE,CAAC,CAAC,GAAyB;SACjC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,wCAAwC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,GAAW,EAAQ,EAAE;QAC5C,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,KAAK,GAAG,SAAS,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC;IACF,MAAM,eAAe,GAAG,CAAC,GAAW,EAAQ,EAAE;QAC5C,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,KAAK,GAAG,QAAQ,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,CAAC,GAAW,EAAQ,EAAE;QAC1C,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,KAAK,GAAG,SAAS,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC;IAEF,eAAe,CAAC,UAAU,CAAC,CAAC;IAC5B,eAAe,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QAClG,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,2BAA2B,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC3G,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8BAA8B,CAAC,CAAC;IAC/D,CAAC;IACD,eAAe,CAAC,aAAa,CAAC,CAAC;IAC/B,eAAe,CAAC,WAAW,CAAC,CAAC;IAC7B,aAAa,CAAC,SAAS,CAAC,CAAC;IACzB,aAAa,CAAC,OAAO,CAAC,CAAC;IACvB,eAAe,CAAC,WAAW,CAAC,CAAC;IAE7B,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,OAAiB;QAC9B,QAAQ,EAAE,GAAG,CAAC,QAA8B;QAC5C,KAAK,EAAE,GAAG,CAAC,KAA2B;QACtC,UAAU,EAAE,GAAG,CAAC,UAAuC;QACvD,aAAa,EAAE,GAAG,CAAC,aAA0C;QAC7D,WAAW,EAAE,GAAG,CAAC,WAAiC;QAClD,SAAS,EAAE,GAAG,CAAC,SAA+B;QAC9C,OAAO,EAAE,GAAG,CAAC,OAA8B;QAC3C,KAAK,EAAE,GAAG,CAAC,KAA4B;QACvC,SAAS,EAAE,GAAG,CAAC,SAA+B;QAC9C,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAkB,EAAE,SAAiB;IAC/D,MAAM,UAAU,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACnF,2FAA2F;IAC3F,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjH,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAEjD,OAAO;QACL,GAAG,MAAM;QACT,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC;QACnC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;QACtE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,CAAC;YACJ,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;YACpH,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS;SAC3C,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}
+{"version":3,"file":"eval-config.js","sourceRoot":"","sources":["../../../src/inputs/eval-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAQ9C,MAAM,WAAW,GAA8B,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;AAExE;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,UAAkB;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACpC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,MAAM,GAAY,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACtD,OAAO,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAA6B;IACjE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ;KACpD,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAe,EAAE,UAAkB;IAC7D,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,WAAW,CAAC,CAAC;IAC5C,CAAC;IACD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAE9C,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,sBAAsB,CAAC,CAAC;IACvD,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,wBAAwB,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAK,GAAG,CAAC,QAAsB,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7D,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,SAAS,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,CAAC,GAAG,GAA8B,CAAC;QACzC,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,iBAAiB,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAsB,CAAC,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CACb,GAAG,UAAU,cAAc,CAAC,yCAAyC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAC/F,CAAC;QACJ,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,qBAAqB,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,CAAC,GAAG,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,cAAc,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,cAAc,CAAC,WAAW,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC;QACvE,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjB,UAAU,GAAG,IAAI,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAsB;YAC9B,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,GAAG,EAAE,CAAC,CAAC,GAAyB;SACjC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,wCAAwC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,GAAW,EAAQ,EAAE;QAC5C,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,KAAK,GAAG,SAAS,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC;IACF,MAAM,eAAe,GAAG,CAAC,GAAW,EAAQ,EAAE;QAC5C,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,KAAK,GAAG,QAAQ,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,CAAC,GAAW,EAAQ,EAAE;QAC1C,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,KAAK,GAAG,SAAS,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC;IAEF,eAAe,CAAC,UAAU,CAAC,CAAC;IAC5B,eAAe,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QAClG,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,2BAA2B,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,GAAG,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;QAC3G,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,8BAA8B,CAAC,CAAC;IAC/D,CAAC;IACD,eAAe,CAAC,aAAa,CAAC,CAAC;IAC/B,eAAe,CAAC,WAAW,CAAC,CAAC;IAC7B,aAAa,CAAC,SAAS,CAAC,CAAC;IACzB,aAAa,CAAC,OAAO,CAAC,CAAC;IACvB,eAAe,CAAC,WAAW,CAAC,CAAC;IAE7B,6FAA6F;IAC7F,IAAI,MAAoD,CAAC;IACzD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC7B,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACvF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,gBAAgB,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,CAAC,GAAG,GAAG,CAAC,MAAiC,CAAC;QAChD,KAAK,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,EAAE,aAAa,CAAC,EAAE,CAAC;YAC5D,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAW,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC3E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,YAAY,CAAC,UAAU,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QACD,MAAM,GAAG;YACP,QAAQ,EAAE,CAAC,CAAC,QAA8B;YAC1C,YAAY,EAAE,CAAC,CAAC,YAAkC;YAClD,WAAW,EAAE,CAAC,CAAC,WAAiC;SACjD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,OAAiB;QAC9B,QAAQ,EAAE,GAAG,CAAC,QAA8B;QAC5C,KAAK,EAAE,GAAG,CAAC,KAA2B;QACtC,UAAU,EAAE,GAAG,CAAC,UAAuC;QACvD,aAAa,EAAE,GAAG,CAAC,aAA0C;QAC7D,WAAW,EAAE,GAAG,CAAC,WAAiC;QAClD,SAAS,EAAE,GAAG,CAAC,SAA+B;QAC9C,OAAO,EAAE,GAAG,CAAC,OAA8B;QAC3C,KAAK,EAAE,GAAG,CAAC,KAA4B;QACvC,SAAS,EAAE,GAAG,CAAC,SAA+B;QAC9C,QAAQ;QACR,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAkB,EAAE,SAAiB;IAC/D,MAAM,UAAU,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACnF,2FAA2F;IAC3F,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjH,MAAM,aAAa,GAAG,CAAC,IAAY,EAAW,EAAE,CAC9C,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAEjD,OAAO;QACL,GAAG,MAAM;QACT,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC;QACnC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;QACtE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,CAAC;YACJ,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;YACpH,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS;SAC3C,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}