oh-my-knowledge 0.19.0 → 0.20.0

package/dist/src/eval-workflows/run-evaluation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run-evaluation.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/run-evaluation.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EACV,QAAQ,EAER,QAAQ,EACR,gBAAgB,EAChB,MAAM,EAGN,YAAY,EAGZ,WAAW,EAGZ,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,UAAU,uBAAuB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAIlB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;yDACqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;uCACmC;IACnC,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAyB,SAAQ,uBAAuB;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,iBAAiB,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CAC9D;AAED,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5E;AAED,UAAU,UAAU;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,MAAM,EAAE,IAAI,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,wBAAsB,aAAa,CAAC,EAClC,WAAW,EACX,QAAQ,EACR,YAAiB,EACjB,SAAS,EACT,KAAqB,EACrB,UAAwB,EACxB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,KAAa,EACb,WAAe,EACf,SAAS,EACT,OAAe,EACf,YAAuB,EACvB,iBAAiB,EACjB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,aAAqB,EACrB,SAAS,EACT,OAAe,EACf,KAAS,EACT,MAAM,EACN,YAAoB,EACpB,MAAM,EACN,IAAI,GACL,EAAE,oBAAoB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAgF5F;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAiB,SAAQ,UAAU;IAClD,IAAI,EAAE,IAAI,CAAC;IACX,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B;AA6CD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,IAAI,CAuErE;AAED,wBAAsB,iBAAiB,CAAC,EACtC,QAAQ,EACR,KAAqB,EACrB,UAAwB,EACxB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,WAAe,EACf,SAAS,EACT,YAAuB,EACvB,iBAAiB,EACjB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,eAAsB,EACtB,aAAqB,EACrB,SAAS,EACT,OAAe,EACf,MAAM,GACP,EAAE,wBAAwB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,GAAG,gBAAgB,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAuDpG;AAED,wBAAsB,WAAW,CAAC,EAAE,MAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAE,EAAE,kBAAkB;;;;GAyBhG"}
+{"version":3,"file":"run-evaluation.d.ts","sourceRoot":"","sources":["../../../src/eval-workflows/run-evaluation.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EACV,QAAQ,EAER,QAAQ,EACR,gBAAgB,EAChB,MAAM,EAIN,YAAY,EAGZ,WAAW,EAGZ,MAAM,aAAa,CAAC;AAIrB,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,UAAU,uBAAuB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACrC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAIlB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;yDACqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;uCACmC;IACnC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;wEACoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;iFAC6E;IAC7E,WAAW,CAAC,EAAE,OAAO,aAAa,EAAE,WAAW,EAAE,CAAC;IAClD,8EAA8E;IAC9E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,2CAA2C;IAC3C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;2DACuD;IACvD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gCAAgC;IAChC,MAAM,CAAC,EAAE,OAAO,aAAa,EAAE,UAAU,CAAC;CAC3C;AAED,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;IAC7B,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAyB,SAAQ,uBAAuB;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,iBAAiB,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CAC9D;AAED,MAAM,WAAW,kBAAmB,SAAQ,oBAAoB;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;CAC5E;AAED,UAAU,UAAU;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACnC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,MAAM,EAAE,IAAI,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAa,SAAQ,UAAU;IAC9C,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,wBAAsB,aAAa,CAAC,EAClC,WAAW,EACX,QAAQ,EACR,YAAiB,EACjB,SAAS,EACT,KAAqB,EACrB,UAAwB,EACxB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,KAAa,EACb,WAAe,EACf,SAAS,EACT,OAAe,EACf,YAAuB,EACvB,iBAAiB,EACjB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,aAAqB,EACrB,SAAS,EACT,OAAe,EACf,KAAS,EACT,MAAM,EACN,YAAoB,EACpB,MAAM,EACN,IAAI,EACJ,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,MAAM,GACP,EAAE,oBAAoB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CAsF5F;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAiB,SAAQ,UAAU;IAClD,IAAI,EAAE,IAAI,CAAC;IACX,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,eAAe,EAAE,CAAC;CAC9B;AAyHD,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,IAAI,CA6ErE;AAED,wBAAsB,iBAAiB,CAAC,EACtC,QAAQ,EACR,KAAqB,EACrB,UAAwB,EACxB,SAA8B,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAe,EACf,MAAc,EACd,WAAe,EACf,SAAS,EACT,YAAuB,EACvB,iBAAiB,EACjB,QAAe,EACf,UAAiB,EACjB,UAAiB,EACjB,eAAsB,EACtB,aAAqB,EACrB,SAAS,EACT,OAAe,EACf,MAAM,EACN,WAAW,EACX,WAAW,EACX,YAAY,GACb,EAAE,wBAAwB,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,GAAG,gBAAgB,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,CA0DpG;AAED,wBAAsB,WAAW,CAAC,EAAE,MAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAE,EAAE,kBAAkB;;;;GAyBhG"}

package/dist/src/eval-workflows/run-evaluation.js

@@ -6,7 +6,9 @@ import { confidenceInterval, tTest, effectSize } from '../eval-core/statistics.j
 import { executeEachEvaluationRuns } from './each-evaluation-workflow.js';
 import { buildDryRunEachArtifacts, buildDryRunTaskReport, prepareEvaluationRun, } from './evaluation-preparation.js';
 import { executeEvaluationPipeline } from './evaluation-pipeline.js';
-export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], artifacts, model = DEFAULT_MODEL, judgeModel = JUDGE_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, blind = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, skipPreflight = false, mcpConfig, verbose = false, retry = 0, resume, layeredStats = false, repeat, each, }) {
+import { findSaturationPoint } from '../analysis/saturation.js';
+import { bootstrapMeanCI } from '../eval-core/bootstrap.js';
+export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], artifacts, model = DEFAULT_MODEL, judgeModel = JUDGE_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, blind = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, skipPreflight = false, mcpConfig, verbose = false, retry = 0, resume, layeredStats = false, repeat, each, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, }) {
     const { samples, artifacts: resolvedArtifacts, tasks, variantNames, requires } = await prepareEvaluationRun({
         samplesPath,
         skillDir,
@@ -83,6 +85,12 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         layeredStats,
         repeat,
         each,
+        judgeRepeat,
+        judgeModels,
+        bootstrap,
+        bootstrapSamples,
+        lengthDebias,
+        budget,
     });
 }
 // Top-level (composite) extractor: feeds the legacy flat-field variance on
@@ -124,6 +132,81 @@ function buildComparisonMetric(scoresA, scoresB, meanA, meanB) {
         effectSize: es,
     };
 }
+/**
+ * Build saturation curve data from a sequence of runs.
+ *
+ * Each run contributes one checkpoint = "all samples up to and including this run".
+ * For each variant we compute (mean, CI) at that cumulative N. When repeat ≥ 5
+ * we additionally run findSaturationPoint to assess whether the curve has
+ * flattened. Below that threshold the data is recorded for plotting but no
+ * verdict is computed — the user still sees the curve, just without the auto
+ * "saturated at N=X" claim.
+ */
+function buildSaturationData(runs) {
+    if (runs.length < 2)
+        return undefined;
+    const variants = runs[0].meta.variants ?? [];
+    if (variants.length === 0)
+        return undefined;
+    // Per-variant: cumulative composite scores after each repeat.
+    const cumulativeByVariant = {};
+    const tracesByVariant = {};
+    const checkpointSampleCounts = [];
+    const acc = Object.fromEntries(variants.map((v) => [v, []]));
+    for (let runIdx = 0; runIdx < runs.length; runIdx++) {
+        const run = runs[runIdx];
+        for (const variant of variants) {
+            const newScores = [];
+            for (const entry of run.results ?? []) {
+                const v = entry.variants?.[variant];
+                if (!v || typeof v.compositeScore !== 'number' || v.compositeScore <= 0)
+                    continue;
+                newScores.push(v.compositeScore);
+            }
+            acc[variant] = acc[variant].concat(newScores);
+        }
+        // Snapshot cumulative state for this checkpoint.
+        const checkpointN = acc[variants[0]]?.length ?? 0;
+        checkpointSampleCounts.push(checkpointN);
+        for (const variant of variants) {
+            if (!cumulativeByVariant[variant])
+                cumulativeByVariant[variant] = [];
+            cumulativeByVariant[variant].push([...acc[variant]]);
+            // Per-checkpoint trace: bootstrap CI on cumulative scores.
+            const ci = bootstrapMeanCI(acc[variant], 0.05, 1000);
+            if (!tracesByVariant[variant])
+                tracesByVariant[variant] = [];
+            tracesByVariant[variant].push({
+                n: acc[variant].length,
+                mean: ci.estimate,
+                ciLow: ci.low,
+                ciHigh: ci.high,
+            });
+        }
+    }
+    const verdicts = {};
+    if (runs.length >= 5) {
+        for (const variant of variants) {
+            const cumulative = cumulativeByVariant[variant];
+            if (!cumulative)
+                continue;
+            const r = findSaturationPoint(cumulative, 'bootstrap-ci-width');
+            verdicts[variant] = {
+                saturated: r.saturated,
+                atN: r.atN,
+                confidence: r.confidence,
+                method: r.method,
+                threshold: r.threshold,
+                reason: r.reason,
+            };
+        }
+    }
+    return {
+        checkpointSampleCounts,
+        perVariant: tracesByVariant,
+        ...(Object.keys(verdicts).length > 0 ? { verdicts } : {}),
+    };
+}
 export function buildVarianceData(runs) {
     if (runs.length <= 1) {
         return null;
@@ -189,9 +272,15 @@ export function buildVarianceData(runs) {
             });
         }
     }
-    return { runs: runs.length, perVariant, comparisons };
+    const saturation = buildSaturationData(runs);
+    return {
+        runs: runs.length,
+        perVariant,
+        comparisons,
+        ...(saturation ? { saturation } : {}),
+    };
 }
-export async function runEachEvaluation({ skillDir, model = DEFAULT_MODEL, judgeModel = JUDGE_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipPreflight = false, mcpConfig, verbose = false, repeat, }) {
+export async function runEachEvaluation({ skillDir, model = DEFAULT_MODEL, judgeModel = JUDGE_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', judgeExecutorName, jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipPreflight = false, mcpConfig, verbose = false, repeat, judgeRepeat, judgeModels, lengthDebias, }) {
     const skillEntries = discoverEachSkills(resolve(skillDir));
     if (skillEntries.length === 0) {
         throw new Error(`未发现带配对 eval-samples 的 skill：${skillDir}`);
@@ -235,13 +324,16 @@ export async function runEachEvaluation({ skillDir, model = DEFAULT_MODEL, judge
         mcpConfig,
         verbose,
         repeat,
+        judgeRepeat,
+        judgeModels,
+        lengthDebias,
         runSingleEvaluation: async (options) => {
             // repeat > 1 时走 runMultiple 做 variance; each=true 标记让 meta.request 如实反映
             if (repeat && repeat > 1) {
-                const multi = await runMultiple({ ...options, repeat, each: true });
+                const multi = await runMultiple({ ...options, repeat, each: true, judgeRepeat, judgeModels, lengthDebias });
                 return { report: multi.report, filePath: multi.filePath };
             }
-            const result = await runEvaluation({ ...options, each: true });
+            const result = await runEvaluation({ ...options, each: true, judgeRepeat, judgeModels, lengthDebias });
             return { report: result.report, filePath: result.filePath };
         },
     });

package/dist/src/eval-workflows/run-evaluation.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-evaluation.js","sourceRoot":"","sources":["../../../src/eval-workflows/run-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACnF,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AA6GrE,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,EAClC,WAAW,EACX,QAAQ,EACR,YAAY,GAAG,EAAE,EACjB,SAAS,EACT,KAAK,GAAG,aAAa,EACrB,UAAU,GAAG,WAAW,EACxB,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,KAAK,EACd,KAAK,GAAG,KAAK,EACb,WAAW,GAAG,CAAC,EACf,SAAS,EACT,OAAO,GAAG,KAAK,EACf,YAAY,GAAG,QAAQ,EACvB,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,EACf,KAAK,GAAG,CAAC,EACT,MAAM,EACN,YAAY,GAAG,KAAK,EACpB,MAAM,EACN,IAAI,GACiB;IACrB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,MAAM,oBAAoB,CAAC;QAC1G,WAAW;QACX,QAAQ;QACR,YAAY;QACZ,SAAS;QACT,MAAM;QACN,SAAS;KACV,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,MAAM,EAAE,qBAAqB,CAAC;gBAC5B,KAAK;gBACL,UAAU;gBACV,YAAY;gBACZ,WAAW;gBACX,QAAQ;gBACR,KAAK;gBACL,YAAY;aACb,CAAC;YACF,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,IAAI,eAAgG,CAAC;IACrG,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,GAAG,EAAE,CAAC;YACrB,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;gBAC3C,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;YACpD,CAAC;YACD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACtH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,QAAQ,KAAK,WAAW,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,cAAc,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAe,cAAc,CAAC,YAAY,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAe,cAAc,CAAC,iBAAiB,IAAI,YAAY,CAAC,CAAC;IACpF,OAAO,yBAAyB,CAAC;QAC/B,WAAW;QACX,QAAQ;QACR,OAAO;QACP,KAAK;QACL,SAAS,EAAE,iBAAiB;QAC5B,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB,EAAE,iBAAiB,IAAI,YAAY;QACpD,QAAQ;QACR,aAAa;QACb,SAAS;QACT,OAAO;QACP,KAAK;QACL,IAAI;QACJ,KAAK;QACL,WAAW;QACX,SAAS;QACT,OAAO;QACP,QAAQ;QACR,UAAU;QACV,UAAU;QACV,aAAa;QACb,OAAO;QACP,KAAK;QACL,eAAe;QACf,QAAQ;QACR,YAAY;QACZ,MAAM;QACN,IAAI;KACL,CAAC,CAAC;AACL,CAAC;AAeD,2EAA2E;AAC3E,0EAA0E;AAC1E,8EAA8E;AAC9E,wEAAwE;AACxE,MAAM,mBAAmB,GAAG,CAAC,CAA6B,EAAsB,EAAE,CAAC,CAAC,EAAE,iBAAiB,CAAC;AAExG,yEAAyE;AACzE,MAAM,iBAAiB,GAAyF;IAC9G,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,gBAAgB;IAChC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,aAAa;CACpC,CAAC;AAEF,4EAA4E;AAC5E,8EAA8E;AAC9E,kFAAkF;AAClF,iEAAiE;AACjE,MAAM,gBAAgB,GAAoF;IACxG,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,YAAY;IAC5B,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,gBAAgB;IACpC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,aAAa;CAC/B,CAAC;AAEF,SAAS,gBAAgB,CAAC,IAAc,EAAE,OAAe,EAAE,SAAgE;IACzH,MAAM,MAAM,GAAG,IAAI;SAChB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;SAC/C,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAiB,EAAE,OAAiB,EAAE,KAAa,EAAE,KAAa;IAC/F,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,OAAO;QACL,QAAQ;QACR,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,UAAU,EAAE,EAAE;KACf,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC7C,MAAM,UAAU,GAAoC,EAAE,CAAC;IACvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,6CAA6C;QAC7C,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC;QACvE,IAAI,CAAC,SAAS;YAAE,SAAS;QAEzB,MAAM,QAAQ,GAA2D,EAAE,CAAC;QAC5E,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACrE,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,CAAC;QACjF,IAAI,UAAU;YAAE,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;QAEjD,uFAAuF;QACvF,MAAM,OAAO,GAAsD,EAAE,CAAC;QACtE,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE,CAAC;YACzD,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1E,IAAI,UAAU;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;QAC5C,CAAC;QAED,UAAU,CAAC,OAAO,CAAC,GAAG;YACpB,GAAG,SAAS;YACZ,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAyB,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE;gBAAE,SAAS;YAEzB,MAAM,aAAa,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YAEpF,MAAM,YAAY,GAAqE,EAAE,CAAC;YAC1F,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,YAAY,CAAU,EAAE,CAAC;gBAClD,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBACb,YAAY,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;YAED,qFAAqF;YACrF,MAAM,WAAW,GAAgE,EAAE,CAAC;YACpF,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE,CAAC;gBACzD,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC7B,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBACb,WAAW,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC;YAED,WAAW,CAAC,IAAI,CAAC;gBACf,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACd,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACd,GAAG,aAAa;gBAChB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3E,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,QAAQ,EACR,KAAK,GAAG,aAAa,EACrB,UAAU,GAAG,WAAW,EACxB,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,KAAK,EACd,WAAW,GAAG,CAAC,EACf,SAAS,EACT,YAAY,GAAG,QAAQ,EACvB,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,eAAe,GAAG,IAAI,EACtB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,EACf,MAAM,GACmB;IACzB,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,wBAAwB,CAAC,YAAY,CAAC,CAAC;QACvF,OAAO;YACL,MAAM,EAAE;gBACN,MAAM,EAAE,IAAI;gBACZ,IAAI,EAAE,IAAI;gBACV,KAAK;gBACL,UAAU;gBACV,QAAQ,EAAE,YAAY;gBACtB,QAAQ;gBACR,cAAc,EAAE,YAAY,CAAC,MAAM;gBACnC,UAAU;gBACV,SAAS,EAAE,YAAY;aACxB;YACD,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IACD,OAAO,yBAAyB,CAAC;QAC/B,QAAQ;QACR,YAAY;QACZ,KAAK;QACL,UAAU;QACV,SAAS;QACT,OAAO;QACP,KAAK;QACL,IAAI;QACJ,OAAO;QACP,WAAW;QACX,SAAS;QACT,YAAY;QACZ,iBAAiB;QACjB,QAAQ;QACR,UAAU;QACV,UAAU;QACV,eAAe;QACf,aAAa;QACb,SAAS;QACT,OAAO;QACP,MAAM;QACN,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACrC,wEAAwE;YACxE,IAAI,MAAM,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBACpE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC5D,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAgB,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxE,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAsB;IAC/F,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,gBAAgB,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC;YACrC,GAAG,MAAM;YACT,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI;YACzC,UAAU,EAAE,MAAM;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,MAAgB,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;QAC7B,gDAAgD;QAChD,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,cAAc,IAAI,kBAAkB,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"run-evaluation.js","sourceRoot":"","sources":["../../../src/eval-workflows/run-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACnF,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAC1E,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAkBrE,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AA4G5D,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,EAClC,WAAW,EACX,QAAQ,EACR,YAAY,GAAG,EAAE,EACjB,SAAS,EACT,KAAK,GAAG,aAAa,EACrB,UAAU,GAAG,WAAW,EACxB,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,KAAK,EACd,KAAK,GAAG,KAAK,EACb,WAAW,GAAG,CAAC,EACf,SAAS,EACT,OAAO,GAAG,KAAK,EACf,YAAY,GAAG,QAAQ,EACvB,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,EACf,KAAK,GAAG,CAAC,EACT,MAAM,EACN,YAAY,GAAG,KAAK,EACpB,MAAM,EACN,IAAI,EACJ,WAAW,EACX,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,YAAY,EACZ,MAAM,GACe;IACrB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,MAAM,oBAAoB,CAAC;QAC1G,WAAW;QACX,QAAQ;QACR,YAAY;QACZ,SAAS;QACT,MAAM;QACN,SAAS;KACV,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,MAAM,EAAE,qBAAqB,CAAC;gBAC5B,KAAK;gBACL,UAAU;gBACV,YAAY;gBACZ,WAAW;gBACX,QAAQ;gBACR,KAAK;gBACL,YAAY;aACb,CAAC;YACF,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IAED,iEAAiE;IACjE,IAAI,eAAgG,CAAC;IACrG,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,GAAG,EAAE,CAAC;YACrB,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;gBAC3C,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;YACpD,CAAC;YACD,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACtH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,QAAQ,KAAK,WAAW,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,cAAc,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAe,cAAc,CAAC,YAAY,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAe,cAAc,CAAC,iBAAiB,IAAI,YAAY,CAAC,CAAC;IACpF,OAAO,yBAAyB,CAAC;QAC/B,WAAW;QACX,QAAQ;QACR,OAAO;QACP,KAAK;QACL,SAAS,EAAE,iBAAiB;QAC5B,KAAK;QACL,UAAU;QACV,OAAO;QACP,YAAY;QACZ,iBAAiB,EAAE,iBAAiB,IAAI,YAAY;QACpD,QAAQ;QACR,aAAa;QACb,SAAS;QACT,OAAO;QACP,KAAK;QACL,IAAI;QACJ,KAAK;QACL,WAAW;QACX,SAAS;QACT,OAAO;QACP,QAAQ;QACR,UAAU;QACV,UAAU;QACV,aAAa;QACb,OAAO;QACP,KAAK;QACL,eAAe;QACf,QAAQ;QACR,YAAY;QACZ,MAAM;QACN,IAAI;QACJ,WAAW;QACX,WAAW;QACX,SAAS;QACT,gBAAgB;QAChB,YAAY;QACZ,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAeD,2EAA2E;AAC3E,0EAA0E;AAC1E,8EAA8E;AAC9E,wEAAwE;AACxE,MAAM,mBAAmB,GAAG,CAAC,CAA6B,EAAsB,EAAE,CAAC,CAAC,EAAE,iBAAiB,CAAC;AAExG,yEAAyE;AACzE,MAAM,iBAAiB,GAAyF;IAC9G,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,gBAAgB;IAChC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,aAAa;CACpC,CAAC;AAEF,4EAA4E;AAC5E,8EAA8E;AAC9E,kFAAkF;AAClF,iEAAiE;AACjE,MAAM,gBAAgB,GAAoF;IACxG,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,YAAY;IAC5B,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,gBAAgB;IACpC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,aAAa;CAC/B,CAAC;AAEF,SAAS,gBAAgB,CAAC,IAAc,EAAE,OAAe,EAAE,SAAgE;IACzH,MAAM,MAAM,GAAG,IAAI;SAChB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;SAC/C,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAiB,EAAE,OAAiB,EAAE,KAAa,EAAE,KAAa;IAC/F,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAClC,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,OAAO;QACL,QAAQ;QACR,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,UAAU,EAAE,EAAE;KACf,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,mBAAmB,CAAC,IAAc;IACzC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC7C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE5C,8DAA8D;IAC9D,MAAM,mBAAmB,GAA+B,EAAE,CAAC;IAC3D,MAAM,eAAe,GAAsF,EAAE,CAAC;IAE9G,MAAM,sBAAsB,GAAa,EAAE,CAAC;IAC5C,MAAM,GAAG,GAA6B,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAEvF,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC;gBACpC,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,CAAC,cAAc,IAAI,CAAC;oBAAE,SAAS;gBAClF,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YACnC,CAAC;YACD,GAAG,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAChD,CAAC;QACD,iDAAiD;QACjD,MAAM,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;QAClD,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC;gBAAE,mBAAmB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YACrE,mBAAmB,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAErD,2DAA2D;YAC3D,MAAM,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YACrD,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC;gBAAE,eAAe,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YAC7D,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;gBAC5B,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM;gBACtB,IAAI,EAAE,EAAE,CAAC,QAAQ;gBACjB,KAAK,EAAE,EAAE,CAAC,GAAG;gBACb,MAAM,EAAE,EAAE,CAAC,IAAI;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAA+B,EAAE,CAAC;IAChD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACrB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAChD,IAAI,CAAC,UAAU;gBAAE,SAAS;YAC1B,MAAM,CAAC,GAAG,mBAAmB,CAAC,UAAU,EAAE,oBAAoB,CAAC,CAAC;YAChE,QAAQ,CAAC,OAAO,CAAC,GAAG;gBAClB,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,GAAG,EAAE,CAAC,CAAC,GAAG;gBACV,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,sBAAsB;QACtB,UAAU,EAAE,eAAe;QAC3B,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IAC7C,MAAM,UAAU,GAAoC,EAAE,CAAC;IACvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,6CAA6C;QAC7C,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC;QACvE,IAAI,CAAC,SAAS;YAAE,SAAS;QAEzB,MAAM,QAAQ,GAA2D,EAAE,CAAC;QAC5E,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACrE,IAAI,IAAI;YAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC;QAC/B,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,CAAC;QACjF,IAAI,UAAU;YAAE,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;QAEjD,uFAAuF;QACvF,MAAM,OAAO,GAAsD,EAAE,CAAC;QACtE,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE,CAAC;YACzD,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1E,IAAI,UAAU;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;QAC5C,CAAC;QAED,UAAU,CAAC,OAAO,CAAC,GAAG;YACpB,GAAG,SAAS;YACZ,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAyB,EAAE,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE;gBAAE,SAAS;YAEzB,MAAM,aAAa,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;YAEpF,MAAM,YAAY,GAAqE,EAAE,CAAC;YAC1F,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,YAAY,CAAU,EAAE,CAAC;gBAClD,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBACb,YAAY,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;YAED,qFAAqF;YACrF,MAAM,WAAW,GAAgE,EAAE,CAAC;YACpF,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAU,EAAE,CAAC;gBACzD,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;gBAC7B,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBACb,WAAW,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBACnF,CAAC;YACH,CAAC;YAED,WAAW,CAAC,IAAI,CAAC;gBACf,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACd,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACd,GAAG,aAAa;gBAChB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3E,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACzE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC7C,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,UAAU;QACV,WAAW;QACX,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,QAAQ,EACR,KAAK,GAAG,aAAa,EACrB,UAAU,GAAG,WAAW,EACxB,SAAS,GAAG,kBAAkB,EAC9B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,KAAK,EACd,WAAW,GAAG,CAAC,EACf,SAAS,EACT,YAAY,GAAG,QAAQ,EACvB,iBAAiB,EACjB,QAAQ,GAAG,IAAI,EACf,UAAU,GAAG,IAAI,EACjB,UAAU,GAAG,IAAI,EACjB,eAAe,GAAG,IAAI,EACtB,aAAa,GAAG,KAAK,EACrB,SAAS,EACT,OAAO,GAAG,KAAK,EACf,MAAM,EACN,WAAW,EACX,WAAW,EACX,YAAY,GACa;IACzB,MAAM,YAAY,GAAG,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,wBAAwB,CAAC,YAAY,CAAC,CAAC;QACvF,OAAO;YACL,MAAM,EAAE;gBACN,MAAM,EAAE,IAAI;gBACZ,IAAI,EAAE,IAAI;gBACV,KAAK;gBACL,UAAU;gBACV,QAAQ,EAAE,YAAY;gBACtB,QAAQ;gBACR,cAAc,EAAE,YAAY,CAAC,MAAM;gBACnC,UAAU;gBACV,SAAS,EAAE,YAAY;aACxB;YACD,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IACD,OAAO,yBAAyB,CAAC;QAC/B,QAAQ;QACR,YAAY;QACZ,KAAK;QACL,UAAU;QACV,SAAS;QACT,OAAO;QACP,KAAK;QACL,IAAI;QACJ,OAAO;QACP,WAAW;QACX,SAAS;QACT,YAAY;QACZ,iBAAiB;QACjB,QAAQ;QACR,UAAU;QACV,UAAU;QACV,eAAe;QACf,aAAa;QACb,SAAS;QACT,OAAO;QACP,MAAM;QACN,WAAW;QACX,WAAW;QACX,YAAY;QACZ,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACrC,wEAAwE;YACxE,IAAI,MAAM,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;gBAC5G,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC5D,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAC;YACvG,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAgB,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxE,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,gBAAgB,EAAE,GAAG,MAAM,EAAsB;IAC/F,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,gBAAgB,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC;YACrC,GAAG,MAAM;YACT,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI;YACzC,UAAU,EAAE,MAAM;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,MAAgB,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,CAAC,QAAQ,GAAG,UAAU,CAAC;QAC7B,gDAAgD;QAChD,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,cAAc,IAAI,kBAAkB,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC"}

package/dist/src/grading/assertions.d.ts

@@ -7,6 +7,22 @@ export interface AsyncAssertionContext {
     samplesDir: string;
 }
 export declare function ratioToScore(ratio: number): number;
+/**
+ * ROUGE-N (recall-oriented). N-gram overlap divided by the reference's
+ * n-gram count. Multi-set form per Lin 2004 — repeated n-grams are clipped
+ * to their reference count, so a candidate that repeats a single matching
+ * n-gram many times can't game the score above 1.
+ */
+export declare function rougeN(candidate: string, reference: string, n: number): number;
+/** Standard Levenshtein edit distance with O(min(m,n)) memory. */
+export declare function levenshtein(a: string, b: string): number;
+/**
+ * BLEU score (corpus-level form, single reference). Geometric mean of
+ * precision at n=1..maxN, multiplied by the brevity penalty for short
+ * candidates. Unsmoothed — returns 0 if any n-gram precision is 0, which
+ * is a known sharpness of the original BLEU; threshold accordingly.
+ */
+export declare function bleu(candidate: string, reference: string, maxN?: number): number;
 export declare function validateJsonSchema(data: unknown, schema: Record<string, unknown>): boolean;
 export declare function runAssertions(output: string, assertions: Assertion[], context?: {
     costUSD?: number;

package/dist/src/grading/assertions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../../src/grading/assertions.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAmB,gBAAgB,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMlH,eAAO,MAAM,qBAAqB,aAA6C,CAAC;AAEhF,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,UAAU,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAwBD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAQ1F;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,YAAY,EAAE,CAAA;CAAO,GAAG,gBAAgB,CAqI/L;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAkFhL"}
+{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../../src/grading/assertions.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAmB,gBAAgB,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMlH,eAAO,MAAM,qBAAqB,aAQhC,CAAC;AAEH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,UAAU,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAwBD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAElD;AAkCD;;;;;GAKG;AACH,wBAAgB,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAK9E;AAED,kEAAkE;AAClE,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAmBxD;AAED;;;;;GAKG;AACH,wBAAgB,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,SAAI,GAAG,MAAM,CAgB3E;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAQ1F;AAoHD,wBAAgB,aAAa,CAC3B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,SAAS,EAAE,EACvB,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,YAAY,EAAE,CAAA;CAAO,GACrG,gBAAgB,CA8BlB;AAED,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA2FhL"}