oh-my-knowledge 0.18.0 → 0.20.0

package/dist/src/grading/debias-validate.d.ts

@@ -0,0 +1,84 @@
+/**
+ * `omk bench debias-validate length <reportId>` — measure how much the judge's
+ * scores shift when the length-debias instruction is toggled.
+ *
+ * What it actually measures
+ * -------------------------
+ * Given a finished report, this command re-judges every (sample, variant) pair
+ * using the OPPOSITE length-debias setting from the original run. It then
+ * compares the two score distributions with a bootstrap CI on the mean
+ * difference.
+ *
+ * If the original report ran with debias-on (v3-cot-length), we re-judge with
+ * v2-cot (legacy). If the original ran with debias-off, we re-judge with
+ * v3-cot-length. Significant difference → the prompt change moves scores → the
+ * judge is sensitive to the length-debias instruction. That's *consistent with*
+ * length bias being present, but it doesn't prove it directly — a perfectly
+ * length-neutral judge could in principle also be sensitive to the wording for
+ * other reasons. We label the verdict accordingly.
+ *
+ * Cost
+ * ----
+ * Re-judging is a full second pass over all (sample, variant) cells. Cost
+ * doubles vs the original run. The CLI surfaces this so users running on
+ * large/expensive evaluations can opt in deliberately.
+ */
+import type { ExecutorFn, Report, Sample } from '../types.js';
+import { type BootstrapDiffCI } from '../eval-core/bootstrap.js';
+export interface DebiasValidateInput {
+    report: Report;
+    samples: Sample[];
+    judgeExecutor: ExecutorFn;
+    judgeModel: string;
+    /** Variant to validate. Defaults to first variant. */
+    variant?: string;
+    /** Bootstrap iterations for the diff CI. Default 1000. */
+    bootstrapSamples?: number;
+    seed?: number;
+    /** Progress hook. */
+    onProgress?: (info: {
+        sample_id: string;
+        completed: number;
+        total: number;
+    }) => void;
+}
+export interface DebiasValidateResult {
+    variant: string;
+    /** Original lengthDebias setting (true if debias-on at run time). */
+    originalLengthDebias: boolean;
+    /** Pairs of (originalScore, alternateScore) per sample. */
+    pairs: Array<{
+        sample_id: string;
+        originalScore: number;
+        alternateScore: number;
+    }>;
+    meanOriginal: number;
+    meanAlternate: number;
+    /** Mean of (alternate - original). Positive = alternate prompt scored higher. */
+    diffCI: BootstrapDiffCI;
+    /** Verdict in the {未检测, 弱, 中, 强} bucket plus an English shadow. */
+    verdict: {
+        zh: string;
+        en: string;
+        level: 'none' | 'weak' | 'medium' | 'strong';
+    };
+    /** Total cost burned re-judging. */
+    alternateJudgeCostUSD: number;
+    /** Sample_ids that the report had but lacked judge scores. */
+    unscored: string[];
+    /** Sample_ids in the samples file that are missing from the report. */
+    missing: string[];
+}
+/**
+ * Re-judge every sample of `variant` in the given report with the OPPOSITE
+ * lengthDebias setting and compute the bootstrap CI on the mean difference.
+ *
+ * The judge call uses the rubric from the samples file and the output stored
+ * in the report — we do NOT re-execute the model. Only judging is repeated.
+ *
+ * Multi-dimensional samples currently use the rubric as fallback when there's
+ * no top-level rubric. Per-dimension validation can be added later if needed.
+ */
+export declare function validateLengthDebias(input: DebiasValidateInput): Promise<DebiasValidateResult>;
+export declare function formatDebiasValidate(result: DebiasValidateResult): string;
+//# sourceMappingURL=debias-validate.d.ts.map

package/dist/src/grading/debias-validate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"debias-validate.d.ts","sourceRoot":"","sources":["../../../src/grading/debias-validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE9D,OAAO,EAAmB,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAElF,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,aAAa,EAAE,UAAU,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qBAAqB;IACrB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACtF;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,oBAAoB,EAAE,OAAO,CAAC;IAC9B,2DAA2D;IAC3D,KAAK,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnF,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,MAAM,EAAE,eAAe,CAAC;IACxB,mEAAmE;IACnE,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAA;KAAE,CAAC;IAClF,oCAAoC;IACpC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,uEAAuE;IACvE,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAqBD;;;;;;;;;GASG;AACH,wBAAsB,oBAAoB,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAuFpG;AAOD,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,oBAAoB,GAAG,MAAM,CA8BzE"}

package/dist/src/grading/debias-validate.js

@@ -0,0 +1,173 @@
+/**
+ * `omk bench debias-validate length <reportId>` — measure how much the judge's
+ * scores shift when the length-debias instruction is toggled.
+ *
+ * What it actually measures
+ * -------------------------
+ * Given a finished report, this command re-judges every (sample, variant) pair
+ * using the OPPOSITE length-debias setting from the original run. It then
+ * compares the two score distributions with a bootstrap CI on the mean
+ * difference.
+ *
+ * If the original report ran with debias-on (v3-cot-length), we re-judge with
+ * v2-cot (legacy). If the original ran with debias-off, we re-judge with
+ * v3-cot-length. Significant difference → the prompt change moves scores → the
+ * judge is sensitive to the length-debias instruction. That's *consistent with*
+ * length bias being present, but it doesn't prove it directly — a perfectly
+ * length-neutral judge could in principle also be sensitive to the wording for
+ * other reasons. We label the verdict accordingly.
+ *
+ * Cost
+ * ----
+ * Re-judging is a full second pass over all (sample, variant) cells. Cost
+ * doubles vs the original run. The CLI surfaces this so users running on
+ * large/expensive evaluations can opt in deliberately.
+ */
+import { llmJudge } from './judge.js';
+import { bootstrapDiffCI } from '../eval-core/bootstrap.js';
+/**
+ * Map a bootstrap diff CI to a verdict bucket. The ranges are deliberately
+ * conservative: we only label "strong" when the CI fully sits >= |0.5| away
+ * from zero (about half a point on a 1-5 scale).
+ */
+function classifyVerdict(diff) {
+    if (!diff.significant) {
+        return { zh: '未检测到显著差异', en: 'no significant shift', level: 'none' };
+    }
+    const mag = Math.min(Math.abs(diff.low), Math.abs(diff.high));
+    if (mag >= 0.5) {
+        return { zh: '强差异 — prompt 改动对评分影响大', en: 'strong shift', level: 'strong' };
+    }
+    if (mag >= 0.2) {
+        return { zh: '中等差异 — 校正对结论有实质影响', en: 'medium shift', level: 'medium' };
+    }
+    return { zh: '弱差异 — 显著但幅度小', en: 'weak shift', level: 'weak' };
+}
+/**
+ * Re-judge every sample of `variant` in the given report with the OPPOSITE
+ * lengthDebias setting and compute the bootstrap CI on the mean difference.
+ *
+ * The judge call uses the rubric from the samples file and the output stored
+ * in the report — we do NOT re-execute the model. Only judging is repeated.
+ *
+ * Multi-dimensional samples currently use the rubric as fallback when there's
+ * no top-level rubric. Per-dimension validation can be added later if needed.
+ */
+export async function validateLengthDebias(input) {
+    const { report, samples, judgeExecutor, judgeModel, bootstrapSamples = 1000, seed, onProgress } = input;
+    const variant = input.variant ?? report.meta.variants?.[0];
+    if (!variant)
+        throw new Error('report has no variants — nothing to validate');
+    // Detect original lengthDebias setting from meta. Default to true (v0.21+).
+    // Older reports without debiasMode set are treated as legacy (debias-off).
+    const debiasModeList = report.meta.debiasMode ?? [];
+    const originalLengthDebias = debiasModeList.includes('length');
+    const alternateLengthDebias = !originalLengthDebias;
+    const sampleById = new Map();
+    for (const s of samples)
+        sampleById.set(s.sample_id, s);
+    // Pre-pass: collect (sample_id, output, originalScore) tuples.
+    const tasks = [];
+    const unscored = [];
+    const missing = [];
+    for (const entry of report.results ?? []) {
+        const v = entry.variants?.[variant];
+        if (!v || !v.fullOutput)
+            continue;
+        const sample = sampleById.get(entry.sample_id);
+        if (!sample) {
+            missing.push(entry.sample_id);
+            continue;
+        }
+        if (typeof v.llmScore !== 'number' || v.llmScore <= 0) {
+            unscored.push(entry.sample_id);
+            continue;
+        }
+        tasks.push({ sample, output: v.fullOutput, originalScore: v.llmScore });
+    }
+    // Re-judge with the opposite debias setting. We use the simplest path:
+    // single-rubric judge. Multi-dim samples fall back to the explicit rubric
+    // string if available. Samples without a rubric are skipped — we have
+    // nothing to feed the judge.
+    const pairs = [];
+    let alternateJudgeCostUSD = 0;
+    let completed = 0;
+    for (const t of tasks) {
+        completed++;
+        onProgress?.({ sample_id: t.sample.sample_id, completed, total: tasks.length });
+        const rubric = t.sample.rubric
+            ?? (t.sample.dimensions ? Object.values(t.sample.dimensions).join('\n') : '');
+        if (!rubric)
+            continue;
+        const altResult = await llmJudge({
+            output: t.output,
+            rubric,
+            prompt: t.sample.prompt,
+            executor: judgeExecutor,
+            model: judgeModel,
+            lengthDebias: alternateLengthDebias,
+        });
+        if (altResult.judgeCostUSD)
+            alternateJudgeCostUSD += altResult.judgeCostUSD;
+        if (altResult.score > 0) {
+            pairs.push({
+                sample_id: t.sample.sample_id,
+                originalScore: t.originalScore,
+                alternateScore: altResult.score,
+            });
+        }
+    }
+    const meanOriginal = avg(pairs.map((p) => p.originalScore));
+    const meanAlternate = avg(pairs.map((p) => p.alternateScore));
+    const diffCI = bootstrapDiffCI(pairs.map((p) => p.originalScore), pairs.map((p) => p.alternateScore), 0.05, bootstrapSamples, seed);
+    const verdict = classifyVerdict(diffCI);
+    return {
+        variant,
+        originalLengthDebias,
+        pairs,
+        meanOriginal: Number(meanOriginal.toFixed(3)),
+        meanAlternate: Number(meanAlternate.toFixed(3)),
+        diffCI,
+        verdict,
+        alternateJudgeCostUSD: Number(alternateJudgeCostUSD.toFixed(6)),
+        unscored,
+        missing,
+    };
+}
+function avg(arr) {
+    if (arr.length === 0)
+        return 0;
+    return arr.reduce((s, x) => s + x, 0) / arr.length;
+}
+export function formatDebiasValidate(result) {
+    const lines = [];
+    const dirOrig = result.originalLengthDebias ? 'on (v3-cot-length)' : 'off (v2-cot)';
+    const dirAlt = result.originalLengthDebias ? 'off (v2-cot)' : 'on (v3-cot-length)';
+    lines.push(`\n  Length-debias 灵敏度验证 (variant: ${result.variant})\n`);
+    lines.push(`  原始 prompt:    ${dirOrig}`);
+    lines.push(`  对照 prompt:    ${dirAlt}`);
+    lines.push(`  样本数:         ${result.pairs.length}`);
+    if (result.pairs.length === 0) {
+        lines.push('  无可比对样本 — 检查报告是否含 fullOutput / rubric。');
+        return lines.join('\n');
+    }
+    lines.push(`  原均值:         ${result.meanOriginal.toFixed(3)}`);
+    lines.push(`  对照均值:       ${result.meanAlternate.toFixed(3)}`);
+    const ci = result.diffCI;
+    lines.push(`  差值 (alt-orig): ${ci.estimate >= 0 ? '+' : ''}${ci.estimate}`);
+    lines.push(`  95% CI:         [${ci.low}, ${ci.high}]   (${ci.significant ? '显著' : '不显著'})`);
+    lines.push('');
+    lines.push(`  结论:           ${result.verdict.zh}`);
+    lines.push('');
+    lines.push(`  注: 该结论反映"prompt 切换是否改变评分"。差异显著 = 评分对 length-debias 指令敏感,`);
+    lines.push(`      间接支持 length bias 存在;但 prompt 文本变化也可能因其他原因影响评分。`);
+    lines.push(`      重判 cost: ${result.alternateJudgeCostUSD.toFixed(6)} USD`);
+    if (result.missing.length) {
+        lines.push(`  缺样本 ID:      ${result.missing.length} 条`);
+    }
+    if (result.unscored.length) {
+        lines.push(`  无 LLM 分:      ${result.unscored.length} 条`);
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=debias-validate.js.map

package/dist/src/grading/debias-validate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"debias-validate.js","sourceRoot":"","sources":["../../../src/grading/debias-validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAwB,MAAM,2BAA2B,CAAC;AAoClF;;;;GAIG;AACH,SAAS,eAAe,CAAC,IAAqB;IAC5C,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QACtB,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,sBAAsB,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACvE,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9D,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACf,OAAO,EAAE,EAAE,EAAE,uBAAuB,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAC9E,CAAC;IACD,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACf,OAAO,EAAE,EAAE,EAAE,mBAAmB,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAC1E,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACjE,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAA0B;IACnE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,gBAAgB,GAAG,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC;IACxG,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3D,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAE9E,4EAA4E;IAC5E,2EAA2E;IAC3E,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IACpD,MAAM,oBAAoB,GAAG,cAAc,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC/D,MAAM,qBAAqB,GAAG,CAAC,oBAAoB,CAAC;IAEpD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IAExD,+DAA+D;IAC/D,MAAM,KAAK,GAAqE,EAAE,CAAC;IACnF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;YAAE,SAAS;QAClC,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC;YACtD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAC/B,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,aAAa,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,uEAAuE;IACvE,0EAA0E;IAC1E,sEAAsE;IACtE,6BAA6B;IAC7B,MAAM,KAAK,GAAkC,EAAE,CAAC;IAChD,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM;eACzB,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChF,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;YAC/B,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,MAAM;YACN,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM;YACvB,QAAQ,EAAE,aAAa;YACvB,KAAK,EAAE,UAAU;YACjB,YAAY,EAAE,qBAAqB;SACpC,CAAC,CAAC;QACH,IAAI,SAAS,CAAC,YAAY;YAAE,qBAAqB,IAAI,SAAS,CAAC,YAAY,CAAC;QAC5E,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC;gBACT,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS;gBAC7B,aAAa,EAAE,CAAC,CAAC,aAAa;gBAC9B,cAAc,EAAE,SAAS,CAAC,KAAK;aAChC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;IAC5D,MAAM,aAAa,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,eAAe,CAC5B,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,EACjC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,EAClC,IAAI,EACJ,gBAAgB,EAChB,IAAI,CACL,CAAC;IACF,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAExC,OAAO;QACL,OAAO;QACP,oBAAoB;QACpB,KAAK;QACL,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7C,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/C,MAAM;QACN,OAAO;QACP,qBAAqB,EAAE,MAAM,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/D,QAAQ;QACR,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAS,GAAG,CAAC,GAAa;IACxB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA4B;IAC/D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAC;IACpF,MAAM,MAAM,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,oBAAoB,CAAC;IACnF,KAAK,CAAC,IAAI,CAAC,qCAAqC,MAAM,CAAC,OAAO,KAAK,CAAC,CAAC;IACrE,KAAK,CAAC,IAAI,CAAC,mBAAmB,OAAO,EAAE,CAAC,CAAC;IACzC,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,EAAE,CAAC,CAAC;IACxC,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACpD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;QACtD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/D,KAAK,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/D,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,QAAQ,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;IAC7F,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;IACnD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;IACzE,KAAK,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;IACnE,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5E,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/src/grading/gold-cli.d.ts

@@ -0,0 +1,88 @@
+/**
+ * `omk bench gold` subcommands: init / validate / compare.
+ *
+ * Kept out of cli.ts to keep that file from growing further; cli.ts only
+ * dispatches subcommand strings to the handlers exported here.
+ */
+import type { Report, ReportHumanAgreement } from '../types.js';
+import { type GoldDataset } from './gold-dataset.js';
+import { type AgreementResult } from './human-gold.js';
+export interface GoldCompareInput {
+    report: Report;
+    gold: GoldDataset;
+    /** Variant to compare against gold. Default: first variant in report.meta.variants. */
+    variant?: string;
+    /** Bootstrap parameters. */
+    samples?: number;
+    seed?: number;
+}
+export interface GoldCompareResult {
+    agreement: AgreementResult;
+    /** Sample-level comparison rows for the renderer / per-row UI. */
+    rows: Array<{
+        sample_id: string;
+        goldScore: number;
+        judgeScore: number;
+        diff: number;
+    }>;
+    /** Sample IDs in the gold set that were missing from the report. */
+    missing: string[];
+    /** Sample IDs in the gold set whose judge score was missing/non-numeric. */
+    unscored: string[];
+    /** Contamination warning string if annotator id matches any judge model id. */
+    contaminationWarning?: string;
+    /** Variant actually used. */
+    variant: string;
+}
+/**
+ * Compute agreement between gold annotations and the LLM judge's scores from
+ * a finished report.
+ *
+ * Pure function — does not touch fs or stdout. The CLI handler wraps it.
+ */
+export declare function compareGoldToReport(input: GoldCompareInput): GoldCompareResult;
+/**
+ * Format the comparison result as a human-readable terminal report.
+ */
+export declare function formatGoldCompare(result: GoldCompareResult, gold: GoldDataset): string;
+/**
+ * Generate a starter dataset directory with a metadata.yaml and an
+ * annotations.yaml stub. Errors out if the directory already has yaml files
+ * (refuses to clobber).
+ */
+export declare function initGoldDataset(targetDir: string, options?: {
+    annotator?: string;
+}): string[];
+/**
+ * Convert an in-memory comparison result into the persisted ReportMeta shape.
+ */
+export declare function toPersistedAgreement(result: GoldCompareResult, gold: GoldDataset): ReportHumanAgreement;
+/**
+ * Side-effecting helper for `omk bench run --gold-dir`: load gold, compute
+ * agreement, mutate report.meta.humanAgreement, re-persist the report file,
+ * print the human-readable comparison to stderr.
+ *
+ * Returns the comparison result so the CLI can also surface contamination
+ * warnings programmatically.
+ */
+export declare function attachGoldAgreementToReport(input: {
+    report: Report;
+    goldDir: string;
+    outputDir: string;
+    samples?: number;
+    seed?: number;
+    variant?: string;
+}): {
+    result?: GoldCompareResult;
+    gold?: GoldDataset;
+    loadIssues: string[];
+};
+/**
+ * Validate-only entry point. Returns issues; CLI exits non-zero if any.
+ */
+export declare function validateGoldDataset(dir: string): {
+    ok: boolean;
+    issues: string[];
+    sampleCount: number;
+};
+//# sourceMappingURL=gold-cli.d.ts.map

package/dist/src/grading/gold-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gold-cli.d.ts","sourceRoot":"","sources":["../../../src/grading/gold-cli.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,MAAM,EAAE,oBAAoB,EAA8B,MAAM,aAAa,CAAC;AAE5F,OAAO,EAA6B,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChF,OAAO,EAGL,KAAK,eAAe,EACrB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,WAAW,CAAC;IAClB,uFAAuF;IACvF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,eAAe,CAAC;IAC3B,kEAAkE;IAClE,IAAI,EAAE,KAAK,CAAC;QACV,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC,CAAC;IACH,oEAAoE;IACpE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,4EAA4E;IAC5E,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,+EAA+E;IAC/E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,6BAA6B;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,gBAAgB,GAAG,iBAAiB,CAqE9E;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,WAAW,GAAG,MAAM,CA8BtF;AAgBD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,MAAM,EAAE,CAwDjG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,WAAW,GAChB,oBAAoB,CAetB;AAED;;;;;;;GAOG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG;IAAE,MAAM,CAAC,EAAE,iBAAiB,CAAC;IAAC,IAAI,CAAC,EAAE,WAAW,CAAC;IAAC,UAAU,EAAE,MAAM,EAAE,CAAA;CAAE,CAU3E;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAWvG"}

package/dist/src/grading/gold-cli.js

@@ -0,0 +1,251 @@
+/**
+ * `omk bench gold` subcommands: init / validate / compare.
+ *
+ * Kept out of cli.ts to keep that file from growing further; cli.ts only
+ * dispatches subcommand strings to the handlers exported here.
+ */
+import { existsSync, mkdirSync, writeFileSync, readdirSync } from 'node:fs';
+import { resolve, join } from 'node:path';
+import { persistReport } from '../eval-core/evaluation-reporting.js';
+import { loadGoldDataset, dumpYaml } from './gold-dataset.js';
+import { computeAgreementWithCI, } from './human-gold.js';
+/**
+ * Compute agreement between gold annotations and the LLM judge's scores from
+ * a finished report.
+ *
+ * Pure function — does not touch fs or stdout. The CLI handler wraps it.
+ */
+export function compareGoldToReport(input) {
+    const { report, gold, samples = 1000, seed } = input;
+    const variants = report.meta?.variants ?? [];
+    const variant = input.variant ?? variants[0];
+    if (!variant) {
+        throw new Error('report has no variants — cannot compare to gold');
+    }
+    // Build sample_id → judge score lookup. We use llmScore (composite mean
+    // across dimensions if multi-dim, single score otherwise) — that's the
+    // number a single human gold rater is most directly comparable to.
+    const judgeScoreById = new Map();
+    for (const entry of report.results ?? []) {
+        const v = entry.variants?.[variant];
+        if (!v)
+            continue;
+        const s = v.llmScore;
+        if (typeof s === 'number' && Number.isFinite(s)) {
+            judgeScoreById.set(entry.sample_id, s);
+        }
+    }
+    const pairs = [];
+    const rows = [];
+    const missing = [];
+    const unscored = [];
+    for (const anno of gold.annotations) {
+        const judge = judgeScoreById.get(anno.sample_id);
+        if (judge === undefined) {
+            // Differentiate "sample doesn't exist in this report" vs "sample exists
+            // but had no score" (e.g. assertions-only sample, judge skipped).
+            const hasEntry = (report.results ?? []).some((r) => r.sample_id === anno.sample_id);
+            if (hasEntry)
+                unscored.push(anno.sample_id);
+            else
+                missing.push(anno.sample_id);
+            continue;
+        }
+        pairs.push({ unitId: anno.sample_id, coderA: anno.score, coderB: judge });
+        rows.push({
+            sample_id: anno.sample_id,
+            goldScore: anno.score,
+            judgeScore: judge,
+            diff: Number((judge - anno.score).toFixed(4)),
+        });
+    }
+    const scaleMeta = gold.metadata.scale ?? { min: 1, max: 5 };
+    const agreement = computeAgreementWithCI(pairs, { samples, seed, scale: scaleMeta });
+    // Contamination check: gold annotator id matching any judge model id is a
+    // statistical hazard (model-judges-itself inflates agreement). We only warn
+    // — users may have one model available and proceed knowingly.
+    const annotator = gold.metadata.annotator.trim().toLowerCase();
+    const judgeIds = [
+        report.meta?.judgeModel ?? '',
+        ...(report.meta?.judgeModels ?? []),
+    ].filter(Boolean).map((s) => s.toLowerCase());
+    const contaminated = judgeIds.find((j) => j.includes(annotator) || annotator.includes(j));
+    const contaminationWarning = contaminated
+        ? `gold annotator "${gold.metadata.annotator}" overlaps with judge model "${contaminated}" — agreement is inflated; treat α as upper bound only`
+        : undefined;
+    return {
+        agreement,
+        rows,
+        missing,
+        unscored,
+        contaminationWarning,
+        variant,
+    };
+}
+/**
+ * Format the comparison result as a human-readable terminal report.
+ */
+export function formatGoldCompare(result, gold) {
+    const lines = [];
+    const a = result.agreement;
+    lines.push(`\n  人工锚点对比 (variant: ${result.variant})\n`);
+    lines.push(`  样本数:           ${a.sampleCount}`);
+    if (a.sampleCount === 0) {
+        lines.push('  无可比对样本 — 检查 sample_id 是否对应。');
+        if (result.missing.length)
+            lines.push(`  报告缺失:         ${result.missing.join(', ')}`);
+        return lines.join('\n');
+    }
+    lines.push(`  Krippendorff α:   ${fmt(a.alpha)}   (主指标，序数加权)`);
+    lines.push(`  α 95% CI:         [${fmt(a.alphaCI.low)}, ${fmt(a.alphaCI.high)}]`);
+    lines.push(`  加权 κ:           ${fmt(a.weightedKappa)}   (副指标)`);
+    lines.push(`  Pearson r:        ${fmt(a.pearson)}   (仅查 rank order)`);
+    lines.push('');
+    lines.push(`  解读:             ${interpret(a.alpha)}`);
+    if (result.missing.length) {
+        lines.push(`  报告缺 sample:    ${result.missing.length} 条 (${result.missing.slice(0, 5).join(', ')}${result.missing.length > 5 ? '...' : ''})`);
+    }
+    if (result.unscored.length) {
+        lines.push(`  缺 LLM 分:        ${result.unscored.length} 条 (${result.unscored.slice(0, 5).join(', ')}${result.unscored.length > 5 ? '...' : ''})`);
+    }
+    if (result.contaminationWarning) {
+        lines.push('');
+        lines.push(`  ⚠ 污染警告: ${result.contaminationWarning}`);
+    }
+    lines.push('');
+    lines.push(`  gold annotator:   ${gold.metadata.annotator} (${gold.metadata.annotatedAt}, v${gold.metadata.version})`);
+    lines.push('');
+    return lines.join('\n');
+}
+function fmt(x) {
+    if (Number.isNaN(x))
+        return 'NaN';
+    return x.toFixed(3);
+}
+function interpret(alpha) {
+    if (Number.isNaN(alpha))
+        return 'α 未定义（数据无方差，需要更多评分多样性）';
+    if (alpha >= 0.8)
+        return 'α ≥ 0.80 — 高度一致，结论可放心使用';
+    if (alpha >= 0.667)
+        return 'α ∈ [0.67, 0.80) — 可接受，但谨慎结论';
+    if (alpha >= 0.4)
+        return 'α ∈ [0.40, 0.67) — 较弱一致，结论需配合 CI 与人工抽检';
+    if (alpha >= 0)
+        return 'α < 0.40 — 评委与 gold 偏差较大，先排查 rubric / prompt';
+    return 'α < 0 — 评委与 gold 系统性反向，必须重新审视判分逻辑';
+}
+/**
+ * Generate a starter dataset directory with a metadata.yaml and an
+ * annotations.yaml stub. Errors out if the directory already has yaml files
+ * (refuses to clobber).
+ */
+export function initGoldDataset(targetDir, options = {}) {
+    const abs = resolve(targetDir);
+    if (existsSync(abs)) {
+        const present = readdirSync(abs).filter((f) => /\.ya?ml$/.test(f));
+        if (present.length > 0) {
+            throw new Error(`目标目录已存在 YAML 文件 (${present.join(', ')})，避免覆盖请换目录`);
+        }
+    }
+    else {
+        mkdirSync(abs, { recursive: true });
+    }
+    const today = new Date().toISOString().slice(0, 10);
+    const metadata = {
+        annotator: options.annotator ?? 'YOUR-MODEL-OR-TEAM-ID',
+        annotatedAt: today,
+        version: '0.1',
+        scale: { min: 1, max: 5 },
+        notes: 'omk gold dataset — 标注者 id 与 omk 默认 judge 不同时（如 sonnet judge + opus gold），CLI 自动检查污染。',
+    };
+    const annotations = {
+        annotations: [
+            { sample_id: 'EXAMPLE_001', score: 4, reason: '示例：替换为真实标注或删除' },
+            { sample_id: 'EXAMPLE_002', score: 2 },
+        ],
+    };
+    const metaPath = join(abs, 'metadata.yaml');
+    const annoPath = join(abs, 'annotations.yaml');
+    writeFileSync(metaPath, dumpYaml({ metadata }));
+    writeFileSync(annoPath, dumpYaml(annotations));
+    const readme = [
+        '# Gold dataset',
+        '',
+        'omk 用此目录作为人工锚点 (human gold) 与 LLM judge 对比。',
+        '',
+        '## 文件说明',
+        '- `metadata.yaml`: annotator / 时间 / 版本 / 评分量程',
+        '- `annotations.yaml`: 每条 `{ sample_id, score, reason? }`，按 sample_id 拼接',
+        '',
+        '## 重要约束',
+        '1. annotator 不应与 omk judge 模型同名 — CLI 检测到会警告',
+        '2. 单标注者只反映一个视角；30 条以下的 demo 不构成基准',
+        '3. score 必须在 metadata.scale 范围内 (默认 1-5)',
+        '',
+        '## 校验',
+        '```bash',
+        'omk bench gold validate <this-dir>',
+        'omk bench gold compare <reportId> --gold-dir <this-dir>',
+        '```',
+        '',
+    ].join('\n');
+    const readmePath = join(abs, 'README.md');
+    if (!existsSync(readmePath))
+        writeFileSync(readmePath, readme);
+    return [metaPath, annoPath, readmePath];
+}
+/**
+ * Convert an in-memory comparison result into the persisted ReportMeta shape.
+ */
+export function toPersistedAgreement(result, gold) {
+    const a = result.agreement;
+    return {
+        alpha: a.alpha,
+        alphaCI: a.alphaCI,
+        weightedKappa: a.weightedKappa,
+        pearson: a.pearson,
+        sampleCount: a.sampleCount,
+        variant: result.variant,
+        goldAnnotator: gold.metadata.annotator,
+        goldVersion: gold.metadata.version,
+        contaminationWarning: result.contaminationWarning,
+        missingCount: result.missing.length,
+        unscoredCount: result.unscored.length,
+    };
+}
+/**
+ * Side-effecting helper for `omk bench run --gold-dir`: load gold, compute
+ * agreement, mutate report.meta.humanAgreement, re-persist the report file,
+ * print the human-readable comparison to stderr.
+ *
+ * Returns the comparison result so the CLI can also surface contamination
+ * warnings programmatically.
+ */
+export function attachGoldAgreementToReport(input) {
+    const { report, goldDir, outputDir, samples, seed, variant } = input;
+    const { dataset, issues } = loadGoldDataset(goldDir);
+    const loadIssues = issues.map((i) => i.message);
+    if (!dataset)
+        return { loadIssues };
+    const result = compareGoldToReport({ report, gold: dataset, samples, seed, variant });
+    report.meta.humanAgreement = toPersistedAgreement(result, dataset);
+    persistReport(report, outputDir);
+    return { result, gold: dataset, loadIssues };
+}
+/**
+ * Validate-only entry point. Returns issues; CLI exits non-zero if any.
+ */
+export function validateGoldDataset(dir) {
+    const { dataset, issues } = loadGoldDataset(dir);
+    const formatted = issues.map((i) => {
+        const at = i.path ? ` (${i.path}${i.index !== undefined ? `:${i.index}` : ''})` : '';
+        return `${i.message}${at}`;
+    });
+    return {
+        ok: !!dataset && issues.length === 0,
+        issues: formatted,
+        sampleCount: dataset?.annotations.length ?? 0,
+    };
+}
+//# sourceMappingURL=gold-cli.js.map

package/dist/src/grading/gold-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gold-cli.js","sourceRoot":"","sources":["../../../src/grading/gold-cli.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC5E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAoB,MAAM,mBAAmB,CAAC;AAChF,OAAO,EACL,sBAAsB,GAGvB,MAAM,iBAAiB,CAAC;AA+BzB;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAuB;IACzD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IACrD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IAED,wEAAwE;IACxE,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACzC,MAAM,CAAC,GAA8B,KAAK,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC;QAC/D,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACrB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YAChD,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,IAAI,GAA8B,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,wEAAwE;YACxE,kEAAkE;YAClE,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC;YACjG,IAAI,QAAQ;gBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;;gBACvC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAClC,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1E,IAAI,CAAC,IAAI,CAAC;YACR,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,KAAK;YACrB,UAAU,EAAE,KAAK;YACjB,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;SAC9C,CAAC,CAAC;IACL,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IAC5D,MAAM,SAAS,GAAG,sBAAsB,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAErF,0EAA0E;IAC1E,4EAA4E;IAC5E,8DAA8D;IAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/D,MAAM,QAAQ,GAAa;QACzB,MAAM,CAAC,IAAI,EAAE,UAAU,IAAI,EAAE;QAC7B,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;KACpC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAC9C,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,oBAAoB,GAAG,YAAY;QACvC,CAAC,CAAC,mBAAmB,IAAI,CAAC,QAAQ,CAAC,SAAS,gCAAgC,YAAY,wDAAwD;QAChJ,CAAC,CAAC,SAAS,CAAC;IAEd,OAAO;QACL,SAAS;QACT,IAAI;QACJ,OAAO;QACP,QAAQ;QACR,oBAAoB;QACpB,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAyB,EAAE,IAAiB;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,OAAO,KAAK,CAAC,CAAC;IACxD,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IAChD,IAAI,CAAC,CAAC,WAAW,KAAK,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC5C,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IAC/D,KAAK,CAAC,IAAI,CAAC,wBAAwB,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClF,KAAK,CAAC,IAAI,CAAC,qBAAqB,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IAChE,KAAK,CAAC,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACtE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,qBAAqB,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtD,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,OAAO,CAAC,MAAM,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAChJ,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,qBAAqB,MAAM,CAAC,QAAQ,CAAC,MAAM,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACpJ,CAAC;IACD,IAAI,MAAM,CAAC,oBAAoB,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,oBAAoB,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,QAAQ,CAAC,WAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,GAAG,CAAC,CAAC;IACvH,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,wBAAwB,CAAC;IACzD,IAAI,KAAK,IAAI,GAAG;QAAE,OAAO,yBAAyB,CAAC;IACnD,IAAI,KAAK,IAAI,KAAK;QAAE,OAAO,8BAA8B,CAAC;IAC1D,IAAI,KAAK,IAAI,GAAG;QAAE,OAAO,wCAAwC,CAAC;IAClE,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,8CAA8C,CAAC;IACtE,OAAO,mCAAmC,CAAC;AAC7C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,SAAiB,EAAE,UAAkC,EAAE;IACrF,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/B,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG;QACf,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,uBAAuB;QACvD,WAAW,EAAE,KAAK;QAClB,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;QACzB,KAAK,EAAE,sFAAsF;KAC9F,CAAC;IACF,MAAM,WAAW,GAAG;QAClB,WAAW,EAAE;YACX,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE;YAC/D,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,EAAE;SACvC;KACF,CAAC;IAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;IAC/C,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IAChD,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;IAE/C,MAAM,MAAM,GAAG;QACb,gBAAgB;QAChB,EAAE;QACF,6CAA6C;QAC7C,EAAE;QACF,SAAS;QACT,+CAA+C;QAC/C,yEAAyE;QACzE,EAAE;QACF,SAAS;QACT,8CAA8C;QAC9C,mCAAmC;QACnC,0CAA0C;QAC1C,EAAE;QACF,OAAO;QACP,SAAS;QACT,oCAAoC;QACpC,yDAAyD;QACzD,KAAK;QACL,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAC1C,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,aAAa,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAE/D,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,MAAyB,EACzB,IAAiB;IAEjB,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC;IAC3B,OAAO;QACL,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;QACtC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;QAClC,oBAAoB,EAAE,MAAM,CAAC,oBAAoB;QACjD,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;QACnC,aAAa,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;KACtC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,2BAA2B,CAAC,KAO3C;IACC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IACrE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,UAAU,EAAE,CAAC;IAEpC,MAAM,MAAM,GAAG,mBAAmB,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACtF,MAAM,CAAC,IAAI,CAAC,cAAc,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnE,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAW;IAC7C,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACrF,OAAO,GAAG,CAAC,CAAC,OAAO,GAAG,EAAE,EAAE,CAAC;IAC7B,CAAC,CAAC,CAAC;IACH,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QACpC,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,IAAI,CAAC;KAC9C,CAAC;AACJ,CAAC"}