oh-my-knowledge 0.18.0 → 0.20.0

package/dist/src/grading/gold-dataset.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Gold dataset loader and validator.
+ *
+ * The "gold" file is a YAML document containing human (or stronger-model)
+ * annotations for a fixed set of sample_ids. omk loads it and compares the
+ * annotations against the judge's scores from a finished evaluation report
+ * to compute α / κ / Pearson agreement (see ./human-gold.ts).
+ *
+ * Schema is deliberately small:
+ *
+ *   metadata:
+ *     annotator: <model id or person identifier>     # required, used for contamination check
+ *     annotatedAt: <YYYY-MM-DD>                       # required
+ *     version: <string>                               # required
+ *     scale: { min: 1, max: 5 }                       # optional, default {1,5}
+ *     notes: <free text>                              # optional
+ *
+ *   annotations:
+ *     - sample_id: <must match a sample_id in the eval samples file>
+ *       score: <number on the scale>                  # required
+ *       reason: <free text>                           # optional, kept for audit only
+ *
+ * v0.21 supports single-score gold only. Multi-dimensional gold (per-rubric
+ * scores) is a candidate for v0.22 once we have real users asking for it —
+ * the schema is forwards-compatible (we can add a `dimensions` field next to
+ * `score` later without breaking existing files).
+ */
+export interface GoldMetadata {
+    /** Identifier of who/what produced the annotations — model id, person handle, or team name. */
+    annotator: string;
+    annotatedAt: string;
+    version: string;
+    scale?: {
+        min: number;
+        max: number;
+    };
+    notes?: string;
+}
+export interface GoldAnnotation {
+    sample_id: string;
+    score: number;
+    reason?: string;
+}
+export interface GoldDataset {
+    metadata: GoldMetadata;
+    annotations: GoldAnnotation[];
+    /** Source file paths — surfaced for diagnostics (e.g. error messages). */
+    sourcePaths: string[];
+}
+export interface ValidationIssue {
+    /** File the issue came from — empty when issue is structural (e.g. missing metadata). */
+    path: string;
+    /** Annotation index within the file, when applicable. */
+    index?: number;
+    message: string;
+}
+export interface LoadResult {
+    dataset?: GoldDataset;
+    issues: ValidationIssue[];
+}
+/**
+ * Load a gold dataset from a directory.
+ *
+ * Convention: the directory contains exactly one `metadata.yaml` and one or
+ * more `*.yaml` annotation files. Annotation files may declare top-level
+ * `annotations: [...]` — multiple files are concatenated. This shape lets
+ * users split annotations by topic (code.yaml, writing.yaml, ...) without
+ * fighting one giant file in code review.
+ */
+export declare function loadGoldDataset(dir: string): LoadResult;
+/** Re-export YAML serializer so the `gold init` CLI can write a template. */
+export declare function dumpYaml(value: unknown): string;
+//# sourceMappingURL=gold-dataset.d.ts.map

package/dist/src/grading/gold-dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gold-dataset.d.ts","sourceRoot":"","sources":["../../../src/grading/gold-dataset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAOH,MAAM,WAAW,YAAY;IAC3B,+FAA+F;IAC/F,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,YAAY,CAAC;IACvB,WAAW,EAAE,cAAc,EAAE,CAAC;IAC9B,0EAA0E;IAC1E,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC9B,yFAAyF;IACzF,IAAI,EAAE,MAAM,CAAC;IACb,yDAAyD;IACzD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,CAyFvD;AAmCD,6EAA6E;AAC7E,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAE/C"}

package/dist/src/grading/gold-dataset.js

@@ -0,0 +1,161 @@
+/**
+ * Gold dataset loader and validator.
+ *
+ * The "gold" file is a YAML document containing human (or stronger-model)
+ * annotations for a fixed set of sample_ids. omk loads it and compares the
+ * annotations against the judge's scores from a finished evaluation report
+ * to compute α / κ / Pearson agreement (see ./human-gold.ts).
+ *
+ * Schema is deliberately small:
+ *
+ *   metadata:
+ *     annotator: <model id or person identifier>     # required, used for contamination check
+ *     annotatedAt: <YYYY-MM-DD>                       # required
+ *     version: <string>                               # required
+ *     scale: { min: 1, max: 5 }                       # optional, default {1,5}
+ *     notes: <free text>                              # optional
+ *
+ *   annotations:
+ *     - sample_id: <must match a sample_id in the eval samples file>
+ *       score: <number on the scale>                  # required
+ *       reason: <free text>                           # optional, kept for audit only
+ *
+ * v0.21 supports single-score gold only. Multi-dimensional gold (per-rubric
+ * scores) is a candidate for v0.22 once we have real users asking for it —
+ * the schema is forwards-compatible (we can add a `dimensions` field next to
+ * `score` later without breaking existing files).
+ */
+import { readFileSync, readdirSync } from 'node:fs';
+import { resolve, join } from 'node:path';
+import yaml from 'js-yaml';
+import { parseYaml } from '../inputs/load-samples.js';
+/**
+ * Load a gold dataset from a directory.
+ *
+ * Convention: the directory contains exactly one `metadata.yaml` and one or
+ * more `*.yaml` annotation files. Annotation files may declare top-level
+ * `annotations: [...]` — multiple files are concatenated. This shape lets
+ * users split annotations by topic (code.yaml, writing.yaml, ...) without
+ * fighting one giant file in code review.
+ */
+export function loadGoldDataset(dir) {
+    const issues = [];
+    const absDir = resolve(dir);
+    let entries;
+    try {
+        entries = readdirSync(absDir);
+    }
+    catch (err) {
+        issues.push({
+            path: absDir,
+            message: `gold dataset directory not found or unreadable: ${err.message}`,
+        });
+        return { issues };
+    }
+    const yamlFiles = entries.filter((f) => f.endsWith('.yaml') || f.endsWith('.yml')).sort();
+    if (yamlFiles.length === 0) {
+        issues.push({ path: absDir, message: 'no .yaml files found in gold dataset directory' });
+        return { issues };
+    }
+    let metadata;
+    const annotations = [];
+    const sourcePaths = [];
+    const seenIds = new Set();
+    for (const fname of yamlFiles) {
+        const fpath = join(absDir, fname);
+        sourcePaths.push(fpath);
+        let parsed;
+        try {
+            parsed = parseYaml(readFileSync(fpath, 'utf-8'));
+        }
+        catch (err) {
+            issues.push({ path: fpath, message: err.message });
+            continue;
+        }
+        if (typeof parsed !== 'object' || parsed === null) {
+            issues.push({ path: fpath, message: 'top-level YAML must be an object' });
+            continue;
+        }
+        const obj = parsed;
+        if (obj.metadata) {
+            const metaIssue = validateMetadata(obj.metadata, fpath);
+            if (metaIssue)
+                issues.push(metaIssue);
+            else if (metadata) {
+                issues.push({ path: fpath, message: 'metadata declared in multiple files; keep it in one place' });
+            }
+            else {
+                metadata = obj.metadata;
+            }
+        }
+        if (Array.isArray(obj.annotations)) {
+            for (const [i, raw] of obj.annotations.entries()) {
+                const annoIssue = validateAnnotation(raw, fpath, i);
+                if (annoIssue) {
+                    issues.push(annoIssue);
+                    continue;
+                }
+                const anno = raw;
+                if (seenIds.has(anno.sample_id)) {
+                    issues.push({
+                        path: fpath,
+                        index: i,
+                        message: `duplicate sample_id "${anno.sample_id}" — each gold annotation must be unique`,
+                    });
+                    continue;
+                }
+                seenIds.add(anno.sample_id);
+                annotations.push(anno);
+            }
+        }
+    }
+    if (!metadata) {
+        issues.push({ path: absDir, message: 'no metadata block found in any file (need annotator/annotatedAt/version)' });
+    }
+    if (annotations.length === 0) {
+        issues.push({ path: absDir, message: 'no annotations found' });
+    }
+    if (!metadata || annotations.length === 0) {
+        return { issues };
+    }
+    return {
+        dataset: { metadata, annotations, sourcePaths },
+        issues,
+    };
+}
+function validateMetadata(raw, path) {
+    if (typeof raw !== 'object' || raw === null) {
+        return { path, message: 'metadata must be an object' };
+    }
+    const m = raw;
+    for (const k of ['annotator', 'annotatedAt', 'version']) {
+        if (typeof m[k] !== 'string' || !m[k]) {
+            return { path, message: `metadata.${k} is required and must be a non-empty string` };
+        }
+    }
+    if (m.scale !== undefined) {
+        const s = m.scale;
+        if (typeof s !== 'object' || typeof s.min !== 'number' || typeof s.max !== 'number' || s.max <= s.min) {
+            return { path, message: 'metadata.scale must be { min: number, max: number } with max > min' };
+        }
+    }
+    return null;
+}
+function validateAnnotation(raw, path, index) {
+    if (typeof raw !== 'object' || raw === null) {
+        return { path, index, message: 'annotation must be an object' };
+    }
+    const a = raw;
+    if (typeof a.sample_id !== 'string' || !a.sample_id) {
+        return { path, index, message: 'sample_id is required and must be a non-empty string' };
+    }
+    if (typeof a.score !== 'number' || !Number.isFinite(a.score)) {
+        return { path, index, message: 'score is required and must be a finite number' };
+    }
+    return null;
+}
+/** Re-export YAML serializer so the `gold init` CLI can write a template. */
+export function dumpYaml(value) {
+    return yaml.dump(value, { lineWidth: 100, noRefs: true });
+}
+//# sourceMappingURL=gold-dataset.js.map

package/dist/src/grading/gold-dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gold-dataset.js","sourceRoot":"","sources":["../../../src/grading/gold-dataset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAqCtD;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAE5B,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,mDAAoD,GAAa,CAAC,OAAO,EAAE;SACrF,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,CAAC;IACpB,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1F,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gDAAgD,EAAE,CAAC,CAAC;QACzF,OAAO,EAAE,MAAM,EAAE,CAAC;IACpB,CAAC;IAED,IAAI,QAAkC,CAAC;IACvC,MAAM,WAAW,GAAqB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAClC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9D,SAAS;QACX,CAAC;QACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YAClD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,kCAAkC,EAAE,CAAC,CAAC;YAC1E,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,MAAiC,CAAC;QAE9C,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACxD,IAAI,SAAS;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBACjC,IAAI,QAAQ,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,2DAA2D,EAAE,CAAC,CAAC;YACrG,CAAC;iBAAM,CAAC;gBACN,QAAQ,GAAG,GAAG,CAAC,QAAwB,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,KAAK,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAK,GAAG,CAAC,WAAyB,CAAC,OAAO,EAAE,EAAE,CAAC;gBAChE,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;gBACpD,IAAI,SAAS,EAAE,CAAC;oBACd,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACvB,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,GAAG,GAAqB,CAAC;gBACnC,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;oBAChC,MAAM,CAAC,IAAI,CAAC;wBACV,IAAI,EAAE,KAAK;wBACX,KAAK,EAAE,CAAC;wBACR,OAAO,EAAE,wBAAwB,IAAI,CAAC,SAAS,yCAAyC;qBACzF,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC5B,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,0EAA0E,EAAE,CAAC,CAAC;IACrH,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,sBAAsB,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1C,OAAO,EAAE,MAAM,EAAE,CAAC;IACpB,CAAC;IAED,OAAO;QACL,OAAO,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE;QAC/C,MAAM;KACP,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAY,EAAE,IAAY;IAClD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC;IACzD,CAAC;IACD,MAAM,CAAC,GAAG,GAA8B,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,aAAa,EAAE,SAAS,CAAC,EAAE,CAAC;QACxD,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,6CAA6C,EAAE,CAAC;QACvF,CAAC;IACH,CAAC;IACD,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAgC,CAAC;QAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,IAAK,CAAC,CAAC,GAAc,IAAK,CAAC,CAAC,GAAc,EAAE,CAAC;YAC9H,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,oEAAoE,EAAE,CAAC;QACjG,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAY,EAAE,IAAY,EAAE,KAAa;IACnE,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,8BAA8B,EAAE,CAAC;IAClE,CAAC;IACD,MAAM,CAAC,GAAG,GAA8B,CAAC;IACzC,IAAI,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,sDAAsD,EAAE,CAAC;IAC1F,CAAC;IACD,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,+CAA+C,EAAE,CAAC;IACnF,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AAC5D,CAAC"}

package/dist/src/grading/human-gold.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Human gold dataset agreement metrics.
+ *
+ * Why this module exists
+ * ----------------------
+ * Bootstrap CI (Phase 1) gives us *precision* — how stable the judge is across
+ * resampled evaluations. It does not give us *validity* — whether the judge is
+ * scoring the right thing at all. A judge can be extremely consistent (CI very
+ * narrow) and yet systematically biased; that's an undetected failure mode.
+ *
+ * Human gold (or stronger-model gold as a proxy) provides an external anchor.
+ * We compute agreement between the LLM judge and the gold annotator, and report
+ * the result alongside the Bootstrap CI. Both numbers must be acceptable for a
+ * conclusion to be trustworthy.
+ *
+ * Three metrics are exported:
+ *
+ *  - **Krippendorff's α (interval)** — primary. Distribution-free, supports
+ *    ordinal/interval scales naturally, doesn't assume coders are exchangeable
+ *    (good fit when one "coder" is a model and the other a human annotator).
+ *  - **Quadratic-weighted Cohen's κ** — secondary. Familiar to many readers,
+ *    useful as a sanity check. Reports lower than α when marginals diverge.
+ *  - **Pearson r** — tertiary. Captures rank-order agreement only; doesn't
+ *    penalize systematic offset (a judge that always scores 1 lower than gold
+ *    has Pearson 1 but α < 1). Listed for completeness.
+ *
+ * All three are wrapped with a bootstrap CI on α (the primary) so the user can
+ * see uncertainty on the agreement number itself when N is small.
+ */
+import { type BootstrapCI } from '../eval-core/bootstrap.js';
+export interface RatingPair {
+    /** Per-sample identifier; used only for diagnostics. */
+    unitId: string;
+    /** Score from coder A — convention: gold annotator goes here. */
+    coderA: number;
+    /** Score from coder B — convention: LLM judge goes here. */
+    coderB: number;
+}
+export interface AgreementResult {
+    /** Krippendorff α (interval weights). 1 = perfect, 0 = chance, < 0 = worse than chance. */
+    alpha: number;
+    /** 95% bootstrap CI on α — width signals uncertainty due to sample size. */
+    alphaCI: BootstrapCI;
+    /** Quadratic-weighted κ. */
+    weightedKappa: number;
+    /** Pearson product-moment correlation. NaN if either coder has zero variance. */
+    pearson: number;
+    /** How many rating pairs went into the calculation. */
+    sampleCount: number;
+}
+/**
+ * Krippendorff's α with interval weights for two coders.
+ *
+ * Implementation follows Krippendorff (2011), "Computing Krippendorff's
+ * Alpha-Reliability". For interval scales the metric is δ²(c, k) = (c − k)².
+ *
+ * For two coders with one rating per unit:
+ *   - Coincidence matrix entries o_{c,k} count both (a_u, b_u) and (b_u, a_u),
+ *     so total mass n_·· = 2N.
+ *   - D_o = Σ o_{c,k} · δ²(c,k) / n_··
+ *   - D_e = Σ n_c · n_k · δ²(c,k) / (n_·· (n_·· − 1))
+ *   - α   = 1 − D_o / D_e
+ *
+ * Returns NaN when D_e = 0 (e.g. all ratings identical across both coders) —
+ * agreement is undefined when there is no variance to disagree about.
+ */
+export declare function computeKrippendorffAlpha(pairs: RatingPair[]): number;
+/**
+ * Cohen's quadratic-weighted κ for two coders on a numeric scale.
+ *
+ *   w(i,j)   = (i − j)² / (K − 1)²    (disagreement weight, scaled to [0,1])
+ *   po_w     = mean over units of (1 − w(a_u, b_u))
+ *   pe_w     = Σ_{i,j} marg_a(i) · marg_b(j) · (1 − w(i,j)) / N²
+ *   κ_w      = (po_w − pe_w) / (1 − pe_w)
+ *
+ * For continuous (non-integer) scores we still need a scale range; the caller
+ * passes (min, max). Default 1..5 fits omk's standard rubric.
+ */
+export declare function computeWeightedKappa(pairs: RatingPair[], scale?: {
+    min: number;
+    max: number;
+}): number;
+/**
+ * Pearson product-moment correlation. NaN when either coder has zero variance.
+ */
+export declare function computePearson(pairs: RatingPair[]): number;
+/**
+ * Compute α + κ + Pearson, with bootstrap CI on α.
+ *
+ * Bootstrap resamples pairs (units), not individual ratings — pairs are the
+ * unit of replication; resampling individual ratings would break the (a,b) tie.
+ */
+export declare function computeAgreementWithCI(pairs: RatingPair[], options?: {
+    samples?: number;
+    seed?: number;
+    alpha?: number;
+    scale?: {
+        min: number;
+        max: number;
+    };
+}): AgreementResult;
+//# sourceMappingURL=human-gold.d.ts.map

package/dist/src/grading/human-gold.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"human-gold.d.ts","sourceRoot":"","sources":["../../../src/grading/human-gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAuB,KAAK,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAElF,MAAM,WAAW,UAAU;IACzB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,2FAA2F;IAC3F,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,OAAO,EAAE,WAAW,CAAC;IACrB,4BAA4B;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,CAoCpE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,UAAU,EAAE,EACnB,KAAK,GAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAuB,GACvD,MAAM,CAgCR;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,CAuB1D;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,UAAU,EAAE,EACnB,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;CAAO,GACtG,eAAe,CA8BjB"}

package/dist/src/grading/human-gold.js

@@ -0,0 +1,188 @@
+/**
+ * Human gold dataset agreement metrics.
+ *
+ * Why this module exists
+ * ----------------------
+ * Bootstrap CI (Phase 1) gives us *precision* — how stable the judge is across
+ * resampled evaluations. It does not give us *validity* — whether the judge is
+ * scoring the right thing at all. A judge can be extremely consistent (CI very
+ * narrow) and yet systematically biased; that's an undetected failure mode.
+ *
+ * Human gold (or stronger-model gold as a proxy) provides an external anchor.
+ * We compute agreement between the LLM judge and the gold annotator, and report
+ * the result alongside the Bootstrap CI. Both numbers must be acceptable for a
+ * conclusion to be trustworthy.
+ *
+ * Three metrics are exported:
+ *
+ *  - **Krippendorff's α (interval)** — primary. Distribution-free, supports
+ *    ordinal/interval scales naturally, doesn't assume coders are exchangeable
+ *    (good fit when one "coder" is a model and the other a human annotator).
+ *  - **Quadratic-weighted Cohen's κ** — secondary. Familiar to many readers,
+ *    useful as a sanity check. Reports lower than α when marginals diverge.
+ *  - **Pearson r** — tertiary. Captures rank-order agreement only; doesn't
+ *    penalize systematic offset (a judge that always scores 1 lower than gold
+ *    has Pearson 1 but α < 1). Listed for completeness.
+ *
+ * All three are wrapped with a bootstrap CI on α (the primary) so the user can
+ * see uncertainty on the agreement number itself when N is small.
+ */
+import { bootstrapWithMetric } from '../eval-core/bootstrap.js';
+/**
+ * Krippendorff's α with interval weights for two coders.
+ *
+ * Implementation follows Krippendorff (2011), "Computing Krippendorff's
+ * Alpha-Reliability". For interval scales the metric is δ²(c, k) = (c − k)².
+ *
+ * For two coders with one rating per unit:
+ *   - Coincidence matrix entries o_{c,k} count both (a_u, b_u) and (b_u, a_u),
+ *     so total mass n_·· = 2N.
+ *   - D_o = Σ o_{c,k} · δ²(c,k) / n_··
+ *   - D_e = Σ n_c · n_k · δ²(c,k) / (n_·· (n_·· − 1))
+ *   - α   = 1 − D_o / D_e
+ *
+ * Returns NaN when D_e = 0 (e.g. all ratings identical across both coders) —
+ * agreement is undefined when there is no variance to disagree about.
+ */
+export function computeKrippendorffAlpha(pairs) {
+    if (pairs.length === 0)
+        return NaN;
+    // Marginal counts: how many times each value appears across both coders.
+    const marginal = new Map();
+    for (const p of pairs) {
+        marginal.set(p.coderA, (marginal.get(p.coderA) ?? 0) + 1);
+        marginal.set(p.coderB, (marginal.get(p.coderB) ?? 0) + 1);
+    }
+    const totalMass = 2 * pairs.length; // n_··
+    // Observed disagreement: average squared distance within units, ×2 because
+    // each unit contributes (a,b) and (b,a). Equivalent to mean (a-b)² over units.
+    let observedSum = 0;
+    for (const p of pairs) {
+        observedSum += 2 * (p.coderA - p.coderB) ** 2;
+    }
+    const Do = observedSum / totalMass;
+    // Expected disagreement under chance: pair every value with every other value
+    // proportional to marginals.
+    let expectedSum = 0;
+    const values = [...marginal.keys()];
+    for (const c of values) {
+        const nc = marginal.get(c);
+        for (const k of values) {
+            const nk = marginal.get(k);
+            expectedSum += nc * nk * (c - k) ** 2;
+        }
+    }
+    const denom = totalMass * (totalMass - 1);
+    if (denom === 0)
+        return NaN;
+    const De = expectedSum / denom;
+    if (De === 0)
+        return NaN; // no variance => agreement is undefined
+    return 1 - Do / De;
+}
+/**
+ * Cohen's quadratic-weighted κ for two coders on a numeric scale.
+ *
+ *   w(i,j)   = (i − j)² / (K − 1)²    (disagreement weight, scaled to [0,1])
+ *   po_w     = mean over units of (1 − w(a_u, b_u))
+ *   pe_w     = Σ_{i,j} marg_a(i) · marg_b(j) · (1 − w(i,j)) / N²
+ *   κ_w      = (po_w − pe_w) / (1 − pe_w)
+ *
+ * For continuous (non-integer) scores we still need a scale range; the caller
+ * passes (min, max). Default 1..5 fits omk's standard rubric.
+ */
+export function computeWeightedKappa(pairs, scale = { min: 1, max: 5 }) {
+    if (pairs.length === 0)
+        return NaN;
+    const range = scale.max - scale.min;
+    if (range <= 0)
+        return NaN;
+    const denom2 = range * range;
+    const margA = new Map();
+    const margB = new Map();
+    for (const p of pairs) {
+        margA.set(p.coderA, (margA.get(p.coderA) ?? 0) + 1);
+        margB.set(p.coderB, (margB.get(p.coderB) ?? 0) + 1);
+    }
+    const N = pairs.length;
+    let poWeighted = 0;
+    for (const p of pairs) {
+        const w = ((p.coderA - p.coderB) ** 2) / denom2;
+        poWeighted += 1 - w;
+    }
+    poWeighted /= N;
+    let peWeighted = 0;
+    for (const [i, ni] of margA) {
+        for (const [j, nj] of margB) {
+            const w = ((i - j) ** 2) / denom2;
+            peWeighted += (ni * nj) * (1 - w);
+        }
+    }
+    peWeighted /= N * N;
+    if (peWeighted === 1)
+        return NaN;
+    return (poWeighted - peWeighted) / (1 - peWeighted);
+}
+/**
+ * Pearson product-moment correlation. NaN when either coder has zero variance.
+ */
+export function computePearson(pairs) {
+    if (pairs.length < 2)
+        return NaN;
+    const n = pairs.length;
+    let sumA = 0;
+    let sumB = 0;
+    for (const p of pairs) {
+        sumA += p.coderA;
+        sumB += p.coderB;
+    }
+    const meanA = sumA / n;
+    const meanB = sumB / n;
+    let cov = 0;
+    let varA = 0;
+    let varB = 0;
+    for (const p of pairs) {
+        const da = p.coderA - meanA;
+        const db = p.coderB - meanB;
+        cov += da * db;
+        varA += da * da;
+        varB += db * db;
+    }
+    if (varA === 0 || varB === 0)
+        return NaN;
+    return cov / Math.sqrt(varA * varB);
+}
+/**
+ * Compute α + κ + Pearson, with bootstrap CI on α.
+ *
+ * Bootstrap resamples pairs (units), not individual ratings — pairs are the
+ * unit of replication; resampling individual ratings would break the (a,b) tie.
+ */
+export function computeAgreementWithCI(pairs, options = {}) {
+    const { samples = 1000, seed, alpha = 0.05, scale } = options;
+    if (pairs.length === 0) {
+        return {
+            alpha: NaN,
+            alphaCI: { low: NaN, high: NaN, estimate: NaN, samples: 0 },
+            weightedKappa: NaN,
+            pearson: NaN,
+            sampleCount: 0,
+        };
+    }
+    // Encode pairs as indices so bootstrapWithMetric can resample them.
+    const indices = pairs.map((_, i) => i);
+    const alphaCI = bootstrapWithMetric(indices, (resampledIdx) => computeKrippendorffAlpha(resampledIdx.map((i) => pairs[i])), alpha, samples, seed);
+    return {
+        alpha: roundOrNaN(computeKrippendorffAlpha(pairs)),
+        alphaCI,
+        weightedKappa: roundOrNaN(computeWeightedKappa(pairs, scale)),
+        pearson: roundOrNaN(computePearson(pairs)),
+        sampleCount: pairs.length,
+    };
+}
+function roundOrNaN(x) {
+    if (Number.isNaN(x))
+        return NaN;
+    return Math.round(x * 10000) / 10000;
+}
+//# sourceMappingURL=human-gold.js.map

package/dist/src/grading/human-gold.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"human-gold.js","sourceRoot":"","sources":["../../../src/grading/human-gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,mBAAmB,EAAoB,MAAM,2BAA2B,CAAC;AAwBlF;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAmB;IAC1D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAEnC,yEAAyE;IACzE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1D,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO;IAE3C,2EAA2E;IAC3E,+EAA+E;IAC/E,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,WAAW,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChD,CAAC;IACD,MAAM,EAAE,GAAG,WAAW,GAAG,SAAS,CAAC;IAEnC,8EAA8E;IAC9E,6BAA6B;IAC7B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;YAC5B,WAAW,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,SAAS,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAC5B,MAAM,EAAE,GAAG,WAAW,GAAG,KAAK,CAAC;IAC/B,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC,CAAC,wCAAwC;IAElE,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;AACrB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAAmB,EACnB,QAAsC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;IAExD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IACpC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IAC3B,MAAM,MAAM,GAAG,KAAK,GAAG,KAAK,CAAC;IAE7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpD,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IAEvB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC;QAChD,UAAU,IAAI,CAAC,GAAG,CAAC,CAAC;IACtB,CAAC;IACD,UAAU,IAAI,CAAC,CAAC;IAEhB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;QAC5B,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC;YAClC,UAAU,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,UAAU,IAAI,CAAC,GAAG,CAAC,CAAC;IAEpB,IAAI,UAAU,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IACjC,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,KAAmB;IAChD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACjC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;QACjB,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;IACnB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;IACvB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;IACvB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;QAC5B,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;QAC5B,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IACzC,OAAO,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;AACtC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAAmB,EACnB,UAAqG,EAAE;IAEvG,MAAM,EAAE,OAAO,GAAG,IAAI,EAAE,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IAE9D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO;YACL,KAAK,EAAE,GAAG;YACV,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE;YAC3D,aAAa,EAAE,GAAG;YAClB,OAAO,EAAE,GAAG;YACZ,WAAW,EAAE,CAAC;SACf,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,mBAAmB,CACjC,OAAO,EACP,CAAC,YAAY,EAAE,EAAE,CAAC,wBAAwB,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAC7E,KAAK,EACL,OAAO,EACP,IAAI,CACL,CAAC;IAEF,OAAO;QACL,KAAK,EAAE,UAAU,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAClD,OAAO;QACP,aAAa,EAAE,UAAU,CAAC,oBAAoB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC7D,OAAO,EAAE,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAC1C,WAAW,EAAE,KAAK,CAAC,MAAM;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IAChC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC;AACvC,CAAC"}

package/dist/src/grading/index.d.ts

@@ -1,7 +1,7 @@
 /**
  * Mixed grading: deterministic assertions + LLM judge + multi-dimensional scoring.
  */
-import type { GradeResult, ExecutorFn, Sample, ToolCallInfo, TurnInfo } from '../types.js';
+import type { GradeResult, ExecutorFn, JudgeConfig, Sample, ToolCallInfo, TurnInfo } from '../types.js';
 interface GradeOptions {
     output: string;
     sample: Sample;
@@ -16,11 +16,36 @@ interface GradeOptions {
         turns?: TurnInfo[];
     };
     samplesDir?: string;
+    /**
+     * Run the LLM judge N times per (sample × dimension) pair and report mean + stddev.
+     * Default 1 (single judge call). Useful for measuring judge self-consistency:
+     * a high stddev means the judge isn't stable on this rubric and the score is noisy.
+     */
+    judgeRepeat?: number;
+    /**
+     * Multi-judge ensemble: if provided with >= 2 judges, every (sample × dimension) is
+     * scored by every judge. Each judge can use a different executor (claude/openai/etc)
+     * and model. Output gets per-judge breakdown + Pearson/MAD agreement metrics — used
+     * to refute "Claude judge Claude same-modality bias" critique.
+     */
+    judgeModels?: JudgeConfig[];
+    /**
+     * Map from executor name → ExecutorFn. Required when judgeModels has judges with
+     * different executor strings. Pipeline layer pre-creates these so grade() stays pure.
+     */
+    judgeExecutors?: Record<string, ExecutorFn>;
+    /**
+     * v0.21 length-debias toggle. Defaults to true — judge prompt includes the
+     * "length is not a quality signal" instruction, prompt template version is
+     * v3-cot-length. Set false (via `--no-debias-length`) to revert to the
+     * legacy v2-cot prompt for reproducing historical reports.
+     */
+    lengthDebias?: boolean;
 }
 /**
  * Grade a model output against a sample's criteria.
  */
-export declare function grade({ output, sample, executor, judgeModel, allowLlmJudge, execMetrics, samplesDir }: GradeOptions): Promise<GradeResult>;
+export declare function grade({ output, sample, executor, judgeModel, allowLlmJudge, execMetrics, samplesDir, judgeRepeat, judgeModels, judgeExecutors, lengthDebias }: GradeOptions): Promise<GradeResult>;
 export { runAssertions, validateJsonSchema } from './assertions.js';
 export { buildTraceSummary } from './judge.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/grading/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/grading/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAoB,WAAW,EAAkC,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAK7I,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAA;KAAE,CAAC;IAC3H,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,wBAAsB,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAoB,EAAE,WAAgB,EAAE,UAAgB,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA+FlK;AACD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/grading/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAKxG,UAAU,YAAY;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,UAAU,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,YAAY,EAAE,CAAC;QAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAA;KAAE,CAAC;IAC3H,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,WAAW,EAAE,CAAC;IAC5B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC5C;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,wBAAsB,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAoB,EAAE,WAAgB,EAAE,UAAgB,EAAE,WAAe,EAAE,WAAW,EAAE,cAAc,EAAE,YAAmB,EAAE,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAuGrO;AACD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}