@tangle-network/agent-eval 0.204.0 → 0.205.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/dist/analyst/index.d.ts +1 -1
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +1 -1
- package/dist/{analyze-runs-BrIibNd0.js → analyze-runs-d5dcMlxi.js} +9 -9
- package/dist/{analyze-runs-BrIibNd0.js.map → analyze-runs-d5dcMlxi.js.map} +1 -1
- package/dist/baseline-Dm8966du.js +259 -0
- package/dist/baseline-Dm8966du.js.map +1 -0
- package/dist/{benchmark-command-Jv_puUGM.js → benchmark-command-BQhi_BVO.js} +6 -3
- package/dist/benchmark-command-BQhi_BVO.js.map +1 -0
- package/dist/benchmarks/index.js +2 -2
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.js +9 -9
- package/dist/{campaign-DYhg4SjN.js → campaign-0lmkAMpE.js} +11 -11
- package/dist/{campaign-DYhg4SjN.js.map → campaign-0lmkAMpE.js.map} +1 -1
- package/dist/{campaign-evidence-Dt_rWPk2.js → campaign-evidence-C__cUpCt.js} +2 -2
- package/dist/{campaign-evidence-Dt_rWPk2.js.map → campaign-evidence-C__cUpCt.js.map} +1 -1
- package/dist/cli.js +5 -5
- package/dist/contract/index.js +9 -9
- package/dist/{define-agent-eval-B25Cc3ki.js → define-agent-eval-CeYrXpl7.js} +4 -4
- package/dist/{define-agent-eval-B25Cc3ki.js.map → define-agent-eval-CeYrXpl7.js.map} +1 -1
- package/dist/{descriptive-D9LUFk1-.js → descriptive-CnU8Yq29.js} +2 -2
- package/dist/{descriptive-D9LUFk1-.js.map → descriptive-CnU8Yq29.js.map} +1 -1
- package/dist/diagnosis/index.js +1 -1
- package/dist/{eval-campaign-ha_aWtk7.js → eval-campaign-DsRmYsQI.js} +2 -2
- package/dist/{eval-campaign-ha_aWtk7.js.map → eval-campaign-DsRmYsQI.js.map} +1 -1
- package/dist/experiment/index.js +9 -9
- package/dist/index.d.ts +60 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +96 -19
- package/dist/index.js.map +1 -1
- package/dist/{internal-BOe3zmjQ.js → internal-CRHwS9iz.js} +3 -3
- package/dist/internal-CRHwS9iz.js.map +1 -0
- package/dist/jev.js +1 -1
- package/dist/judge-calibration-DFtEMlde.d.ts.map +1 -1
- package/dist/{judge-calibration-C5zgtiJT.js → judge-calibration-N44V04Se.js} +7 -17
- package/dist/judge-calibration-N44V04Se.js.map +1 -0
- package/dist/{llm-judge-DyZdrp6I.js → llm-judge-Bka5q-ds.js} +8 -8
- package/dist/{llm-judge-DyZdrp6I.js.map → llm-judge-Bka5q-ds.js.map} +1 -1
- package/dist/matrix/index.js +1 -1
- package/dist/{matrix-DnlHn5x1.js → matrix-_BV9-Lk5.js} +12 -9
- package/dist/matrix-_BV9-Lk5.js.map +1 -0
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/golden/index.js +1 -1
- package/dist/multishot/index.js +1 -1
- package/dist/normal-DsJZtrJc.js +25 -0
- package/dist/normal-DsJZtrJc.js.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/{paired-arms-DpS_h8ON.js → paired-arms-Dsu-r6xd.js} +6 -27
- package/dist/paired-arms-Dsu-r6xd.js.map +1 -0
- package/dist/{paired-binary-OMBVRtc-.js → paired-binary-DpwwK1fU.js} +2 -2
- package/dist/{paired-binary-OMBVRtc-.js.map → paired-binary-DpwwK1fU.js.map} +1 -1
- package/dist/{paired-promotion-decision-DhrmjdV8.js → paired-promotion-decision-CO0kA0bq.js} +3 -3
- package/dist/{paired-promotion-decision-DhrmjdV8.js.map → paired-promotion-decision-CO0kA0bq.js.map} +1 -1
- package/dist/{paired-tests-Ccr16gcW.js → paired-tests-jbwPdSz5.js} +3 -3
- package/dist/{paired-tests-Ccr16gcW.js.map → paired-tests-jbwPdSz5.js.map} +1 -1
- package/dist/pipelines/index.d.ts +1 -1
- package/dist/pipelines/index.js +2 -2
- package/dist/{power-and-mde-bie6i965.js → power-and-mde-fK5Qocwo.js} +3 -3
- package/dist/{power-and-mde-bie6i965.js.map → power-and-mde-fK5Qocwo.js.map} +1 -1
- package/dist/{produced-state-DLToUXrD.js → produced-state-FdbE-88t.js} +3 -3
- package/dist/{produced-state-DLToUXrD.js.map → produced-state-FdbE-88t.js.map} +1 -1
- package/dist/{promotion-policy-BI1RxzwD.js → promotion-policy-BSbTAMtE.js} +4 -4
- package/dist/{promotion-policy-BI1RxzwD.js.map → promotion-policy-BSbTAMtE.js.map} +1 -1
- package/dist/quantile--0yb07pD.js +15 -0
- package/dist/quantile--0yb07pD.js.map +1 -0
- package/dist/{report-command-D0XL2Cvi.js → report-command-JrjiWReS.js} +2 -2
- package/dist/{report-command-D0XL2Cvi.js.map → report-command-JrjiWReS.js.map} +1 -1
- package/dist/reporting.js +4 -4
- package/dist/{reward-hacking-C-a__ShZ.js → reward-hacking-BFcQgkux.js} +2 -2
- package/dist/{reward-hacking-C-a__ShZ.js.map → reward-hacking-BFcQgkux.js.map} +1 -1
- package/dist/rl.js +7 -7
- package/dist/{rubric-predictive-validity-B0Pg-3I5.js → rubric-predictive-validity-BsgkBfT0.js} +3 -3
- package/dist/{rubric-predictive-validity-B0Pg-3I5.js.map → rubric-predictive-validity-BsgkBfT0.js.map} +1 -1
- package/dist/search/index.js +2 -2
- package/dist/{search-kernel-Dpniloks.js → search-kernel-CsISc5t8.js} +5 -5
- package/dist/{search-kernel-Dpniloks.js.map → search-kernel-CsISc5t8.js.map} +1 -1
- package/dist/{search-summary-C1nAHgkB.js → search-summary-BZkoLwbB.js} +4 -4
- package/dist/{search-summary-C1nAHgkB.js.map → search-summary-BZkoLwbB.js.map} +1 -1
- package/dist/{sequential-DWn_np6P.js → sequential-7uxpO0Vm.js} +2 -2
- package/dist/{sequential-DWn_np6P.js.map → sequential-7uxpO0Vm.js.map} +1 -1
- package/dist/{skillopt-optimization-method-Dz5CGZb8.js → skillopt-optimization-method-bEm4jYV4.js} +4 -4
- package/dist/{skillopt-optimization-method-Dz5CGZb8.js.map → skillopt-optimization-method-bEm4jYV4.js.map} +1 -1
- package/dist/{statistical-heldout-CNLOvRq_.js → statistical-heldout-C26hOFWD.js} +3 -3
- package/dist/{statistical-heldout-CNLOvRq_.js.map → statistical-heldout-C26hOFWD.js.map} +1 -1
- package/dist/{student-t-BBZYka0E.js → student-t-BQeKJx2N.js} +2 -2
- package/dist/{student-t-BBZYka0E.js.map → student-t-BQeKJx2N.js.map} +1 -1
- package/dist/{summary-report-DLzptLSA.js → summary-report-BG6-hOkm.js} +6 -6
- package/dist/{summary-report-DLzptLSA.js.map → summary-report-BG6-hOkm.js.map} +1 -1
- package/dist/supervisor-run/index.js +1 -1
- package/dist/{tool-waste-CP5DO383.d.ts → tool-waste-DxlebQkx.d.ts} +42 -2
- package/dist/tool-waste-DxlebQkx.d.ts.map +1 -0
- package/dist/{tool-waste-B-u3rzCz.js → tool-waste-x_E8h9Ih.js} +3 -3
- package/dist/{tool-waste-B-u3rzCz.js.map → tool-waste-x_E8h9Ih.js.map} +1 -1
- package/dist/trace-repair/index.js +1 -1
- package/dist/{tree-D48IJUK0.js → tree-CAQflhLR.js} +5 -5
- package/dist/{tree-D48IJUK0.js.map → tree-CAQflhLR.js.map} +1 -1
- package/docs/concepts.md +21 -0
- package/package.json +1 -1
- package/dist/baseline-CVKsfWTg.js +0 -150
- package/dist/baseline-CVKsfWTg.js.map +0 -1
- package/dist/benchmark-command-Jv_puUGM.js.map +0 -1
- package/dist/internal-BOe3zmjQ.js.map +0 -1
- package/dist/judge-calibration-C5zgtiJT.js.map +0 -1
- package/dist/matrix-DnlHn5x1.js.map +0 -1
- package/dist/paired-arms-DpS_h8ON.js.map +0 -1
- package/dist/tool-waste-CP5DO383.d.ts.map +0 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { s as ValidationError } from "./errors-Dngq5h35.js";
|
|
2
|
+
import { t as interpolatedQuantile } from "./quantile--0yb07pD.js";
|
|
2
3
|
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
3
4
|
//#region src/math/special-functions.ts
|
|
4
5
|
/** Lanczos approximation to ln Gamma(z). */
|
|
@@ -173,8 +174,7 @@ function zQuantile(p) {
|
|
|
173
174
|
function medianInPlace(xs) {
|
|
174
175
|
if (xs.length === 0) return 0;
|
|
175
176
|
xs.sort((a, b) => a - b);
|
|
176
|
-
|
|
177
|
-
return xs.length % 2 === 0 ? (xs[mid - 1] + xs[mid]) / 2 : xs[mid];
|
|
177
|
+
return interpolatedQuantile(xs, .5);
|
|
178
178
|
}
|
|
179
179
|
/**
|
|
180
180
|
* PRNG for every resampling path in this module.
|
|
@@ -211,4 +211,4 @@ function symmetricTwoSampleSeed(a, b) {
|
|
|
211
211
|
//#endregion
|
|
212
212
|
export { medianInPlace as a, lnGamma as c, makeRng as i, regularizedIncompleteBeta as l, binomialHalfUpperTail as n, symmetricTwoSampleSeed as o, binomialSignTwoSided as r, zQuantile as s, assertFiniteSample as t };
|
|
213
213
|
|
|
214
|
-
//# sourceMappingURL=internal-
|
|
214
|
+
//# sourceMappingURL=internal-CRHwS9iz.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"internal-CRHwS9iz.js","names":[],"sources":["../src/math/special-functions.ts","../src/statistics/internal.ts"],"sourcesContent":["/** Lanczos approximation to ln Gamma(z). */\nexport function lnGamma(z: number): number {\n const g = 7\n const coefficients = [\n 0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313,\n -176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6,\n 1.5056327351493116e-7,\n ]\n if (z < 0.5) {\n return Math.log(Math.PI / Math.sin(Math.PI * z)) - lnGamma(1 - z)\n }\n z -= 1\n let x = coefficients[0]!\n for (let i = 1; i < g + 2; i++) x += coefficients[i]! / (z + i)\n const t = z + g + 0.5\n return 0.5 * Math.log(2 * Math.PI) + (z + 0.5) * Math.log(t) - t + Math.log(x)\n}\n\n/**\n * Regularized incomplete beta function I_x(a, b).\n *\n * The Lentz continued fraction converges only for `x < (a+1)/(a+b+2)`; outside\n * that domain it must be reached through the symmetry `I_x(a,b) = 1 −\n * I_{1−x}(b,a)`. `studentTCdf` drives `x → 1` as `|t| → 0`, so the mirrored\n * branch is the one every near-null t-statistic takes.\n */\nexport function regularizedIncompleteBeta(x: number, a: number, b: number): number {\n if (x <= 0) return 0\n if (x >= 1) return 1\n const logBeta = lnGamma(a) + lnGamma(b) - lnGamma(a + b)\n // x^a·(1−x)^b / B(a,b) — symmetric under (x,a,b) → (1−x,b,a), so the\n // mirrored branch reuses it unchanged.\n const front = Math.exp(Math.log(x) * a + Math.log(1 - x) * b - logBeta)\n if (x < (a + 1) / (a + b + 2)) {\n return (front * betaContinuedFraction(x, a, b)) / a\n }\n return 1 - (front * betaContinuedFraction(1 - x, b, a)) / b\n}\n\n/** Modified Lentz evaluation of the beta continued fraction at `x`. */\nfunction betaContinuedFraction(x: number, a: number, b: number): number {\n const maxIterations = 300\n const epsilon = 3e-15\n let c = 1\n let d = 1 - ((a + b) * x) / (a + 1)\n if (Math.abs(d) < 1e-30) d = 1e-30\n d = 1 / d\n let fraction = d\n for (let m = 1; m <= maxIterations; m++) {\n const m2 = 2 * m\n let numerator = (m * (b - m) * x) / ((a + m2 - 1) * (a + m2))\n d = 1 + numerator * d\n if (Math.abs(d) < 1e-30) d = 1e-30\n c = 1 + numerator / c\n if (Math.abs(c) < 1e-30) c = 1e-30\n d = 1 / d\n fraction *= d * c\n numerator = -((a + m) * (a + b + m) * x) / ((a + m2) * (a + m2 + 1))\n d = 1 + numerator * d\n if (Math.abs(d) < 1e-30) d = 1e-30\n c = 1 + numerator / c\n if (Math.abs(c) < 1e-30) c = 1e-30\n d = 1 / d\n const delta = d * c\n fraction *= delta\n if (Math.abs(delta - 1) < epsilon) break\n }\n return fraction\n}\n","/**\n * Shared internals for the statistics modules. Nothing here is part of the\n * public statistics surface — `./index` does not re-export this module.\n */\n\nimport { ValidationError } from '../errors'\nimport { interpolatedQuantile } from '../math/quantile'\nimport { lnGamma } from '../math/special-functions'\nimport { mulberry32 } from './random'\n\n/** Every rank test refuses non-finite input. Beyond the arithmetic being\n * undefined, the tie-grouping scan compares values with `===`, and\n * `NaN === NaN` is false, so a NaN would leave the group boundary unable to\n * advance and spin the loop forever. */\nexport function assertFiniteSample(fn: string, label: string, xs: readonly number[]): void {\n for (let i = 0; i < xs.length; i++) {\n if (!Number.isFinite(xs[i])) {\n throw new ValidationError(`${fn}: ${label}[${i}] must be finite, got ${xs[i]}`)\n }\n }\n}\n\n/**\n * Two-sided exact p-value for b successes out of (b + c) Bernoulli(0.5) trials —\n * the exact-binomial core of {@link mcnemar}. `min(1, 2·P(X ≤ min(b,c)))`. No\n * discordant pairs ⇒ no evidence ⇒ p = 1. Summed in log space (lnGamma) so it\n * stays exact at large discordant counts without overflow.\n */\nexport function binomialSignTwoSided(b: number, c: number): number {\n const nd = b + c\n if (nd === 0) return 1\n return Math.min(1, 2 * binomialHalfLowerTail(Math.min(b, c), nd))\n}\n\n/** P(X >= successes) for X ~ Binomial(n, 0.5). */\nexport function binomialHalfUpperTail(successes: number, n: number): number {\n if (successes <= 0) return 1\n if (successes > n) return 0\n\n // Use the smaller side of the distribution. For lower thresholds the\n // complement sums at most half the mass; for upper thresholds symmetry\n // maps the upper tail to a lower tail without subtraction.\n if (successes <= n / 2) {\n return Math.max(0, 1 - binomialHalfLowerTail(successes - 1, n))\n }\n return binomialHalfLowerTail(n - successes, n)\n}\n\n/** P(X <= maxSuccesses) for X ~ Binomial(n, 0.5), accumulated in log space. */\nfunction binomialHalfLowerTail(maxSuccesses: number, n: number): number {\n if (maxSuccesses < 0) return 0\n if (maxSuccesses >= n) return 1\n if (maxSuccesses === 0) return 2 ** -n\n\n const logHalfN = n * Math.log(0.5)\n let logTail = Number.NEGATIVE_INFINITY\n for (let i = 0; i <= maxSuccesses; i++) {\n const logChoose = lnGamma(n + 1) - lnGamma(i + 1) - lnGamma(n - i + 1)\n logTail = logAddExp(logTail, logChoose + logHalfN)\n }\n return Math.min(1, Math.exp(logTail))\n}\n\nfunction logAddExp(a: number, b: number): number {\n if (a === Number.NEGATIVE_INFINITY) return b\n if (b === Number.NEGATIVE_INFINITY) return a\n const max = Math.max(a, b)\n const min = Math.min(a, b)\n return max + Math.log1p(Math.exp(min - max))\n}\n\n/** Standard-normal inverse CDF (Acklam approximation). */\nexport function zQuantile(p: number): number {\n if (p <= 0 || p >= 1) {\n if (p === 0) return -Infinity\n if (p === 1) return Infinity\n return NaN\n }\n const a = [\n -3.969683028665376e1, 2.209460984245205e2, -2.759285104469687e2, 1.38357751867269e2,\n -3.066479806614716e1, 2.506628277459239,\n ]\n const b = [\n -5.447609879822406e1, 1.615858368580409e2, -1.556989798598866e2, 6.680131188771972e1,\n -1.328068155288572e1,\n ]\n const c = [\n -7.784894002430293e-3, -3.223964580411365e-1, -2.400758277161838, -2.549732539343734,\n 4.374664141464968, 2.938163982698783,\n ]\n const d = [7.784695709041462e-3, 3.224671290700398e-1, 2.445134137142996, 3.754408661907416]\n const pLow = 0.02425\n const pHigh = 1 - pLow\n let q: number\n let r: number\n if (p < pLow) {\n q = Math.sqrt(-2 * Math.log(p))\n return (\n (((((c[0]! * q + c[1]!) * q + c[2]!) * q + c[3]!) * q + c[4]!) * q + c[5]!) /\n ((((d[0]! * q + d[1]!) * q + d[2]!) * q + d[3]!) * q + 1)\n )\n }\n if (p <= pHigh) {\n q = p - 0.5\n r = q * q\n return (\n ((((((a[0]! * r + a[1]!) * r + a[2]!) * r + a[3]!) * r + a[4]!) * r + a[5]!) * q) /\n (((((b[0]! * r + b[1]!) * r + b[2]!) * r + b[3]!) * r + b[4]!) * r + 1)\n )\n }\n q = Math.sqrt(-2 * Math.log(1 - p))\n return (\n -(((((c[0]! * q + c[1]!) * q + c[2]!) * q + c[3]!) * q + c[4]!) * q + c[5]!) /\n ((((d[0]! * q + d[1]!) * q + d[2]!) * q + d[3]!) * q + 1)\n )\n}\n\nexport function medianInPlace(xs: number[]): number {\n if (xs.length === 0) return 0\n xs.sort((a, b) => a - b)\n return interpolatedQuantile(xs, 0.5)\n}\n\n/**\n * PRNG for every resampling path in this module.\n *\n * With no caller seed the seed is DERIVED FROM THE DATA rather than taken from\n * `Math.random`, so re-running the same input reproduces the same interval —\n * a gate verdict that cannot be re-derived is not evidence. Distinct data\n * still gets a distinct stream. Same pattern as `promotion-gate.ts`.\n */\nexport function makeRng(\n seed: number | undefined,\n ...series: readonly (readonly number[])[]\n): () => number {\n return mulberry32(seed ?? seedFromData(series))\n}\n\n/** FNV-1a over the IEEE-754 bytes of every observation. */\nfunction seedFromData(series: readonly (readonly number[])[]): number {\n const view = new DataView(new ArrayBuffer(8))\n // 32-bit FNV-1a over the IEEE-754 BYTES of each observation, not over a\n // string. Frozen: it seeds the unseeded bootstrap from the data itself, so a\n // change moves every interval this package has already published. The\n // string-input variants elsewhere cannot be substituted here.\n let hash = 0x811c9dc5\n for (const xs of series) {\n for (const x of xs) {\n view.setFloat64(0, x)\n for (let byte = 0; byte < 8; byte++) {\n hash = Math.imul(hash ^ view.getUint8(byte), 0x01000193)\n }\n }\n // Separator, so ([1],[2]) and ([1,2],[]) do not collide.\n hash = Math.imul(hash ^ 0xff, 0x01000193)\n }\n return hash | 0\n}\n\n/** Order-independent seed for a symmetric two-sample statistic. */\nexport function symmetricTwoSampleSeed(a: readonly number[], b: readonly number[]): number {\n const sortedA = [...a].sort((left, right) => left - right)\n const sortedB = [...b].sort((left, right) => left - right)\n const forward = seedFromData([sortedA, sortedB]) >>> 0\n const reversed = seedFromData([sortedB, sortedA]) >>> 0\n return Math.min(forward, reversed)\n}\n"],"mappings":";;;;;AACA,SAAgB,QAAQ,GAAmB;CACzC,MAAM,IAAI;CACV,MAAM,eAAe;EACnB;EAAqB;EAAmB;EAAqB;EAC7D;EAAqB;EAAoB;EAAsB;EAC/D;CACF;CACA,IAAI,IAAI,IACN,OAAO,KAAK,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,QAAQ,IAAI,CAAC;CAElE,KAAK;CACL,IAAI,IAAI,aAAa;CACrB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAO,KAAK,KAAK,aAAa,MAAO,IAAI;CAC7D,MAAM,IAAI,IAAI,IAAI;CAClB,OAAO,KAAM,KAAK,IAAI,IAAI,KAAK,EAAE,KAAK,IAAI,MAAO,KAAK,IAAI,CAAC,IAAI,IAAI,KAAK,IAAI,CAAC;AAC/E;;;;;;;;;AAUA,SAAgB,0BAA0B,GAAW,GAAW,GAAmB;CACjF,IAAI,KAAK,GAAG,OAAO;CACnB,IAAI,KAAK,GAAG,OAAO;CACnB,MAAM,UAAU,QAAQ,CAAC,IAAI,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC;CAGvD,MAAM,QAAQ,KAAK,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO;CACtE,IAAI,KAAK,IAAI,MAAM,IAAI,IAAI,IACzB,OAAQ,QAAQ,sBAAsB,GAAG,GAAG,CAAC,IAAK;CAEpD,OAAO,IAAK,QAAQ,sBAAsB,IAAI,GAAG,GAAG,CAAC,IAAK;AAC5D;;AAGA,SAAS,sBAAsB,GAAW,GAAW,GAAmB;CACtE,MAAM,gBAAgB;CACtB,MAAM,UAAU;CAChB,IAAI,IAAI;CACR,IAAI,IAAI,KAAM,IAAI,KAAK,KAAM,IAAI;CACjC,IAAI,KAAK,IAAI,CAAC,IAAI,OAAO,IAAI;CAC7B,IAAI,IAAI;CACR,IAAI,WAAW;CACf,KAAK,IAAI,IAAI,GAAG,KAAK,eAAe,KAAK;EACvC,MAAM,KAAK,IAAI;EACf,IAAI,YAAa,KAAK,IAAI,KAAK,MAAO,IAAI,KAAK,MAAM,IAAI;EACzD,IAAI,IAAI,YAAY;EACpB,IAAI,KAAK,IAAI,CAAC,IAAI,OAAO,IAAI;EAC7B,IAAI,IAAI,YAAY;EACpB,IAAI,KAAK,IAAI,CAAC,IAAI,OAAO,IAAI;EAC7B,IAAI,IAAI;EACR,YAAY,IAAI;EAChB,YAAY,GAAG,IAAI,MAAM,IAAI,IAAI,KAAK,OAAO,IAAI,OAAO,IAAI,KAAK;EACjE,IAAI,IAAI,YAAY;EACpB,IAAI,KAAK,IAAI,CAAC,IAAI,OAAO,IAAI;EAC7B,IAAI,IAAI,YAAY;EACpB,IAAI,KAAK,IAAI,CAAC,IAAI,OAAO,IAAI;EAC7B,IAAI,IAAI;EACR,MAAM,QAAQ,IAAI;EAClB,YAAY;EACZ,IAAI,KAAK,IAAI,QAAQ,CAAC,IAAI,SAAS;CACrC;CACA,OAAO;AACT;;;;;;;;;;;ACtDA,SAAgB,mBAAmB,IAAY,OAAe,IAA6B;CACzF,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,QAAQ,KAC7B,IAAI,CAAC,OAAO,SAAS,GAAG,EAAE,GACxB,MAAM,IAAI,gBAAgB,GAAG,GAAG,IAAI,MAAM,GAAG,EAAE,wBAAwB,GAAG,IAAI;AAGpF;;;;;;;AAQA,SAAgB,qBAAqB,GAAW,GAAmB;CACjE,MAAM,KAAK,IAAI;CACf,IAAI,OAAO,GAAG,OAAO;CACrB,OAAO,KAAK,IAAI,GAAG,IAAI,sBAAsB,KAAK,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;AAClE;;AAGA,SAAgB,sBAAsB,WAAmB,GAAmB;CAC1E,IAAI,aAAa,GAAG,OAAO;CAC3B,IAAI,YAAY,GAAG,OAAO;CAK1B,IAAI,aAAa,IAAI,GACnB,OAAO,KAAK,IAAI,GAAG,IAAI,sBAAsB,YAAY,GAAG,CAAC,CAAC;CAEhE,OAAO,sBAAsB,IAAI,WAAW,CAAC;AAC/C;;AAGA,SAAS,sBAAsB,cAAsB,GAAmB;CACtE,IAAI,eAAe,GAAG,OAAO;CAC7B,IAAI,gBAAgB,GAAG,OAAO;CAC9B,IAAI,iBAAiB,GAAG,OAAO,KAAK,CAAC;CAErC,MAAM,WAAW,IAAI,KAAK,IAAI,EAAG;CACjC,IAAI,UAAU,OAAO;CACrB,KAAK,IAAI,IAAI,GAAG,KAAK,cAAc,KAAK;EACtC,MAAM,YAAY,QAAQ,IAAI,CAAC,IAAI,QAAQ,IAAI,CAAC,IAAI,QAAQ,IAAI,IAAI,CAAC;EACrE,UAAU,UAAU,SAAS,YAAY,QAAQ;CACnD;CACA,OAAO,KAAK,IAAI,GAAG,KAAK,IAAI,OAAO,CAAC;AACtC;AAEA,SAAS,UAAU,GAAW,GAAmB;CAC/C,IAAI,MAAM,OAAO,mBAAmB,OAAO;CAC3C,IAAI,MAAM,OAAO,mBAAmB,OAAO;CAC3C,MAAM,MAAM,KAAK,IAAI,GAAG,CAAC;CAEzB,OAAO,MAAM,KAAK,MAAM,KAAK,IADjB,KAAK,IAAI,GAAG,CACW,IAAI,GAAG,CAAC;AAC7C;;AAGA,SAAgB,UAAU,GAAmB;CAC3C,IAAI,KAAK,KAAK,KAAK,GAAG;EACpB,IAAI,MAAM,GAAG,OAAO;EACpB,IAAI,MAAM,GAAG,OAAO;EACpB,OAAO;CACT;CACA,MAAM,IAAI;EACR;EAAsB;EAAqB;EAAsB;EACjE;EAAsB;CACxB;CACA,MAAM,IAAI;EACR;EAAsB;EAAqB;EAAsB;EACjE;CACF;CACA,MAAM,IAAI;EACR;EAAuB;EAAuB;EAAoB;EAClE;EAAmB;CACrB;CACA,MAAM,IAAI;EAAC;EAAsB;EAAsB;EAAmB;CAAiB;CAC3F,MAAM,OAAO;CACb,MAAM,QAAQ;CACd,IAAI;CACJ,IAAI;CACJ,IAAI,IAAI,MAAM;EACZ,IAAI,KAAK,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC;EAC9B,YACO,EAAE,KAAM,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,UACnE,EAAE,KAAM,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI;CAE3D;CACA,IAAI,KAAK,OAAO;EACd,IAAI,IAAI;EACR,IAAI,IAAI;EACR,YACQ,EAAE,KAAM,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,SAC1E,EAAE,KAAM,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI;CAEzE;CACA,IAAI,KAAK,KAAK,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC;CAClC,OACE,MAAM,EAAE,KAAM,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,UACpE,EAAE,KAAM,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI,EAAE,MAAO,IAAI;AAE3D;AAEA,SAAgB,cAAc,IAAsB;CAClD,IAAI,GAAG,WAAW,GAAG,OAAO;CAC5B,GAAG,MAAM,GAAG,MAAM,IAAI,CAAC;CACvB,OAAO,qBAAqB,IAAI,EAAG;AACrC;;;;;;;;;AAUA,SAAgB,QACd,MACA,GAAG,QACW;CACd,OAAO,WAAW,QAAQ,aAAa,MAAM,CAAC;AAChD;;AAGA,SAAS,aAAa,QAAgD;CACpE,MAAM,uBAAO,IAAI,yBAAS,IAAI,YAAY,CAAC,CAAC;CAK5C,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,KAAK,MAAM,KAAK,IAAI;GAClB,KAAK,WAAW,GAAG,CAAC;GACpB,KAAK,IAAI,OAAO,GAAG,OAAO,GAAG,QAC3B,OAAO,KAAK,KAAK,OAAO,KAAK,SAAS,IAAI,GAAG,QAAU;EAE3D;EAEA,OAAO,KAAK,KAAK,OAAO,KAAM,QAAU;CAC1C;CACA,OAAO,OAAO;AAChB;;AAGA,SAAgB,uBAAuB,GAAsB,GAA8B;CACzF,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,MAAM,UAAU,OAAO,KAAK;CACzD,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,MAAM,UAAU,OAAO,KAAK;CACzD,MAAM,UAAU,aAAa,CAAC,SAAS,OAAO,CAAC,MAAM;CACrD,MAAM,WAAW,aAAa,CAAC,SAAS,OAAO,CAAC,MAAM;CACtD,OAAO,KAAK,IAAI,SAAS,QAAQ;AACnC"}
|
package/dist/jev.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { o as jsonDocument } from "./canonical-BsZVgBhW.js";
|
|
2
|
-
import { s as weightedComposite } from "./descriptive-
|
|
2
|
+
import { s as weightedComposite } from "./descriptive-CnU8Yq29.js";
|
|
3
3
|
import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
|
|
4
4
|
import { asAnalyst, asJudge, createEvaluator } from "./evaluation.js";
|
|
5
5
|
import { JevResponseError, jevUsage, parseJevQuestions, parseJevRecordedRequest, parseJevRequest, parseJevResult } from "./jev-protocol.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge-calibration-DFtEMlde.d.ts","names":[],"sources":["../src/judge-calibration.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"judge-calibration-DFtEMlde.d.ts","names":[],"sources":["../src/judge-calibration.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;UAyBiB;EACf;EACA;;EAEA;;UAGe;EACf;EACA;;EAEA;;UAGe;EACf;EACA;;EAEA;;EAEA;;EAEA,YAAY;IAAQ;IAAgB;IAAe;IAAe;;;;;;iBAMpD,eACd,QAAQ,cACR,WAAW,mBACV;UA0Bc;;;;;EAKf;EACA;;;;;;iBAOc,eAAe,QAAQ,mBAAmB;UAgBzC;;EAEf;EACA;;iBAGc,cACd,SAAS;EAAQ;EAAmB;KACnC;UAYc;;EAEf;EACA;EACA;EACA;;;;;;;iBAQc,eACd,SAAS;EAAQ;EAAe;KAC/B;UA8Ec;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;IACE;IACA;;;EAGF;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;;;;;;;;iBAUc,oBACd,oBACA,OAAM,6BACL;UAqEc,oCAAoC;;EAEnD;;EAEA;EACA;EACA;IACE;IACA;;;;;;;iBAQY,yBACd,QAAQ,cACR,WAAW,kBACX,OAAM,6BACL"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { t as interpolatedQuantile } from "./quantile--0yb07pD.js";
|
|
1
2
|
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
2
|
-
import { a as spearmanR, r as pearsonR } from "./descriptive-
|
|
3
|
+
import { a as spearmanR, r as pearsonR } from "./descriptive-CnU8Yq29.js";
|
|
3
4
|
//#region src/judge-calibration.ts
|
|
4
5
|
/**
|
|
5
6
|
* Judge calibration — measure judge quality against human gold + bias.
|
|
@@ -192,13 +193,13 @@ function continuousAgreement(scores, opts = {}) {
|
|
|
192
193
|
const [lo, hi] = percentileBounds(ciLevel);
|
|
193
194
|
if (iccs.length > 0) {
|
|
194
195
|
iccs.sort((a, b) => a - b);
|
|
195
|
-
ciIcc[0] =
|
|
196
|
-
ciIcc[1] =
|
|
196
|
+
ciIcc[0] = interpolatedQuantile(iccs, lo);
|
|
197
|
+
ciIcc[1] = interpolatedQuantile(iccs, hi);
|
|
197
198
|
}
|
|
198
199
|
if (kappas.length > 0) {
|
|
199
200
|
kappas.sort((a, b) => a - b);
|
|
200
|
-
ciKappa[0] =
|
|
201
|
-
ciKappa[1] =
|
|
201
|
+
ciKappa[0] = interpolatedQuantile(kappas, lo);
|
|
202
|
+
ciKappa[1] = interpolatedQuantile(kappas, hi);
|
|
202
203
|
}
|
|
203
204
|
}
|
|
204
205
|
return {
|
|
@@ -336,18 +337,7 @@ function percentileBounds(ciLevel) {
|
|
|
336
337
|
const tail = (1 - ciLevel) / 2;
|
|
337
338
|
return [tail, 1 - tail];
|
|
338
339
|
}
|
|
339
|
-
/** Linear-interpolated quantile of a pre-sorted ascending array. */
|
|
340
|
-
function quantile(sorted, q) {
|
|
341
|
-
if (sorted.length === 0) return NaN;
|
|
342
|
-
if (sorted.length === 1) return sorted[0];
|
|
343
|
-
const pos = q * (sorted.length - 1);
|
|
344
|
-
const lo = Math.floor(pos);
|
|
345
|
-
const hi = Math.ceil(pos);
|
|
346
|
-
if (lo === hi) return sorted[lo];
|
|
347
|
-
const frac = pos - lo;
|
|
348
|
-
return sorted[lo] * (1 - frac) + sorted[hi] * frac;
|
|
349
|
-
}
|
|
350
340
|
//#endregion
|
|
351
341
|
export { selfPreference as a, positionalBias as i, calibrateJudgeContinuous as n, verbosityBias as o, continuousAgreement as r, calibrateJudge as t };
|
|
352
342
|
|
|
353
|
-
//# sourceMappingURL=judge-calibration-
|
|
343
|
+
//# sourceMappingURL=judge-calibration-N44V04Se.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-calibration-N44V04Se.js","names":["quantile"],"sources":["../src/judge-calibration.ts"],"sourcesContent":["/**\n * Judge calibration — measure judge quality against human gold + bias.\n *\n * Workflow:\n * 1. Build a golden set: {itemId, humanScore}[].\n * 2. Run candidate judges; each produces {itemId, score}.\n * 3. `calibrateJudge(golden, candidate)` reports κ + Pearson + MAE.\n * 4. `calibrateJudgeContinuous(golden, candidate)` adds quadratic-weighted\n * κ over the un-rounded [0,1] scores plus ICC(2,1), Pearson, Spearman,\n * and bootstrap CIs — use this for fine-grained judges where rounding\n * to int discards information (e.g. 0.78 vs 0.81 both round to 1 and\n * look \"perfectly agreed\" to integer κ).\n * 5. Run bias probes (positional, verbosity, self-preference) to\n * detect systematic score inflation.\n * 6. For N≥2 judges on the same items, `continuousAgreement(scores)`\n * reports ICC(2,1) + κ_w + Pearson + Spearman with bootstrap CIs.\n *\n * Returns actionable diagnostics, not a single number. Consumers then\n * decide whether to trust the judge, retrain it, or add a tie-breaker.\n */\n\nimport { interpolatedQuantile as quantile } from './math/quantile'\nimport { pearsonR, spearmanR } from './statistics/descriptive'\nimport { mulberry32 } from './statistics/random'\n\nexport interface GoldenItem {\n itemId: string\n humanScore: number\n /** Optional group used for per-group bias audits (e.g. model-of-output family). */\n group?: string\n}\n\nexport interface CandidateScore {\n itemId: string\n score: number\n /** Optional — enables positional-bias analysis (did order matter?). */\n positionOfAInput?: 'first' | 'second'\n}\n\nexport interface CalibrationResult {\n n: number\n pearson: number\n /** Cohen's κ with quadratic weights over integer-rounded scores. */\n kappa: number\n /** Mean absolute error vs human. */\n mae: number\n /** Worst-5 miscalibrations (largest |judge - human|). */\n worstItems: Array<{ itemId: string; judge: number; human: number; delta: number }>\n}\n\n/**\n * Measure judge quality against human gold labels: computes Cohen's κ, Pearson correlation, and MAE over matched item ids.\n */\nexport function calibrateJudge(\n golden: GoldenItem[],\n candidate: CandidateScore[],\n): CalibrationResult {\n const map = new Map<string, { h: number; j: number }>()\n for (const g of golden) map.set(g.itemId, { h: g.humanScore, j: NaN })\n for (const c of candidate) {\n const entry = map.get(c.itemId)\n if (entry) entry.j = c.score\n }\n const common = [...map.values()].filter((v) => Number.isFinite(v.j))\n const n = common.length\n if (n < 2) {\n return { n, pearson: NaN, kappa: NaN, mae: NaN, worstItems: [] }\n }\n const humans = common.map((c) => c.h)\n const judges = common.map((c) => c.j)\n const pearson = pearsonR(humans, judges)\n const kappa = weightedKappa(humans.map(Math.round), judges.map(Math.round))\n const absDiffs = common.map((c) => Math.abs(c.j - c.h))\n const mae = absDiffs.reduce((a, b) => a + b, 0) / n\n const worst = [...map.entries()]\n .filter(([, v]) => Number.isFinite(v.j))\n .map(([itemId, v]) => ({ itemId, judge: v.j, human: v.h, delta: Math.abs(v.j - v.h) }))\n .sort((a, b) => b.delta - a.delta)\n .slice(0, 5)\n return { n, pearson, kappa, mae, worstItems: worst }\n}\n\nexport interface PositionalBiasResult {\n /**\n * Score delta (first-position - second-position) averaged across items\n * presented in both positions. Non-zero = positional bias.\n */\n avgDelta: number\n n: number\n}\n\n/**\n * Feed the same items to the judge twice with A/B swapped and pass all\n * results here. Items that don't appear in both positions are ignored.\n */\nexport function positionalBias(scores: CandidateScore[]): PositionalBiasResult {\n const pairs = new Map<string, { first?: number; second?: number }>()\n for (const s of scores) {\n const slot = pairs.get(s.itemId) ?? {}\n if (s.positionOfAInput === 'first') slot.first = s.score\n else if (s.positionOfAInput === 'second') slot.second = s.score\n pairs.set(s.itemId, slot)\n }\n const deltas: number[] = []\n for (const { first, second } of pairs.values()) {\n if (first !== undefined && second !== undefined) deltas.push(first - second)\n }\n if (deltas.length === 0) return { avgDelta: 0, n: 0 }\n return { avgDelta: deltas.reduce((a, b) => a + b, 0) / deltas.length, n: deltas.length }\n}\n\nexport interface VerbosityBiasResult {\n /** Pearson correlation between output length and score. Strong positive = verbosity bias. */\n pearson: number\n n: number\n}\n\nexport function verbosityBias(\n samples: Array<{ outputLen: number; score: number }>,\n): VerbosityBiasResult {\n const n = samples.length\n if (n < 3) return { pearson: NaN, n }\n return {\n pearson: pearsonR(\n samples.map((s) => s.outputLen),\n samples.map((s) => s.score),\n ),\n n,\n }\n}\n\nexport interface SelfPreferenceResult {\n /** Mean judge score when judge's family matches output's family. */\n inFamilyMean: number\n outOfFamilyMean: number\n deltaMean: number\n n: number\n}\n\n/**\n * Pass the same scenarios scored with judge-model X grading outputs from\n * model X (in-family) and model Y (out-of-family). Non-zero delta\n * indicates self-preference.\n */\nexport function selfPreference(\n samples: Array<{ score: number; inFamily: boolean }>,\n): SelfPreferenceResult {\n const inF = samples.filter((s) => s.inFamily).map((s) => s.score)\n const outF = samples.filter((s) => !s.inFamily).map((s) => s.score)\n if (inF.length === 0 || outF.length === 0)\n return { inFamilyMean: 0, outOfFamilyMean: 0, deltaMean: 0, n: 0 }\n const inMean = inF.reduce((a, b) => a + b, 0) / inF.length\n const outMean = outF.reduce((a, b) => a + b, 0) / outF.length\n return {\n inFamilyMean: inMean,\n outOfFamilyMean: outMean,\n deltaMean: inMean - outMean,\n n: samples.length,\n }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\n/** Quadratic weighted Cohen's κ over bounded integer scores. */\nfunction weightedKappa(a: number[], b: number[]): number {\n if (a.length !== b.length || a.length === 0) return NaN\n const min = Math.min(...a, ...b)\n const max = Math.max(...a, ...b)\n const K = max - min + 1\n if (K < 2) return 1\n const observed: number[][] = Array.from({ length: K }, () => new Array(K).fill(0))\n const rowMarg = new Array(K).fill(0)\n const colMarg = new Array(K).fill(0)\n for (let i = 0; i < a.length; i++) {\n const ai = a[i]! - min\n const bi = b[i]! - min\n const row = observed[ai]!\n row[bi] = (row[bi] ?? 0) + 1\n rowMarg[ai]++\n colMarg[bi]++\n }\n let num = 0\n let den = 0\n for (let i = 0; i < K; i++) {\n for (let j = 0; j < K; j++) {\n const w = (i - j) ** 2 / (K - 1) ** 2\n const expected = (rowMarg[i] * colMarg[j]) / a.length\n num += w * observed[i]![j]!\n den += w * expected\n }\n }\n if (den === 0) return 1\n return 1 - num / den\n}\n\n// ── Continuous-value inter-rater agreement ──────────────────────────\n//\n// `weightedKappa` above quantises to integers. For [0,1] judges that is\n// lossy: 0.78 and 0.81 both round to 1 and look identical to integer κ.\n// `continuousAgreement` keeps the original magnitudes:\n//\n// - κ_w : Cohen's weighted κ generalised to continuous scores. For two\n// raters, weighted κ = 1 − Σ w_ij·o_ij / Σ w_ij·e_ij collapses\n// into the closed form below when each \"category\" is the score\n// itself (Janson & Olsson 2001; Cohen 1968 quadratic weights).\n// - ICC : Intraclass Correlation Coefficient, ICC(2,1) per Shrout &\n// Fleiss (1979) — two-way random effects, absolute agreement,\n// single rater. The right reliability coefficient when judges\n// are a random sample of the judge population.\n// - r, ρ : Pearson + Spearman for sanity-check. Pearson catches linear\n// association regardless of scale; Spearman catches monotone\n// association regardless of distribution.\n// - CIs : Bootstrap percentile CIs over items (rows). Bias is preserved\n// because we resample the full row, i.e. a judge's vector\n// stays internally consistent within each bootstrap replicate.\n//\n// References:\n// - Shrout, P. E. & Fleiss, J. L. (1979). Intraclass correlations: uses\n// in assessing rater reliability. Psychological Bulletin, 86(2), 420.\n// - McGraw, K. O. & Wong, S. P. (1996). Forming inferences about some\n// intraclass correlation coefficients. Psychological Methods, 1, 30.\n// - Janson, H. & Olsson, U. (2001). A measure of agreement for interval\n// or nominal multivariate observations. Educ. Psychol. Meas., 61, 277.\n\nexport interface ContinuousAgreement {\n /** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */\n weightedKappa: number\n /** ICC(2,1): two-way random effects, absolute agreement, single rater. */\n icc: number\n /** Pearson product-moment correlation (averaged over rater pairs if N>2). */\n pearson: number\n /** Spearman rank correlation (averaged over rater pairs if N>2). */\n spearman: number\n /** 95% bootstrap percentile CIs over items. */\n ci: {\n icc: [number, number]\n weightedKappa: [number, number]\n }\n /** Number of complete items (no NaN across raters). */\n n: number\n /** Number of raters. */\n raters: number\n}\n\nexport interface ContinuousAgreementOptions {\n /** Bootstrap iterations. Default 1000. Set to 0 to skip CIs (CI = [NaN, NaN]). */\n bootstrap?: number\n /** κ weighting scheme. Default 'quadratic'. */\n weights?: 'linear' | 'quadratic'\n /** PRNG seed for reproducible bootstrap. Default 0xC0FFEE. */\n seed?: number\n /** Confidence level for percentile CI. Default 0.95. */\n ciLevel?: number\n}\n\n/**\n * Inter-rater agreement on continuous (typically [0,1]) scores.\n *\n * `scores` has shape [n_items][n_raters]. Rows with any non-finite entry\n * are dropped. Returns NaN metrics if fewer than 2 raters or 2 complete\n * items remain.\n */\nexport function continuousAgreement(\n scores: number[][],\n opts: ContinuousAgreementOptions = {},\n): ContinuousAgreement {\n const bootstrap = opts.bootstrap ?? 1000\n const weights = opts.weights ?? 'quadratic'\n const seed = opts.seed ?? 0xc0ffee\n const ciLevel = opts.ciLevel ?? 0.95\n\n const matrix = scores.filter((row) => row.length >= 2 && row.every((v) => Number.isFinite(v)))\n const raters = matrix[0]?.length ?? 0\n // All rows must have the same rater count, else drop the offenders.\n const clean = matrix.filter((row) => row.length === raters)\n const nClean = clean.length\n\n if (nClean < 2 || raters < 2) {\n return {\n weightedKappa: NaN,\n icc: NaN,\n pearson: NaN,\n spearman: NaN,\n ci: { icc: [NaN, NaN], weightedKappa: [NaN, NaN] },\n n: nClean,\n raters,\n }\n }\n\n const kappa = continuousWeightedKappa(clean, weights)\n const icc = icc21(clean)\n const pearson = avgPairwise(clean, pearsonR)\n const spearman = avgPairwise(clean, spearmanR)\n\n const ciIcc: [number, number] = [NaN, NaN]\n const ciKappa: [number, number] = [NaN, NaN]\n if (bootstrap > 0) {\n const rng = mulberry32(seed)\n const iccs: number[] = []\n const kappas: number[] = []\n for (let b = 0; b < bootstrap; b++) {\n const sample: number[][] = new Array(nClean)\n for (let i = 0; i < nClean; i++) {\n sample[i] = clean[Math.floor(rng() * nClean)]!\n }\n const iccB = icc21(sample)\n const kB = continuousWeightedKappa(sample, weights)\n if (Number.isFinite(iccB)) iccs.push(iccB)\n if (Number.isFinite(kB)) kappas.push(kB)\n }\n const [lo, hi] = percentileBounds(ciLevel)\n if (iccs.length > 0) {\n iccs.sort((a, b) => a - b)\n ciIcc[0] = quantile(iccs, lo)\n ciIcc[1] = quantile(iccs, hi)\n }\n if (kappas.length > 0) {\n kappas.sort((a, b) => a - b)\n ciKappa[0] = quantile(kappas, lo)\n ciKappa[1] = quantile(kappas, hi)\n }\n }\n\n return {\n weightedKappa: kappa,\n icc,\n pearson,\n spearman,\n ci: { icc: ciIcc, weightedKappa: ciKappa },\n n: nClean,\n raters,\n }\n}\n\nexport interface ContinuousCalibrationResult extends CalibrationResult {\n /** Cohen's κ_w computed on raw (un-rounded) scores. */\n weightedKappaContinuous: number\n /** ICC(2,1) treating golden + candidate as two raters. */\n icc: number\n spearman: number\n ci: {\n icc: [number, number]\n weightedKappa: [number, number]\n }\n}\n\n/**\n * Extends `calibrateJudge` with continuous-value agreement metrics while\n * retaining its base calibration summary.\n */\nexport function calibrateJudgeContinuous(\n golden: GoldenItem[],\n candidate: CandidateScore[],\n opts: ContinuousAgreementOptions = {},\n): ContinuousCalibrationResult {\n const base = calibrateJudge(golden, candidate)\n const map = new Map<string, { h: number; j: number }>()\n for (const g of golden) map.set(g.itemId, { h: g.humanScore, j: NaN })\n for (const c of candidate) {\n const entry = map.get(c.itemId)\n if (entry) entry.j = c.score\n }\n const rows: number[][] = []\n for (const v of map.values()) {\n if (Number.isFinite(v.j)) rows.push([v.h, v.j])\n }\n const agreement = continuousAgreement(rows, opts)\n return {\n ...base,\n weightedKappaContinuous: agreement.weightedKappa,\n icc: agreement.icc,\n spearman: agreement.spearman,\n ci: agreement.ci,\n }\n}\n\n// ── Continuous-agreement internals ──────────────────────────────────\n\n/**\n * Quadratic-weighted κ on continuous scores. With weights w(x,y) = (x-y)^2\n * (or |x-y| for linear) the formula collapses to:\n *\n * κ_w = 1 − E_obs[w] / E_exp[w]\n *\n * where E_obs averages w over paired (a_i, b_i) and E_exp averages w over\n * the independent product distribution (sum_{i,j} w(a_i, b_j) / n^2).\n * The normalisation by (max-min)^2 in the integer version cancels in the\n * ratio, so we don't need it here. Generalises to N raters by averaging κ_w\n * over all rater pairs (mean pairwise weighted agreement).\n */\nfunction continuousWeightedKappa(rows: number[][], scheme: 'linear' | 'quadratic'): number {\n if (rows.length === 0) return NaN\n const raters = rows[0]!.length\n if (raters < 2) return NaN\n const wFn =\n scheme === 'linear'\n ? (x: number, y: number) => Math.abs(x - y)\n : (x: number, y: number) => (x - y) ** 2\n let sum = 0\n let pairs = 0\n for (let r1 = 0; r1 < raters; r1++) {\n for (let r2 = r1 + 1; r2 < raters; r2++) {\n const a = rows.map((row) => row[r1]!)\n const b = rows.map((row) => row[r2]!)\n const n = a.length\n let obs = 0\n for (let i = 0; i < n; i++) obs += wFn(a[i]!, b[i]!)\n obs /= n\n // Expected under independence: average of w over all i,j cross pairs.\n let exp = 0\n for (let i = 0; i < n; i++) {\n for (let j = 0; j < n; j++) exp += wFn(a[i]!, b[j]!)\n }\n exp /= n * n\n if (exp === 0) {\n // Degenerate: at least one rater is constant. Convention: perfect\n // agreement iff observed disagreement is also zero, else 0.\n sum += obs === 0 ? 1 : 0\n } else {\n sum += 1 - obs / exp\n }\n pairs++\n }\n }\n return pairs === 0 ? NaN : sum / pairs\n}\n\n/**\n * ICC(2,1) — two-way random effects, absolute agreement, single rater.\n *\n * ICC(2,1) = (MSR − MSE) / (MSR + (k−1)·MSE + k·(MSC − MSE)/n)\n *\n * where MSR = between-rows MS, MSC = between-columns MS, MSE = residual MS,\n * n = rows (items), k = columns (raters).\n */\nfunction icc21(rows: number[][]): number {\n const n = rows.length\n if (n < 2) return NaN\n const k = rows[0]!.length\n if (k < 2) return NaN\n\n // Row means.\n const rowMeans = rows.map((row) => row.reduce((s, v) => s + v, 0) / k)\n // Column means.\n const colMeans = new Array(k).fill(0)\n for (let j = 0; j < k; j++) {\n let s = 0\n for (let i = 0; i < n; i++) s += rows[i]![j]!\n colMeans[j] = s / n\n }\n // Grand mean.\n let grand = 0\n for (let i = 0; i < n; i++) grand += rowMeans[i]!\n grand /= n\n\n let ssR = 0\n for (let i = 0; i < n; i++) ssR += (rowMeans[i]! - grand) ** 2\n ssR *= k\n let ssC = 0\n for (let j = 0; j < k; j++) ssC += (colMeans[j]! - grand) ** 2\n ssC *= n\n let ssT = 0\n for (let i = 0; i < n; i++) {\n for (let j = 0; j < k; j++) ssT += (rows[i]![j]! - grand) ** 2\n }\n const ssE = ssT - ssR - ssC\n\n const dfR = n - 1\n const dfC = k - 1\n const dfE = (n - 1) * (k - 1)\n const msR = ssR / dfR\n const msC = ssC / dfC\n const msE = dfE > 0 ? ssE / dfE : 0\n\n const denom = msR + (k - 1) * msE + (k * (msC - msE)) / n\n if (denom === 0) {\n // Degenerate (all values equal): convention = 1 if rows identical,\n // else 0. Identical rows ⇒ msR = 0 too, so msR − msE near 0.\n return msR === 0 && msE === 0 ? 1 : 0\n }\n return (msR - msE) / denom\n}\n\n/** Average pairwise statistic over all rater pairs. */\nfunction avgPairwise(rows: number[][], fn: (a: number[], b: number[]) => number): number {\n const k = rows[0]?.length ?? 0\n if (k < 2) return NaN\n let sum = 0\n let pairs = 0\n for (let i = 0; i < k; i++) {\n for (let j = i + 1; j < k; j++) {\n const a = rows.map((row) => row[i]!)\n const b = rows.map((row) => row[j]!)\n const r = fn(a, b)\n if (Number.isFinite(r)) {\n sum += r\n pairs++\n }\n }\n }\n return pairs === 0 ? NaN : sum / pairs\n}\n\nfunction percentileBounds(ciLevel: number): [number, number] {\n const tail = (1 - ciLevel) / 2\n return [tail, 1 - tail]\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,SAAgB,eACd,QACA,WACmB;CACnB,MAAM,sBAAM,IAAI,IAAsC;CACtD,KAAK,MAAM,KAAK,QAAQ,IAAI,IAAI,EAAE,QAAQ;EAAE,GAAG,EAAE;EAAY,GAAG;CAAI,CAAC;CACrE,KAAK,MAAM,KAAK,WAAW;EACzB,MAAM,QAAQ,IAAI,IAAI,EAAE,MAAM;EAC9B,IAAI,OAAO,MAAM,IAAI,EAAE;CACzB;CACA,MAAM,SAAS,CAAC,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,OAAO,SAAS,EAAE,CAAC,CAAC;CACnE,MAAM,IAAI,OAAO;CACjB,IAAI,IAAI,GACN,OAAO;EAAE;EAAG,SAAS;EAAK,OAAO;EAAK,KAAK;EAAK,YAAY,CAAC;CAAE;CAEjE,MAAM,SAAS,OAAO,KAAK,MAAM,EAAE,CAAC;CACpC,MAAM,SAAS,OAAO,KAAK,MAAM,EAAE,CAAC;CAUpC,OAAO;EAAE;EAAG,SATI,SAAS,QAAQ,MASf;EAAG,OARP,cAAc,OAAO,IAAI,KAAK,KAAK,GAAG,OAAO,IAAI,KAAK,KAAK,CAQhD;EAAG,KAPX,OAAO,KAAK,MAAM,KAAK,IAAI,EAAE,IAAI,EAAE,CAAC,CAClC,CAAC,CAAC,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;EAMjB,YALnB,CAAC,GAAG,IAAI,QAAQ,CAAC,CAAC,CAC7B,QAAQ,GAAG,OAAO,OAAO,SAAS,EAAE,CAAC,CAAC,CAAC,CACvC,KAAK,CAAC,QAAQ,QAAQ;GAAE;GAAQ,OAAO,EAAE;GAAG,OAAO,EAAE;GAAG,OAAO,KAAK,IAAI,EAAE,IAAI,EAAE,CAAC;EAAE,EAAE,CAAC,CACtF,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,MAAM,GAAG,CACqC;CAAE;AACrD;;;;;AAeA,SAAgB,eAAe,QAAgD;CAC7E,MAAM,wBAAQ,IAAI,IAAiD;CACnE,KAAK,MAAM,KAAK,QAAQ;EACtB,MAAM,OAAO,MAAM,IAAI,EAAE,MAAM,KAAK,CAAC;EACrC,IAAI,EAAE,qBAAqB,SAAS,KAAK,QAAQ,EAAE;OAC9C,IAAI,EAAE,qBAAqB,UAAU,KAAK,SAAS,EAAE;EAC1D,MAAM,IAAI,EAAE,QAAQ,IAAI;CAC1B;CACA,MAAM,SAAmB,CAAC;CAC1B,KAAK,MAAM,EAAE,OAAO,YAAY,MAAM,OAAO,GAC3C,IAAI,UAAU,KAAA,KAAa,WAAW,KAAA,GAAW,OAAO,KAAK,QAAQ,MAAM;CAE7E,IAAI,OAAO,WAAW,GAAG,OAAO;EAAE,UAAU;EAAG,GAAG;CAAE;CACpD,OAAO;EAAE,UAAU,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;EAAQ,GAAG,OAAO;CAAO;AACzF;AAQA,SAAgB,cACd,SACqB;CACrB,MAAM,IAAI,QAAQ;CAClB,IAAI,IAAI,GAAG,OAAO;EAAE,SAAS;EAAK;CAAE;CACpC,OAAO;EACL,SAAS,SACP,QAAQ,KAAK,MAAM,EAAE,SAAS,GAC9B,QAAQ,KAAK,MAAM,EAAE,KAAK,CAC5B;EACA;CACF;AACF;;;;;;AAeA,SAAgB,eACd,SACsB;CACtB,MAAM,MAAM,QAAQ,QAAQ,MAAM,EAAE,QAAQ,CAAC,CAAC,KAAK,MAAM,EAAE,KAAK;CAChE,MAAM,OAAO,QAAQ,QAAQ,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC,KAAK,MAAM,EAAE,KAAK;CAClE,IAAI,IAAI,WAAW,KAAK,KAAK,WAAW,GACtC,OAAO;EAAE,cAAc;EAAG,iBAAiB;EAAG,WAAW;EAAG,GAAG;CAAE;CACnE,MAAM,SAAS,IAAI,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI;CACpD,MAAM,UAAU,KAAK,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK;CACvD,OAAO;EACL,cAAc;EACd,iBAAiB;EACjB,WAAW,SAAS;EACpB,GAAG,QAAQ;CACb;AACF;;AAKA,SAAS,cAAc,GAAa,GAAqB;CACvD,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,GAAG,OAAO;CACpD,MAAM,MAAM,KAAK,IAAI,GAAG,GAAG,GAAG,CAAC;CAE/B,MAAM,IADM,KAAK,IAAI,GAAG,GAAG,GAAG,CAClB,IAAI,MAAM;CACtB,IAAI,IAAI,GAAG,OAAO;CAClB,MAAM,WAAuB,MAAM,KAAK,EAAE,QAAQ,EAAE,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;CACjF,MAAM,UAAU,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;CACnC,MAAM,UAAU,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;CACnC,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;EACjC,MAAM,KAAK,EAAE,KAAM;EACnB,MAAM,KAAK,EAAE,KAAM;EACnB,MAAM,MAAM,SAAS;EACrB,IAAI,OAAO,IAAI,OAAO,KAAK;EAC3B,QAAQ,GAAG;EACX,QAAQ,GAAG;CACb;CACA,IAAI,MAAM;CACV,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,MAAM,KAAK,IAAI,MAAM,KAAK,IAAI,MAAM;EACpC,MAAM,WAAY,QAAQ,KAAK,QAAQ,KAAM,EAAE;EAC/C,OAAO,IAAI,SAAS,EAAE,CAAE;EACxB,OAAO,IAAI;CACb;CAEF,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO,IAAI,MAAM;AACnB;;;;;;;;AAqEA,SAAgB,oBACd,QACA,OAAmC,CAAC,GACf;CACrB,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,UAAU,KAAK,WAAW;CAChC,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,UAAU,KAAK,WAAW;CAEhC,MAAM,SAAS,OAAO,QAAQ,QAAQ,IAAI,UAAU,KAAK,IAAI,OAAO,MAAM,OAAO,SAAS,CAAC,CAAC,CAAC;CAC7F,MAAM,SAAS,OAAO,EAAE,EAAE,UAAU;CAEpC,MAAM,QAAQ,OAAO,QAAQ,QAAQ,IAAI,WAAW,MAAM;CAC1D,MAAM,SAAS,MAAM;CAErB,IAAI,SAAS,KAAK,SAAS,GACzB,OAAO;EACL,eAAe;EACf,KAAK;EACL,SAAS;EACT,UAAU;EACV,IAAI;GAAE,KAAK,CAAC,KAAK,GAAG;GAAG,eAAe,CAAC,KAAK,GAAG;EAAE;EACjD,GAAG;EACH;CACF;CAGF,MAAM,QAAQ,wBAAwB,OAAO,OAAO;CACpD,MAAM,MAAM,MAAM,KAAK;CACvB,MAAM,UAAU,YAAY,OAAO,QAAQ;CAC3C,MAAM,WAAW,YAAY,OAAO,SAAS;CAE7C,MAAM,QAA0B,CAAC,KAAK,GAAG;CACzC,MAAM,UAA4B,CAAC,KAAK,GAAG;CAC3C,IAAI,YAAY,GAAG;EACjB,MAAM,MAAM,WAAW,IAAI;EAC3B,MAAM,OAAiB,CAAC;EACxB,MAAM,SAAmB,CAAC;EAC1B,KAAK,IAAI,IAAI,GAAG,IAAI,WAAW,KAAK;GAClC,MAAM,SAAqB,IAAI,MAAM,MAAM;GAC3C,KAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,KAC1B,OAAO,KAAK,MAAM,KAAK,MAAM,IAAI,IAAI,MAAM;GAE7C,MAAM,OAAO,MAAM,MAAM;GACzB,MAAM,KAAK,wBAAwB,QAAQ,OAAO;GAClD,IAAI,OAAO,SAAS,IAAI,GAAG,KAAK,KAAK,IAAI;GACzC,IAAI,OAAO,SAAS,EAAE,GAAG,OAAO,KAAK,EAAE;EACzC;EACA,MAAM,CAAC,IAAI,MAAM,iBAAiB,OAAO;EACzC,IAAI,KAAK,SAAS,GAAG;GACnB,KAAK,MAAM,GAAG,MAAM,IAAI,CAAC;GACzB,MAAM,KAAKA,qBAAS,MAAM,EAAE;GAC5B,MAAM,KAAKA,qBAAS,MAAM,EAAE;EAC9B;EACA,IAAI,OAAO,SAAS,GAAG;GACrB,OAAO,MAAM,GAAG,MAAM,IAAI,CAAC;GAC3B,QAAQ,KAAKA,qBAAS,QAAQ,EAAE;GAChC,QAAQ,KAAKA,qBAAS,QAAQ,EAAE;EAClC;CACF;CAEA,OAAO;EACL,eAAe;EACf;EACA;EACA;EACA,IAAI;GAAE,KAAK;GAAO,eAAe;EAAQ;EACzC,GAAG;EACH;CACF;AACF;;;;;AAkBA,SAAgB,yBACd,QACA,WACA,OAAmC,CAAC,GACP;CAC7B,MAAM,OAAO,eAAe,QAAQ,SAAS;CAC7C,MAAM,sBAAM,IAAI,IAAsC;CACtD,KAAK,MAAM,KAAK,QAAQ,IAAI,IAAI,EAAE,QAAQ;EAAE,GAAG,EAAE;EAAY,GAAG;CAAI,CAAC;CACrE,KAAK,MAAM,KAAK,WAAW;EACzB,MAAM,QAAQ,IAAI,IAAI,EAAE,MAAM;EAC9B,IAAI,OAAO,MAAM,IAAI,EAAE;CACzB;CACA,MAAM,OAAmB,CAAC;CAC1B,KAAK,MAAM,KAAK,IAAI,OAAO,GACzB,IAAI,OAAO,SAAS,EAAE,CAAC,GAAG,KAAK,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;CAEhD,MAAM,YAAY,oBAAoB,MAAM,IAAI;CAChD,OAAO;EACL,GAAG;EACH,yBAAyB,UAAU;EACnC,KAAK,UAAU;EACf,UAAU,UAAU;EACpB,IAAI,UAAU;CAChB;AACF;;;;;;;;;;;;;AAgBA,SAAS,wBAAwB,MAAkB,QAAwC;CACzF,IAAI,KAAK,WAAW,GAAG,OAAO;CAC9B,MAAM,SAAS,KAAK,EAAE,CAAE;CACxB,IAAI,SAAS,GAAG,OAAO;CACvB,MAAM,MACJ,WAAW,YACN,GAAW,MAAc,KAAK,IAAI,IAAI,CAAC,KACvC,GAAW,OAAe,IAAI,MAAM;CAC3C,IAAI,MAAM;CACV,IAAI,QAAQ;CACZ,KAAK,IAAI,KAAK,GAAG,KAAK,QAAQ,MAC5B,KAAK,IAAI,KAAK,KAAK,GAAG,KAAK,QAAQ,MAAM;EACvC,MAAM,IAAI,KAAK,KAAK,QAAQ,IAAI,GAAI;EACpC,MAAM,IAAI,KAAK,KAAK,QAAQ,IAAI,GAAI;EACpC,MAAM,IAAI,EAAE;EACZ,IAAI,MAAM;EACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,OAAO,IAAI,EAAE,IAAK,EAAE,EAAG;EACnD,OAAO;EAEP,IAAI,MAAM;EACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,OAAO,IAAI,EAAE,IAAK,EAAE,EAAG;EAErD,OAAO,IAAI;EACX,IAAI,QAAQ,GAGV,OAAO,QAAQ,IAAI,IAAI;OAEvB,OAAO,IAAI,MAAM;EAEnB;CACF;CAEF,OAAO,UAAU,IAAI,MAAM,MAAM;AACnC;;;;;;;;;AAUA,SAAS,MAAM,MAA0B;CACvC,MAAM,IAAI,KAAK;CACf,IAAI,IAAI,GAAG,OAAO;CAClB,MAAM,IAAI,KAAK,EAAE,CAAE;CACnB,IAAI,IAAI,GAAG,OAAO;CAGlB,MAAM,WAAW,KAAK,KAAK,QAAQ,IAAI,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC;CAErE,MAAM,WAAW,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;CACpC,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI,IAAI;EACR,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,KAAK,KAAK,EAAE,CAAE;EAC1C,SAAS,KAAK,IAAI;CACpB;CAEA,IAAI,QAAQ;CACZ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,SAAS,SAAS;CAC9C,SAAS;CAET,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,QAAQ,SAAS,KAAM,UAAU;CAC7D,OAAO;CACP,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,QAAQ,SAAS,KAAM,UAAU;CAC7D,OAAO;CACP,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,QAAQ,KAAK,EAAE,CAAE,KAAM,UAAU;CAE/D,MAAM,MAAM,MAAM,MAAM;CAExB,MAAM,MAAM,IAAI;CAChB,MAAM,MAAM,IAAI;CAChB,MAAM,OAAO,IAAI,MAAM,IAAI;CAC3B,MAAM,MAAM,MAAM;CAClB,MAAM,MAAM,MAAM;CAClB,MAAM,MAAM,MAAM,IAAI,MAAM,MAAM;CAElC,MAAM,QAAQ,OAAO,IAAI,KAAK,MAAO,KAAK,MAAM,OAAQ;CACxD,IAAI,UAAU,GAGZ,OAAO,QAAQ,KAAK,QAAQ,IAAI,IAAI;CAEtC,QAAQ,MAAM,OAAO;AACvB;;AAGA,SAAS,YAAY,MAAkB,IAAkD;CACvF,MAAM,IAAI,KAAK,EAAE,EAAE,UAAU;CAC7B,IAAI,IAAI,GAAG,OAAO;CAClB,IAAI,MAAM;CACV,IAAI,QAAQ;CACZ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KACrB,KAAK,IAAI,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAG9B,MAAM,IAAI,GAFA,KAAK,KAAK,QAAQ,IAAI,EAEnB,GADH,KAAK,KAAK,QAAQ,IAAI,EAChB,CAAC;EACjB,IAAI,OAAO,SAAS,CAAC,GAAG;GACtB,OAAO;GACP;EACF;CACF;CAEF,OAAO,UAAU,IAAI,MAAM,MAAM;AACnC;AAEA,SAAS,iBAAiB,SAAmC;CAC3D,MAAM,QAAQ,IAAI,WAAW;CAC7B,OAAO,CAAC,MAAM,IAAI,IAAI;AACxB"}
|
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
import { i as JudgeError, s as ValidationError } from "./errors-Dngq5h35.js";
|
|
2
2
|
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-BsZVgBhW.js";
|
|
3
3
|
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
4
|
-
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-
|
|
5
|
-
import { r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-
|
|
6
|
-
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-
|
|
4
|
+
import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-CnU8Yq29.js";
|
|
5
|
+
import { r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-CO0kA0bq.js";
|
|
6
|
+
import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-C26hOFWD.js";
|
|
7
7
|
import { i as modelHasSnapshot, l as searchCellRunId } from "./run-record-BRlM99rX.js";
|
|
8
8
|
import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
|
|
9
9
|
import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-BB290Y9A.js";
|
|
10
10
|
import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-_IbkejCF.js";
|
|
11
|
-
import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-
|
|
11
|
+
import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-C__cUpCt.js";
|
|
12
12
|
import { m as mapConcurrent } from "./journal-DvKCvjty.js";
|
|
13
13
|
import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
|
|
14
14
|
import { S as fsCampaignStorage, g as isRecord, h as isExternalTextCandidate, x as createRunCostLedger } from "./external-optimizer-subprocess-BmKtH3T6.js";
|
|
15
15
|
import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-BXcZw-jo.js";
|
|
16
16
|
import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-Dk1ZdJEm.js";
|
|
17
|
-
import { t as detectRewardHacking } from "./reward-hacking-
|
|
17
|
+
import { t as detectRewardHacking } from "./reward-hacking-BFcQgkux.js";
|
|
18
18
|
import { t as assertProposalFindings } from "./proposal-findings-CXblsskH.js";
|
|
19
|
-
import { D as paretoFrontier, O as paretoFrontierWithCrowding, a as searchPolicyView, b as estimateNode, i as searchExpansionIndex, n as runSearch, p as EXPANSION_OPERATORS, t as SEARCH_KERNEL_SOURCE } from "./search-kernel-
|
|
19
|
+
import { D as paretoFrontier, O as paretoFrontierWithCrowding, a as searchPolicyView, b as estimateNode, i as searchExpansionIndex, n as runSearch, p as EXPANSION_OPERATORS, t as SEARCH_KERNEL_SOURCE } from "./search-kernel-CsISc5t8.js";
|
|
20
20
|
import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, u as searchNodeId } from "./search-ledger-CXQPwdWz.js";
|
|
21
|
-
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-
|
|
21
|
+
import { n as searchProposerView, t as renderSearchSummary } from "./search-summary-BZkoLwbB.js";
|
|
22
22
|
import { C as combineAbortSignals, d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-DmCNzrc9.js";
|
|
23
23
|
import { createHash } from "node:crypto";
|
|
24
24
|
import { writeFileSync } from "node:fs";
|
|
@@ -6174,4 +6174,4 @@ function firstString(value) {
|
|
|
6174
6174
|
//#endregion
|
|
6175
6175
|
export { captureFinalEvidencePolicy as $, assertCompleteSearchHistory as A, searchReceiptAccounting as B, incumbentWithOperatorBandit as C, surfaceDiff as D, searchEdgeId as E, verifySearchHistoryArtifact as F, captureJudge as G, uniform as H, verifySearchHistoryReceipt as I, redTeamDataset as J, defaultProductionGate as K, campaignCellSearchResult as L, assertSearchHistoryMatchesState as M, createSearchHistoryReceipt as N, surfaceNode as O, searchHistoryCoverageRow as P, assertIndependentEvaluationSplit as Q, searchModelIdentity as R, incumbent as S, developmentClaim as T, runFinalComparison as U, asha as V, openAutoPr as W, scoreRedTeamOutput as X, redTeamReport as Y, runCanaries as Z, isProposedCandidate as _, transientDispatchFailure as a, resolveRunDir as at, beam as b, compareOptimizationMethods as c, buildCellSchedule as ct, combineComparisonCosts as d, evaluationUnitMap as et, costFromLedgerSummary as f, runOptimization as g, runImprovementLoop as h, quotaExhaustedUntil as i, planCampaignRun as it, assertSearchHistoryAdmissionOptions as j, SearchHistoryRequiredError as k, optimizationTokenUsageFromSummary as l, cellCachePath as lt, readGepaCandidatePopulationArtifact as m, JudgeParseError as n, runEval as nt, aggregateRunScore as o, tangleTracesRoot as ot, assertGepaCandidatePopulationSummary as p, DEFAULT_RED_TEAM_CORPUS as q, isTransientTransportFailure as r, runCampaign as rt, clamp01 as s, readCachedCell as st, llmJudge as t, reserveFinalEvidence as tt, executeOptimizationMethod as u, computeManifestHash as ut, labelTrustRank as v, SearchRecorder as w, crowdedFrontierParent as x, aide as y, searchProposalExecution as z };
|
|
6176
6176
|
|
|
6177
|
-
//# sourceMappingURL=llm-judge-
|
|
6177
|
+
//# sourceMappingURL=llm-judge-Bka5q-ds.js.map
|