@tangle-network/agent-eval 0.220.2 → 0.221.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-D01Io2Xw.d.ts → agent-profile-Bc7ADJY4.d.ts} +2 -2
- package/dist/{agent-profile-D01Io2Xw.d.ts.map → agent-profile-Bc7ADJY4.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +77 -78
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +204 -25
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DqPJaJ4X.js → analyze-runs-CHmsykNJ.js} +4 -4
- package/dist/{analyze-runs-DqPJaJ4X.js.map → analyze-runs-CHmsykNJ.js.map} +1 -1
- package/dist/{analyze-runs-BWAHBDAw.d.ts → analyze-runs-D7In4c3z.d.ts} +2 -2
- package/dist/{analyze-runs-BWAHBDAw.d.ts.map → analyze-runs-D7In4c3z.d.ts.map} +1 -1
- package/dist/{benchmark-command-Dx0E4a6z.js → benchmark-command-DiGqnGUt.js} +13 -11
- package/dist/benchmark-command-DiGqnGUt.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/calibration-CTwxpJ-t.js +197 -0
- package/dist/calibration-CTwxpJ-t.js.map +1 -0
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-BVPC6ODQ.js → campaign-Bt6rbWIU.js} +9 -8
- package/dist/{campaign-BVPC6ODQ.js.map → campaign-Bt6rbWIU.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{completion-verifier-Cigap8CG.js → completion-verifier-YwoeujPl.js} +2 -2
- package/dist/completion-verifier-YwoeujPl.js.map +1 -0
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +7 -7
- package/dist/decision-point-Co3u3W8k.js +831 -0
- package/dist/decision-point-Co3u3W8k.js.map +1 -0
- package/dist/default-registry-BMHqNR6_.d.ts +257 -0
- package/dist/default-registry-BMHqNR6_.d.ts.map +1 -0
- package/dist/{default-registry-ovrIBGmH.js → default-registry-BbFCkqvB.js} +607 -14
- package/dist/default-registry-BbFCkqvB.js.map +1 -0
- package/dist/{define-agent-eval-C2nFhHPu.d.ts → define-agent-eval-Bsoi2GUV.d.ts} +7 -7
- package/dist/{define-agent-eval-C2nFhHPu.d.ts.map → define-agent-eval-Bsoi2GUV.d.ts.map} +1 -1
- package/dist/{define-agent-eval-UQY3VbJ2.js → define-agent-eval-CJhECJNL.js} +4 -4
- package/dist/{define-agent-eval-UQY3VbJ2.js.map → define-agent-eval-CJhECJNL.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +2 -2
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-Bn_Mt_LW.js → dspy-rlm-engine-BTy7u0jQ.js} +3 -3
- package/dist/{dspy-rlm-engine-Bn_Mt_LW.js.map → dspy-rlm-engine-BTy7u0jQ.js.map} +1 -1
- package/dist/{engine-BpGGw6cd.d.ts → engine-BF95CWzH.d.ts} +2 -2
- package/dist/{engine-BpGGw6cd.d.ts.map → engine-BF95CWzH.d.ts.map} +1 -1
- package/dist/{eval-campaign-BUMIxKTE.js → eval-campaign-DoX5vz3E.js} +2 -2
- package/dist/{eval-campaign-BUMIxKTE.js.map → eval-campaign-DoX5vz3E.js.map} +1 -1
- package/dist/{evaluation-CBdQt_kr.d.ts → evaluation-D1PGzkPR.d.ts} +3 -3
- package/dist/{evaluation-CBdQt_kr.d.ts.map → evaluation-D1PGzkPR.d.ts.map} +1 -1
- package/dist/evaluation-wdGyZ7jV.js +147 -0
- package/dist/evaluation-wdGyZ7jV.js.map +1 -0
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -136
- package/dist/{exact-types-BYFmYq44.d.ts → exact-types-CVn1QxFO.d.ts} +2 -2
- package/dist/{exact-types-BYFmYq44.d.ts.map → exact-types-CVn1QxFO.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -3
- package/dist/experiment/index.js +2 -2
- package/dist/{external-optimizer-process-wOLfzsCC.js → external-optimizer-process-E5yY2SYS.js} +2 -2
- package/dist/{external-optimizer-process-wOLfzsCC.js.map → external-optimizer-process-E5yY2SYS.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-D670Oxf2.js → external-optimizer-subprocess-DA02c_L6.js} +12 -19
- package/dist/{external-optimizer-subprocess-D670Oxf2.js.map → external-optimizer-subprocess-DA02c_L6.js.map} +1 -1
- package/dist/{feedback-trajectory-DhB9WzYg.d.ts → feedback-trajectory-BZhVCWuz.d.ts} +3 -3
- package/dist/{feedback-trajectory-DhB9WzYg.d.ts.map → feedback-trajectory-BZhVCWuz.d.ts.map} +1 -1
- package/dist/finding-composition-8n0fFgVI.js +131 -0
- package/dist/finding-composition-8n0fFgVI.js.map +1 -0
- package/dist/{findings-store-CvNo712L.js → findings-store-DRiFdoid.js} +4 -4
- package/dist/{findings-store-CvNo712L.js.map → findings-store-DRiFdoid.js.map} +1 -1
- package/dist/{index-CVFStrQD.d.ts → index-BJZ7s-sv.d.ts} +7 -58
- package/dist/index-BJZ7s-sv.d.ts.map +1 -0
- package/dist/{index-Q6DvdcwI.d.ts → index-BvX3CFNS.d.ts} +6 -6
- package/dist/{index-Q6DvdcwI.d.ts.map → index-BvX3CFNS.d.ts.map} +1 -1
- package/dist/index-DWd1gUwR.d.ts +1086 -0
- package/dist/index-DWd1gUwR.d.ts.map +1 -0
- package/dist/{index-DAXfpVf7.d.ts → index-y7i8l6Y_.d.ts} +8 -8
- package/dist/{index-DAXfpVf7.d.ts.map → index-y7i8l6Y_.d.ts.map} +1 -1
- package/dist/index.d.ts +18 -17
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -15
- package/dist/{kind-factory-BhxbtUsQ.js → kind-factory-Dl0PmNQL.js} +2 -2
- package/dist/{kind-factory-BhxbtUsQ.js.map → kind-factory-Dl0PmNQL.js.map} +1 -1
- package/dist/{llm-judge-Cc8vDJh_.js → llm-judge-3uAeRawE.js} +4 -4
- package/dist/{llm-judge-Cc8vDJh_.js.map → llm-judge-3uAeRawE.js.map} +1 -1
- package/dist/{matrix-Cf-AGxGh.d.ts → matrix-BXo3PSqM.d.ts} +2 -2
- package/dist/{matrix-Cf-AGxGh.d.ts.map → matrix-BXo3PSqM.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +3 -638
- package/dist/meta-eval/index.js +3 -193
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/outcome-observations-FLmRpgYa.js +78 -0
- package/dist/outcome-observations-FLmRpgYa.js.map +1 -0
- package/dist/{paired-arms-Ct4LTKc1.js → paired-arms-B2ttLc0_.js} +108 -31
- package/dist/paired-arms-B2ttLc0_.js.map +1 -0
- package/dist/{power-and-mde-N5nU4_Ly.js → power-and-mde-CQX-0weG.js} +2 -2
- package/dist/{power-and-mde-N5nU4_Ly.js.map → power-and-mde-CQX-0weG.js.map} +1 -1
- package/dist/{prime-protocol-CckHBxGu.d.ts → prime-protocol-BEaqBoQE.d.ts} +2 -2
- package/dist/{prime-protocol-CckHBxGu.d.ts.map → prime-protocol-BEaqBoQE.d.ts.map} +1 -1
- package/dist/{proposal-findings-E1aSC_cU.js → proposal-findings-D0_zkQ2l.js} +2 -2
- package/dist/{proposal-findings-E1aSC_cU.js.map → proposal-findings-D0_zkQ2l.js.map} +1 -1
- package/dist/{registry-B_yG-1jh.d.ts → registry-DDKaUrmM.d.ts} +3 -3
- package/dist/{registry-B_yG-1jh.d.ts.map → registry-DDKaUrmM.d.ts.map} +1 -1
- package/dist/reporting.d.ts +1 -1
- package/dist/reporting.js +3 -3
- package/dist/rl.d.ts +4 -5
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +5 -4
- package/dist/rl.js.map +1 -1
- package/dist/{rubric-predictive-validity-noY7uE03.d.ts → rubric-predictive-validity-Cd5SCtAf.d.ts} +58 -3
- package/dist/rubric-predictive-validity-Cd5SCtAf.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-D3_oI72N.js → rubric-predictive-validity-CeKW2GQN.js} +3 -77
- package/dist/rubric-predictive-validity-CeKW2GQN.js.map +1 -0
- package/dist/search/index.d.ts +2 -2
- package/dist/{search-kernel-Dloxd6gs.d.ts → search-kernel-C6__V0IS.d.ts} +2 -2
- package/dist/{search-kernel-Dloxd6gs.d.ts.map → search-kernel-C6__V0IS.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-DLKqdhcR.js → skillopt-optimization-method-D21dH_Bh.js} +4 -4
- package/dist/{skillopt-optimization-method-DLKqdhcR.js.map → skillopt-optimization-method-D21dH_Bh.js.map} +1 -1
- package/dist/{statistical-heldout-3z3I-kJN.d.ts → statistical-heldout-pD1Yah-0.d.ts} +58 -8
- package/dist/statistical-heldout-pD1Yah-0.d.ts.map +1 -0
- package/dist/{store-otlp-dyWE3wAG.js → store-otlp-BVZfR168.js} +2 -2
- package/dist/{store-otlp-dyWE3wAG.js.map → store-otlp-BVZfR168.js.map} +1 -1
- package/dist/{store-tool-spans-DIUgZbzu.js → store-tool-spans-CjYiF4fF.js} +3 -3
- package/dist/{store-tool-spans-DIUgZbzu.js.map → store-tool-spans-CjYiF4fF.js.map} +1 -1
- package/dist/{store-tool-spans-DPwXHQkH.d.ts → store-tool-spans-DfnwWMFR.d.ts} +4 -4
- package/dist/{store-tool-spans-DPwXHQkH.d.ts.map → store-tool-spans-DfnwWMFR.d.ts.map} +1 -1
- package/dist/{summary-report-BsXJoYX5.js → summary-report-DHDWxe6p.js} +3 -3
- package/dist/{summary-report-BsXJoYX5.js.map → summary-report-DHDWxe6p.js.map} +1 -1
- package/dist/systemone-D3MrW63C.d.ts +400 -0
- package/dist/systemone-D3MrW63C.d.ts.map +1 -0
- package/dist/{systemone-protocol-DKoi1NdK.d.ts → systemone-protocol-Bl2EI2KQ.d.ts} +4 -2
- package/dist/{systemone-protocol-DKoi1NdK.d.ts.map → systemone-protocol-Bl2EI2KQ.d.ts.map} +1 -1
- package/dist/systemone-protocol.d.ts +2 -2
- package/dist/systemone-protocol.js +7 -7
- package/dist/systemone-protocol.js.map +1 -1
- package/dist/{systemone-review-CUBMuTLi.d.ts → systemone-review-C8DRiT_-.d.ts} +3 -3
- package/dist/{systemone-review-CUBMuTLi.d.ts.map → systemone-review-C8DRiT_-.d.ts.map} +1 -1
- package/dist/{systemone-review-DzdOJ8I6.js → systemone-review-DFGtIUcf.js} +2 -2
- package/dist/{systemone-review-DzdOJ8I6.js.map → systemone-review-DFGtIUcf.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -343
- package/dist/{tool-groups-LISbBWCg.d.ts → tool-groups-DK1bu5TD.d.ts} +3 -3
- package/dist/tool-groups-DK1bu5TD.d.ts.map +1 -0
- package/dist/{trace-questions-C42fYEE4.d.ts → trace-questions-D_OKYZ6u.d.ts} +3 -3
- package/dist/{trace-questions-C42fYEE4.d.ts.map → trace-questions-D_OKYZ6u.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +3 -3
- package/dist/traces.js +3 -3
- package/dist/{types-DqFMNTgK.js → types-CnWmj8NW.js} +19 -6
- package/dist/types-CnWmj8NW.js.map +1 -0
- package/dist/{types-CHfn1zg5.d.ts → types-DDKfH-N6.d.ts} +2 -2
- package/dist/{types-CHfn1zg5.d.ts.map → types-DDKfH-N6.d.ts.map} +1 -1
- package/dist/{types-8yQzNm8u.d.ts → types-DGImvHIr.d.ts} +150 -4
- package/dist/types-DGImvHIr.d.ts.map +1 -0
- package/dist/{types-D8GXK9Z5.d.ts → types-IERw_HxR.d.ts} +2 -2
- package/dist/{types-D8GXK9Z5.d.ts.map → types-IERw_HxR.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/docs/behavioral-trace-questions.md +99 -0
- package/docs/experiment.md +27 -0
- package/docs/finding-layers.md +54 -1
- package/docs/systemone.md +107 -0
- package/package.json +4 -4
- package/dist/benchmark-command-Dx0E4a6z.js.map +0 -1
- package/dist/completion-verifier-Cigap8CG.js.map +0 -1
- package/dist/default-registry-Cwg_QN3d.d.ts +0 -70
- package/dist/default-registry-Cwg_QN3d.d.ts.map +0 -1
- package/dist/default-registry-ovrIBGmH.js.map +0 -1
- package/dist/evaluation.js.map +0 -1
- package/dist/index-CVFStrQD.d.ts.map +0 -1
- package/dist/judge-gate-Dc_mXQsL.d.ts +0 -453
- package/dist/judge-gate-Dc_mXQsL.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/outcome-store-CNt4iZ67.d.ts +0 -58
- package/dist/outcome-store-CNt4iZ67.d.ts.map +0 -1
- package/dist/paired-arms-Ct4LTKc1.js.map +0 -1
- package/dist/rubric-predictive-validity-D3_oI72N.js.map +0 -1
- package/dist/rubric-predictive-validity-noY7uE03.d.ts.map +0 -1
- package/dist/statistical-heldout-3z3I-kJN.d.ts.map +0 -1
- package/dist/systemone-DVA7t76V.d.ts +0 -135
- package/dist/systemone-DVA7t76V.d.ts.map +0 -1
- package/dist/systemone.js.map +0 -1
- package/dist/tool-groups-LISbBWCg.d.ts.map +0 -1
- package/dist/types-8yQzNm8u.d.ts.map +0 -1
- package/dist/types-DqFMNTgK.js.map +0 -1
|
@@ -0,0 +1,1086 @@
|
|
|
1
|
+
import { a as JudgeError } from "./errors-DEE6u6ot.js";
|
|
2
|
+
import { f as Run } from "./schema-D6EVKzTB.js";
|
|
3
|
+
import { o as DimensionJudgeScore } from "./types-eMhdacVp.js";
|
|
4
|
+
import { a as ContinuousCalibrationResult, i as ContinuousAgreementOptions, n as CandidateScore, o as GoldenItem, r as ContinuousAgreement, t as CalibrationResult } from "./judge-calibration-DFtEMlde.js";
|
|
5
|
+
import { r as LedgerHash } from "./canonical-CFpojCN5.js";
|
|
6
|
+
import { s as TraceStore } from "./store-B3nWRy3B.js";
|
|
7
|
+
import { h as OutcomeStore, m as OutcomeFilter } from "./rubric-predictive-validity-Cd5SCtAf.js";
|
|
8
|
+
import { t as ComputedInterval } from "./interval-ohuOJurX.js";
|
|
9
|
+
import { z } from "zod";
|
|
10
|
+
//#region src/statistics/agreement-irr.d.ts
|
|
11
|
+
/**
|
|
12
|
+
* Inter-rater reliability — Krippendorff's α under the squared-difference
|
|
13
|
+
* metric, pooled across dimensions.
|
|
14
|
+
*
|
|
15
|
+
* Each inner array is one judge's scores. Items are matched by position
|
|
16
|
+
* WITHIN a dimension: the k-th score a judge supplies carrying dimension
|
|
17
|
+
* `d` is item k of `d`, and the ratings compared against each other are
|
|
18
|
+
* the ones different judges gave to the same item. Every judge that scores
|
|
19
|
+
* a dimension at all must supply the same number of scores for it —
|
|
20
|
+
* ragged input cannot be aligned into items and throws rather than
|
|
21
|
+
* comparing mismatched items.
|
|
22
|
+
*
|
|
23
|
+
* α = 1 − D_observed / D_expected: D_observed averages the squared
|
|
24
|
+
* difference over within-item judge pairs, D_expected over every pair of
|
|
25
|
+
* ratings irrespective of item. α = 1 is perfect agreement, 0 is chance,
|
|
26
|
+
* negative is systematic disagreement.
|
|
27
|
+
*/
|
|
28
|
+
declare function interRaterReliability(judgeScores: DimensionJudgeScore[][]): number;
|
|
29
|
+
interface CorpusScoreRecord {
|
|
30
|
+
/** Stable identifier for the rated item (scenario, span, turn, …). */
|
|
31
|
+
itemId: string;
|
|
32
|
+
/** Identifier for the judge that produced this score. */
|
|
33
|
+
judgeName: string;
|
|
34
|
+
/** Dimension name (matches `DimensionJudgeScore.dimension`). */
|
|
35
|
+
dimension: string;
|
|
36
|
+
/** Numeric score; must be finite. */
|
|
37
|
+
score: number;
|
|
38
|
+
}
|
|
39
|
+
interface CorpusAgreementPerDimension extends ContinuousAgreement {
|
|
40
|
+
dimension: string;
|
|
41
|
+
/** Item IDs that contributed to this dimension's matrix (every judge scored them). */
|
|
42
|
+
itemIds: string[];
|
|
43
|
+
/** Judge IDs that contributed to this dimension's matrix. */
|
|
44
|
+
judgeIds: string[];
|
|
45
|
+
}
|
|
46
|
+
interface CorpusAgreementReport {
|
|
47
|
+
/** Per-dimension ICC(2,1) + κ_w + Pearson + Spearman + bootstrap CIs. */
|
|
48
|
+
perDimension: CorpusAgreementPerDimension[];
|
|
49
|
+
/** Mean ICC across dimensions (NaN if no dimension yielded a finite ICC). */
|
|
50
|
+
overallIcc: number;
|
|
51
|
+
/** Mean weighted κ across dimensions (NaN if none finite). */
|
|
52
|
+
overallWeightedKappa: number;
|
|
53
|
+
/** Dimensions evaluated (sorted). */
|
|
54
|
+
dimensions: string[];
|
|
55
|
+
/** Judges seen across the corpus (sorted). */
|
|
56
|
+
judgeIds: string[];
|
|
57
|
+
}
|
|
58
|
+
interface CorpusAgreementOptions extends ContinuousAgreementOptions {
|
|
59
|
+
/**
|
|
60
|
+
* Restrict the audit to these dimensions. Default = every dimension
|
|
61
|
+
* that appears in the input. A dimension named here but absent from
|
|
62
|
+
* the input throws — silent omission would corrupt the overall metric.
|
|
63
|
+
*/
|
|
64
|
+
dimensions?: string[];
|
|
65
|
+
/**
|
|
66
|
+
* Restrict the audit to these judges. Default = every judge that
|
|
67
|
+
* appears in the input. A judge named here but absent from a
|
|
68
|
+
* dimension throws (see "fail loud" below).
|
|
69
|
+
*/
|
|
70
|
+
judges?: string[];
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Corpus-wide inter-rater agreement across N items × M judges × D dimensions.
|
|
74
|
+
*
|
|
75
|
+
* For each dimension, builds the [n_items][n_judges] matrix of scores
|
|
76
|
+
* (keeping only items every judge rated on that dimension), then runs
|
|
77
|
+
* `continuousAgreement` to get ICC(2,1), κ_w, Pearson, Spearman, and
|
|
78
|
+
* bootstrap CIs. Reports a pooled mean across dimensions as a single
|
|
79
|
+
* "is this judge panel reliable on this corpus?" number.
|
|
80
|
+
*
|
|
81
|
+
* Fail-loud contract:
|
|
82
|
+
* - Empty input throws.
|
|
83
|
+
* - Fewer than 2 judges or fewer than 2 items per dimension throws.
|
|
84
|
+
* - A judge present in some dimensions but with zero scored items on
|
|
85
|
+
* another dimension throws (would silently shrink the matrix).
|
|
86
|
+
* - Duplicate (itemId, judgeName, dimension) records throw.
|
|
87
|
+
*/
|
|
88
|
+
declare function corpusInterRaterAgreement(records: CorpusScoreRecord[], opts?: CorpusAgreementOptions): CorpusAgreementReport;
|
|
89
|
+
/**
|
|
90
|
+
* Convenience adapter for `DimensionJudgeScore[]` data keyed externally by item.
|
|
91
|
+
*
|
|
92
|
+
* Use when you have per-item arrays of `DimensionJudgeScore[]` (e.g. one
|
|
93
|
+
* `ScenarioResult.judgeScores` per scenario) and want corpus-wide
|
|
94
|
+
* agreement without manually flattening. `itemId` must be unique per
|
|
95
|
+
* row of `itemsScores`.
|
|
96
|
+
*/
|
|
97
|
+
declare function corpusInterRaterAgreementFromJudgeScores(itemsScores: Array<{
|
|
98
|
+
itemId: string;
|
|
99
|
+
scores: DimensionJudgeScore[];
|
|
100
|
+
}>, opts?: CorpusAgreementOptions): CorpusAgreementReport;
|
|
101
|
+
//#endregion
|
|
102
|
+
//#region src/meta-eval/correlation-study.d.ts
|
|
103
|
+
interface EvalMetricSpec {
|
|
104
|
+
id: string;
|
|
105
|
+
/** Extract a scalar from a run. Omit it and `id` must name one of
|
|
106
|
+
* `RUN_METRICS`; any other `id` is refused. */
|
|
107
|
+
extract?: (run: Run, store: TraceStore) => Promise<number | null>;
|
|
108
|
+
}
|
|
109
|
+
interface OutcomePair {
|
|
110
|
+
evalMetric: string;
|
|
111
|
+
outcomeMetric: string;
|
|
112
|
+
}
|
|
113
|
+
interface CorrelationResult {
|
|
114
|
+
evalMetric: string;
|
|
115
|
+
outcomeMetric: string;
|
|
116
|
+
n: number;
|
|
117
|
+
pearson: number;
|
|
118
|
+
spearman: number;
|
|
119
|
+
/** 95% bootstrap CI for Pearson. */
|
|
120
|
+
pearsonCi95: {
|
|
121
|
+
lower: number;
|
|
122
|
+
upper: number;
|
|
123
|
+
} | null;
|
|
124
|
+
/** 95% bootstrap CI for Spearman; null when no resample is estimable. */
|
|
125
|
+
spearmanCi95: {
|
|
126
|
+
lower: number;
|
|
127
|
+
upper: number;
|
|
128
|
+
} | null;
|
|
129
|
+
/** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
|
|
130
|
+
verdict: 'strong' | 'moderate' | 'weak';
|
|
131
|
+
}
|
|
132
|
+
interface CorrelationStudyResult {
|
|
133
|
+
pairs: CorrelationResult[];
|
|
134
|
+
/** Declared pairs without an estimable correlation, including their usable sample count. */
|
|
135
|
+
excludedPairs: Array<OutcomePair & {
|
|
136
|
+
n: number;
|
|
137
|
+
reason: 'insufficient_samples' | 'constant_eval_metric' | 'constant_outcome';
|
|
138
|
+
}>;
|
|
139
|
+
joinedSamples: number;
|
|
140
|
+
skippedRuns: number;
|
|
141
|
+
}
|
|
142
|
+
interface CorrelationStudyOptions {
|
|
143
|
+
/** Only join outcomes captured within this window after run.startedAt. */
|
|
144
|
+
maxCaptureLagMs?: number;
|
|
145
|
+
/** Restrict to a subset of outcomes (cohort, region, source). */
|
|
146
|
+
outcomeFilter?: OutcomeFilter;
|
|
147
|
+
/** Which outcome per run to use when multiple exist. Default 'latest'. */
|
|
148
|
+
reduction?: 'latest' | 'mean' | 'max';
|
|
149
|
+
/** Bootstrap iterations for the CI. Default 500. */
|
|
150
|
+
bootstrapIterations?: number;
|
|
151
|
+
/** Seed for the bootstrap resampler. Absent, the seed is derived from the
|
|
152
|
+
* paired observations, so the same study reproduces the same interval. */
|
|
153
|
+
seed?: number;
|
|
154
|
+
}
|
|
155
|
+
declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;
|
|
156
|
+
//#endregion
|
|
157
|
+
//#region src/meta-eval/calibration.d.ts
|
|
158
|
+
interface CalibrationBin {
|
|
159
|
+
lower: number;
|
|
160
|
+
upper: number;
|
|
161
|
+
n: number;
|
|
162
|
+
evalMean: number;
|
|
163
|
+
outcomeMean: number;
|
|
164
|
+
/** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
|
|
165
|
+
gap: number;
|
|
166
|
+
}
|
|
167
|
+
interface CalibrationReport {
|
|
168
|
+
evalMetric: string;
|
|
169
|
+
outcomeMetric: string;
|
|
170
|
+
n: number;
|
|
171
|
+
bins: CalibrationBin[];
|
|
172
|
+
/** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
|
|
173
|
+
ece: number;
|
|
174
|
+
/** Largest observed difference between a bin's mean score and mean outcome. */
|
|
175
|
+
maxGap: number;
|
|
176
|
+
}
|
|
177
|
+
interface CalibrationOptions {
|
|
178
|
+
/** Positive integer; empty bins are omitted. Default 10. */
|
|
179
|
+
bins?: number;
|
|
180
|
+
/** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
|
|
181
|
+
binning?: 'equal-width' | 'equal-frequency';
|
|
182
|
+
/** Clip eval values to [lo, hi] before binning. */
|
|
183
|
+
range?: {
|
|
184
|
+
lo: number;
|
|
185
|
+
hi: number;
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
interface CalibrationPair {
|
|
189
|
+
evalScore: number;
|
|
190
|
+
outcome: number;
|
|
191
|
+
}
|
|
192
|
+
declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
|
|
193
|
+
/** Measure already joined observations without constructing trace and outcome stores. */
|
|
194
|
+
declare function calibrationFromPairs(inputPairs: readonly CalibrationPair[], evalMetric: string, outcomeMetric: string, options?: CalibrationOptions): CalibrationReport | null;
|
|
195
|
+
/**
|
|
196
|
+
* Mean squared error of probability forecasts against binary outcomes, Σ(p − y)²/N; always
|
|
197
|
+
* forecasting 0.5 scores 0.25. For the reliability table and ECE use `calibrationFromPairs`
|
|
198
|
+
* with `range: { lo: 0, hi: 1 }` (`binning: 'equal-frequency'` gives equal-mass bins).
|
|
199
|
+
*/
|
|
200
|
+
declare function brierScore(pairs: readonly CalibrationPair[]): number;
|
|
201
|
+
/** One categorical forecast: a distribution over named outcomes and the outcome that occurred. */
|
|
202
|
+
interface CategoricalForecast {
|
|
203
|
+
/** Probability per outcome name. Must sum to 1 within 1e-6 and include `outcome`. */
|
|
204
|
+
probabilities: Readonly<Record<string, number>>;
|
|
205
|
+
outcome: string;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Multiclass Brier score in its sum form, mean over forecasts of Σₖ(pₖ − yₖ)², range [0, 2].
|
|
209
|
+
* With two outcomes it is twice the binary `brierScore`. A uniform forecast over K outcomes
|
|
210
|
+
* scores 1 − 1/K. The distribution is not renormalized here: repair provider rounding before
|
|
211
|
+
* scoring (`normalizeSystemOneResult` in `/systemone/protocol`), so a defective answer stays
|
|
212
|
+
* visible instead of being scored as if it were valid.
|
|
213
|
+
*/
|
|
214
|
+
declare function multiclassBrierScore(forecasts: readonly CategoricalForecast[]): number;
|
|
215
|
+
/** One ordinal forecast: probabilities in level order, lowest level first. */
|
|
216
|
+
interface OrdinalForecast {
|
|
217
|
+
probabilities: readonly number[];
|
|
218
|
+
/** Index into `probabilities` of the level that occurred. */
|
|
219
|
+
outcome: number;
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Ranked probability score, mean over forecasts of Σₖ₌₁ᴷ⁻¹(Fₖ − Oₖ)² / (K − 1), where F and O are
|
|
223
|
+
* the cumulative forecast and observation, range [0, 1]. Unlike multiclass Brier it charges
|
|
224
|
+
* less for mass on a level next to the outcome than for mass far from it, which is the right
|
|
225
|
+
* penalty for score questions. With two levels it equals the binary Brier score.
|
|
226
|
+
*/
|
|
227
|
+
declare function rankedProbabilityScore(forecasts: readonly OrdinalForecast[]): number;
|
|
228
|
+
//#endregion
|
|
229
|
+
//#region src/meta-eval/cluster-bootstrap.d.ts
|
|
230
|
+
/** One observation and the source cluster it belongs to. */
|
|
231
|
+
interface ClusteredObservation {
|
|
232
|
+
cluster: string;
|
|
233
|
+
value: number;
|
|
234
|
+
}
|
|
235
|
+
interface ClusterBootstrapOptions {
|
|
236
|
+
/** Confidence level. Default 0.95. */
|
|
237
|
+
confidence?: number;
|
|
238
|
+
/** Bootstrap resample count. Default 2000. */
|
|
239
|
+
resamples?: number;
|
|
240
|
+
/** Deterministic seed. If omitted, derived from the values so the interval is reproducible. */
|
|
241
|
+
seed?: number;
|
|
242
|
+
}
|
|
243
|
+
interface ClusterBootstrapResult {
|
|
244
|
+
n: number;
|
|
245
|
+
clusters: number;
|
|
246
|
+
/** Mean over observations, each observation weighted equally. */
|
|
247
|
+
mean: number;
|
|
248
|
+
low: number;
|
|
249
|
+
high: number;
|
|
250
|
+
confidence: number;
|
|
251
|
+
resamples: number;
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Percentile cluster bootstrap of an observation mean. Whole clusters are resampled with
|
|
255
|
+
* replacement and each resample's statistic is Σ values / Σ observations over the drawn
|
|
256
|
+
* clusters, so observations that share a source move together and a large cluster cannot
|
|
257
|
+
* pose as many independent results. For a paired comparison pass the per-observation
|
|
258
|
+
* difference (candidate − baseline) under the observation's cluster.
|
|
259
|
+
*
|
|
260
|
+
* With one cluster the interval collapses to the point estimate: there is no between-cluster
|
|
261
|
+
* spread to measure, which callers must treat as no interval, not as certainty.
|
|
262
|
+
*/
|
|
263
|
+
declare function clusterBootstrapMean(observations: readonly ClusteredObservation[], options?: ClusterBootstrapOptions): ClusterBootstrapResult;
|
|
264
|
+
//#endregion
|
|
265
|
+
//#region src/meta-eval/evaluator-admission.d.ts
|
|
266
|
+
declare const evaluatorAdmissionPolicySchema: z.ZodObject<{
|
|
267
|
+
confidence: z.ZodNumber;
|
|
268
|
+
maxFalseAcceptanceRate: z.ZodNumber;
|
|
269
|
+
maxFalseRejectionRate: z.ZodNumber;
|
|
270
|
+
}, z.core.$strict>;
|
|
271
|
+
/** Register both error limits before inspecting audit judgments. */
|
|
272
|
+
type EvaluatorAdmissionPolicy = z.infer<typeof evaluatorAdmissionPolicySchema>;
|
|
273
|
+
declare const evaluatorAuditObservationSchema: z.ZodObject<{
|
|
274
|
+
id: z.ZodString;
|
|
275
|
+
independentUnitId: z.ZodString;
|
|
276
|
+
evidenceRef: z.ZodString;
|
|
277
|
+
expected: z.ZodEnum<{
|
|
278
|
+
accept: "accept";
|
|
279
|
+
reject: "reject";
|
|
280
|
+
}>;
|
|
281
|
+
observed: z.ZodEnum<{
|
|
282
|
+
accept: "accept";
|
|
283
|
+
reject: "reject";
|
|
284
|
+
unknown: "unknown";
|
|
285
|
+
}>;
|
|
286
|
+
exposure: z.ZodEnum<{
|
|
287
|
+
development: "development";
|
|
288
|
+
fresh: "fresh";
|
|
289
|
+
}>;
|
|
290
|
+
}, z.core.$strict>;
|
|
291
|
+
/** Actual control judgments. Variants from one source retain one independent identity. */
|
|
292
|
+
type EvaluatorAuditObservation = z.infer<typeof evaluatorAuditObservationSchema>;
|
|
293
|
+
declare const inputSchema: z.ZodObject<{
|
|
294
|
+
evaluatorDigest: z.ZodString;
|
|
295
|
+
population: z.ZodString;
|
|
296
|
+
samplingFrame: z.ZodString;
|
|
297
|
+
authority: z.ZodObject<{
|
|
298
|
+
evaluatorAuthorId: z.ZodString;
|
|
299
|
+
auditorId: z.ZodString;
|
|
300
|
+
independenceEvidenceRef: z.ZodString;
|
|
301
|
+
}, z.core.$strict>;
|
|
302
|
+
policy: z.ZodObject<{
|
|
303
|
+
confidence: z.ZodNumber;
|
|
304
|
+
maxFalseAcceptanceRate: z.ZodNumber;
|
|
305
|
+
maxFalseRejectionRate: z.ZodNumber;
|
|
306
|
+
}, z.core.$strict>;
|
|
307
|
+
observations: z.ZodArray<z.ZodObject<{
|
|
308
|
+
id: z.ZodString;
|
|
309
|
+
independentUnitId: z.ZodString;
|
|
310
|
+
evidenceRef: z.ZodString;
|
|
311
|
+
expected: z.ZodEnum<{
|
|
312
|
+
accept: "accept";
|
|
313
|
+
reject: "reject";
|
|
314
|
+
}>;
|
|
315
|
+
observed: z.ZodEnum<{
|
|
316
|
+
accept: "accept";
|
|
317
|
+
reject: "reject";
|
|
318
|
+
unknown: "unknown";
|
|
319
|
+
}>;
|
|
320
|
+
exposure: z.ZodEnum<{
|
|
321
|
+
development: "development";
|
|
322
|
+
fresh: "fresh";
|
|
323
|
+
}>;
|
|
324
|
+
}, z.core.$strict>>;
|
|
325
|
+
}, z.core.$strict>;
|
|
326
|
+
type EvaluatorAuditInput = z.infer<typeof inputSchema>;
|
|
327
|
+
interface EvaluatorErrorRate {
|
|
328
|
+
/** Eligible control judgments in this class, including unknown judgments. */
|
|
329
|
+
cases: number;
|
|
330
|
+
independentUnits: number;
|
|
331
|
+
/** Units with at least one observed mistake. */
|
|
332
|
+
errorUnits: number;
|
|
333
|
+
/** Units with unknown judgments and no observed mistake yet. */
|
|
334
|
+
unresolvedUnits: number;
|
|
335
|
+
unknownCases: number;
|
|
336
|
+
/** Missing judgments do not become measured zeros. */
|
|
337
|
+
errorRate: number | null;
|
|
338
|
+
/** Exact bounds conservatively include every possible outcome of unknown judgments. */
|
|
339
|
+
interval: ComputedInterval | null;
|
|
340
|
+
limit: number;
|
|
341
|
+
verdict: 'pass' | 'fail' | 'inconclusive';
|
|
342
|
+
}
|
|
343
|
+
interface EvaluatorAdmissionReport {
|
|
344
|
+
evaluatorDigest: LedgerHash;
|
|
345
|
+
policyDigest: LedgerHash;
|
|
346
|
+
inputDigest: LedgerHash;
|
|
347
|
+
reportDigest: LedgerHash;
|
|
348
|
+
population: string;
|
|
349
|
+
samplingFrame: string;
|
|
350
|
+
authority: EvaluatorAuditInput['authority'];
|
|
351
|
+
policy: EvaluatorAdmissionPolicy;
|
|
352
|
+
/** Joint coverage for the two reported error-rate intervals. */
|
|
353
|
+
confidence: number;
|
|
354
|
+
intervalConfidence: number;
|
|
355
|
+
verdict: 'admit' | 'reject' | 'inconclusive';
|
|
356
|
+
reasons: string[];
|
|
357
|
+
observations: EvaluatorAuditObservation[];
|
|
358
|
+
coverage: {
|
|
359
|
+
cases: number;
|
|
360
|
+
independentUnits: number;
|
|
361
|
+
eligibleCases: number;
|
|
362
|
+
eligibleIndependentUnits: number;
|
|
363
|
+
excludedCases: number;
|
|
364
|
+
unknownCases: number;
|
|
365
|
+
};
|
|
366
|
+
exclusions: Array<{
|
|
367
|
+
id: string;
|
|
368
|
+
independentUnitId: string;
|
|
369
|
+
reason: 'development-exposure';
|
|
370
|
+
}>;
|
|
371
|
+
falseAcceptance: EvaluatorErrorRate;
|
|
372
|
+
falseRejection: EvaluatorErrorRate;
|
|
373
|
+
}
|
|
374
|
+
declare const capacitySchema: z.ZodObject<{
|
|
375
|
+
policy: z.ZodObject<{
|
|
376
|
+
confidence: z.ZodNumber;
|
|
377
|
+
maxFalseAcceptanceRate: z.ZodNumber;
|
|
378
|
+
maxFalseRejectionRate: z.ZodNumber;
|
|
379
|
+
}, z.core.$strict>;
|
|
380
|
+
controls: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
381
|
+
id: z.ZodString;
|
|
382
|
+
independentUnitId: z.ZodString;
|
|
383
|
+
expected: z.ZodEnum<{
|
|
384
|
+
accept: "accept";
|
|
385
|
+
reject: "reject";
|
|
386
|
+
}>;
|
|
387
|
+
exposure: z.ZodEnum<{
|
|
388
|
+
development: "development";
|
|
389
|
+
fresh: "fresh";
|
|
390
|
+
}>;
|
|
391
|
+
}, z.core.$strict>>>;
|
|
392
|
+
}, z.core.$strict>;
|
|
393
|
+
type EvaluatorAuditPlanInput = z.input<typeof capacitySchema>;
|
|
394
|
+
/**
|
|
395
|
+
* Can the fixed control roster meet the policy even with zero errors?
|
|
396
|
+
* Planning uses the admission rule's exact bounds and source-unit exclusions.
|
|
397
|
+
* It accepts no judgments or authority and returns no admission report.
|
|
398
|
+
*/
|
|
399
|
+
declare function planEvaluatorAudit(input: EvaluatorAuditPlanInput): {
|
|
400
|
+
sufficient: boolean;
|
|
401
|
+
falseAcceptance: {
|
|
402
|
+
independentUnits: number;
|
|
403
|
+
bestPossibleUpperBound: number | null;
|
|
404
|
+
limit: number;
|
|
405
|
+
minimumIndependentUnits: number | null;
|
|
406
|
+
additionalIndependentUnits: number | null;
|
|
407
|
+
};
|
|
408
|
+
falseRejection: {
|
|
409
|
+
independentUnits: number;
|
|
410
|
+
bestPossibleUpperBound: number | null;
|
|
411
|
+
limit: number;
|
|
412
|
+
minimumIndependentUnits: number | null;
|
|
413
|
+
additionalIndependentUnits: number | null;
|
|
414
|
+
};
|
|
415
|
+
};
|
|
416
|
+
type EvaluatorAuditPlan = ReturnType<typeof planEvaluatorAudit>;
|
|
417
|
+
/**
|
|
418
|
+
* Audit frozen judgments using independent source units and exact binomial bounds.
|
|
419
|
+
* A unit fails a class when any control in that class is misjudged.
|
|
420
|
+
* The execution owner enforces auditor separation, fresh sampling, and evidence authenticity.
|
|
421
|
+
*/
|
|
422
|
+
declare function auditEvaluator(input: EvaluatorAuditInput): EvaluatorAdmissionReport;
|
|
423
|
+
declare const probabilityAuditSchema: z.ZodObject<{
|
|
424
|
+
evaluatorDigest: z.ZodString;
|
|
425
|
+
population: z.ZodString;
|
|
426
|
+
samplingFrame: z.ZodString;
|
|
427
|
+
authority: z.ZodObject<{
|
|
428
|
+
evaluatorAuthorId: z.ZodString;
|
|
429
|
+
auditorId: z.ZodString;
|
|
430
|
+
independenceEvidenceRef: z.ZodString;
|
|
431
|
+
}, z.core.$strict>;
|
|
432
|
+
policy: z.ZodObject<{
|
|
433
|
+
confidence: z.ZodNumber;
|
|
434
|
+
maxFalseAcceptanceRate: z.ZodNumber;
|
|
435
|
+
maxFalseRejectionRate: z.ZodNumber;
|
|
436
|
+
}, z.core.$strict>;
|
|
437
|
+
thresholds: z.ZodObject<{
|
|
438
|
+
rejectAtOrBelow: z.ZodNumber;
|
|
439
|
+
acceptAtOrAbove: z.ZodNumber;
|
|
440
|
+
}, z.core.$strict>;
|
|
441
|
+
observations: z.ZodArray<z.ZodObject<{
|
|
442
|
+
id: z.ZodString;
|
|
443
|
+
independentUnitId: z.ZodString;
|
|
444
|
+
evidenceRef: z.ZodString;
|
|
445
|
+
expected: z.ZodEnum<{
|
|
446
|
+
accept: "accept";
|
|
447
|
+
reject: "reject";
|
|
448
|
+
}>;
|
|
449
|
+
exposure: z.ZodEnum<{
|
|
450
|
+
development: "development";
|
|
451
|
+
fresh: "fresh";
|
|
452
|
+
}>;
|
|
453
|
+
acceptProbability: z.ZodNullable<z.ZodNumber>;
|
|
454
|
+
}, z.core.$strict>>;
|
|
455
|
+
}, z.core.$strict>;
|
|
456
|
+
/** A frozen caller-owned rule over P(accept), not the provider's entropy/confidence statistic. */
|
|
457
|
+
type ProbabilityPolicyAuditInput = z.infer<typeof probabilityAuditSchema>;
|
|
458
|
+
interface ProbabilityPolicyAuditReport extends EvaluatorAdmissionReport {
|
|
459
|
+
probabilityPolicy: {
|
|
460
|
+
sourceEvaluatorDigest: LedgerHash;
|
|
461
|
+
thresholds: ProbabilityPolicyAuditInput['thresholds'];
|
|
462
|
+
/** Retain probabilities by observation id so the reported mapping can be reproduced. */
|
|
463
|
+
observations: Array<{
|
|
464
|
+
id: string;
|
|
465
|
+
acceptProbability: number | null;
|
|
466
|
+
}>;
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
/**
|
|
470
|
+
* Audit a fixed probability-to-decision rule with the existing independent-unit error bounds.
|
|
471
|
+
* No threshold search, model calls, promotion, or assumption of calibrated probabilities.
|
|
472
|
+
* Null scores and the open interval between thresholds remain unknown, never correct by default.
|
|
473
|
+
*/
|
|
474
|
+
declare function auditProbabilityPolicy(input: ProbabilityPolicyAuditInput): ProbabilityPolicyAuditReport;
|
|
475
|
+
//#endregion
|
|
476
|
+
//#region src/series-convergence.d.ts
|
|
477
|
+
/**
|
|
478
|
+
* Series convergence — detects whether a sequence of scalar measurements
|
|
479
|
+
* is stabilizing, drifting, or noisy. It reads drift *across* runs, e.g.
|
|
480
|
+
* "are my nightly eval scores stabilizing?".
|
|
481
|
+
*
|
|
482
|
+
* Three signals:
|
|
483
|
+
* - stabilized: last K values have low variance (< epsilon) — done
|
|
484
|
+
* - drifting: recent trend is monotonic and beyond noise — regressing or improving
|
|
485
|
+
* - noisy: neither — keep iterating, but flag as untrustworthy for gating
|
|
486
|
+
*/
|
|
487
|
+
interface SeriesConvergenceOptions {
|
|
488
|
+
/** Window size for "recent" analysis (default 5). */
|
|
489
|
+
window?: number;
|
|
490
|
+
/** Coefficient-of-variation threshold below which the window is stabilized (default 0.05 = 5%). */
|
|
491
|
+
stableCv?: number;
|
|
492
|
+
/** Minimum monotone run length to call drift (default 3). */
|
|
493
|
+
driftRun?: number;
|
|
494
|
+
}
|
|
495
|
+
interface SeriesConvergenceResult {
|
|
496
|
+
state: 'stabilized' | 'drifting-up' | 'drifting-down' | 'noisy' | 'insufficient-data';
|
|
497
|
+
windowMean: number;
|
|
498
|
+
windowCv: number;
|
|
499
|
+
/** Longest monotonic run at the tail of the series (positive for up, negative for down). */
|
|
500
|
+
tailRun: number;
|
|
501
|
+
/** True when n ≥ window AND windowCv ≤ stableCv. */
|
|
502
|
+
stable: boolean;
|
|
503
|
+
}
|
|
504
|
+
declare function analyzeSeries(values: number[], options?: SeriesConvergenceOptions): SeriesConvergenceResult;
|
|
505
|
+
//#endregion
|
|
506
|
+
//#region src/meta-eval/sentinel.d.ts
|
|
507
|
+
declare const SENTINEL_METRIC_NAMES: readonly ['irr', 'calibrationKappa', 'sentinelPassRate'];
|
|
508
|
+
type SentinelMetricName = (typeof SENTINEL_METRIC_NAMES)[number];
|
|
509
|
+
interface SentinelMetrics {
|
|
510
|
+
/** Inter-rater reliability — ICC(2,1) from agreement instruments. */
|
|
511
|
+
irr?: number;
|
|
512
|
+
/** Weighted κ vs the human golden set (`calibrateJudge*`). */
|
|
513
|
+
calibrationKappa?: number;
|
|
514
|
+
/** Fraction of a fixed sentinel golden set the judge still scores within tolerance. */
|
|
515
|
+
sentinelPassRate?: number;
|
|
516
|
+
}
|
|
517
|
+
interface SentinelSnapshot {
|
|
518
|
+
/** Caller-supplied ISO-8601 timestamp — the module never reads a clock. */
|
|
519
|
+
at: string;
|
|
520
|
+
judgeId: string;
|
|
521
|
+
/** Exact model identity behind the judge (pin the full version string). */
|
|
522
|
+
judgeModel: string;
|
|
523
|
+
/**
|
|
524
|
+
* Metrics measured at `at`. May be empty: an empty-metrics snapshot is a
|
|
525
|
+
* model-change marker — it records the new `judgeModel` immediately and
|
|
526
|
+
* the silent-upgrade alarm stays raised until a golden-grounded snapshot
|
|
527
|
+
* (calibrationKappa or sentinelPassRate) follows.
|
|
528
|
+
*/
|
|
529
|
+
metrics: SentinelMetrics;
|
|
530
|
+
}
|
|
531
|
+
/** Identity + timestamp the caller supplies alongside an instrument output. */
|
|
532
|
+
interface SnapshotMeta {
|
|
533
|
+
at: string;
|
|
534
|
+
judgeId: string;
|
|
535
|
+
judgeModel: string;
|
|
536
|
+
}
|
|
537
|
+
/** Shape guard for snapshots crossing a boundary (store append, JSONL load, report input). */
|
|
538
|
+
declare function validateSentinelSnapshot(snapshot: SentinelSnapshot, source?: string): void;
|
|
539
|
+
/**
|
|
540
|
+
* From `calibrateJudge` / `calibrateJudgeContinuous` output. Prefers the
|
|
541
|
+
* un-rounded continuous κ_w when the report carries one (integer κ
|
|
542
|
+
* discards information for [0,1] judges).
|
|
543
|
+
*/
|
|
544
|
+
declare function snapshotFromCalibration(report: CalibrationResult | ContinuousCalibrationResult, meta: SnapshotMeta): SentinelSnapshot;
|
|
545
|
+
/**
|
|
546
|
+
* From `corpusInterRaterAgreement` (statistics.ts) or `continuousAgreement`
|
|
547
|
+
* (judge-calibration.ts) output. IRR = ICC(2,1) — the reliability
|
|
548
|
+
* coefficient both instruments compute. For ensemble-level agreement,
|
|
549
|
+
* `meta.judgeId` names the ensemble and `meta.judgeModel` pins its
|
|
550
|
+
* composition (e.g. a joined list of member model strings).
|
|
551
|
+
*/
|
|
552
|
+
declare function snapshotFromAgreement(report: CorpusAgreementReport | ContinuousAgreement, meta: SnapshotMeta): SentinelSnapshot;
|
|
553
|
+
interface SentinelSetOptions {
|
|
554
|
+
/** Max |judge − human| for an item to count as passed. Default 0.1 (tuned for [0,1] scores). */
|
|
555
|
+
tolerance?: number;
|
|
556
|
+
}
|
|
557
|
+
/**
|
|
558
|
+
* From a rerun of a fixed sentinel golden set: the same `GoldenItem`s the
|
|
559
|
+
* judge was originally calibrated on, re-scored periodically. Pass rate =
|
|
560
|
+
* fraction of joined items where the judge stays within `tolerance` of the
|
|
561
|
+
* human score. Items the judge didn't score are excluded from the
|
|
562
|
+
* denominator; zero joined items is an error, not a 0% pass rate.
|
|
563
|
+
*/
|
|
564
|
+
declare function snapshotFromSentinelSet(scores: CandidateScore[], golden: GoldenItem[], meta: SnapshotMeta, options?: SentinelSetOptions): SentinelSnapshot;
|
|
565
|
+
interface SentinelStore {
|
|
566
|
+
append(snapshot: SentinelSnapshot): Promise<void>;
|
|
567
|
+
/** All snapshots (optionally for one judge), in append order. */
|
|
568
|
+
history(judgeId?: string): Promise<SentinelSnapshot[]>;
|
|
569
|
+
}
|
|
570
|
+
/**
|
|
571
|
+
* JSONL store — one snapshot per line, appended atomically per call.
|
|
572
|
+
* A corrupt or shape-invalid line is a loud error naming the file and
|
|
573
|
+
* line number: a sentinel history that silently drops records would
|
|
574
|
+
* defeat the drift detection it exists to provide.
|
|
575
|
+
*/
|
|
576
|
+
declare function fileSentinelStore(path: string): SentinelStore;
|
|
577
|
+
interface SentinelThresholds {
|
|
578
|
+
/** Floor for irr — below it the judge ensemble is unreliable regardless of trend. Default 0.6. */
|
|
579
|
+
minIrr?: number;
|
|
580
|
+
/** Max tolerated calibrationKappa drop vs the series baseline. Default 0.15. */
|
|
581
|
+
maxKappaDrop?: number;
|
|
582
|
+
/** Max tolerated sentinelPassRate drop vs the series baseline. Default 0.1. */
|
|
583
|
+
maxSentinelDrop?: number;
|
|
584
|
+
/** Newest snapshot older than this (days, vs asOf) = the sentinel itself went blind. Default 30. */
|
|
585
|
+
staleAfterDays?: number;
|
|
586
|
+
}
|
|
587
|
+
interface JudgeSentinelOptions {
|
|
588
|
+
/** Caller-supplied ISO timestamp staleness is measured against. */
|
|
589
|
+
asOf: string;
|
|
590
|
+
thresholds?: SentinelThresholds;
|
|
591
|
+
/** Forwarded to `analyzeSeries` (window, stableCv, driftRun). */
|
|
592
|
+
convergence?: SeriesConvergenceOptions;
|
|
593
|
+
}
|
|
594
|
+
interface SentinelTrend {
|
|
595
|
+
judgeId: string;
|
|
596
|
+
metric: SentinelMetricName;
|
|
597
|
+
/** Verbatim `analyzeSeries` state for this judge×metric history. */
|
|
598
|
+
state: SeriesConvergenceResult['state'];
|
|
599
|
+
/** Newest value in the series. */
|
|
600
|
+
current: number;
|
|
601
|
+
/** Oldest value in the series — the reference the drop thresholds compare against. */
|
|
602
|
+
baseline: number;
|
|
603
|
+
/** current − baseline (signed; negative = decayed). */
|
|
604
|
+
drift: number;
|
|
605
|
+
alarmed: boolean;
|
|
606
|
+
reason?: string;
|
|
607
|
+
}
|
|
608
|
+
interface SentinelReport {
|
|
609
|
+
perJudge: SentinelTrend[];
|
|
610
|
+
alarms: string[];
|
|
611
|
+
healthy: boolean;
|
|
612
|
+
/**
|
|
613
|
+
* `judgeId:metric` pairs with too few snapshots for the convergence
|
|
614
|
+
* machine. Named, not counted — a blind spot you can't see is a blind
|
|
615
|
+
* spot you won't fix. Floor/drop checks still apply to thin series, so
|
|
616
|
+
* insufficient history alone never masks an alarm.
|
|
617
|
+
*/
|
|
618
|
+
insufficientHistory: string[];
|
|
619
|
+
}
|
|
620
|
+
declare function judgeSentinelReport(history: SentinelSnapshot[], opts: JudgeSentinelOptions): SentinelReport;
|
|
621
|
+
interface EvalHealthStamp {
|
|
622
|
+
healthy: boolean;
|
|
623
|
+
alarms: string[];
|
|
624
|
+
}
|
|
625
|
+
/**
|
|
626
|
+
* The object a campaign attaches to its verdicts. Compute it from the
|
|
627
|
+
* sentinel report in a post-campaign hook (or a nightly job feeding the
|
|
628
|
+
* next day's campaigns) and store it alongside the verdict payload.
|
|
629
|
+
* `healthy: false` means the judges that produced those verdicts have an
|
|
630
|
+
* unresolved drift / decay / silent-upgrade / staleness alarm — treat the
|
|
631
|
+
* verdicts as carrying an integrity warning until recalibration clears it.
|
|
632
|
+
*/
|
|
633
|
+
declare function evalHealthStamp(report: SentinelReport): EvalHealthStamp;
|
|
634
|
+
//#endregion
|
|
635
|
+
//#region src/meta-eval/judge-gate.d.ts
|
|
636
|
+
type OwnerVerdict = 'pass' | 'fail';
|
|
637
|
+
/** One item the owner decided, with where that decision is recorded. */
|
|
638
|
+
interface CalibrationExample {
|
|
639
|
+
/** Stable id of the decided item; a judge run names it in `exampleId`. */
|
|
640
|
+
id: string;
|
|
641
|
+
verdict: OwnerVerdict;
|
|
642
|
+
/** The person whose verdict this is. Never a model. */
|
|
643
|
+
labeledBy: string;
|
|
644
|
+
/** ISO-8601 time the verdict was given. */
|
|
645
|
+
labeledAt: string;
|
|
646
|
+
/** Where the verdict is recorded, e.g. `gtm:asset_decision:<id>`. */
|
|
647
|
+
source: string;
|
|
648
|
+
/** What the owner judged (an image digest, a run id), so a replay judges the same thing. */
|
|
649
|
+
evidenceRef?: string;
|
|
650
|
+
note?: string;
|
|
651
|
+
}
|
|
652
|
+
interface JudgeCalibrationSet {
|
|
653
|
+
judgeId: string;
|
|
654
|
+
/** Whose verdicts the judge stands in for. */
|
|
655
|
+
owner: string;
|
|
656
|
+
examples: readonly CalibrationExample[];
|
|
657
|
+
}
|
|
658
|
+
/**
|
|
659
|
+
* Validate a calibration set and return a normalized copy ordered by example
|
|
660
|
+
* id. Every verdict needs its person, time and source; ids are unique.
|
|
661
|
+
*/
|
|
662
|
+
declare function registerCalibrationSet(set: JudgeCalibrationSet): JudgeCalibrationSet;
|
|
663
|
+
interface CalibrationMerge {
|
|
664
|
+
set: JudgeCalibrationSet;
|
|
665
|
+
/** Ids that joined the set. */
|
|
666
|
+
added: string[];
|
|
667
|
+
/** Ids whose verdict a later decision replaced. */
|
|
668
|
+
superseded: string[];
|
|
669
|
+
/** Ids whose incoming verdict was already recorded, or older than the recorded one. */
|
|
670
|
+
unchanged: string[];
|
|
671
|
+
}
|
|
672
|
+
/**
|
|
673
|
+
* Join new owner verdicts to a set. A verdict already recorded from the same
|
|
674
|
+
* source is unchanged, so ingestion can rerun. A later verdict on an item
|
|
675
|
+
* replaces an earlier one: the owner changed their mind, and the set keeps
|
|
676
|
+
* what they decided last.
|
|
677
|
+
*/
|
|
678
|
+
declare function addCalibrationVerdicts(set: JudgeCalibrationSet, verdicts: readonly CalibrationExample[]): CalibrationMerge;
|
|
679
|
+
/** One gate decision the judge made on a calibration example. */
|
|
680
|
+
interface JudgeRun {
|
|
681
|
+
exampleId: string;
|
|
682
|
+
/** The judge's decision after its own sample aggregation (median, majority). */
|
|
683
|
+
verdict: OwnerVerdict;
|
|
684
|
+
/** The per-sample verdicts behind that decision, when the judge samples. */
|
|
685
|
+
samples?: readonly OwnerVerdict[];
|
|
686
|
+
}
|
|
687
|
+
interface JudgeRunMeta {
|
|
688
|
+
judgeId: string;
|
|
689
|
+
/** Exact model identity, or `deterministic` for a grader without one. */
|
|
690
|
+
judgeModel: string;
|
|
691
|
+
rubricVersion: string;
|
|
692
|
+
measuredAt: string;
|
|
693
|
+
}
|
|
694
|
+
interface JudgeConfusion {
|
|
695
|
+
/** Judge pass, owner pass. */
|
|
696
|
+
truePass: number;
|
|
697
|
+
/** Judge fail, owner fail. */
|
|
698
|
+
trueFail: number;
|
|
699
|
+
/** Judge pass, owner fail: the judge let through what the owner rejected. */
|
|
700
|
+
falsePass: number;
|
|
701
|
+
/** Judge fail, owner pass: the judge blocked what the owner accepted. */
|
|
702
|
+
falseFail: number;
|
|
703
|
+
}
|
|
704
|
+
interface JudgeExampleAgreement {
|
|
705
|
+
id: string;
|
|
706
|
+
verdict: OwnerVerdict;
|
|
707
|
+
runs: number;
|
|
708
|
+
/** Share of runs that passed. */
|
|
709
|
+
passRate: number;
|
|
710
|
+
/** Share of runs that reached the owner's verdict. */
|
|
711
|
+
agreement: number;
|
|
712
|
+
}
|
|
713
|
+
interface JudgeAgreement extends JudgeRunMeta {
|
|
714
|
+
owner: string;
|
|
715
|
+
/** Examples in the calibration set. */
|
|
716
|
+
examples: number;
|
|
717
|
+
/** Examples the judge ran on at least once. */
|
|
718
|
+
judged: number;
|
|
719
|
+
/** Judged examples per owner verdict. */
|
|
720
|
+
judgedByVerdict: Record<OwnerVerdict, number>;
|
|
721
|
+
runs: number;
|
|
722
|
+
/** Runs that reached the owner's verdict, over all runs; null with no runs. */
|
|
723
|
+
accuracy: number | null;
|
|
724
|
+
/**
|
|
725
|
+
* Exact (Clopper-Pearson) 95% interval for the share of judged examples on
|
|
726
|
+
* which every run reached the owner's verdict. Examples, not runs, are the
|
|
727
|
+
* independent units.
|
|
728
|
+
*/
|
|
729
|
+
exampleAccuracy: {
|
|
730
|
+
agreed: number;
|
|
731
|
+
judged: number;
|
|
732
|
+
interval: ComputedInterval;
|
|
733
|
+
} | null;
|
|
734
|
+
/**
|
|
735
|
+
* Cohen's κ between the judge and the owner over runs. Null when undefined:
|
|
736
|
+
* no runs, or both sides used a single verdict.
|
|
737
|
+
*/
|
|
738
|
+
kappa: number | null;
|
|
739
|
+
/** Runs that failed what the owner passed, over runs on owner passes; null with none. A veto judge is held to this. */
|
|
740
|
+
falseFailRate: number | null;
|
|
741
|
+
/** Runs that failed what the owner failed, over runs on owner fails; null with none. */
|
|
742
|
+
catchRate: number | null;
|
|
743
|
+
confusion: JudgeConfusion;
|
|
744
|
+
variance: {
|
|
745
|
+
/** Mean over judged examples of p(1 − p), p the example's run pass rate. 0 = no run changed its mind. */
|
|
746
|
+
betweenRuns: number | null;
|
|
747
|
+
/** Mean over runs that reported samples of p(1 − p) across those samples; null when none did. */
|
|
748
|
+
withinRun: number | null;
|
|
749
|
+
/** Examples whose runs disagree with each other. */
|
|
750
|
+
unstable: string[];
|
|
751
|
+
};
|
|
752
|
+
/** Examples with at least one run against the owner, least agreement first. */
|
|
753
|
+
disagreements: JudgeExampleAgreement[];
|
|
754
|
+
/** Examples without any run. */
|
|
755
|
+
unjudged: string[];
|
|
756
|
+
}
|
|
757
|
+
/**
|
|
758
|
+
* Agreement between a judge's runs and the owner's verdicts on one
|
|
759
|
+
* calibration set. A run that names an example outside the set is an error:
|
|
760
|
+
* a silently dropped run would overstate agreement.
|
|
761
|
+
*/
|
|
762
|
+
declare function measureJudgeAgreement(set: JudgeCalibrationSet, runs: readonly JudgeRun[], meta: JudgeRunMeta): JudgeAgreement;
|
|
763
|
+
interface JudgeGatePolicy {
|
|
764
|
+
/**
|
|
765
|
+
* `verdict`: the judge passes and blocks, so accuracy and κ must hold.
|
|
766
|
+
* `veto`: it only blocks, so its false-fail rate on owner passes must hold.
|
|
767
|
+
*/
|
|
768
|
+
decides: 'verdict' | 'veto';
|
|
769
|
+
/** Judged examples required before the judge may gate; for a veto judge, owner passes. */
|
|
770
|
+
minExamples: number;
|
|
771
|
+
/** Judged examples required of each owner verdict, so κ is defined. */
|
|
772
|
+
minPerVerdict: number;
|
|
773
|
+
/** Floor on run accuracy. */
|
|
774
|
+
minAccuracy: number;
|
|
775
|
+
/** Floor on Cohen's κ. */
|
|
776
|
+
minKappa: number;
|
|
777
|
+
/** Ceiling on a veto judge's false-fail rate. */
|
|
778
|
+
maxFalseFailRate: number;
|
|
779
|
+
/** A report older than this (days, vs `asOf`) no longer calibrates the judge. */
|
|
780
|
+
maxAgeDays: number;
|
|
781
|
+
/** Drift thresholds forwarded to `judgeSentinelReport` over the history. */
|
|
782
|
+
drift?: SentinelThresholds;
|
|
783
|
+
}
|
|
784
|
+
declare const DEFAULT_JUDGE_GATE_POLICY: Readonly<JudgeGatePolicy>;
|
|
785
|
+
type JudgeGateStatus = 'calibrated' | 'uncalibrated' | 'below-threshold' | 'drifted';
|
|
786
|
+
interface JudgeGateDecision {
|
|
787
|
+
judgeId: string;
|
|
788
|
+
judgeModel: string;
|
|
789
|
+
rubricVersion: string;
|
|
790
|
+
/** True only for `calibrated`. */
|
|
791
|
+
mayGate: boolean;
|
|
792
|
+
status: JudgeGateStatus;
|
|
793
|
+
/** Why the judge may not gate; empty when it may. */
|
|
794
|
+
reasons: string[];
|
|
795
|
+
agreement: JudgeAgreement | null;
|
|
796
|
+
policy: JudgeGatePolicy;
|
|
797
|
+
}
|
|
798
|
+
interface JudgeGateInput {
|
|
799
|
+
judgeId: string;
|
|
800
|
+
/** The model the judge runs now. */
|
|
801
|
+
judgeModel: string;
|
|
802
|
+
/** The rubric the judge runs now. */
|
|
803
|
+
rubricVersion: string;
|
|
804
|
+
/** The latest calibration report; null when the judge was never measured. */
|
|
805
|
+
agreement: JudgeAgreement | null;
|
|
806
|
+
asOf: string;
|
|
807
|
+
/** Sentinel snapshots for this judge, oldest first or in any order. */
|
|
808
|
+
history?: readonly SentinelSnapshot[];
|
|
809
|
+
policy?: Partial<JudgeGatePolicy>;
|
|
810
|
+
}
|
|
811
|
+
/** Whether a judge may gate now, and every reason it may not. */
|
|
812
|
+
declare function judgeGateDecision(input: JudgeGateInput): JudgeGateDecision;
|
|
813
|
+
/** Thrown by `assertJudgeMayGate`; carries the refused decision. */
|
|
814
|
+
declare class JudgeGateRefusedError extends JudgeError {
|
|
815
|
+
readonly decision: JudgeGateDecision;
|
|
816
|
+
constructor(decision: JudgeGateDecision);
|
|
817
|
+
}
|
|
818
|
+
/** Throw unless the decision lets the judge gate. */
|
|
819
|
+
declare function assertJudgeMayGate(decision: JudgeGateDecision): void;
|
|
820
|
+
/**
|
|
821
|
+
* The sentinel snapshot for one calibration run. A deciding judge records
|
|
822
|
+
* accuracy as `sentinelPassRate` and κ as `calibrationKappa` when it is
|
|
823
|
+
* defined; a veto judge records the share of owner passes it let through as
|
|
824
|
+
* `sentinelPassRate`. Append it to the judge's `SentinelStore`; the gate reads
|
|
825
|
+
* drift from that history.
|
|
826
|
+
*/
|
|
827
|
+
declare function snapshotFromJudgeAgreement(agreement: JudgeAgreement, options?: {
|
|
828
|
+
decides?: JudgeGatePolicy['decides'];
|
|
829
|
+
}): SentinelSnapshot;
|
|
830
|
+
//#endregion
|
|
831
|
+
//#region src/meta-eval/plants.d.ts
|
|
832
|
+
/**
|
|
833
|
+
* How a plant item was authored wrong. The class is reported separately in
|
|
834
|
+
* {@link CatchRateReport.byKind} because a grader is routinely sharp on one
|
|
835
|
+
* and blind to another.
|
|
836
|
+
*/
|
|
837
|
+
type PlantKind =
|
|
838
|
+
/** A load-bearing value is altered: a number off by one, a comparison flipped. */
|
|
839
|
+
'wrong-value' |
|
|
840
|
+
/** The item carries its own check, and that check passes without testing the claim. */
|
|
841
|
+
'self-certifying' |
|
|
842
|
+
/** The check names an input that does not exist, so it cannot run at all. */
|
|
843
|
+
'unreachable-input' |
|
|
844
|
+
/** A copy of an item already in the set, which is owed a duplicate flag rather than a second grade. */
|
|
845
|
+
'duplicate';
|
|
846
|
+
/** What a working grader owes a seeded item. */
|
|
847
|
+
type PlantExpectation = 'reject' | 'accept';
|
|
848
|
+
interface Plant {
|
|
849
|
+
/** Name of the plant record. Reported in `missedIds` and `missingIds`. */
|
|
850
|
+
id: string;
|
|
851
|
+
kind: PlantKind;
|
|
852
|
+
/** The seeded item, indistinguishable from a real one once mixed. */
|
|
853
|
+
item: GoldenItem;
|
|
854
|
+
/** The verdict a working grader owes this item. */
|
|
855
|
+
expectedVerdict: PlantExpectation;
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Build one plant record and refuse an incoherent one.
|
|
859
|
+
*
|
|
860
|
+
* The refusal that matters is the last: a record whose `expectedVerdict`
|
|
861
|
+
* disagrees with `item.humanScore` inverts the measurement silently, because
|
|
862
|
+
* the same item then reads as wrong here and as correct to every calibration
|
|
863
|
+
* instrument that joins on the id.
|
|
864
|
+
*
|
|
865
|
+
* `item` is copied field by field so a later mutation of the caller's object
|
|
866
|
+
* cannot change what the manifest sealed, and `group` is dropped when it is
|
|
867
|
+
* absent so the record always has a canonical JSON form.
|
|
868
|
+
*/
|
|
869
|
+
declare function definePlant(input: {
|
|
870
|
+
id: string;
|
|
871
|
+
kind: PlantKind;
|
|
872
|
+
item: GoldenItem;
|
|
873
|
+
expectedVerdict: PlantExpectation;
|
|
874
|
+
}): Plant;
|
|
875
|
+
/**
|
|
876
|
+
* A claim's executable evidence: the command a grader runs and the output it
|
|
877
|
+
* must produce.
|
|
878
|
+
*/
|
|
879
|
+
interface PlantEvidence {
|
|
880
|
+
/** The command a grader runs to test the claim. */
|
|
881
|
+
check?: string;
|
|
882
|
+
/** The output that command must produce for the claim to stand. */
|
|
883
|
+
expect?: string;
|
|
884
|
+
}
|
|
885
|
+
/** Which evidence field {@link perturbEvidence} altered. */
|
|
886
|
+
type PerturbedField = 'check' | 'expect';
|
|
887
|
+
/** How {@link perturbEvidence} altered it. */
|
|
888
|
+
type PerturbationKind = 'number-off-by-one' | 'comparison-flipped';
|
|
889
|
+
interface EvidencePerturbation {
|
|
890
|
+
/** The evidence with exactly one value altered. */
|
|
891
|
+
evidence: PlantEvidence;
|
|
892
|
+
/** Which field changed. */
|
|
893
|
+
field: PerturbedField;
|
|
894
|
+
/** How it changed. */
|
|
895
|
+
how: PerturbationKind;
|
|
896
|
+
/**
|
|
897
|
+
* The exact substring that was replaced. A comparison token carries its
|
|
898
|
+
* surrounding spaces, because the spaces are what make ` -ge ` a comparison
|
|
899
|
+
* and not part of a word.
|
|
900
|
+
*/
|
|
901
|
+
original: string;
|
|
902
|
+
/** What replaced it. */
|
|
903
|
+
perturbed: string;
|
|
904
|
+
}
|
|
905
|
+
/**
|
|
906
|
+
* Derive a known-wrong version of a claim's evidence by altering exactly one
|
|
907
|
+
* value, or return null when no value can be altered safely.
|
|
908
|
+
*
|
|
909
|
+
* Exactly one value changes. A perturbation that moves two does not name the
|
|
910
|
+
* defect it seeded, so a grader that catches it says nothing about which defect
|
|
911
|
+
* the grader can see.
|
|
912
|
+
*
|
|
913
|
+
* The preference order IS the safety order:
|
|
914
|
+
*
|
|
915
|
+
* 1. the last standalone number in `expect` — it falsifies the claim while the
|
|
916
|
+
* command still runs exactly as it did;
|
|
917
|
+
* 2. a flipped comparison in `check` — it inverts a test that was passing;
|
|
918
|
+
* 3. the last standalone number in `check` — last because a number in a command
|
|
919
|
+
* can name an input (a port, a width, a version) rather than a threshold,
|
|
920
|
+
* and renaming an input breaks execution, not the claim.
|
|
921
|
+
*
|
|
922
|
+
* "Standalone" excludes a digit run glued to a word, a dot, a slash, or a
|
|
923
|
+
* hyphen: `python3`, `file42.txt`, `1.5`, `utf-8`. That exclusion is why this
|
|
924
|
+
* helper exists. Hand-rolled perturbation bumps one of those, the check stops
|
|
925
|
+
* executing, and the plant measures the environment instead of the grader.
|
|
926
|
+
*
|
|
927
|
+
* A number is bumped with BigInt, so a value wider than
|
|
928
|
+
* `Number.MAX_SAFE_INTEGER` moves by exactly one. Bumped as a `number` it
|
|
929
|
+
* rounds back to itself, and the plant is then a correct claim the grader is
|
|
930
|
+
* right to verify.
|
|
931
|
+
*/
|
|
932
|
+
declare function perturbEvidence(evidence: PlantEvidence): EvidencePerturbation | null;
|
|
933
|
+
interface PerturbedPlant {
|
|
934
|
+
plant: Plant;
|
|
935
|
+
/** The perturbed evidence to write into the item the grader is handed. */
|
|
936
|
+
evidence: PlantEvidence;
|
|
937
|
+
field: PerturbedField;
|
|
938
|
+
how: PerturbationKind;
|
|
939
|
+
original: string;
|
|
940
|
+
perturbed: string;
|
|
941
|
+
}
|
|
942
|
+
/**
|
|
943
|
+
* Derive a reject-direction plant from a claim a grader verified, by perturbing
|
|
944
|
+
* one value.
|
|
945
|
+
*
|
|
946
|
+
* {@link definePlant} takes a {@link GoldenItem} and never authors a wrong item
|
|
947
|
+
* from a right one, so until now every caller hand-rolled the perturbation.
|
|
948
|
+
* This is that step, done once and the same way every time: the claim's
|
|
949
|
+
* evidence is altered by {@link perturbEvidence}, the result is a `wrong-value`
|
|
950
|
+
* plant the grader owes a `reject`, and the record itself is built by
|
|
951
|
+
* `definePlant` so every refusal there still applies — in particular the one
|
|
952
|
+
* that catches an `expectedVerdict` its `humanScore` contradicts.
|
|
953
|
+
*
|
|
954
|
+
* Returns null when nothing in the evidence can be perturbed. That is not a
|
|
955
|
+
* failure: a claim with no executable evidence cannot be made wrong in a way a
|
|
956
|
+
* grader could execute, and seeding it would measure prose.
|
|
957
|
+
*
|
|
958
|
+
* Refuses an empty `id` or `itemId`. Both name the record — one in the manifest
|
|
959
|
+
* and in `missedIds`, the other in the join with the results — and a miss
|
|
960
|
+
* nobody can look up is not a measurement.
|
|
961
|
+
*/
|
|
962
|
+
declare function plantByPerturbation(input: {
|
|
963
|
+
id: string;
|
|
964
|
+
itemId: string;
|
|
965
|
+
evidence: PlantEvidence;
|
|
966
|
+
/** Score for the perturbed item. Must sit below the run's accept threshold. Default 0. */
|
|
967
|
+
humanScore?: number;
|
|
968
|
+
group?: string;
|
|
969
|
+
}): PerturbedPlant | null;
|
|
970
|
+
interface PlantManifest {
|
|
971
|
+
/**
|
|
972
|
+
* Digest over the seeded order, the plants, and the threshold. Publish it
|
|
973
|
+
* before the grading run: a manifest edited afterwards to match the results
|
|
974
|
+
* no longer matches its seal, and {@link catchRate} refuses it.
|
|
975
|
+
*/
|
|
976
|
+
seal: LedgerHash;
|
|
977
|
+
/** The seed that fixed the mix order. */
|
|
978
|
+
seed: number;
|
|
979
|
+
/** The grade at or above which the graded policy's own gate accepts an item. */
|
|
980
|
+
acceptThreshold: number;
|
|
981
|
+
/** Every item id in the seeded set, in the order handed out. */
|
|
982
|
+
itemIds: string[];
|
|
983
|
+
/** The seeded plants. This is the answer key; keep it out of the graded workspace. */
|
|
984
|
+
plants: Plant[];
|
|
985
|
+
}
|
|
986
|
+
interface SeededGradingSet {
|
|
987
|
+
/**
|
|
988
|
+
* The mixed set in seeded order. Operator-side: it still carries every
|
|
989
|
+
* item's `humanScore`, so hand the graded policy the payload each `itemId`
|
|
990
|
+
* names, never this array.
|
|
991
|
+
*/
|
|
992
|
+
items: GoldenItem[];
|
|
993
|
+
manifest: PlantManifest;
|
|
994
|
+
}
|
|
995
|
+
interface SeedPlantsOptions {
|
|
996
|
+
/**
|
|
997
|
+
* Fixes the mix order. The same dataset, plants, threshold, and seed always
|
|
998
|
+
* produce the same seeded order and the same seal.
|
|
999
|
+
*/
|
|
1000
|
+
seed?: number;
|
|
1001
|
+
/**
|
|
1002
|
+
* The grade at or above which the graded policy's own gate accepts an item.
|
|
1003
|
+
* Supply the threshold your gate uses; the default suits a judge scoring in
|
|
1004
|
+
* [0, 1] with a pass at the midpoint.
|
|
1005
|
+
*/
|
|
1006
|
+
acceptThreshold?: number;
|
|
1007
|
+
}
|
|
1008
|
+
/**
|
|
1009
|
+
* Mix plants into a grading set and seal which items they are.
|
|
1010
|
+
*
|
|
1011
|
+
* Refusals: a duplicate id anywhere in the mixed set (the join is by id, so a
|
|
1012
|
+
* repeat makes one of the two unscoreable), a plant whose item id collides
|
|
1013
|
+
* with a dataset item (it would shadow real work and grade it as a plant), a
|
|
1014
|
+
* dataset item with no usable label, and a plant whose expectation the run's
|
|
1015
|
+
* `acceptThreshold` contradicts.
|
|
1016
|
+
*/
|
|
1017
|
+
declare function seedPlants(dataset: readonly GoldenItem[], plants: readonly Plant[], options?: SeedPlantsOptions): SeededGradingSet;
|
|
1018
|
+
/**
|
|
1019
|
+
* One grader outcome for one item. `score: null` says the grader ran and
|
|
1020
|
+
* declined to decide — the check never tested the item's defect. Any
|
|
1021
|
+
* `CandidateScore` from a judge run is already a valid outcome.
|
|
1022
|
+
*/
|
|
1023
|
+
interface PlantOutcome {
|
|
1024
|
+
itemId: string;
|
|
1025
|
+
score: number | null;
|
|
1026
|
+
}
|
|
1027
|
+
/**
|
|
1028
|
+
* `evaluated` — a rate stands. `incomplete` — a seeded id had no result.
|
|
1029
|
+
* `not_evaluated` — nothing was seeded, or nothing seeded was decided.
|
|
1030
|
+
*/
|
|
1031
|
+
type CatchRateStatus = 'evaluated' | 'incomplete' | 'not_evaluated';
|
|
1032
|
+
interface PlantKindCounts {
|
|
1033
|
+
seeded: number;
|
|
1034
|
+
caught: number;
|
|
1035
|
+
missed: number;
|
|
1036
|
+
indecisive: number;
|
|
1037
|
+
/** caught / (caught + missed), or null when the report is not `evaluated`. */
|
|
1038
|
+
rate: number | null;
|
|
1039
|
+
}
|
|
1040
|
+
/**
|
|
1041
|
+
* How the same grader treated the unseeded items of the same set. No labels
|
|
1042
|
+
* are needed to read it: a grader that refuses everything scores `rate` 1.0
|
|
1043
|
+
* on reject-plants, and a `rejectionRate` of 1.0 here is what separates that
|
|
1044
|
+
* reflex from discrimination.
|
|
1045
|
+
*/
|
|
1046
|
+
interface UnseededRejection {
|
|
1047
|
+
n: number;
|
|
1048
|
+
decided: number;
|
|
1049
|
+
rejected: number;
|
|
1050
|
+
/** rejected / decided, or null when nothing unseeded was decided. */
|
|
1051
|
+
rejectionRate: number | null;
|
|
1052
|
+
}
|
|
1053
|
+
interface CatchRateReport {
|
|
1054
|
+
status: CatchRateStatus;
|
|
1055
|
+
/** Why the status is not `evaluated`. Absent when it is. */
|
|
1056
|
+
reason?: string;
|
|
1057
|
+
seeded: number;
|
|
1058
|
+
caught: number;
|
|
1059
|
+
missed: number;
|
|
1060
|
+
/**
|
|
1061
|
+
* The grader returned a result and declined to decide. Counted apart: it
|
|
1062
|
+
* enters neither side of `rate`.
|
|
1063
|
+
*/
|
|
1064
|
+
indecisive: number;
|
|
1065
|
+
/** caught / (caught + missed), or null unless the status is `evaluated`. */
|
|
1066
|
+
rate: number | null;
|
|
1067
|
+
/** One entry per kind actually seeded. A kind nobody seeded is absent, never zero. */
|
|
1068
|
+
byKind: Partial<Record<PlantKind, PlantKindCounts>>;
|
|
1069
|
+
/** Plant ids the grader graded as the seed says it must not. */
|
|
1070
|
+
missedIds: string[];
|
|
1071
|
+
/** Plant ids with no result at all — the reason a status is `incomplete`. */
|
|
1072
|
+
missingIds: string[];
|
|
1073
|
+
unseeded: UnseededRejection;
|
|
1074
|
+
}
|
|
1075
|
+
/**
|
|
1076
|
+
* Score a grading run against its sealed manifest.
|
|
1077
|
+
*
|
|
1078
|
+
* Refusals: a manifest whose contents no longer match its seal, a result for
|
|
1079
|
+
* an id the manifest never handed out, and a repeated result id. Each says
|
|
1080
|
+
* the results and the manifest describe different runs, and a rate computed
|
|
1081
|
+
* across two runs is a fabrication.
|
|
1082
|
+
*/
|
|
1083
|
+
declare function catchRate(results: readonly PlantOutcome[], manifest: PlantManifest): CatchRateReport;
|
|
1084
|
+
//#endregion
|
|
1085
|
+
export { SnapshotMeta as $, JudgeGateInput as A, CalibrationPair as At, measureJudgeAgreement as B, CorrelationStudyOptions as Bt, CalibrationMerge as C, planEvaluatorAudit as Ct, JudgeConfusion as D, clusterBootstrapMean as Dt, JudgeCalibrationSet as E, ClusteredObservation as Et, JudgeRunMeta as F, calibrationCurve as Ft, SentinelMetricName as G, CorpusAgreementOptions as Gt, snapshotFromJudgeAgreement as H, EvalMetricSpec as Ht, OwnerVerdict as I, calibrationFromPairs as It, SentinelSetOptions as J, CorpusScoreRecord as Jt, SentinelMetrics as K, CorpusAgreementPerDimension as Kt, addCalibrationVerdicts as L, multiclassBrierScore as Lt, JudgeGateRefusedError as M, CategoricalForecast as Mt, JudgeGateStatus as N, OrdinalForecast as Nt, JudgeExampleAgreement as O, CalibrationBin as Ot, JudgeRun as P, brierScore as Pt, SentinelTrend as Q, assertJudgeMayGate as R, rankedProbabilityScore as Rt, CalibrationExample as S, evaluatorAuditObservationSchema as St, JudgeAgreement as T, ClusterBootstrapResult as Tt, EvalHealthStamp as U, OutcomePair as Ut, registerCalibrationSet as V, CorrelationStudyResult as Vt, JudgeSentinelOptions as W, correlationStudy as Wt, SentinelStore as X, corpusInterRaterAgreementFromJudgeScores as Xt, SentinelSnapshot as Y, corpusInterRaterAgreement as Yt, SentinelThresholds as Z, interRaterReliability as Zt, catchRate as _, ProbabilityPolicyAuditInput as _t, PerturbedField as a, snapshotFromSentinelSet as at, plantByPerturbation as b, auditProbabilityPolicy as bt, PlantEvidence as c, SeriesConvergenceResult as ct, PlantKindCounts as d, EvaluatorAdmissionReport as dt, evalHealthStamp as et, PlantManifest as f, EvaluatorAuditInput as ft, UnseededRejection as g, EvaluatorErrorRate as gt, SeededGradingSet as h, EvaluatorAuditPlanInput as ht, PerturbationKind as i, snapshotFromCalibration as it, JudgeGatePolicy as j, CalibrationReport as jt, JudgeGateDecision as k, CalibrationOptions as kt, PlantExpectation as l, analyzeSeries as lt, SeedPlantsOptions as m, EvaluatorAuditPlan as mt, CatchRateStatus as n, judgeSentinelReport as nt, PerturbedPlant as o, validateSentinelSnapshot as ot, PlantOutcome as p, EvaluatorAuditObservation as pt, SentinelReport as q, CorpusAgreementReport as qt, EvidencePerturbation as r, snapshotFromAgreement as rt, Plant as s, SeriesConvergenceOptions as st, CatchRateReport as t, fileSentinelStore as tt, PlantKind as u, EvaluatorAdmissionPolicy as ut, definePlant as v, ProbabilityPolicyAuditReport as vt, DEFAULT_JUDGE_GATE_POLICY as w, ClusterBootstrapOptions as wt, seedPlants as x, evaluatorAdmissionPolicySchema as xt, perturbEvidence as y, auditEvaluator as yt, judgeGateDecision as z, CorrelationResult as zt };
|
|
1086
|
+
//# sourceMappingURL=index-DWd1gUwR.d.ts.map
|