@tangle-network/agent-eval 0.221.5 → 0.221.7

package/dist/{matrix-BXo3PSqM.d.ts → matrix-DtkZLD_3.d.ts}

@@ -1,5 +1,5 @@
 import { u as CostProvenance } from "./cost-ledger-C1W3ICmP.js";
-import { C as JudgeDimension, w as JudgeScore } from "./types-DDKfH-N6.js";
+import { C as JudgeDimension, w as JudgeScore } from "./types-M6eF9m-r.js";
 import { o as MatrixResult } from "./index-CHxwfW5g.js";
 import { AgentProfile } from "@tangle-network/agent-interface";
 //#region src/multishot/types.d.ts
@@ -389,4 +389,4 @@ interface RunMultishotMatrixResult {
 declare function runMultishotMatrix<TPersona extends MultishotPersona>(opts: RunMultishotMatrixOptions<TPersona>): Promise<RunMultishotMatrixResult>;
 //#endregion
 export { MultishotTransportRequest as A, MultishotMessage as C, MultishotToolDefinition as D, MultishotShape as E, MultishotTransportToolCall as M, assertMultishotShotResult as N, MultishotToolExecutor as O, MultishotFatalToolError as S, MultishotResult as T, renderDimensions as _, MultishotCellOutput as a, MultishotArtifact as b, RunMultishotMatrixResult as c, MultishotShot as d, RunMultishotOptions as f, MultishotJudgeConfig as g, JudgeRunResult as h, ConversationJudgeInput as i, MultishotTransportResponse as j, MultishotTransport as k, computeCellComposite as l, DEFAULT_JUDGE_MODEL as m, CellCompositeInput as n, MultishotJudges as o, runMultishot as p, CellCompositeScore as r, RunMultishotMatrixOptions as s, ArtifactJudgeInput as t, runMultishotMatrix as u, renderJsonFooter as v, MultishotPersona as w, MultishotDriverEmptyError as x, runJudge as y };
-//# sourceMappingURL=matrix-BXo3PSqM.d.ts.map
+//# sourceMappingURL=matrix-DtkZLD_3.d.ts.map

package/dist/{matrix-BXo3PSqM.d.ts.map → matrix-DtkZLD_3.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"matrix-BXo3PSqM.d.ts","names":[],"sources":["../src/multishot/types.ts","../src/multishot/judges.ts","../src/multishot/multishot.ts","../src/multishot/matrix.ts"],"mappings":";;;;;UAIiB;EACf;EACA;EACA;EACA,YAAY;IAAQ;IAAY;IAAc,MAAM;;;UAGrC;EACf;EACA;EACA;IAAc;IAAc,MAAM;;EAClC;;UAGe;EACf,YAAY;EACZ,WAAW;EACX;EACA;;;EAGA;;;;;;;;EAQA,iBAAiB;;UAGF;EACf;EACA;IACE;IACA;IACA,YAAY;;;;;;UAOC;EACf;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;;EAGlC;;;;EAIA;;;;;;;;KASU,sBACV,KAAK,8BACF,QAAQ;KAED,yBACV,MAAM,yBACN;;EAEE,WAAW;EACX,SAAS;MAER;EAAU;EAAiB;;UAEf;;EAEf;;GAEC;;;;;;;;UASc,eAAe,iBAAiB;;EAE/C,eAAe,SAAS;;;EAGxB,2BAA2B,SAAS;;cAGzB,kCAAkC;WACjB;EAA5B,YAA4B;;cAMjB,gCAAgC;EAC3C,YAAY;;;;;;;;;;;;;;;;;;;iBAgCE,0BAA0B,yBAAyB,SAAS;;;cCvI/D;UAEI,qBAAqB;;EAEpC;;;EAGA,WAAW;;EAEX;;EAEA,YAAY;;EAEZ;;;EAGA,cAAc,OAAO;;EAErB;;UAGe;;EAEf,OAAO;;EAEP,MAAM;;iBAGc,SAAS,QAC7B,OAAO,qBAAqB,SAC5B,OAAO,SACN,QAAQ;;;iBAsIK,iBAAiB,eAAe;;iBAKhC,iBAAiB,eAAe;;;UCjK/B,oBAAoB,iBAAiB;EACpD,SAAS;EACT,SAAS;;;EAGT,QAAQ,eAAe;;EAEvB,QAAQ;;EAER,gBAAgB,eAAe;;;;EAI/B,mBAAmB;EACnB;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA,gBAAgB;;;;EAIhB,iBAAiB;;;EAGjB,gBAAgB;EAChB,SAAS;;;;;;;;;;;KAYC,cAAc,iBAAiB,qBACzC,MAAM,oBAAoB,cACvB,QAAQ;;;;;;;;;;;;;;;;;iBA2BS,aAAa,iBAAiB,kBAClD,MAAM,oBAAoB,YACzB,QAAQ;;;UClFM,uBAAuB,iBAAiB;EACvD,YAAY;EACZ,SAAS;;UAGM,mBAAmB,iBAAiB;EACnD,UAAU;EACV,SAAS;;UAGM,gBAAgB,iBAAiB;;EAEhD,cAAc,qBAAqB,uBAAuB;;EAE1D,aAAa,qBAAqB,mBAAmB;;EAErD,iBAAiB,qBAAqB,mBAAmB;;EAEzD;;EAEA;;UAGe;EACf;EACA,cAAc;EACd;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;EAEF;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;;UAIa,0BAA0B,iBAAiB;;EAE1D,UAAU;IAAQ;IAAY,OAAO;;;EAErC,UAAU;;;EAGV,QAAQ,eAAe;;EAEvB,QAAQ,gBAAgB;;EAExB,QAAQ;;EAER,gBAAgB,eAAe;;EAE/B,mBAAmB;;EAEnB;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,gBAAgB;;EAEhB,iBAAiB;;;EAGjB,gBAAgB;;;;;;;;;;;;;;;EAehB,UAAU,cAAc;;;;;UAMT;EACf;EACA;EACA;;UAoBe;EACf,cAAc;;EAEd,cAAc,cAAc;;EAE5B,iBAAiB,cAAc;;;;;;;iBAQjB,qBAAqB,OAAO;EAC1C;EACA;EACA;EACA;;UAuBe;EACf,QAAQ,aAAa;;iBAGD,mBAAmB,iBAAiB,kBACxD,MAAM,0BAA0B,YAC/B,QAAQ"}
+{"version":3,"file":"matrix-DtkZLD_3.d.ts","names":[],"sources":["../src/multishot/types.ts","../src/multishot/judges.ts","../src/multishot/multishot.ts","../src/multishot/matrix.ts"],"mappings":";;;;;UAIiB;EACf;EACA;EACA;EACA,YAAY;IAAQ;IAAY;IAAc,MAAM;;;UAGrC;EACf;EACA;EACA;IAAc;IAAc,MAAM;;EAClC;;UAGe;EACf,YAAY;EACZ,WAAW;EACX;EACA;;;EAGA;;;;;;;;EAQA,iBAAiB;;UAGF;EACf;EACA;IACE;IACA;IACA,YAAY;;;;;;UAOC;EACf;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;;EAGlC;;;;EAIA;;;;;;;;KASU,sBACV,KAAK,8BACF,QAAQ;KAED,yBACV,MAAM,yBACN;;EAEE,WAAW;EACX,SAAS;MAER;EAAU;EAAiB;;UAEf;;EAEf;;GAEC;;;;;;;;UASc,eAAe,iBAAiB;;EAE/C,eAAe,SAAS;;;EAGxB,2BAA2B,SAAS;;cAGzB,kCAAkC;WACjB;EAA5B,YAA4B;;cAMjB,gCAAgC;EAC3C,YAAY;;;;;;;;;;;;;;;;;;;iBAgCE,0BAA0B,yBAAyB,SAAS;;;cCvI/D;UAEI,qBAAqB;;EAEpC;;;EAGA,WAAW;;EAEX;;EAEA,YAAY;;EAEZ;;;EAGA,cAAc,OAAO;;EAErB;;UAGe;;EAEf,OAAO;;EAEP,MAAM;;iBAGc,SAAS,QAC7B,OAAO,qBAAqB,SAC5B,OAAO,SACN,QAAQ;;;iBAsIK,iBAAiB,eAAe;;iBAKhC,iBAAiB,eAAe;;;UCxK/B,oBAAoB,iBAAiB;EACpD,SAAS;EACT,SAAS;;;EAGT,QAAQ,eAAe;;EAEvB,QAAQ;;EAER,gBAAgB,eAAe;;;;EAI/B,mBAAmB;EACnB;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA,gBAAgB;;;;EAIhB,iBAAiB;;;EAGjB,gBAAgB;EAChB,SAAS;;;;;;;;;;;KAYC,cAAc,iBAAiB,qBACzC,MAAM,oBAAoB,cACvB,QAAQ;;;;;;;;;;;;;;;;;iBAkBS,aAAa,iBAAiB,kBAClD,MAAM,oBAAoB,YACzB,QAAQ;;;UClEM,uBAAuB,iBAAiB;EACvD,YAAY;EACZ,SAAS;;UAGM,mBAAmB,iBAAiB;EACnD,UAAU;EACV,SAAS;;UAGM,gBAAgB,iBAAiB;;EAEhD,cAAc,qBAAqB,uBAAuB;;EAE1D,aAAa,qBAAqB,mBAAmB;;EAErD,iBAAiB,qBAAqB,mBAAmB;;EAEzD;;EAEA;;UAGe;EACf;EACA,cAAc;EACd;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;EAEF;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;;UAIa,0BAA0B,iBAAiB;;EAE1D,UAAU;IAAQ;IAAY,OAAO;;;EAErC,UAAU;;;EAGV,QAAQ,eAAe;;EAEvB,QAAQ,gBAAgB;;EAExB,QAAQ;;EAER,gBAAgB,eAAe;;EAE/B,mBAAmB;;EAEnB;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,gBAAgB;;EAEhB,iBAAiB;;;EAGjB,gBAAgB;;;;;;;;;;;;;;;EAehB,UAAU,cAAc;;;;;UAMT;EACf;EACA;EACA;;UAoBe;EACf,cAAc;;EAEd,cAAc,cAAc;;EAE5B,iBAAiB,cAAc;;;;;;;iBAQjB,qBAAqB,OAAO;EAC1C;EACA;EACA;EACA;;UAuBe;EACf,QAAQ,aAAa;;iBAGD,mBAAmB,iBAAiB,kBACxD,MAAM,0BAA0B,YAC/B,QAAQ"}

package/dist/meta-eval/index.d.ts

@@ -1,4 +1,4 @@
 import { a as ContinuousCalibrationResult, c as SelfPreferenceResult, d as calibrateJudgeContinuous, f as continuousAgreement, h as verbosityBias, i as ContinuousAgreementOptions, l as VerbosityBiasResult, m as selfPreference, n as CandidateScore, o as GoldenItem, p as positionalBias, r as ContinuousAgreement, s as PositionalBiasResult, t as CalibrationResult, u as calibrateJudge } from "../judge-calibration-DFtEMlde.js";
-import { $ as SnapshotMeta, A as JudgeGateInput, At as CalibrationPair, B as measureJudgeAgreement, Bt as CorrelationStudyOptions, C as CalibrationMerge, Ct as planEvaluatorAudit, D as JudgeConfusion, Dt as clusterBootstrapMean, E as JudgeCalibrationSet, Et as ClusteredObservation, F as JudgeRunMeta, Ft as calibrationCurve, G as SentinelMetricName, H as snapshotFromJudgeAgreement, Ht as EvalMetricSpec, I as OwnerVerdict, It as calibrationFromPairs, J as SentinelSetOptions, K as SentinelMetrics, L as addCalibrationVerdicts, Lt as multiclassBrierScore, M as JudgeGateRefusedError, Mt as CategoricalForecast, N as JudgeGateStatus, Nt as OrdinalForecast, O as JudgeExampleAgreement, Ot as CalibrationBin, P as JudgeRun, Pt as brierScore, Q as SentinelTrend, R as assertJudgeMayGate, Rt as rankedProbabilityScore, S as CalibrationExample, St as evaluatorAuditObservationSchema, T as JudgeAgreement, Tt as ClusterBootstrapResult, U as EvalHealthStamp, Ut as OutcomePair, V as registerCalibrationSet, Vt as CorrelationStudyResult, W as JudgeSentinelOptions, Wt as correlationStudy, X as SentinelStore, Y as SentinelSnapshot, Z as SentinelThresholds, _ as catchRate, _t as ProbabilityPolicyAuditInput, a as PerturbedField, at as snapshotFromSentinelSet, b as plantByPerturbation, bt as auditProbabilityPolicy, c as PlantEvidence, d as PlantKindCounts, dt as EvaluatorAdmissionReport, et as evalHealthStamp, f as PlantManifest, ft as EvaluatorAuditInput, g as UnseededRejection, gt as EvaluatorErrorRate, h as SeededGradingSet, ht as EvaluatorAuditPlanInput, i as PerturbationKind, it as snapshotFromCalibration, j as JudgeGatePolicy, jt as CalibrationReport, k as JudgeGateDecision, kt as CalibrationOptions, l as PlantExpectation, m as SeedPlantsOptions, mt as EvaluatorAuditPlan, n as CatchRateStatus, nt as judgeSentinelReport, o as PerturbedPlant, ot as validateSentinelSnapshot, p as PlantOutcome, pt as EvaluatorAuditObservation, q as SentinelReport, r as EvidencePerturbation, rt as snapshotFromAgreement, s as Plant, t as CatchRateReport, tt as fileSentinelStore, u as PlantKind, ut as EvaluatorAdmissionPolicy, v as definePlant, vt as ProbabilityPolicyAuditReport, w as DEFAULT_JUDGE_GATE_POLICY, wt as ClusterBootstrapOptions, x as seedPlants, xt as evaluatorAdmissionPolicySchema, y as perturbEvidence, yt as auditEvaluator, z as judgeGateDecision, zt as CorrelationResult } from "../index-DWd1gUwR.js";
+import { $ as measureJudgeAgreement, $t as CorrelationStudyOptions, A as JudgePanelOutput, At as ProbabilityPolicyAuditInput, B as JudgeConfusion, Bt as clusterBootstrapMean, C as JudgeCalibrationCellVerdict, Ct as EvaluatorAdmissionPolicy, D as JudgeCalibrationGoldItem, Dt as EvaluatorAuditPlan, E as JudgeCalibrationGateConfig, Et as EvaluatorAuditObservation, F as CalibrationExample, Ft as evaluatorAuditObservationSchema, G as JudgeGateRefusedError, Gt as CategoricalForecast, H as JudgeGateDecision, Ht as CalibrationOptions, I as CalibrationMerge, It as planEvaluatorAudit, J as JudgeRunMeta, Jt as calibrationCurve, K as JudgeGateStatus, Kt as OrdinalForecast, L as DEFAULT_JUDGE_GATE_POLICY, Lt as ClusterBootstrapOptions, M as RubricDimensionGuide, Mt as auditEvaluator, N as calibrateJudgePanel, Nt as auditProbabilityPolicy, O as JudgeDimensionCalibration, Ot as EvaluatorAuditPlanInput, P as gateJudgeCalibration, Pt as evaluatorAdmissionPolicySchema, Q as judgeGateDecision, Qt as CorrelationResult, R as JudgeAgreement, Rt as ClusterBootstrapResult, S as CalibrateJudgePanelInput, T as JudgeCalibrationGateCell, Tt as EvaluatorAuditInput, U as JudgeGateInput, Ut as CalibrationPair, V as JudgeExampleAgreement, Vt as CalibrationBin, W as JudgeGatePolicy, Wt as CalibrationReport, X as addCalibrationVerdicts, Xt as multiclassBrierScore, Y as OwnerVerdict, Yt as calibrationFromPairs, Z as assertJudgeMayGate, Zt as rankedProbabilityScore, _ as catchRate, _t as snapshotFromCalibration, a as PerturbedField, at as SentinelMetrics, b as plantByPerturbation, c as PlantEvidence, ct as SentinelSnapshot, d as PlantKindCounts, dt as SentinelTrend, en as CorrelationStudyResult, et as registerCalibrationSet, f as PlantManifest, ft as SnapshotMeta, g as UnseededRejection, gt as snapshotFromAgreement, h as SeededGradingSet, ht as judgeSentinelReport, i as PerturbationKind, it as SentinelMetricName, j as JudgePanelPrompt, jt as ProbabilityPolicyAuditReport, k as JudgePanelCalibration, kt as EvaluatorErrorRate, l as PlantExpectation, lt as SentinelStore, m as SeedPlantsOptions, mt as fileSentinelStore, n as CatchRateStatus, nn as OutcomePair, nt as EvalHealthStamp, o as PerturbedPlant, ot as SentinelReport, p as PlantOutcome, pt as evalHealthStamp, q as JudgeRun, qt as brierScore, r as EvidencePerturbation, rn as correlationStudy, rt as JudgeSentinelOptions, s as Plant, st as SentinelSetOptions, t as CatchRateReport, tn as EvalMetricSpec, tt as snapshotFromJudgeAgreement, u as PlantKind, ut as SentinelThresholds, v as definePlant, vt as snapshotFromSentinelSet, w as JudgeCalibrationGate, wt as EvaluatorAdmissionReport, x as seedPlants, y as perturbEvidence, yt as validateSentinelSnapshot, z as JudgeCalibrationSet, zt as ClusteredObservation } from "../index-CfJULcW5.js";
 import { a as RubricPredictiveValidityReport, c as CorrelationInterval, d as FileSystemOutcomeStore, f as FileSystemOutcomeStoreOptions, g as OutcomeStoreError, h as OutcomeStore, i as RubricPredictiveValidityInput, l as OutcomeReduction, m as OutcomeFilter, n as RubricOutcomeExclusion, o as RubricRanking, p as InMemoryOutcomeStore, r as RubricOutcomePair, s as rubricPredictiveValidity, t as OutcomeMetricSpec, u as DeploymentOutcome } from "../rubric-predictive-validity-Cd5SCtAf.js";
-export { CalibrationBin, CalibrationExample, CalibrationMerge, CalibrationOptions, CalibrationPair, CalibrationReport, type CalibrationResult, type CandidateScore, CatchRateReport, CatchRateStatus, CategoricalForecast, ClusterBootstrapOptions, ClusterBootstrapResult, ClusteredObservation, type ContinuousAgreement, type ContinuousAgreementOptions, type ContinuousCalibrationResult, type CorrelationInterval, CorrelationResult, CorrelationStudyOptions, CorrelationStudyResult, DEFAULT_JUDGE_GATE_POLICY, DeploymentOutcome, type EvalHealthStamp, EvalMetricSpec, type EvaluatorAdmissionPolicy, type EvaluatorAdmissionReport, type EvaluatorAuditInput, type EvaluatorAuditObservation, type EvaluatorAuditPlan, type EvaluatorAuditPlanInput, type EvaluatorErrorRate, EvidencePerturbation, FileSystemOutcomeStore, FileSystemOutcomeStoreOptions, type GoldenItem, InMemoryOutcomeStore, JudgeAgreement, JudgeCalibrationSet, JudgeConfusion, JudgeExampleAgreement, JudgeGateDecision, JudgeGateInput, JudgeGatePolicy, JudgeGateRefusedError, JudgeGateStatus, JudgeRun, JudgeRunMeta, type JudgeSentinelOptions, OrdinalForecast, OutcomeFilter, OutcomeMetricSpec, OutcomePair, type OutcomeReduction, OutcomeStore, OutcomeStoreError, OwnerVerdict, PerturbationKind, PerturbedField, PerturbedPlant, Plant, PlantEvidence, PlantExpectation, PlantKind, PlantKindCounts, PlantManifest, PlantOutcome, type PositionalBiasResult, type ProbabilityPolicyAuditInput, type ProbabilityPolicyAuditReport, RubricOutcomeExclusion, RubricOutcomePair, RubricPredictiveValidityInput, RubricPredictiveValidityReport, RubricRanking, SeedPlantsOptions, SeededGradingSet, type SelfPreferenceResult, type SentinelMetricName, type SentinelMetrics, type SentinelReport, type SentinelSetOptions, type SentinelSnapshot, type SentinelStore, type SentinelThresholds, type SentinelTrend, type SnapshotMeta, UnseededRejection, type VerbosityBiasResult, addCalibrationVerdicts, assertJudgeMayGate, auditEvaluator, auditProbabilityPolicy, brierScore, calibrateJudge, calibrateJudgeContinuous, calibrationCurve, calibrationFromPairs, catchRate, clusterBootstrapMean, continuousAgreement, correlationStudy, definePlant, evalHealthStamp, evaluatorAdmissionPolicySchema, evaluatorAuditObservationSchema, fileSentinelStore, judgeGateDecision, judgeSentinelReport, measureJudgeAgreement, multiclassBrierScore, perturbEvidence, planEvaluatorAudit, plantByPerturbation, positionalBias, rankedProbabilityScore, registerCalibrationSet, rubricPredictiveValidity, seedPlants, selfPreference, snapshotFromAgreement, snapshotFromCalibration, snapshotFromJudgeAgreement, snapshotFromSentinelSet, validateSentinelSnapshot, verbosityBias };
+export { type CalibrateJudgePanelInput, CalibrationBin, CalibrationExample, CalibrationMerge, CalibrationOptions, CalibrationPair, CalibrationReport, type CalibrationResult, type CandidateScore, CatchRateReport, CatchRateStatus, CategoricalForecast, ClusterBootstrapOptions, ClusterBootstrapResult, ClusteredObservation, type ContinuousAgreement, type ContinuousAgreementOptions, type ContinuousCalibrationResult, type CorrelationInterval, CorrelationResult, CorrelationStudyOptions, CorrelationStudyResult, DEFAULT_JUDGE_GATE_POLICY, DeploymentOutcome, type EvalHealthStamp, EvalMetricSpec, type EvaluatorAdmissionPolicy, type EvaluatorAdmissionReport, type EvaluatorAuditInput, type EvaluatorAuditObservation, type EvaluatorAuditPlan, type EvaluatorAuditPlanInput, type EvaluatorErrorRate, EvidencePerturbation, FileSystemOutcomeStore, FileSystemOutcomeStoreOptions, type GoldenItem, InMemoryOutcomeStore, JudgeAgreement, type JudgeCalibrationCellVerdict, type JudgeCalibrationGate, type JudgeCalibrationGateCell, type JudgeCalibrationGateConfig, type JudgeCalibrationGoldItem, JudgeCalibrationSet, JudgeConfusion, type JudgeDimensionCalibration, JudgeExampleAgreement, JudgeGateDecision, JudgeGateInput, JudgeGatePolicy, JudgeGateRefusedError, JudgeGateStatus, type JudgePanelCalibration, type JudgePanelOutput, type JudgePanelPrompt, JudgeRun, JudgeRunMeta, type JudgeSentinelOptions, OrdinalForecast, OutcomeFilter, OutcomeMetricSpec, OutcomePair, type OutcomeReduction, OutcomeStore, OutcomeStoreError, OwnerVerdict, PerturbationKind, PerturbedField, PerturbedPlant, Plant, PlantEvidence, PlantExpectation, PlantKind, PlantKindCounts, PlantManifest, PlantOutcome, type PositionalBiasResult, type ProbabilityPolicyAuditInput, type ProbabilityPolicyAuditReport, type RubricDimensionGuide, RubricOutcomeExclusion, RubricOutcomePair, RubricPredictiveValidityInput, RubricPredictiveValidityReport, RubricRanking, SeedPlantsOptions, SeededGradingSet, type SelfPreferenceResult, type SentinelMetricName, type SentinelMetrics, type SentinelReport, type SentinelSetOptions, type SentinelSnapshot, type SentinelStore, type SentinelThresholds, type SentinelTrend, type SnapshotMeta, UnseededRejection, type VerbosityBiasResult, addCalibrationVerdicts, assertJudgeMayGate, auditEvaluator, auditProbabilityPolicy, brierScore, calibrateJudge, calibrateJudgeContinuous, calibrateJudgePanel, calibrationCurve, calibrationFromPairs, catchRate, clusterBootstrapMean, continuousAgreement, correlationStudy, definePlant, evalHealthStamp, evaluatorAdmissionPolicySchema, evaluatorAuditObservationSchema, fileSentinelStore, gateJudgeCalibration, judgeGateDecision, judgeSentinelReport, measureJudgeAgreement, multiclassBrierScore, perturbEvidence, planEvaluatorAudit, plantByPerturbation, positionalBias, rankedProbabilityScore, registerCalibrationSet, rubricPredictiveValidity, seedPlants, selfPreference, snapshotFromAgreement, snapshotFromCalibration, snapshotFromJudgeAgreement, snapshotFromSentinelSet, validateSentinelSnapshot, verbosityBias };

package/dist/meta-eval/index.js

@@ -3,9 +3,10 @@ import { a as hashCanonical, i as compareCodeUnits } from "../canonical-Dq2hOQ8L
 import { t as mulberry32 } from "../random-DDDcC7YU.js";
 import { a as selfPreference, i as positionalBias, n as calibrateJudgeContinuous, o as verbosityBias, r as continuousAgreement, t as calibrateJudge } from "../judge-calibration-BkXMIWA4.js";
 import { s as runMetricExtractor } from "../query-B_RZ36an.js";
+import { n as callLlmJson } from "../llm-client-b1ysZVIU.js";
 import { a as validateObservationOptions, i as reduceOutcomeMetric, n as correlationSummary, r as hasVariation, t as assertUniqueObservationIds } from "../outcome-observations-FLmRpgYa.js";
 import { a as rankedProbabilityScore, i as multiclassBrierScore, n as calibrationCurve, r as calibrationFromPairs, t as brierScore } from "../calibration-CTwxpJ-t.js";
-import { a as judgeGateDecision, c as snapshotFromJudgeAgreement, d as judgeSentinelReport, f as snapshotFromAgreement, h as validateSentinelSnapshot, i as assertJudgeMayGate, l as evalHealthStamp, m as snapshotFromSentinelSet, n as JudgeGateRefusedError, o as measureJudgeAgreement, p as snapshotFromCalibration, r as addCalibrationVerdicts, s as registerCalibrationSet, t as DEFAULT_JUDGE_GATE_POLICY, u as fileSentinelStore } from "../judge-gate-Co5tuh57.js";
+import { a as judgeGateDecision, c as snapshotFromJudgeAgreement, d as judgeSentinelReport, f as snapshotFromAgreement, g as withJudgeRetry, h as validateSentinelSnapshot, i as assertJudgeMayGate, l as evalHealthStamp, m as snapshotFromSentinelSet, n as JudgeGateRefusedError, o as measureJudgeAgreement, p as snapshotFromCalibration, r as addCalibrationVerdicts, s as registerCalibrationSet, t as DEFAULT_JUDGE_GATE_POLICY, u as fileSentinelStore } from "../judge-gate-BVvmP00P.js";
 import { t as computeInterval } from "../interval-Dq9gt7jx.js";
 import { t as clusterBootstrapMean } from "../cluster-bootstrap-DEwwhl0C.js";
 import { t as rubricPredictiveValidity } from "../rubric-predictive-validity-CeKW2GQN.js";
@@ -372,6 +373,220 @@ function auditProbabilityPolicy(input) {
 	};
 }
 //#endregion
+//#region src/meta-eval/judge-panel-calibration.ts
+/**
+* Judge-panel calibration: score a hand-graded gold set with every judge model
+* on a panel, under one weighted rubric, and measure each (judge × dimension)
+* against the human grades with `calibrateJudge`.
+*
+* The caller owns the domain: the rubric text and weights, the system prompt,
+* the gold items and the judge list. This module owns the mechanics every
+* product repeats — the rubric prompt, the JSON call with retries, the
+* sequential (item × judge) sweep, null-grade handling, and the per-cell
+* calibration — plus `gateJudgeCalibration`, the pass/warn/fail rule over the
+* resulting cells.
+*
+* Calls run one at a time. Subscription bridges serialise per subprocess, and
+* concurrent requests from a full panel get killed under load.
+*
+* A gold cell with a null or missing human grade is still scored (the judge's
+* score lands in `rawScores`, useful for inter-judge agreement) but never
+* enters that dimension's calibration.
+*/
+const DEFAULT_RUBRIC_HEADING = "Rubric (score each dimension in [0,1]; 1 = perfectly satisfies the guide, 0 = fails):";
+function clampUnit(n) {
+	if (!Number.isFinite(n)) return 0;
+	if (n < 0) return 0;
+	if (n > 1) return 1;
+	return n;
+}
+function humanScore(item, d) {
+	const v = item.humanGrade?.[d];
+	return typeof v === "number" && Number.isFinite(v) ? v : null;
+}
+function shortModel(model) {
+	return (model.split("/").pop() ?? model).padEnd(28);
+}
+function judgeUserPrompt(input, dimensions, item) {
+	const rubricLines = dimensions.map((d) => `- ${d} (weight ${input.rubric[d].weight}): ${input.rubric[d].guide}`).join("\n");
+	const jsonShape = `{${dimensions.map((d) => `"${d}":number`).join(",")},"rationale":string}`;
+	return [
+		`Scenario: ${item.scenarioLabel ?? item.id}`,
+		`Input: ${item.scenario}`,
+		"",
+		input.prompt.rubricHeading ?? DEFAULT_RUBRIC_HEADING,
+		rubricLines,
+		"",
+		input.prompt.candidateHeading ?? "Candidate output to score:",
+		"---",
+		item.candidateOutput.slice(0, input.prompt.maxCandidateChars ?? 12e3),
+		"---",
+		"",
+		`Respond ONLY with a JSON object exactly of shape: ${jsonShape}.`,
+		...input.prompt.closingLines ?? []
+	].join("\n");
+}
+/** Run every judge on every gold item and calibrate each (judge × dimension). */
+async function calibrateJudgePanel(input) {
+	const dimensions = [...input.dimensions ?? Object.keys(input.rubric)];
+	const judges = [...input.judges];
+	const log = input.log ?? (() => {});
+	const perDim = (make) => Object.fromEntries(dimensions.map((d) => [d, make()]));
+	const rawScores = {};
+	const outputs = {};
+	const goldenByJudge = {};
+	const scoresByJudge = {};
+	for (const judge of judges) {
+		rawScores[judge] = perDim(() => []);
+		outputs[judge] = {};
+		goldenByJudge[judge] = perDim(() => []);
+		scoresByJudge[judge] = perDim(() => []);
+	}
+	const ungradedItems = perDim(() => []);
+	for (const item of input.items) for (const d of dimensions) if (humanScore(item, d) === null) ungradedItems[d].push(item.id);
+	const failures = [];
+	const startedAt = Date.now();
+	for (let i = 0; i < input.items.length; i++) {
+		const item = input.items[i];
+		log(`  [${(i + 1).toString().padStart(2)}/${input.items.length}] ${item.id}`);
+		const userPrompt = judgeUserPrompt(input, dimensions, item);
+		for (const judge of judges) {
+			const outcome = await withJudgeRetry(async (_model, signal) => (await callLlmJson({
+				model: judge,
+				messages: [{
+					role: "system",
+					content: input.prompt.system
+				}, {
+					role: "user",
+					content: userPrompt
+				}],
+				jsonMode: true,
+				temperature: 0,
+				maxTokens: input.maxTokens ?? 1200,
+				timeoutMs: input.timeoutMs ?? 3e5
+			}, {
+				...input.llm,
+				signal
+			})).value, {
+				maxAttempts: input.maxAttempts ?? 3,
+				timeoutMs: input.timeoutMs ?? 3e5,
+				isRetryable: () => true,
+				backoffMs: (attempt) => 2e3 * (attempt + 1)
+			});
+			const out = outcome.succeeded ? outcome.value : null;
+			outputs[judge][item.id] = out;
+			if (!out) {
+				const errors = outcome.attemptErrors.map((e) => e.error);
+				failures.push({
+					judge,
+					itemId: item.id,
+					errors
+				});
+				log(`        ${shortModel(judge)} - FAILED (${errors.at(-1) ?? "no attempt"})`);
+				continue;
+			}
+			let judged = 0;
+			let expected = 0;
+			let covered = 0;
+			for (const d of dimensions) {
+				const score = clampUnit(Number(out[d]));
+				rawScores[judge][d].push({
+					itemId: item.id,
+					score
+				});
+				const human = humanScore(item, d);
+				if (human === null) continue;
+				goldenByJudge[judge][d].push({
+					itemId: item.id,
+					humanScore: human
+				});
+				scoresByJudge[judge][d].push({
+					itemId: item.id,
+					score
+				});
+				const weight = input.rubric[d].weight;
+				judged += score * weight;
+				expected += human * weight;
+				covered += weight;
+			}
+			const pct = (n) => `${(n * 100).toFixed(0)}%`;
+			log(covered > 0 ? `        ${shortModel(judge)} judged=${pct(judged / covered)} expected=${pct(expected / covered)} delta=${pct((judged - expected) / covered)} cover=${pct(covered)}` : `        ${shortModel(judge)} judged (ungraded item)`);
+		}
+	}
+	const elapsedMs = Date.now() - startedAt;
+	const perJudge = {};
+	for (const judge of judges) {
+		log(`  ${judge}`);
+		perJudge[judge] = {};
+		for (const d of dimensions) {
+			const golden = goldenByJudge[judge][d];
+			const scores = scoresByJudge[judge][d];
+			const calibration = calibrateJudge(golden, scores);
+			perJudge[judge][d] = {
+				calibration,
+				golden,
+				scores
+			};
+			log(`    ${d.padEnd(34)} n=${calibration.n.toString().padStart(2)} kappa=${calibration.kappa.toFixed(2)} r=${calibration.pearson.toFixed(2)} MAE=${calibration.mae.toFixed(3)}`);
+		}
+	}
+	return {
+		judges,
+		dimensions,
+		perJudge,
+		rawScores,
+		outputs,
+		ungradedItems,
+		failures,
+		elapsedMs
+	};
+}
+/** Pass/warn/fail every (judge × dimension) cell; the gate passes with zero fail cells. */
+function gateJudgeCalibration(perJudge, config) {
+	const cells = [];
+	for (const [judge, dims] of Object.entries(perJudge)) for (const [dim, { calibration: cal }] of Object.entries(dims)) {
+		const reasons = [];
+		let verdict = "pass";
+		if (cal.n < config.minSamples) {
+			verdict = "insufficient";
+			reasons.push(`n=${cal.n} < minSamples=${config.minSamples}`);
+		} else {
+			if (Number.isFinite(cal.pearson) && cal.pearson < config.pearsonFloor) {
+				if (cal.pearson < config.pearsonWarn) {
+					verdict = "fail";
+					reasons.push(`pearson=${cal.pearson.toFixed(2)} < floor=${config.pearsonFloor.toFixed(2)} (below warn=${config.pearsonWarn.toFixed(2)})`);
+				} else {
+					verdict = "warn";
+					reasons.push(`pearson=${cal.pearson.toFixed(2)} in warn band [${config.pearsonWarn.toFixed(2)}, ${config.pearsonFloor.toFixed(2)})`);
+				}
+			}
+			if (Number.isFinite(cal.mae) && cal.mae > config.maeCeiling) {
+				verdict = "fail";
+				reasons.push(`mae=${cal.mae.toFixed(3)} > ceiling=${config.maeCeiling.toFixed(3)}`);
+			}
+		}
+		cells.push({
+			judge,
+			dim,
+			n: cal.n,
+			pearson: cal.pearson,
+			mae: cal.mae,
+			verdict,
+			reasons
+		});
+	}
+	const count = (v) => cells.filter((c) => c.verdict === v).length;
+	const failCount = count("fail");
+	return {
+		pass: failCount === 0,
+		config,
+		failCount,
+		warnCount: count("warn"),
+		insufficientCount: count("insufficient"),
+		cells
+	};
+}
+//#endregion
 //#region src/meta-eval/plants.ts
 /**
 * Plants — seeded known-wrong items that measure the grader, not the work.
@@ -853,6 +1068,6 @@ function unseededRejection(manifest, scoreByItemId) {
 	};
 }
 //#endregion
-export { DEFAULT_JUDGE_GATE_POLICY, FileSystemOutcomeStore, InMemoryOutcomeStore, JudgeGateRefusedError, OutcomeStoreError, addCalibrationVerdicts, assertJudgeMayGate, auditEvaluator, auditProbabilityPolicy, brierScore, calibrateJudge, calibrateJudgeContinuous, calibrationCurve, calibrationFromPairs, catchRate, clusterBootstrapMean, continuousAgreement, correlationStudy, definePlant, evalHealthStamp, evaluatorAdmissionPolicySchema, evaluatorAuditObservationSchema, fileSentinelStore, judgeGateDecision, judgeSentinelReport, measureJudgeAgreement, multiclassBrierScore, perturbEvidence, planEvaluatorAudit, plantByPerturbation, positionalBias, rankedProbabilityScore, registerCalibrationSet, rubricPredictiveValidity, seedPlants, selfPreference, snapshotFromAgreement, snapshotFromCalibration, snapshotFromJudgeAgreement, snapshotFromSentinelSet, validateSentinelSnapshot, verbosityBias };
+export { DEFAULT_JUDGE_GATE_POLICY, FileSystemOutcomeStore, InMemoryOutcomeStore, JudgeGateRefusedError, OutcomeStoreError, addCalibrationVerdicts, assertJudgeMayGate, auditEvaluator, auditProbabilityPolicy, brierScore, calibrateJudge, calibrateJudgeContinuous, calibrateJudgePanel, calibrationCurve, calibrationFromPairs, catchRate, clusterBootstrapMean, continuousAgreement, correlationStudy, definePlant, evalHealthStamp, evaluatorAdmissionPolicySchema, evaluatorAuditObservationSchema, fileSentinelStore, gateJudgeCalibration, judgeGateDecision, judgeSentinelReport, measureJudgeAgreement, multiclassBrierScore, perturbEvidence, planEvaluatorAudit, plantByPerturbation, positionalBias, rankedProbabilityScore, registerCalibrationSet, rubricPredictiveValidity, seedPlants, selfPreference, snapshotFromAgreement, snapshotFromCalibration, snapshotFromJudgeAgreement, snapshotFromSentinelSet, validateSentinelSnapshot, verbosityBias };
 
 //# sourceMappingURL=index.js.map