@tangle-network/agent-eval 0.133.2 → 0.134.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +218 -0
- package/dist/analyst/index.d.ts +11 -35
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +4 -53
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-BmX-h_yn.d.ts → analyze-runs-DMo3Lb_y.d.ts} +4 -4
- package/dist/{analyze-runs-BmX-h_yn.d.ts.map → analyze-runs-DMo3Lb_y.d.ts.map} +1 -1
- package/dist/{analyze-runs-B-afTpCv.js → analyze-runs-qk8op0tN.js} +63 -42
- package/dist/analyze-runs-qk8op0tN.js.map +1 -0
- package/dist/baseline-BaPxoROc.js +149 -0
- package/dist/baseline-BaPxoROc.js.map +1 -0
- package/dist/{baseline-hG3K85h4.d.ts → baseline-D_fT6277.d.ts} +43 -11
- package/dist/baseline-D_fT6277.d.ts.map +1 -0
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-CJr1H1_a.js → benchmarks-v5piCeDl.js} +3 -3
- package/dist/{benchmarks-CJr1H1_a.js.map → benchmarks-v5piCeDl.js.map} +1 -1
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +5 -4
- package/dist/campaign/index.js +4 -3
- package/dist/{campaign-BJjn1rhw.js → campaign-DEC_7DLn.js} +12 -6
- package/dist/{campaign-BJjn1rhw.js.map → campaign-DEC_7DLn.js.map} +1 -1
- package/dist/{client-COvaLoQG.d.ts → client-BIyh1RCr.d.ts} +29 -15
- package/dist/client-BIyh1RCr.d.ts.map +1 -0
- package/dist/{client-CYzbdJOZ.js → client-LIuo-KPv.js} +19 -7
- package/dist/client-LIuo-KPv.js.map +1 -0
- package/dist/contract/index.d.ts +12 -11
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +12 -18
- package/dist/contract/index.js.map +1 -1
- package/dist/{default-registry-Cl3pHo4n.d.ts → default-registry-Brxr728w.d.ts} +4 -268
- package/dist/default-registry-Brxr728w.d.ts.map +1 -0
- package/dist/{default-registry-D3T9XbuY.js → default-registry-IjYs7T8l.js} +4 -61
- package/dist/default-registry-IjYs7T8l.js.map +1 -0
- package/dist/{eval-campaign-DXhpZghy.js → eval-campaign-CvPcvqXC.js} +2 -2
- package/dist/{eval-campaign-DXhpZghy.js.map → eval-campaign-CvPcvqXC.js.map} +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-C7Wue8R6.d.ts → index-BoJNQR6n.d.ts} +29 -11
- package/dist/index-BoJNQR6n.d.ts.map +1 -0
- package/dist/{index-BREtv3ZZ.d.ts → index-C21xKtxu.d.ts} +4 -4
- package/dist/{index-BREtv3ZZ.d.ts.map → index-C21xKtxu.d.ts.map} +1 -1
- package/dist/{index-DSC51roc.d.ts → index-DSC51roc2.d.ts} +1 -1
- package/dist/index-DSC51roc2.d.ts.map +1 -0
- package/dist/{index-nhIYz9hn.d.ts → index-DuhJaaiH.d.ts} +68 -7
- package/dist/index-DuhJaaiH.d.ts.map +1 -0
- package/dist/index.d.ts +60 -13
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +134 -27
- package/dist/index.js.map +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +2 -2
- package/dist/{ledger-core-CPZfcrC2.js → ledger-core-DAKFKRzi.js} +136 -18
- package/dist/ledger-core-DAKFKRzi.js.map +1 -0
- package/dist/matrix/index.d.ts +1 -1
- package/dist/meta-eval/index.d.ts +1 -1
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{paired-arms-6XItKzd1.js → paired-arms-CA_8pN01.js} +2 -2
- package/dist/{paired-arms-6XItKzd1.js.map → paired-arms-CA_8pN01.js.map} +1 -1
- package/dist/pipelines/index.d.ts +1 -1
- package/dist/pipelines/index.js +3 -2
- package/dist/pipelines/index.js.map +1 -1
- package/dist/proposal-findings-DCawte-y.js +164 -0
- package/dist/proposal-findings-DCawte-y.js.map +1 -0
- package/dist/{release-report-wuilQkvK.js → release-report-BVZBmRZp.js} +2 -2
- package/dist/{release-report-wuilQkvK.js.map → release-report-BVZBmRZp.js.map} +1 -1
- package/dist/{release-report-CjHWa8Ia.d.ts → release-report-CuULWKyk.d.ts} +2 -2
- package/dist/{release-report-CjHWa8Ia.d.ts.map → release-report-CuULWKyk.d.ts.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +4 -4
- package/dist/{researcher-CbSKhK8z.d.ts → researcher-DVtruQ9U.d.ts} +2 -2
- package/dist/{researcher-CbSKhK8z.d.ts.map → researcher-DVtruQ9U.d.ts.map} +1 -1
- package/dist/{reward-hacking-Dl2UBzej.js → reward-hacking-DCdRK9TY.js} +2 -2
- package/dist/{reward-hacking-Dl2UBzej.js.map → reward-hacking-DCdRK9TY.js.map} +1 -1
- package/dist/rl.d.ts +2 -2
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +18 -5
- package/dist/rl.js.map +1 -1
- package/dist/{rubric-predictive-validity-QG7ydk0s.js → rubric-predictive-validity-D6Q6n9oq.js} +2 -2
- package/dist/{rubric-predictive-validity-QG7ydk0s.js.map → rubric-predictive-validity-D6Q6n9oq.js.map} +1 -1
- package/dist/{semantic-concept-judge-BypLt6Fw.js → semantic-concept-judge-C0P1VTXD.js} +2 -3
- package/dist/{semantic-concept-judge-BypLt6Fw.js.map → semantic-concept-judge-C0P1VTXD.js.map} +1 -1
- package/dist/{skill-usage-BaaxFSJR.d.ts → skill-usage-BDQVPIG1.d.ts} +3 -2
- package/dist/skill-usage-BDQVPIG1.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-CF6a327Q.js → skillopt-optimization-method-BY6vKLJB.js} +169 -53
- package/dist/skillopt-optimization-method-BY6vKLJB.js.map +1 -0
- package/dist/{skillopt-optimization-method-wHF5xsUv.d.ts → skillopt-optimization-method-DJ3l4w8W.d.ts} +20 -18
- package/dist/skillopt-optimization-method-DJ3l4w8W.d.ts.map +1 -0
- package/dist/{statistics-DbvkkDPa.d.ts → statistics-D_4Snl-5.d.ts} +158 -30
- package/dist/statistics-D_4Snl-5.d.ts.map +1 -0
- package/dist/{statistics-DWM_AyLe.js → statistics-RwRNu2__.js} +546 -98
- package/dist/statistics-RwRNu2__.js.map +1 -0
- package/dist/{summary-report-Ci17nIdU.js → summary-report-BxtossFi.js} +3 -3
- package/dist/{summary-report-Ci17nIdU.js.map → summary-report-BxtossFi.js.map} +1 -1
- package/dist/{summary-report-CFnQgNfg.d.ts → summary-report-DGp0-_XO.d.ts} +61 -4
- package/dist/summary-report-DGp0-_XO.d.ts.map +1 -0
- package/dist/{baseline-DcX5hQDv.js → tool-use-metrics-DEGMKycK.js} +2 -114
- package/dist/tool-use-metrics-DEGMKycK.js.map +1 -0
- package/dist/types-DVjczBM9.d.ts +276 -0
- package/dist/types-DVjczBM9.d.ts.map +1 -0
- package/dist/{types-BokuXvOG.d.ts → types-DiWLru6Z.d.ts} +20 -37
- package/dist/types-DiWLru6Z.d.ts.map +1 -0
- package/docs/campaign-proposers.md +5 -0
- package/docs/design/statistics-decisions.md +271 -0
- package/docs/design.md +1 -0
- package/docs/insight-report.md +1 -1
- package/docs/research-report-methodology.md +4 -1
- package/package.json +2 -1
- package/dist/analyze-runs-B-afTpCv.js.map +0 -1
- package/dist/baseline-DcX5hQDv.js.map +0 -1
- package/dist/baseline-hG3K85h4.d.ts.map +0 -1
- package/dist/client-COvaLoQG.d.ts.map +0 -1
- package/dist/client-CYzbdJOZ.js.map +0 -1
- package/dist/default-registry-Cl3pHo4n.d.ts.map +0 -1
- package/dist/default-registry-D3T9XbuY.js.map +0 -1
- package/dist/index-C7Wue8R6.d.ts.map +0 -1
- package/dist/index-DSC51roc.d.ts.map +0 -1
- package/dist/index-nhIYz9hn.d.ts.map +0 -1
- package/dist/ledger-core-CPZfcrC2.js.map +0 -1
- package/dist/run-score-iEEAWiBY.js +0 -41
- package/dist/run-score-iEEAWiBY.js.map +0 -1
- package/dist/skill-usage-BaaxFSJR.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-CF6a327Q.js.map +0 -1
- package/dist/skillopt-optimization-method-wHF5xsUv.d.ts.map +0 -1
- package/dist/statistics-DWM_AyLe.js.map +0 -1
- package/dist/statistics-DbvkkDPa.d.ts.map +0 -1
- package/dist/summary-report-CFnQgNfg.d.ts.map +0 -1
- package/dist/types-BokuXvOG.d.ts.map +0 -1
|
@@ -3,11 +3,12 @@ import { a as RunRecord } from "./run-record-DcObtIGh.js";
|
|
|
3
3
|
import { s as TraceStore } from "./store-CT9YIIve.js";
|
|
4
4
|
import { n as LlmCallMetadata } from "./llm-client-BiK4HW0u.js";
|
|
5
5
|
import { A as ChatClient } from "./types-Cc3qbqzj.js";
|
|
6
|
-
import {
|
|
7
|
-
import { f as
|
|
6
|
+
import { m as ProposalFinding } from "./types-DVjczBM9.js";
|
|
7
|
+
import { C as JudgeDimension, H as SurfaceProposer, N as ParetoParent, R as Scenario, S as JudgeConfig, _ as GateDecision, a as CampaignResult, b as GenerationRecord, c as CampaignTraceWriter, d as DispatchContext, f as DispatchFn, g as GateContribution, h as GateContext, i as CampaignCostMeter, j as MutableSurface, k as LabeledScenarioStore, o as CampaignScenarioIdentity, p as Gate, r as CampaignCellResult, v as GateResult, y as GenerationCandidate } from "./types-DiWLru6Z.js";
|
|
8
|
+
import { _ as PairedBootstrapResult } from "./statistics-D_4Snl-5.js";
|
|
8
9
|
import { a as DatasetScenario, t as Dataset } from "./dataset-BvtnC8Dc.js";
|
|
9
10
|
import { t as DetectRewardHackingInput } from "./reward-hacking-D-QqXvg-.js";
|
|
10
|
-
import { g as TraceSpanEvent, t as HostedClient } from "./client-
|
|
11
|
+
import { g as TraceSpanEvent, t as HostedClient } from "./client-BIyh1RCr.js";
|
|
11
12
|
import { z } from "zod";
|
|
12
13
|
//#region src/llm-judge.d.ts
|
|
13
14
|
/** A rubric dimension as a bare key or the full `{ key, description }` shape. A
|
|
@@ -831,8 +832,8 @@ interface DefaultProductionGateOptions {
|
|
|
831
832
|
/** Fixed bootstrap seed for a deterministic verdict. Default 1337. */
|
|
832
833
|
bootstrapSeed?: number;
|
|
833
834
|
/** Minimum paired holdout observations (scenarios × reps) before a
|
|
834
|
-
* significance claim is allowed
|
|
835
|
-
*
|
|
835
|
+
* significance claim is allowed. The exact small-sample test may require
|
|
836
|
+
* more observations at the selected confidence. Default 3. */
|
|
836
837
|
minProductiveRuns?: number;
|
|
837
838
|
/** Ship statistic for the held-out significance test. Default `'mean'`
|
|
838
839
|
* (tie-robust — see `heldoutSignificance`). Pass `'median'` for
|
|
@@ -880,7 +881,9 @@ interface HeldOutGateOptions<TScenario extends Scenario = Scenario> {
|
|
|
880
881
|
deltaThreshold?: number;
|
|
881
882
|
/** Bootstrap CI confidence. Default 0.95. */
|
|
882
883
|
confidence?: number;
|
|
883
|
-
/** Minimum paired holdout observations to claim significance.
|
|
884
|
+
/** Minimum paired holdout observations to claim significance. The exact
|
|
885
|
+
* small-sample test may require more observations at the selected
|
|
886
|
+
* confidence. Default 3. */
|
|
884
887
|
minProductiveRuns?: number;
|
|
885
888
|
/** Bootstrap resamples. Default 2000. */
|
|
886
889
|
resamples?: number;
|
|
@@ -1084,6 +1087,8 @@ interface AxisEvidence {
|
|
|
1084
1087
|
bootstrap: PairedBootstrapResult;
|
|
1085
1088
|
/** Paired observations contributing to this axis. */
|
|
1086
1089
|
n: number;
|
|
1090
|
+
minimumRequired: number;
|
|
1091
|
+
decisionMethod: 'bootstrap-ci' | 'exact-sign';
|
|
1087
1092
|
gainThreshold: number;
|
|
1088
1093
|
floorTolerance: number;
|
|
1089
1094
|
verdict: AxisVerdict;
|
|
@@ -1107,7 +1112,8 @@ interface EvidenceVector {
|
|
|
1107
1112
|
type PromotionPolicy = (ev: EvidenceVector) => GateResult;
|
|
1108
1113
|
interface BuildEvidenceVectorOptions {
|
|
1109
1114
|
/** Minimum paired observations before an axis can claim significance; below
|
|
1110
|
-
* it the axis is `few_runs`.
|
|
1115
|
+
* it the axis is `few_runs`. The exact small-sample test may require more
|
|
1116
|
+
* observations at the selected confidence. */
|
|
1111
1117
|
minProductiveRuns?: number;
|
|
1112
1118
|
/** Confidence level for every axis bootstrap. Default 0.95. */
|
|
1113
1119
|
confidence?: number;
|
|
@@ -1310,7 +1316,7 @@ interface RunOptimizationBaseOptions<TScenario extends Scenario, TArtifact> exte
|
|
|
1310
1316
|
/** Dispatcher that takes the CURRENT surface + scenario → artifact. */
|
|
1311
1317
|
dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: Parameters<RunCampaignOptions<TScenario, TArtifact>['dispatch']>[1]) => Promise<TArtifact>;
|
|
1312
1318
|
/** The candidate-generation strategy. */
|
|
1313
|
-
proposer: SurfaceProposer
|
|
1319
|
+
proposer: SurfaceProposer<ProposalFinding>;
|
|
1314
1320
|
populationSize: number;
|
|
1315
1321
|
maxGenerations: number;
|
|
1316
1322
|
/** Candidate campaigns run at once. Default 1. Total concurrent cells are
|
|
@@ -1319,14 +1325,8 @@ interface RunOptimizationBaseOptions<TScenario extends Scenario, TArtifact> exte
|
|
|
1319
1325
|
/** DEPTH knob forwarded to the proposer's `propose()` — max iterations the
|
|
1320
1326
|
* agentic generator may take per candidate. */
|
|
1321
1327
|
maxImprovementShots?: number;
|
|
1322
|
-
/**
|
|
1323
|
-
|
|
1324
|
-
report?: unknown;
|
|
1325
|
-
/** Structured findings forwarded to `propose()` as `ctx.findings`. A
|
|
1326
|
-
* findings producer emits these from the
|
|
1327
|
-
* generation's traces; findings-grounded proposers consume them. Opaque here;
|
|
1328
|
-
* the proposer types its `TFindings`. Empty when no producer is wired. */
|
|
1329
|
-
findings?: unknown[];
|
|
1328
|
+
/** Search or observed-production findings forwarded to candidate generation. */
|
|
1329
|
+
findings?: ReadonlyArray<ProposalFinding>;
|
|
1330
1330
|
/** Per-generation findings producer. Runs once on the BASELINE campaign
|
|
1331
1331
|
* (as `generation: -1`, the baseline convention) before generation 0
|
|
1332
1332
|
* proposes — so even a single-generation run proposes with trace context —
|
|
@@ -1349,7 +1349,7 @@ interface RunOptimizationBaseOptions<TScenario extends Scenario, TArtifact> exte
|
|
|
1349
1349
|
/** Shared run spend account and receipt attribution phase. */
|
|
1350
1350
|
costLedger?: CostLedgerHandle;
|
|
1351
1351
|
costPhase?: string;
|
|
1352
|
-
}) => Promise<
|
|
1352
|
+
}) => Promise<ReadonlyArray<ProposalFinding>>;
|
|
1353
1353
|
/**
|
|
1354
1354
|
* Optional override for how the WINNER is selected among coverage-complete
|
|
1355
1355
|
* candidates (and how the incumbent bar is set). Returns a lexicographic rank
|
|
@@ -1379,6 +1379,8 @@ interface RunOptimizationResult<TArtifact, TScenario extends Scenario> {
|
|
|
1379
1379
|
campaign: CampaignResult<TArtifact, TScenario>;
|
|
1380
1380
|
}>;
|
|
1381
1381
|
}>;
|
|
1382
|
+
/** Frozen snapshot of the exact starting surface measured by `baselineCampaign`. */
|
|
1383
|
+
baselineSurface: MutableSurface;
|
|
1382
1384
|
winnerSurface: MutableSurface;
|
|
1383
1385
|
winnerSurfaceHash: string;
|
|
1384
1386
|
/** Proposer label for the promoted surface. Present when the winning
|
|
@@ -1737,4 +1739,4 @@ interface SkillOptOptimizationMethodConfig<TScenario extends Scenario, TArtifact
|
|
|
1737
1739
|
declare function skillOptOptimizationMethod<TScenario extends Scenario, TArtifact>(config: SkillOptOptimizationMethodConfig<TScenario, TArtifact>): OptimizationMethod<TScenario, TArtifact>;
|
|
1738
1740
|
//#endregion
|
|
1739
1741
|
export { Objective as $, optimizationTokenUsageFromSummary as $t, RunEvalOptions as A, CanarySeverity as At, OptimizerModelBudget as B, ComparisonCost as Bt, RunImprovementLoopOptions as C, ReferenceEquivalenceJudgeResult as Cn, scoreRedTeamOutput as Ct, RunOptimizationOptions as D, LlmJudgeDimension as Dn, CanaryKind as Dt, PremeasuredOptimizationBaseline as E, runReferenceEquivalenceJudge as En, CanaryEvaluation as Et, GepaOptimizationMethodConfig as F, ExternalTextOptimizerContext as Ft, ObjectiveSource as G, OptimizationMethodProvenance as Gt, AxisVerdict as H, OptimizationMethodComparison as Ht, GepaOptimizationRecipe as I, ExternalTextOptimizerResult as It, PromotionPolicy as J, OptimizationMethodScore as Jt, ParetoSignificanceGateOptions as K, OptimizationMethodResult as Kt, GepaRunnerCommand as L, ExternalOptimizationExample as Lt, GepaAdaptiveEngineRun as M, composeGate as Mt, GepaEngineOptions as N, externalTextOptimizationMethod as Nt, RunOptimizationResult as O, LlmJudgeOptions as On, CanaryOptions as Ot, GepaEngineRun as P, ExternalTextOptimizationMethodConfig as Pt, Direction as Q, costFromLedgerSummary as Qt, gepaOptimizationMethod as R, ExternalTextEvaluationResponse as Rt, verifyLoopProvenanceRecord as S, ReferenceEquivalenceJudgeOptions as Sn, redTeamReport as St, runImprovementLoop as T, createReferenceEquivalenceJudge as Tn, CanaryAlert as Tt, BuildEvidenceVectorOptions as U, OptimizationMethodInput as Ut, AxisEvidence as V, OptimizationMethod as Vt, EvidenceVector as W, OptimizationMethodPairwise as Wt, paretoPolicy as X, OptimizationTokenUsage as Xt, buildEvidenceVector as Y, OptimizationPackageSource as Yt, paretoSignificanceGate as Z, compareOptimizationMethods as Zt, emitLoopProvenance as _, OpenAutoPrResult as _n, RedTeamCategory as _t, BuildLoopProvenanceArgs as a, planCampaignRun as an, scalarScore as at, provenanceRecordPath as b, REFERENCE_EQUIVALENCE_JUDGE_VERSION as bn, RedTeamReport as bt, LoopProvenanceArgsFromResult as c, createRunCostLedger as cn, powerPreflight as ct, LoopProvenanceEvidence as d, assertCampaignDesign as dn, DefaultProductionGateCheck as dt, CampaignCellFailureReceipt as en, ParetoResult as et, LoopProvenanceOptimizationMethod as f, assertCampaignSplitIdentity as fn, DefaultProductionGateOptions as ft, canonicalDigest as g, OpenAutoPrOptions as gn, RedTeamCase as gt, campaignMeasurementDigest as h, campaignSplitDigestFromIdentities as hn, DEFAULT_RED_TEAM_CORPUS as ht, skillOptOptimizationMethod as i, RunCampaignOptions as in, paretoFrontierWithCrowding as it, runEval as j, runCanaries as jt, runOptimization as k, llmJudge as kn, CanaryReport as kt, LoopProvenanceBackend as l, fsCampaignStorage as ln, HeldOutGateOptions as lt, buildLoopProvenanceRecord as m, campaignSplitDigest as mn, defaultProductionGate as mt, SkillOptRunnerCommand as n, CampaignRunPlanCell as nn, dominates as nt, EmitLoopProvenanceArgs as o, runCampaign as on, PowerPreflight as ot, LoopProvenanceRecord as p, campaignScenarioIdentity as pn, DefaultProductionRewardHackingOptions as pt, PromotionObjective as q, OptimizationMethodRunOptions as qt, SkillOptTrainerConfig as r, PlanCampaignRunOptions as rn, paretoFrontier as rt, EmitLoopProvenanceResult as s, CampaignStorage as sn, PowerPreflightOptions as st, SkillOptOptimizationMethodConfig as t, CampaignRunPlan as tn, crowdingDistance as tt, LoopProvenanceCandidate as u, inMemoryCampaignStorage as un, heldOutGate as ut, loopProvenanceArgsFromResult as v, openAutoPr as vn, RedTeamFinding as vt, RunImprovementLoopResult as w, ReferenceEquivalenceScenario as wn, toolNamesForRun as wt, provenanceSpansPath as x, ReferenceEquivalenceJudgeInput as xn, redTeamDataset as xt, loopProvenanceSpans as y, REFERENCE_EQUIVALENCE_INPUT_LIMITS as yn, RedTeamPayload as yt, OpenAICompatibleOptimizerModel as z, CompareOptimizationMethodsOptions as zt };
|
|
1740
|
-
//# sourceMappingURL=skillopt-optimization-method-
|
|
1742
|
+
//# sourceMappingURL=skillopt-optimization-method-DJ3l4w8W.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"skillopt-optimization-method-DJ3l4w8W.d.ts","names":[],"sources":["../src/llm-judge.ts","../src/reference-equivalence-judge.ts","../src/campaign/auto-pr.ts","../src/campaign/coverage.ts","../src/campaign/external-optimizer-contracts.ts","../src/campaign/storage.ts","../src/campaign/run-campaign.ts","../src/campaign/presets/compare-optimization-methods.ts","../src/campaign/external-text-evaluation.ts","../src/campaign/external-text-optimization-contract.ts","../src/campaign/external-text-optimization.ts","../src/campaign/gates/compose.ts","../src/canary.ts","../src/red-team.ts","../src/campaign/gates/default-production-gate.ts","../src/campaign/gates/heldout-gate.ts","../src/campaign/gates/power-preflight.ts","../src/pareto.ts","../src/campaign/gates/promotion-policy.ts","../src/campaign/optimizer-model.ts","../src/campaign/gepa-optimization-method.ts","../src/campaign/presets/run-eval.ts","../src/campaign/presets/run-optimization.ts","../src/campaign/presets/run-improvement-loop.ts","../src/campaign/provenance.ts","../src/campaign/skillopt-optimization-method.ts"],"mappings":";;;;;;;;;;;;;;;KA6CY,6BAA6B;UAExB,gBAAgB,WAAW,kBAAkB,WAAW;;;;EAIvE,MAAM;;;EAGN,aAAa;;EAEb;;EAEA;EACA;EACA;;;EAGA,UAAU;;;;;EAKV;;EAEA,aAAa,UAAU;;;EAGvB,cAAc;IAAS,UAAU;IAAW,UAAU;;;EAEtD,aAAa;EACb;IAAmB;IAAc,QAAQ,EAAE;;;;;;;;;;;;;iBAoB7B,SAAS,qBAAqB,kBAAkB,WAAW,UACzE,cACA,gBACA,MAAM,gBAAgB,WAAW,aAChC,YAAY,WAAW;;;cC7Fb;cAEA;WACX;WACA;WACA;;UAGe,qCAAqC;EACpD;EACA;;UAGe;EACf;EACA;EACA;;UAGe;;EAEf,MAAM;;EAEN;;EAEA,SAAS;;EAET,aAAa;;UAGE,wCAAwC;EACvD;EACA;EACA;EACA;;;iBA0Bc,gCACd,SAAS,mCACR,oBAAoB;;iBA2CD,6BACpB,OAAO,gCACP,SAAS,mCACR,QAAQ;;;UCjGM,kBAAkB,WAAW,kBAAkB;;EAE9D,QAAQ,eAAe,WAAW;;;EAGlC,MAAM;;;EAGN;;EAEA;EACA;;EAEA;;EAEA;;;EAGA;;EAEA,UAAU;IAAqB;IAAgB;IAAgB;;;UAGhD;EACf;EACA;EACA;EACA;;;;;iBAMc,WAAW,WAAW,kBAAkB,UACtD,SAAS,kBAAkB,WAAW,aACrC;;;;iBC1Ca,qBAAqB,kBAAkB,UACrD,oBAAoB,aACpB;;iBA2Bc,yBAAyB,kBAAkB,UACzD,UAAU,YACT,2BAA2B,KAAK;;iBAUnB,kCACd,oBAAoB,4BACpB;;iBAgBc,oBAAoB,kBAAkB,UACpD,oBAAoB,aACpB;;iBAOc,4BACd,oBAAoB,4BACpB,cACA;;;UC7Ee;EACf;EACA;EACA,MAAM,OAAO;;KAGH;KAEA,iCAAiC;UAE5B;EACf,WAAW;EACX;;UAUe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,SAAS;;EAET;;;;;;;;;;;;;;;;;;;UCrBe;;EAEf,UAAU;;EAEV,OAAO;;EAEP,KAAK;;EAEL,MAAM,cAAc,kBAAkB;;;EAGtC,OAAO,cAAc,iBAAiB;;;;;;;;;iBAUxB,qBAAqB;;;;iBA0CrB,2BAA2B;;iBAmC3B,oBAAoB;EAClC,SAAS;EACT;EACA;IACE;;;UC/Ea,mBAAmB,kBAAkB,UAAU;EAC9D,WAAW;EACX,UAAU,WAAW,WAAW;;EAEhC,SAAS;;;;;;EAMT;EACA,SAAS,YAAY,WAAW;;EAEhC;;;EAGA;;;EAGA;;;;EAIA,eAAe;EACf;EACA;;EAEA;;;EAGA,aAAa;;EAEb;;EAEA,WAAW,SAAS;;EAEpB;;;;;;;;EAQA;;;;;;;;;;EAUA;;;;;EAKA;;;;EAIA;;;EAGA;;;;EAIA;;;;;;;;;;;EAWA;;EAEA,YAAY;;EAEZ,oBAAoB,gBAAgB,gBAAgB;;;;;;EAMpD,UAAU;;;;;;;;;;;;EAYV,iBAAiB;IACf,UAAU;IACV;IACA;;;;;;;UAQa,2BAA2B;EAC1C;EACA;EACA;EACA;IACE;IACA;IACA;MACE;MACA;MACA;;;EAGJ,MAAM,mBAAmB;EACzB,MAAM;;;;;iBAMc,YAAY,kBAAkB,UAAU,WAC5D,MAAM,mBAAmB,WAAW,aACnC,QAAQ,eAAe,WAAW;UAyjBpB;EACf;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA,OAAO;;UAGQ,uBAAuB,kBAAkB,UAAU;EAClE,WAAW;EACX,WAAW,WAAW,WAAW;EACjC;EACA,SAAS,YAAY,WAAW;EAChC;EACA;EACA;EACA;;EAEA;EACA,UAAU;;;;;;iBAOI,gBAAgB,kBAAkB,UAAU,WAC1D,MAAM,uBAAuB,WAAW,aACvC;;;;KCxvBS,6BAA6B,kBAAkB,UAAU,aAAa,KAChF,mBAAmB,WAAW;;UAKf;EACf;EACA;EACA;;UAGe;EACf;;EAEA;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;;EAEA;EACA;EACA;;UAGe;;EAEf,QAAQ;;EAER,SAAS;;EAET,UAAU;;EAEV,SAAS;;EAET;EACA;;EAEA;EACA;EACA;EACA;EACA,aAAa;;;UAIE,wBAAwB,kBAAkB,UAAU;;WAE1D,iBAAiB;;WAEjB,yBAAyB;;WAEzB,6BAA6B;;WAE7B,sBACP,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;;WAEJ,iBAAiB,YAAY,WAAW;;WAExC;WACA;;WAEA,YAAY,SAAS,6BAA6B,WAAW;;WAE7D,YAAY;;UAGN;;EAEf,eAAe;;EAEf,MAAM;;EAEN;;EAEA,aAAa;;;UAIE,mBAAmB,kBAAkB,WAAW,UAAU;;EAEzE;EACA,WACE,OAAO,wBAAwB,WAAW,eACvC,QAAQ;;UAGE;EACf;;EAEA;;EAEA;;EAEA;;;EAGA;IAAU;IAAa;;;EAEvB,kBAAkB;;EAElB;;EAEA,aAAa;;EAEb,gBAAgB;IACd;IACA;IACA;IACA;;EAEF,eAAe;;EAEf;;UAGe;;EAEf;EACA;;EAEA;EACA;EACA;;EAEA;;UAGe;;EAEf,QAAQ;EACR,MAAM;;EAEN,UAAU;EACV;;EAEA,kBAAkB;;EAElB,UAAU;;EAEV,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,kCAAkC,kBAAkB,UAAU,mBACrE,KAAK,mBAAmB,WAAW;EAC3C,SAAS,mBAAmB,WAAW;EACvC,iBAAiB;;EAEjB,gBAAgB;;EAEhB,oBAAoB;;EAEpB,eAAe;;EAEf,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;EACb,QAAQ,YAAY,WAAW;;;EAG/B;;EAEA,yBAAyB,6BAA6B,WAAW;;EAEjE;;;EAGA;;EAEA;;;;;iBAMoB,2BAA2B,kBAAkB,UAAU,WAC3E,MAAM,kCAAkC,WAAW,aAClD,QAAQ;;iBAyrBK,sBAAsB,SAAS,oBAAoB;;iBAWnD,kCACd,SAAS,mBACT,mBAAmB,gBAClB;;;UC76Bc;EACf;EACA;;UAGe;EACf;EACA;IACE;IACA,YAAY;IACZ;IACA;;;;;UCba;WACN;WACA;WACA;WACA;WACA;WACA,eAAe;WACf,mBAAmB;WACnB,uBAAuB;WACvB;WACA;;WAEA;WACA;WACA;WACA,QAAQ;;WAER,MAAM;WACN,WACP,SAFa,kCAGV,QAAQ;;UAGE;EACf,eAAe;EACf;EACA;IACM;;IACA;;IACA;IAAkB;;;;;;;;;UAqBT,qCACf,kBAAkB,UAClB;EAEA;EACA,QAAQ,KAAK;EACb;EACA;EACA;EACA;;EAEA;;EAEA;;EAEA,SAAS;EACT;EACA;EACA,oBAAoB,UAAU;EAC9B,oBAAoB,UAAU,WAAW,UAAU;EACnD,MAAM,SAAS,iCAAiC,QAAQ;;;;;;;;;;iBCP1C,+BAA+B,kBAAkB,UAAU,WACzE,QAAQ,qCAAqC,WAAW,aACvD,mBAAmB,WAAW;;;;;;iBCjEjB,YAAY,qBAAqB,kBAAkB,WAAW,aACzE,OAAO,MAAM,KAAK,WAAW,cAC/B,KAAK,WAAW;;;KCkBP;KAEA;UAEK;EACf,MAAM;EACN,UAAU;EACV;;;EAGA,UAAU;;UAGK;EACf,QAAQ;;EAER,QAAQ,OAAO;;EAEf,aAAa;;UAGE;EACf,MAAM;EACN;EACA;EACA;;UAGe;;;;;;;;;;EAUf;IACE;IACA;;IAEA;;;;;;;;;;;;;EAcF;IACE;IACA;IACA;IACA;;;;;;;;;;EAWF;IACE,WAAW,KAAK;IAChB;IACA;IACA;IACA;;;;;;;;iBASY,YAAY,MAAM,aAAa,OAAM,gBAAqB;;;KClG9D;UAUK;EACf,UAAU;;EAEV;;;;;EAKA;;EAEA;;EAEA;;UAGe,oBAAoB;EACnC,SAAS;;UAGM;EACf;EACA,UAAU;EACV;EACA;EACA;;UAGe;EACf,UAAU;EACV,oBAAoB,OAAO;EAC3B;;;cA0DW,yBAAyB;iBA6FtB,eAAe,aAAY,gBAAqB;;;;;iBAkBhD,mBACd,gBACA,qBACA,QAAQ,cACP;;iBA4Fa,cAAc,UAAU,mBAAmB;;;;;iBAqBrC,gBAAgB,OAAO,YAAY,gBAAgB;;;KC9T7D;KAOA,wCAAwC,KAClD;EAGA,SAAS,YAAY;;UAGN;;;EAGf,kBAAkB;;;;;EAKlB;;EAEA;;EAEA;;EAEA;;;;EAIA;;;;EAIA;;;;;EAKA;;;;EAIA;;;EAGA;;;;EAIA,iBAAiB;;;EAGjB,aAAa;;EAEb,gBAAgB;;EAEhB,SAAS;;;;;EAKT,iBAAiB;;;;;iBAMH,sBAAsB,WAAW,kBAAkB,UACjE,SAAS,+BACR,KAAK,WAAW;;;UCnFF,mBAAmB,kBAAkB,WAAW;EAC/D,WAAW;;;EAGX;;EAEA;;;;EAIA;;EAEA;;EAEA;;;;;;iBAOc,YAAY,WAAW,kBAAkB,UACvD,SAAS,mBAAmB,aAC3B,KAAK,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;UCfF;;EAEf;;;EAGA;;EAEA;;EAEA;;;;;;;EAOA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;;EAGA;EACA;EACA;;;EAGA;;EAEA;;;;;;iBAec,eAAe,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;KClEjD;UAEK,UAAU;;EAEzB;EACA,WAAW;EACX,QAAQ,WAAW;;UAGJ,aAAa;EAC5B,UAAU;EACV,WAAW;;EAEX,cAAc;IAAQ,WAAW;IAAG,WAAW;;;;iBAIjC,UAAU,GAAG,GAAG,GAAG,GAAG,GAAG,YAAY,UAAU;;;;;;iBAmB/C,eAAe,GAAG,YAAY,KAAK,YAAY,UAAU,OAAO,aAAa;;;;;;;;;;iBA4B7E,YAAY,GAC1B,YAAY,KACZ,YAAY,UAAU,MACtB;EAAW,UAAU,QAAQ;IAC5B;EAAQ,WAAW;EAAG;;;;;;;;;;;;;;iBAyCT,iBAAiB,GAC/B,YAAY,KACZ,YAAY,UAAU,OACrB;EAAQ,WAAW;EAAG;;;;;;;iBA6BT,2BAA2B,GACzC,YAAY,KACZ,YAAY,UAAU,OACrB;EAAQ,WAAW;EAAG;;;;;;KC/Hb;EAAoB;;EAAwB;EAAmB;;UAE1D;;EAEf;EACA,QAAQ;;;EAGR,WAAW;;;;EAIX;;;;EAIA;;;KAIU;UAEK;EACf;EACA,QAAQ;EACR,WAAW;;;EAGX,WAAW;;EAEX;EACA;EACA;EACA;EACA;EACA,SAAS;;UAGM;;EAEf,MAAM;;;EAGN;;;EAGA;IAAQ;IAAmB;;;;;;KAMjB,mBAAmB,IAAI,mBAAmB;UAErC;;;;EAIf;;EAEA;;EAEA;;EAEA;;;;;;;;iBASc,oBAAoB,WAAW,kBAAkB,UAC/D,KAAK,YAAY,WAAW,YAC5B,YAAY,sBACZ,OAAM,6BACL;;;;;;;;cAmFU,cAAc;UA2EV,sCAAsC;;EAErD,YAAY;;;EAGZ,SAAS;;EAET;;;;;;;iBAQc,uBAAuB,qBAAqB,kBAAkB,WAAW,UACvF,SAAS,gCACR,KAAK,WAAW;;;KCxRP,uBAAuB;;UAGlB;EACf;EACA;EACA;EACA,QAAQ;;;;;UC6CO;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;;;EAKA,eAAe;;;UAIA,sBAAsB;;EAErC;;;KAIU,wBAAwB;;;;;;;KAQxB;EAEN;EACA,KAAK;;EAGL;EACA,eAAe;;EAGf;EACA,eAAe;;EAEf;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;;EAGA;EACA,eAAe;EACf;;EAGA;EACA,eAAe;EACf;;EAGA;EACA,kBAAkB;EAClB,cAAc;EACd;;;KAIM,oBAAoB;UAEf,6BAA6B,kBAAkB,UAAU;;EAExE;;EAEA,QAAQ;;EAER;;EAEA;;EAEA;;;;;EAKA;;EAEA;;EAEA;;EAEA;;;;;;EAMA,YAAY;;;;;;EAMZ,oBAAoB,UAAU;;EAE9B,oBAAoB,UAAU,WAAW,UAAU;;EAEnD,SAAS;;;;;EAKT;EACA,SAAS;;;;;;;;;;iBAWK,uBAAuB,kBAAkB,UAAU,WACjE,QAAQ,6BAA6B,WAAW,aAC/C,mBAAmB,WAAW;;;UClLhB,eAAe,kBAAkB,UAAU,mBAClD,KAAK,mBAAmB,WAAW;EAC3C;;;;;iBAMoB,QAAQ,kBAAkB,UAAU,WACxD,MAAM,eAAe,WAAW,aAC/B,QAAQ,eAAe,WAAW;;;UC0BpB,gCAAgC,WAAW,kBAAkB;;EAE5E;;EAEA,UAAU,eAAe,WAAW;;UAGrB,2BAA2B,kBAAkB,UAAU,mBAC9D,KAAK,mBAAmB,WAAW;;EAE3C,iBAAiB;;;;;;;;;EASjB,sBAAsB,gCAAgC,WAAW;;EAEjE,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,WAAW,mBAAmB,WAAW,+BAC3C,QAAQ;;EAEb,UAAU,gBAAgB;EAC1B;EACA;;;EAGA;;;EAGA;;EAEA,WAAW,cAAc;;;;;;;;;;;EAWzB,qBAAqB;IACnB;IACA;IACA,YAAY;MACV;MACA,UAAU,eAAe,WAAW;MACpC;;IAEF,SAAS;;IAET,aAAa;IACb;QACI,QAAQ,cAAc;;;;;;;;;;;;;;;;;;EAkB5B,oBAAoB,UAAU,eAAe,WAAW;;KAG9C,uBACV,kBAAkB,UAClB,aACE,2BAA2B,WAAW;UAEzB,sBAAsB,WAAW,kBAAkB;EAClE,aAAa;IACX,QAAQ;IACR,UAAU;MACR;MACA,SAAS;MACT,UAAU,eAAe,WAAW;;;;EAIxC,iBAAiB;EACjB,eAAe;EACf;;;;EAIA;;;;EAIA;EACA,kBAAkB,eAAe,WAAW;;EAE5C,MAAM;;;;;;EAMN,gBAAgB;;;;;iBAMI,gBAAgB,kBAAkB,UAAU,WAChE,MAAM,uBAAuB,WAAW,aACvC,QAAQ,sBAAsB,WAAW;;;KCpJhC,0BACV,kBAAkB,UAClB,aACE,uBAAuB,WAAW;;;EAGpC,kBAAkB;;;;;;;;;EASlB;;;;EAIA,MAAM,KAAK,WAAW;;;;;EAKtB;;EAEA;EACA;;;;;;;;EAQA,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;UAGlE,yBAAyB,WAAW,kBAAkB,kBAC7D,sBAAsB,WAAW;EACzC,mBAAmB,eAAe,WAAW;EAC7C,iBAAiB,eAAe,WAAW;EAC3C,uBAAuB,eAAe,WAAW;EACjD,qBAAqB;EACrB,YAAY,QAAQ,WAAW,KAAK,WAAW;;;;EAI/C;;;;;EAKA;EACA,WAAW,kBAAkB;;;;;iBAMT,mBAAmB,kBAAkB,UAAU,WACnE,MAAM,0BAA0B,WAAW,aAC1C,QAAQ,yBAAyB,WAAW;;;UC1B9B;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,UAAU,YAAY;;EAEtB;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;EACA;EACA;EACA;;UAGe;EACf;IACE;IACA;;EAEF;IACE;IACA;IACA;IACA;MACE;MACA;MACA;MACA;;;EAGJ;;UAGe;EACf;EACA,MAAM;EACN;EACA,aAAa;;;;;;;UAQE;EACf;;EAEA;EACA;EACA;EACA;;EAEA;EACA;;EAEA;EACA;;EAEA;;EAEA,YAAY;;EAEZ,qBAAqB;;EAErB,UAAU;;EAEV;;EAEA;IACE,UAAU;IACV;IACA;IACA,mBAAmB;;;;;;EAMrB;;EAEA;;EAEA;;;EAGA;;EAEA,SAAS;EACT;EACA;;UAGe,wBAAwB,WAAW,kBAAkB;EACpE;EACA;EACA;EACA,iBAAiB;EACjB,eAAe;EACf;EACA;;EAEA,wBAAwB,eAAe,WAAW;;EAElD,aAAa;IACX;IACA,YAAY;IACZ;;;IAGA,UAAU;MACR;MACA,SAAS;MACT,UAAU,eAAe,WAAW;;;EAGxC,MAAM;;;;EAIN;EACA,mBAAmB,eAAe,WAAW;EAC7C,iBAAiB,eAAe,WAAW;EAC3C,qBAAqB;EACrB,uBAAuB,eAAe,WAAW;;EAEjD,cAAc,cAAc;EAC5B;EACA;EACA,qBAAqB;;UAGN,6BAA6B,WAAW,kBAAkB;EACzE;EACA;EACA;EACA,iBAAiB;EACjB,QAAQ,yBAAyB,WAAW;EAC5C,cAAc,cAAc;EAC5B;EACA;;;iBAIc,6BAA6B,WAAW,kBAAkB,UACxE,OAAO,6BAA6B,WAAW,aAC9C,wBAAwB,WAAW;;iBA4CtB,0BAA0B,WAAW,kBAAkB,UACrE,MAAM,wBAAwB,WAAW,aACxC;;iBAoRa,0BAA0B,WAAW,kBAAkB,UACrE,UAAU,eAAe,WAAW;;iBAoCtB,2BAA2B,QAAQ,uBAAuB;;iBAa1D,gBAAgB;;;;;;;;;;;;iBA2KhB,oBACd,QAAQ,sBACR;EAAQ;IACP;;iBAiJa,qBAAqB;;;;iBAMrB,oBAAoB;UAInB;EACf,QAAQ;EACR,OAAO;;EAEP;EACA;;UAGe,uBAAuB,WAAW,kBAAkB,kBAC3D,wBAAwB,WAAW;;EAE3C,SAAS;;;EAGT,eAAe;;;;;;;;;;;;iBA+EK,mBAAmB,WAAW,kBAAkB,UACpE,MAAM,uBAAuB,WAAW,aACvC,QAAQ;;;UC/8BM;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;;;;EAKA,YAAY;;KAGF,wBAAwB;UAEnB,iCAAiC,kBAAkB,UAAU;EAC5E;;EAEA;EACA;;EAEA;EACA,SAAS;;;;;EAKT,WAAW;;EAEX;;EAEA;EACA;;EAEA;EACA;EACA,oBAAoB,UAAU;EAC9B,oBAAoB,UAAU,WAAW,UAAU;EACnD,SAAS;EACT,SAAS;;;iBAIK,2BAA2B,kBAAkB,UAAU,WACrE,QAAQ,iCAAiC,WAAW,aACnD,mBAAmB,WAAW"}
|
|
@@ -9,7 +9,14 @@ declare function weightedMean(scores: {
|
|
|
9
9
|
score: number;
|
|
10
10
|
weight?: number;
|
|
11
11
|
}[]): number;
|
|
12
|
-
/**
|
|
12
|
+
/**
|
|
13
|
+
* Percentile bootstrap confidence interval on the mean of `scores`.
|
|
14
|
+
*
|
|
15
|
+
* Descriptive spread. It is not a significance test, and at small n its bounds
|
|
16
|
+
* are anti-conservative in the same way {@link pairedBootstrap}'s are — see
|
|
17
|
+
* {@link BOOTSTRAP_GATE_MIN_N}. With no `seed` the resampling is seeded from
|
|
18
|
+
* the scores themselves, so the interval is reproducible either way.
|
|
19
|
+
*/
|
|
13
20
|
declare function confidenceInterval(scores: number[], confidence?: number, opts?: {
|
|
14
21
|
seed?: number;
|
|
15
22
|
resamples?: number;
|
|
@@ -19,47 +26,136 @@ declare function confidenceInterval(scores: number[], confidence?: number, opts?
|
|
|
19
26
|
upper: number;
|
|
20
27
|
};
|
|
21
28
|
/**
|
|
22
|
-
* Inter-rater reliability —
|
|
29
|
+
* Inter-rater reliability — Krippendorff's α under the squared-difference
|
|
30
|
+
* metric, pooled across dimensions.
|
|
23
31
|
*
|
|
24
|
-
* Each inner array is one judge's scores
|
|
25
|
-
*
|
|
32
|
+
* Each inner array is one judge's scores. Items are matched by position
|
|
33
|
+
* WITHIN a dimension: the k-th score a judge supplies carrying dimension
|
|
34
|
+
* `d` is item k of `d`, and the ratings compared against each other are
|
|
35
|
+
* the ones different judges gave to the same item. Every judge that scores
|
|
36
|
+
* a dimension at all must supply the same number of scores for it —
|
|
37
|
+
* ragged input cannot be aligned into items and throws rather than
|
|
38
|
+
* comparing mismatched items.
|
|
39
|
+
*
|
|
40
|
+
* α = 1 − D_observed / D_expected: D_observed averages the squared
|
|
41
|
+
* difference over within-item judge pairs, D_expected over every pair of
|
|
42
|
+
* ratings irrespective of item. α = 1 is perfect agreement, 0 is chance,
|
|
43
|
+
* negative is systematic disagreement.
|
|
26
44
|
*/
|
|
27
45
|
declare function interRaterReliability(judgeScores: JudgeScore[][]): number;
|
|
46
|
+
/** How a rank test's p-value was actually computed. */
|
|
47
|
+
type RankTestMethod = 'exact' | 'permutation' | 'asymptotic';
|
|
28
48
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
49
|
+
* What the caller asks for. `'auto'` selects `'exact'` inside the enumeration
|
|
50
|
+
* threshold and `'permutation'` above it, and never selects `'asymptotic'`.
|
|
31
51
|
*/
|
|
32
|
-
|
|
52
|
+
type RankTestMethodRequest = 'auto' | 'exact' | 'asymptotic';
|
|
53
|
+
interface RankTestOptions {
|
|
54
|
+
/** Default `'auto'`. `'asymptotic'` inside the exact-feasible range throws. */
|
|
55
|
+
method?: RankTestMethodRequest;
|
|
56
|
+
/** Resamples on the Monte Carlo permutation path. Default 100000. */
|
|
57
|
+
permutations?: number;
|
|
58
|
+
/** Seed for the permutation path. Omitted ⇒ derived from the data itself, so
|
|
59
|
+
* the result is reproducible either way. */
|
|
60
|
+
seed?: number;
|
|
61
|
+
}
|
|
62
|
+
/** Maximum dynamic-programming cells used by an exact two-sample rank test. */
|
|
63
|
+
declare const MANN_WHITNEY_EXACT_MAX_STATES = 8192;
|
|
64
|
+
/** Maximum inner-loop transitions used by an exact two-sample rank test. */
|
|
65
|
+
declare const MANN_WHITNEY_EXACT_MAX_WORK = 250000;
|
|
66
|
+
/** Non-zero differences up to which the signed-rank null is enumerated exactly. */
|
|
67
|
+
declare const WILCOXON_EXACT_MAX_N = 20;
|
|
68
|
+
/** Resamples used when a rank test falls back to Monte Carlo permutation. */
|
|
69
|
+
declare const DEFAULT_PERMUTATIONS = 100000;
|
|
70
|
+
interface MannWhitneyResult {
|
|
71
|
+
/** `min(U_a, U_b)` — the conventional reported statistic. */
|
|
33
72
|
u: number;
|
|
73
|
+
/** U for sample `a`. Carries the direction of the effect, which `u` discards. */
|
|
74
|
+
uA: number;
|
|
75
|
+
/** Two-sided p-value. */
|
|
34
76
|
p: number;
|
|
35
|
-
|
|
77
|
+
/** How `p` was computed. */
|
|
78
|
+
method: RankTestMethod;
|
|
79
|
+
/** Smallest two-sided p this design can produce. `p` can never be below it. */
|
|
80
|
+
pFloor: number;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Mann-Whitney U — two independent samples, no distributional assumption.
|
|
84
|
+
*
|
|
85
|
+
* Exact conditional (permutation) p by default when the dynamic program fits
|
|
86
|
+
* {@link MANN_WHITNEY_EXACT_MAX_STATES} cells and
|
|
87
|
+
* {@link MANN_WHITNEY_EXACT_MAX_WORK} transitions, seeded Monte Carlo
|
|
88
|
+
* permutation above those limits. This keeps imbalanced designs such as 1+24
|
|
89
|
+
* exact without admitting expensive balanced designs merely because they have
|
|
90
|
+
* the same total size. Throws on non-finite input and on `method:
|
|
91
|
+
* 'asymptotic'` where an exact answer is available. Empty input yields `p = 1,
|
|
92
|
+
* pFloor = 1` — no design, no attainable evidence.
|
|
93
|
+
*/
|
|
94
|
+
declare function mannWhitneyU(a: number[], b: number[], opts?: RankTestOptions): MannWhitneyResult;
|
|
36
95
|
/** Partial credit: returns 0-1 ratio of current toward target */
|
|
37
96
|
declare function partialCredit(current: number, target: number): number;
|
|
97
|
+
interface PairedTTestResult {
|
|
98
|
+
/** Null when the statistic is undefined — see {@link pairedTTest}. */
|
|
99
|
+
t: number | null;
|
|
100
|
+
df: number;
|
|
101
|
+
/** Null exactly when `t` is null. */
|
|
102
|
+
p: number | null;
|
|
103
|
+
}
|
|
38
104
|
/**
|
|
39
105
|
* Paired t-test — before/after measurements on the SAME items.
|
|
40
106
|
* Pairing removes inter-item variance, giving tighter significance than
|
|
41
107
|
* an unpaired test when comparing prompt v1 vs prompt v2 on identical
|
|
42
108
|
* scenarios.
|
|
109
|
+
*
|
|
110
|
+
* Returns `t = p = null` where the statistic is undefined: fewer than two
|
|
111
|
+
* pairs, or a non-zero constant delta whose observed variance is zero. A
|
|
112
|
+
* constant shift carries no information about the variance it would have to
|
|
113
|
+
* be compared against, so the honest answer is "undefined", not `p = 0` —
|
|
114
|
+
* three observations cannot buy absolute certainty. This is the same contract
|
|
115
|
+
* {@link pairedCohensDz} states for the same condition. An all-zero delta is
|
|
116
|
+
* different: it is a measured null, and returns `t = 0, p = 1`.
|
|
43
117
|
*/
|
|
44
|
-
declare function pairedTTest(before: number[], after: number[]):
|
|
45
|
-
|
|
46
|
-
|
|
118
|
+
declare function pairedTTest(before: number[], after: number[]): PairedTTestResult;
|
|
119
|
+
interface WilcoxonSignedRankResult {
|
|
120
|
+
/** W⁺, the rank sum of the positive differences. (scipy reports
|
|
121
|
+
* `min(W⁺, W⁻)`; compare statistics only after converting.) */
|
|
122
|
+
w: number;
|
|
123
|
+
/** Two-sided p-value. */
|
|
47
124
|
p: number;
|
|
48
|
-
|
|
125
|
+
/** How `p` was computed. */
|
|
126
|
+
method: RankTestMethod;
|
|
127
|
+
/** Smallest two-sided p this design can produce. */
|
|
128
|
+
pFloor: number;
|
|
129
|
+
/** Non-zero differences — zero differences are dropped and carry no rank. */
|
|
130
|
+
nNonZero: number;
|
|
131
|
+
}
|
|
49
132
|
/**
|
|
50
|
-
* Wilcoxon signed-rank
|
|
51
|
-
*
|
|
133
|
+
* Wilcoxon signed-rank — paired, no distributional assumption on the deltas.
|
|
134
|
+
*
|
|
135
|
+
* Exact conditional (sign-flip) p by default at `n ≤
|
|
136
|
+
* {@link WILCOXON_EXACT_MAX_N}` non-zero differences, seeded Monte Carlo
|
|
137
|
+
* permutation above it. Throws on non-finite input and on `method:
|
|
138
|
+
* 'asymptotic'` where an exact answer is available.
|
|
139
|
+
*
|
|
140
|
+
* `n` is the count of NON-ZERO differences: exact ties are dropped before
|
|
141
|
+
* ranking, so a run of tied pairs shrinks the design and raises `pFloor`.
|
|
142
|
+
* All-tied input yields `p = 1, pFloor = 1` — no attainable evidence, which
|
|
143
|
+
* `pFloor` states rather than leaving `p = 1` to be read as a measured null.
|
|
52
144
|
*/
|
|
53
|
-
declare function wilcoxonSignedRank(before: number[], after: number[]):
|
|
54
|
-
w: number;
|
|
55
|
-
p: number;
|
|
56
|
-
};
|
|
145
|
+
declare function wilcoxonSignedRank(before: number[], after: number[], opts?: RankTestOptions): WilcoxonSignedRankResult;
|
|
57
146
|
/**
|
|
58
147
|
* Cohen's d — standardized effect size for two independent groups.
|
|
59
148
|
* Positive d means group b has higher mean than group a.
|
|
60
149
|
* Rule of thumb: |d| < 0.2 negligible, 0.2–0.5 small, 0.5–0.8 medium, > 0.8 large.
|
|
150
|
+
*
|
|
151
|
+
* Returns null where the standardized effect is undefined: fewer than two
|
|
152
|
+
* observations in either group, or a zero pooled standard deviation with
|
|
153
|
+
* unequal means. Null is NOT "no effect" — zero within-group spread across a
|
|
154
|
+
* real mean gap is an unbounded effect, the opposite of negligible. Equal
|
|
155
|
+
* means with zero spread is a genuine 0. Same contract as
|
|
156
|
+
* {@link pairedCohensDz}.
|
|
61
157
|
*/
|
|
62
|
-
declare function cohensD(a: number[], b: number[]): number;
|
|
158
|
+
declare function cohensD(a: number[], b: number[]): number | null;
|
|
63
159
|
/**
|
|
64
160
|
* Cohen's dz for paired observations: mean(after - before) divided by the
|
|
65
161
|
* sample standard deviation of those within-pair deltas.
|
|
@@ -215,6 +311,11 @@ declare function requiredSampleSize(opts: {
|
|
|
215
311
|
/**
|
|
216
312
|
* Required number of paired observations for a target Cohen's dz.
|
|
217
313
|
* Unlike the independent-groups formula, this has no two-arm factor of two.
|
|
314
|
+
*
|
|
315
|
+
* Normal quantiles with no t correction, so treat the result as a LOWER bound:
|
|
316
|
+
* it returns 32 where the exact t-based answer is 34 at dz = 0.5, and 13 where
|
|
317
|
+
* it is 15 at dz = 0.8 — a 6–13 % shortfall precisely in the range a caller
|
|
318
|
+
* consults to decide whether 3–10 repetitions suffice.
|
|
218
319
|
*/
|
|
219
320
|
declare function requiredPairedSampleSize(opts: {
|
|
220
321
|
effect: number;
|
|
@@ -267,8 +368,14 @@ declare function mcnemarPower(opts: {
|
|
|
267
368
|
alpha?: number;
|
|
268
369
|
twoSided?: boolean;
|
|
269
370
|
}): number;
|
|
270
|
-
/**
|
|
271
|
-
|
|
371
|
+
/**
|
|
372
|
+
* Bonferroni adjustment: multiply every p-value by the test count, clamp at 1.
|
|
373
|
+
*
|
|
374
|
+
* Rejects at `p_adjusted ≤ alpha` — the boundary is inclusive, matching
|
|
375
|
+
* {@link holm}, which uniformly dominates this correction and must therefore
|
|
376
|
+
* never reject less. Validates its inputs on the same terms.
|
|
377
|
+
*/
|
|
378
|
+
declare function bonferroni(pValues: readonly number[], alpha?: number): {
|
|
272
379
|
adjusted: number[];
|
|
273
380
|
significant: boolean[];
|
|
274
381
|
};
|
|
@@ -287,8 +394,11 @@ declare function holm(pValues: readonly number[], alpha?: number): {
|
|
|
287
394
|
/**
|
|
288
395
|
* Benjamini–Hochberg false discovery rate. Returns adjusted q-values and
|
|
289
396
|
* significance at the target FDR; handles ties and preserves q monotonicity.
|
|
397
|
+
*
|
|
398
|
+
* Rejects at `q ≤ fdr` — the BH rule is inclusive at the boundary, so an
|
|
399
|
+
* exactly-`fdr` q-value is a discovery.
|
|
290
400
|
*/
|
|
291
|
-
declare function benjaminiHochberg(pValues: number[], fdr?: number): {
|
|
401
|
+
declare function benjaminiHochberg(pValues: readonly number[], fdr?: number): {
|
|
292
402
|
qValues: number[];
|
|
293
403
|
significant: boolean[];
|
|
294
404
|
};
|
|
@@ -307,7 +417,20 @@ interface PairedBootstrapResult {
|
|
|
307
417
|
confidence: number;
|
|
308
418
|
/** Number of bootstrap resamples used. */
|
|
309
419
|
resamples: number;
|
|
420
|
+
/** False below {@link BOOTSTRAP_GATE_MIN_N}. See {@link pairedBootstrap}. */
|
|
421
|
+
gateEligible: boolean;
|
|
310
422
|
}
|
|
423
|
+
/**
|
|
424
|
+
* Pairs below which a percentile bootstrap interval is descriptive spread only.
|
|
425
|
+
*
|
|
426
|
+
* `P(low > 0)` under a true null, against a nominal 2.5 %, measured over 4000
|
|
427
|
+
* seeded trials: 13.53 % at n = 3, 3.52 % at n = 10, 3.10 % at n = 20 on the
|
|
428
|
+
* median; 13.85 %, 4.90 %, 3.80 % on the mean. This is intrinsic to resampling
|
|
429
|
+
* three points, not an implementation error — scipy's BCa gives 16.0 % on the
|
|
430
|
+
* same n = 3 data — so no change to the estimator moves it. Below this floor
|
|
431
|
+
* the decision belongs to the exact sign test or exact signed-rank test.
|
|
432
|
+
*/
|
|
433
|
+
declare const BOOTSTRAP_GATE_MIN_N = 20;
|
|
311
434
|
interface PairedBootstrapOptions {
|
|
312
435
|
/** Confidence level. Default 0.95. */
|
|
313
436
|
confidence?: number;
|
|
@@ -315,14 +438,19 @@ interface PairedBootstrapOptions {
|
|
|
315
438
|
resamples?: number;
|
|
316
439
|
/** Statistic to bootstrap. Default 'median'. */
|
|
317
440
|
statistic?: 'median' | 'mean';
|
|
318
|
-
/** Deterministic seed. If omitted,
|
|
441
|
+
/** Deterministic seed. If omitted, derived from the deltas so the interval
|
|
442
|
+
* is reproducible regardless. */
|
|
319
443
|
seed?: number;
|
|
320
444
|
}
|
|
321
445
|
/**
|
|
322
446
|
* Paired bootstrap on (after − before) deltas. Returns a CI on the chosen
|
|
323
|
-
* statistic (median by default); pairs are resampled with replacement.
|
|
324
|
-
*
|
|
325
|
-
*
|
|
447
|
+
* statistic (median by default); pairs are resampled with replacement. Throws
|
|
448
|
+
* on unequal sample sizes.
|
|
449
|
+
*
|
|
450
|
+
* `low > threshold` carries the stated confidence ONLY at `n ≥
|
|
451
|
+
* {@link BOOTSTRAP_GATE_MIN_N}`, which `gateEligible` reports. Below it the
|
|
452
|
+
* check fires under a true null several times more often than nominal, so the
|
|
453
|
+
* interval is descriptive spread and a promotion must not turn on it.
|
|
326
454
|
*/
|
|
327
455
|
declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
|
|
328
456
|
/** Pre-registered direction for a one-sided paired sign test. */
|
|
@@ -506,9 +634,9 @@ interface EProcess {
|
|
|
506
634
|
declare function eProcess(opts?: EProcessOptions): EProcess;
|
|
507
635
|
/** Tiny seedable PRNG (mulberry32) — deterministic resampling/shuffling, not
|
|
508
636
|
* cryptographic. Exported so e-process shuffles and bootstrap resampling
|
|
509
|
-
* share ONE PRNG implementation
|
|
510
|
-
*
|
|
637
|
+
* share ONE PRNG implementation. Every distinct 32-bit seed gives a distinct
|
|
638
|
+
* stream, including 0. */
|
|
511
639
|
declare function mulberry32(seed: number): () => number;
|
|
512
640
|
//#endregion
|
|
513
|
-
export {
|
|
514
|
-
//# sourceMappingURL=statistics-
|
|
641
|
+
export { partialCredit as $, benjaminiHochberg as A, interpretCliffs as B, RankTestOptions as C, WeightedCompositeInput as D, WILCOXON_EXACT_MAX_N as E, corpusInterRaterAgreement as F, mulberry32 as G, mcnemar as H, corpusInterRaterAgreementFromJudgeScores as I, pairedCohensDz as J, normalizeScores as K, eProcess as L, cliffsDelta as M, cohensD as N, WeightedCompositeResult as O, confidenceInterval as P, pairedTTest as Q, holm as R, RankTestMethodRequest as S, SignTestAlternative as T, mcnemarPower as U, mannWhitneyU as V, mcnemarRequiredN as W, pairedRiskDifference as X, pairedMde as Y, pairedSignTest as Z, PairedBootstrapResult as _, CorpusAgreementReport as a, spearmanR as at, ProportionInterval as b, EProcess as c, wilcoxonSignedRank as ct, EProcessStep as d, passAtK as et, MANN_WHITNEY_EXACT_MAX_STATES as f, PairedBootstrapOptions as g, McNemarResult as h, CorpusAgreementPerDimension as i, requiredSampleSize as it, bonferroni as j, WilcoxonSignedRankResult as k, EProcessOptions as l, wilson as lt, MannWhitneyResult as m, CliffsMagnitude as n, ranks as nt, CorpusScoreRecord as o, weightedComposite as ot, MANN_WHITNEY_EXACT_MAX_WORK as p, pairedBootstrap as q, CorpusAgreementOptions as r, requiredPairedSampleSize as rt, DEFAULT_PERMUTATIONS as s, weightedMean as st, BOOTSTRAP_GATE_MIN_N as t, pearsonR as tt, EProcessState as u, PairedSignTestResult as v, RiskDifferenceResult as w, RankTestMethod as x, PairedTTestResult as y, interRaterReliability as z };
|
|
642
|
+
//# sourceMappingURL=statistics-D_4Snl-5.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"statistics-D_4Snl-5.d.ts","names":[],"sources":["../src/statistics.ts"],"mappings":";;;;;cAaa,kBAAe,QAAY,iBAAe;;iBAGvC,aAAa;EAAU;EAAe;;;;;;;;;;iBAoBtC,mBACd,kBACA,qBACA;EAAQ;EAAe;;EACpB;EAAc;EAAe;;;;;;;;;;;;;;;;;;;iBAiDlB,sBAAsB,aAAa;;KAkFvC;;;;;KAMA;UAEK;;EAEf,SAAS;;EAET;;;EAGA;;;cAIW;;cAEA;;cAEA;;cAEA;UAEI;;EAEf;;EAEA;;EAEA;;EAEA,QAAQ;;EAER;;;;;;;;;;;;;;iBAec,aACd,aACA,aACA,OAAM,kBACL;;iBAwFa,cAAc,iBAAiB;UAK9B;;EAEf;EACA;;EAEA;;;;;;;;;;;;;;;;iBAiBc,YAAY,kBAAkB,kBAAkB;UAyB/C;;;EAGf;;EAEA;;EAEA,QAAQ;;EAER;;EAEA;;;;;;;;;;;;;;;iBAgBc,mBACd,kBACA,iBACA,OAAM,kBACL;;;;;;;;;;;;;iBAmFa,QAAQ,aAAa;;;;;;;;;iBAqBrB,eAAe,kBAAkB;KAoBrC;;;;;;;;;;;iBAYI,YAAY,kBAAkB;;;;;;iBAkB9B,gBAAgB,gBAAgB;;;;;iBAuBhC,MAAM;;;;;;iBAmBN,SAAS,aAAa;;;;;iBAuBtB,UAAU,aAAa;UAKtB;;EAEf,MAAM;;;;;EAKN,SAAS;;EAET;;UAGe;EACf;EACA;;;;;;;;;;;iBAYc,kBAAkB,OAAO,yBAAyB;UA4CjD;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe,oCAAoC;EACnD;;EAEA;;EAEA;;UAGe;;EAEf,cAAc;;EAEd;;EAEA;;EAEA;;EAEA;;UAGe,+BAA+B;;;;;;EAM9C;;;;;;EAMA;;;;;;;;;;;;;;;;;;iBAmBc,0BACd,SAAS,qBACT,OAAM,yBACL;;;;;;;;;iBA0Ha,yCACd,aAAa;EAAQ;EAAgB,QAAQ;IAC7C,OAAM,yBACL;;;;;;;iBA8Ba,mBAAmB;EACjC;EACA;EACA;EACA;;;;;;;;;;;iBAsBc,yBAAyB;EACvC;EACA;EACA;EACA;;;;;;;;iBAkBc,UAAU;EACxB;EACA;EACA;EACA;;;;;;;;;;;;;;;;iBAyBc,iBAAiB;EAC/B;EACA;EACA;EACA;EACA;;;;;;;iBAyBc,aAAa;EAC3B;EACA;EACA;EACA;EACA;;;;;;;;;iBAwBc,WACd,4BACA;EACG;EAAoB;;;;;;;;;;iBAgBT,KACd,4BACA;EACG;EAAoB;;;;;;;;;iBA4BT,kBACd,4BACA;EACG;EAAmB;;UAuCP;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;cAaW;UAEI;;EAEf;;EAEA;;EAEA;;;EAGA;;;;;;;;;;;;iBAac,gBACd,kBACA,iBACA,OAAM,yBACL;;KA0DS;;UAGK;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;;;;;;;;;iBAgBc,eACd,gCACA,aAAa,sBACZ;;UAgDc;;EAEf;;EAEA;;EAEA;;;;;;;;;iBAUc,OAAO,mBAAmB,WAAW,sBAAoB;;UAmBxD;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;iBAec,QACd,SAAS,6BACT,WAAW,8BACV;;UAmBc;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;iBAWc,qBACd,SAAS,6BACT,WAAW,6BACX,sBACC;;;;;;;;;;;iBAyCa,QAAQ,WAAW,WAAW;UAgE7B;;;EAGf;;;;EAIA;;;;EAIA;;UAGe;;EAEf;;EAEA;;EAEA;;UAGe,sBAAsB;EACrC;EACA;EACA;;EAEA;;EAEA;;UAGe;;;EAGf,OAAO,YAAY;EACnB,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA8BK,SAAS,OAAM,kBAAuB;;;;;iBAsatC,WAAW"}
|