@tangle-network/agent-eval 0.171.0 → 0.172.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/README.md +3 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +5 -5
- package/dist/analyst/index.js +2 -2
- package/dist/{experiment-tracker-Dm8yQMqb.d.ts → attestation-CJBGmMVh.d.ts} +78 -2
- package/dist/attestation-CJBGmMVh.d.ts.map +1 -0
- package/dist/{experiment-tracker-BKEumQug.js → attestation-XSUpbc4o.js} +96 -2
- package/dist/attestation-XSUpbc4o.js.map +1 -0
- package/dist/{benchmark-command-D8k3Gf0J.js → benchmark-command-DoFcisuM.js} +2 -2
- package/dist/{benchmark-command-D8k3Gf0J.js.map → benchmark-command-DoFcisuM.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +2 -2
- package/dist/bounded-process-CVOC_D3H.js +181 -0
- package/dist/bounded-process-CVOC_D3H.js.map +1 -0
- package/dist/builder-eval/index.js +39 -98
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +8 -7
- package/dist/campaign/index.js +4 -4
- package/dist/{campaign-B72njjHj.js → campaign-Dp35pBbS.js} +4 -4
- package/dist/{campaign-B72njjHj.js.map → campaign-Dp35pBbS.js.map} +1 -1
- package/dist/canonical-CFpojCN5.d.ts +31 -0
- package/dist/canonical-CFpojCN5.d.ts.map +1 -0
- package/dist/{chat-client-DEtybj5i.js → chat-client-DI79OPye.js} +2 -2
- package/dist/{chat-client-DEtybj5i.js.map → chat-client-DI79OPye.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{client-CDtcZ3p9.d.ts → client-Df7wdslk.d.ts} +2 -2
- package/dist/{client-CDtcZ3p9.d.ts.map → client-Df7wdslk.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +9 -9
- package/dist/contract/index.js +4 -4
- package/dist/{default-registry-B0s2zU-s.d.ts → default-registry-XxedTLwu.d.ts} +3 -3
- package/dist/{default-registry-B0s2zU-s.d.ts.map → default-registry-XxedTLwu.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Cjy2yhqP.d.ts → define-agent-eval-0wW7gFhr.d.ts} +4 -4
- package/dist/{define-agent-eval-Cjy2yhqP.d.ts.map → define-agent-eval-0wW7gFhr.d.ts.map} +1 -1
- package/dist/{define-agent-eval-Dy8QgxAI.js → define-agent-eval-jS8xj_Q_.js} +2 -2
- package/dist/{define-agent-eval-Dy8QgxAI.js.map → define-agent-eval-jS8xj_Q_.js.map} +1 -1
- package/dist/descriptive-B2iPaT9J.d.ts +89 -0
- package/dist/descriptive-B2iPaT9J.d.ts.map +1 -0
- package/dist/{engine-CAmTUk52.d.ts → engine-BfRay1qD.d.ts} +2 -2
- package/dist/{engine-CAmTUk52.d.ts.map → engine-BfRay1qD.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -54
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +1 -95
- package/dist/experiment/index.js.map +1 -1
- package/dist/{heldout-gate-Dh2b62w8.d.ts → heldout-gate-JgNRDZwZ.d.ts} +3 -3
- package/dist/{heldout-gate-Dh2b62w8.d.ts.map → heldout-gate-JgNRDZwZ.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +1 -1
- package/dist/{index-lfaSeKSD.d.ts → index-D-UdhAmg.d.ts} +3 -31
- package/dist/index-D-UdhAmg.d.ts.map +1 -0
- package/dist/{index-DT73JraI.d.ts → index-DDAPhUJJ.d.ts} +4 -4
- package/dist/{index-DT73JraI.d.ts.map → index-DDAPhUJJ.d.ts.map} +1 -1
- package/dist/{index-fNXZMCzX.d.ts → index-DnglhM0A.d.ts} +9 -9
- package/dist/{index-fNXZMCzX.d.ts.map → index-DnglhM0A.d.ts.map} +1 -1
- package/dist/{index-8VIogTyS.d.ts → index-_vPrVMRX.d.ts} +6 -6
- package/dist/{index-8VIogTyS.d.ts.map → index-_vPrVMRX.d.ts.map} +1 -1
- package/dist/index.d.ts +116 -15
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -6
- package/dist/index.js.map +1 -1
- package/dist/{integrity-CyWSSoQS.js → integrity-BWywb34E.js} +34 -11
- package/dist/{integrity-CyWSSoQS.js.map → integrity-BWywb34E.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -1
- package/dist/{llm-judge-BtJ2Sfk_.js → llm-judge-aQHIk5_-.js} +16 -10
- package/dist/llm-judge-aQHIk5_-.js.map +1 -0
- package/dist/{matrix-DiHmUobV.d.ts → matrix-Ch8JO1pG.d.ts} +2 -2
- package/dist/{matrix-DiHmUobV.d.ts.map → matrix-Ch8JO1pG.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +162 -2
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +287 -2
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-Cm6DU_Ao.js → produced-state-CxmbFxFd.js} +2 -2
- package/dist/{produced-state-Cm6DU_Ao.js.map → produced-state-CxmbFxFd.js.map} +1 -1
- package/dist/{promotion-policy-WSXtBgBb.d.ts → promotion-policy-BBBcz5_3.d.ts} +2 -2
- package/dist/{promotion-policy-WSXtBgBb.d.ts.map → promotion-policy-BBBcz5_3.d.ts.map} +1 -1
- package/dist/{provenance-CafMdZKM.d.ts → provenance-Dp-vvyrU.d.ts} +13 -5
- package/dist/provenance-Dp-vvyrU.d.ts.map +1 -0
- package/dist/rl.d.ts +1 -1
- package/dist/sandbox-harness-BlSOu4LX.d.ts.map +1 -1
- package/dist/{skillopt-optimization-method-DzlF2RM7.js → skillopt-optimization-method-LHi02MzH.js} +2 -2
- package/dist/{skillopt-optimization-method-DzlF2RM7.js.map → skillopt-optimization-method-LHi02MzH.js.map} +1 -1
- package/dist/{statistical-heldout-UhiexnjU.d.ts → statistical-heldout-Yldkntvy.d.ts} +2 -2
- package/dist/{statistical-heldout-UhiexnjU.d.ts.map → statistical-heldout-Yldkntvy.d.ts.map} +1 -1
- package/dist/{store-tool-spans-D_qMl2__.d.ts → store-tool-spans-BvdUbeOB.d.ts} +3 -3
- package/dist/{store-tool-spans-D_qMl2__.d.ts.map → store-tool-spans-BvdUbeOB.d.ts.map} +1 -1
- package/dist/supervisor-run/index.d.ts +35 -2
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +83 -38
- package/dist/supervisor-run/index.js.map +1 -1
- package/dist/{tool-groups-RGYfVWpc.d.ts → tool-groups-DjwlMBvW.d.ts} +2 -2
- package/dist/tool-groups-DjwlMBvW.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{types-CMyW4GnH.d.ts → types-CCZ34qmV.d.ts} +2 -2
- package/dist/{types-CMyW4GnH.d.ts.map → types-CCZ34qmV.d.ts.map} +1 -1
- package/dist/{types-DeIUdzNd.d.ts → types-CoPUTiXb.d.ts} +23 -92
- package/dist/types-CoPUTiXb.d.ts.map +1 -0
- package/dist/{types-JHMOqZI4.d.ts → types-nokrtr7M.d.ts} +11 -1
- package/dist/types-nokrtr7M.d.ts.map +1 -0
- package/docs/eval-surface-map.md +36 -0
- package/docs/insight-report.md +19 -0
- package/docs/plants.md +123 -0
- package/docs/public-api.md +45 -20
- package/package.json +1 -1
- package/dist/experiment-tracker-BKEumQug.js.map +0 -1
- package/dist/experiment-tracker-Dm8yQMqb.d.ts.map +0 -1
- package/dist/index-lfaSeKSD.d.ts.map +0 -1
- package/dist/llm-judge-BtJ2Sfk_.js.map +0 -1
- package/dist/provenance-CafMdZKM.d.ts.map +0 -1
- package/dist/tool-groups-RGYfVWpc.d.ts.map +0 -1
- package/dist/types-DeIUdzNd.d.ts.map +0 -1
- package/dist/types-JHMOqZI4.d.ts.map +0 -1
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
//#region src/statistics/descriptive.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* Descriptive statistics: means, bootstrap spread, correlation, and the
|
|
4
|
+
* weighted judge-dimension composite. Nothing here is a significance test.
|
|
5
|
+
*/
|
|
6
|
+
/** Weighted mean — falls back to uniform weights when omitted */
|
|
7
|
+
declare function weightedMean(scores: {
|
|
8
|
+
score: number;
|
|
9
|
+
weight?: number;
|
|
10
|
+
}[]): number;
|
|
11
|
+
/**
|
|
12
|
+
* Percentile bootstrap confidence interval on the mean of `scores`.
|
|
13
|
+
*
|
|
14
|
+
* Descriptive spread. It is not a significance test, and at small n its bounds
|
|
15
|
+
* are anti-conservative in the same way {@link pairedBootstrap}'s are — see
|
|
16
|
+
* {@link BOOTSTRAP_GATE_MIN_N}. With no `seed` the resampling is seeded from
|
|
17
|
+
* the scores themselves, so the interval is reproducible either way.
|
|
18
|
+
*/
|
|
19
|
+
declare function confidenceInterval(scores: number[], confidence?: number, opts?: {
|
|
20
|
+
seed?: number;
|
|
21
|
+
resamples?: number;
|
|
22
|
+
}): {
|
|
23
|
+
mean: number;
|
|
24
|
+
lower: number;
|
|
25
|
+
upper: number;
|
|
26
|
+
};
|
|
27
|
+
/** Partial credit: returns 0-1 ratio of current toward target */
|
|
28
|
+
declare function partialCredit(current: number, target: number): number;
|
|
29
|
+
/** Distribution summary of a number series: count, extremes, quantiles, sum. */
|
|
30
|
+
interface SeriesDistribution {
|
|
31
|
+
readonly n: number;
|
|
32
|
+
readonly min: number;
|
|
33
|
+
readonly p50: number;
|
|
34
|
+
readonly p90: number;
|
|
35
|
+
readonly max: number;
|
|
36
|
+
readonly sum: number;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Fold a number series into its distribution summary. Quantiles use the
|
|
40
|
+
* nearest-rank definition — the `ceil(q·n)`-th order statistic — so every
|
|
41
|
+
* reported quantile is a value from the series. Returns `null` for an empty
|
|
42
|
+
* series: an empty series has no distribution, and a zero-filled summary
|
|
43
|
+
* would read as a measured all-zero series.
|
|
44
|
+
*/
|
|
45
|
+
declare function summarizeNumberSeries(values: readonly number[]): SeriesDistribution | null;
|
|
46
|
+
/**
|
|
47
|
+
* Average-rank-with-ties transform (1-indexed). Tied values receive the mean
|
|
48
|
+
* of the ranks they span, the standard correction for Spearman's ρ.
|
|
49
|
+
*/
|
|
50
|
+
declare function ranks(xs: number[]): number[];
|
|
51
|
+
/**
|
|
52
|
+
* Pearson product-moment correlation coefficient r ∈ [-1, 1] between two
|
|
53
|
+
* equal-length series. See the edge-case contract above: NaN for n < 2 or
|
|
54
|
+
* unequal lengths, 1 when both series are constant, 0 when exactly one is.
|
|
55
|
+
*/
|
|
56
|
+
declare function pearsonR(a: number[], b: number[]): number;
|
|
57
|
+
/**
|
|
58
|
+
* Spearman's rank correlation ρ — Pearson over the average-rank-with-ties
|
|
59
|
+
* transform of each series. Same edge-case contract as {@link pearsonR}.
|
|
60
|
+
*/
|
|
61
|
+
declare function spearmanR(a: number[], b: number[]): number;
|
|
62
|
+
interface WeightedCompositeInput {
|
|
63
|
+
/** Per-dimension scores (typically 0..1). */
|
|
64
|
+
dims: Record<string, number>;
|
|
65
|
+
/** Weight per dimension. Every weighted dimension MUST be present in
|
|
66
|
+
* `dims` — a weight for an absent dimension is a config error and throws,
|
|
67
|
+
* because silently dropping it would renormalise the composite onto a
|
|
68
|
+
* different denominator than intended. */
|
|
69
|
+
weights: Record<string, number>;
|
|
70
|
+
/** Optional pass threshold; when set, the result reports `pass`. */
|
|
71
|
+
threshold?: number;
|
|
72
|
+
}
|
|
73
|
+
interface WeightedCompositeResult {
|
|
74
|
+
composite: number;
|
|
75
|
+
pass?: boolean;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Weighted composite over judge dimensions: `Σ(score_d · w_d) / Σ(w_d)` across
|
|
79
|
+
* the weighted dimensions. The canonical replacement for the per-consumer
|
|
80
|
+
* hand-rolled composite math (tax/legal/creative/gtm each ship a copy).
|
|
81
|
+
*
|
|
82
|
+
* Fail-loud: throws if a weighted dimension is missing from `dims`, if any
|
|
83
|
+
* weight is negative, or if the weights sum to 0 — none of which can produce
|
|
84
|
+
* a meaningful composite.
|
|
85
|
+
*/
|
|
86
|
+
declare function weightedComposite(input: WeightedCompositeInput): WeightedCompositeResult;
|
|
87
|
+
//#endregion
|
|
88
|
+
export { partialCredit as a, spearmanR as c, weightedMean as d, confidenceInterval as i, summarizeNumberSeries as l, WeightedCompositeInput as n, pearsonR as o, WeightedCompositeResult as r, ranks as s, SeriesDistribution as t, weightedComposite as u };
|
|
89
|
+
//# sourceMappingURL=descriptive-B2iPaT9J.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"descriptive-B2iPaT9J.d.ts","names":[],"sources":["../src/statistics/descriptive.ts"],"mappings":";;;;;;iBAQgB,aAAa;EAAU;EAAe;;;;;;;;;;iBAoBtC,mBACd,kBACA,qBACA;EAAQ;EAAe;;EACpB;EAAc;EAAe;;;iBAiClB,cAAc,iBAAiB;;UAM9B;WACN;WACA;WACA;WACA;WACA;WACA;;;;;;;;;iBAUK,sBAAsB,4BAA4B;;;;;iBA8BlD,MAAM;;;;;;iBAmBN,SAAS,aAAa;;;;;iBAuBtB,UAAU,aAAa;UAKtB;;EAEf,MAAM;;;;;EAKN,SAAS;;EAET;;UAGe;EACf;EACA;;;;;;;;;;;iBAYc,kBAAkB,OAAO,yBAAyB"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { c as CostLedgerHandle } from "./cost-ledger-DbQdN3nO.js";
|
|
2
2
|
import { a as RunRecord } from "./run-record-DQjRcYwA.js";
|
|
3
3
|
import { S as BoundedTraceAnalysisStoreOptions, p as EvidenceRef, w as TraceAnalysisStore } from "./types-DMoNFDWi.js";
|
|
4
|
-
import "./types-
|
|
4
|
+
import "./types-nokrtr7M.js";
|
|
5
5
|
import { r as DatasetScenario } from "./dataset-DQqhOCPt.js";
|
|
6
6
|
import "./summary-report-DRstQNBX.js";
|
|
7
7
|
import { o as InsightReport, r as ExecutionInsight, t as CostProvenanceSummary } from "./insight-report-08F022xN.js";
|
|
@@ -212,4 +212,4 @@ interface TraceAnalysisEngine {
|
|
|
212
212
|
declare function resolveTraceAnalystLimits(limits: Partial<TraceAnalystLimits> | undefined): TraceAnalystLimits;
|
|
213
213
|
//#endregion
|
|
214
214
|
export { AnalyzeRunsOptions as _, resolveTraceAnalystLimits as a, analyzeRuns as b, RawAnalystFinding as c, parseRawFinding as d, BuildTraceAnalysisToolsOptions as f, traceAnalystFunctionGroup as g, buildTraceAnalysisToolDescriptors as h, TraceAnalystLimits as i, RawAnalystFindingSchema as l, TraceAnalysisToolDescriptor as m, TraceAnalysisEngineRequest as n, RAW_FINDING_SCHEMA_PROMPT as o, TRACE_ANALYST_TOOL_NAMESPACE as p, TraceAnalysisEngineResult as r, RawAnalystEvidence as s, TraceAnalysisEngine as t, evidenceRefsFromRawFinding as u, ExecutionReport as v, summarizeExecution as x, SummarizeExecutionOptions as y };
|
|
215
|
-
//# sourceMappingURL=engine-
|
|
215
|
+
//# sourceMappingURL=engine-BfRay1qD.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"engine-
|
|
1
|
+
{"version":3,"file":"engine-BfRay1qD.d.ts","names":[],"sources":["../src/contract/analyze-runs.ts","../src/eval-tools.ts","../src/trace-analyst/tools.ts","../src/analyst/finding-signature.ts","../src/analyst/engine.ts"],"mappings":";;;;;;;;;;UAoEiB;;EAEf,MAAM;;;EAGN;;;;;EAKA;EACA;;;EAGA,kBAAkB;;;EAGlB,UAAU;;;;EAIV;IACE;IACA,cAAc;;;;;EAKhB,cAAc;IAAQ;IAAe;IAAe;;;EAEpD;;;;EAIA;;;;;;;;;EASA,eAAe;;;EAGf;;UAGe;EACf,MAAM;EACN;;UAGe;EACf,WAAW;EACX,gBAAgB;;;iBAIF,mBAAmB,MAAM,4BAA4B;iBAgB/C,YAAY,MAAM,qBAAqB,QAAQ;;;;UCpHpD;EACf;EACA;EACA,YAAY;EACZ,UAAU,eAAe;IAAQ,SAAS;QAAkB;;;;cClBjD;UAEI,uCAAuC;EACtD,OAAO;;UAGQ,oCAAoC;EACnD,kBAAkB;;;iBAIJ,kCACd,SAAS,iCACR;iBAkHa,0BAA0B,SAAS;EACjD;EACA;EACA;EACA;EACA,WAAW;;;;cCtIP,0BAAwB,EAAA;;;GAKnB,EAAA,KAAA;KAEC,qBAAqB,EAAE,aAAa;cAiBnC,yBAAuB,EAAA;;;;;;;;;;;;;;;;;GAKzB,EAAA,KAAA;KAEC,oBAAoB,EAAE,aAAa;;;;;;cAOlC;;iBAYG,2BAA2B,SAAS,oBAAoB;iBAQxD,gBACd,cACA,OAAO,aAAa,SAAS,mCAC5B;;;;UCpEc;;EAEf;;EAEA;;EAEA;;EAEA;;UAUe;EACf;EACA;EACA;EACA,gBAAgB;;;;;;;;;;;EAWhB,aAAa,SAAS;EACtB,QAAQ;EACR,YAAY;EACZ;EACA,WAAW;EACX,SAAS;EACT,OAAO,iBAAiB,SAAS;;;UAIlB;EACf;EACA,UAAU;;EAEV;;;;;EAKA;;EAEA;EACA,SAAS;;;;;;;;;UAUM;WACN;WACA;WACA;;WAEA;;;;;;;;WAQA,iBAAiB,SAAS;EACnC,QAAQ,SAAS,6BAA6B,QAAQ;;iBAGxC,0BACd,QAAQ,QAAQ,kCACf"}
|
|
@@ -2,10 +2,10 @@ import { c as ValidationError, r as CaptureIntegrityError } from "../errors-DEE6
|
|
|
2
2
|
import { n as bonferroni, r as holm, t as benjaminiHochberg } from "../multiplicity-DIWHvysC.js";
|
|
3
3
|
import { A as ProportionInterval, B as wilson, C as EProcess, D as eProcess, E as EProcessStep, I as pairedRiskDifference, L as pairedRiskDifferenceExact, P as mcnemar, R as pairedRiskDifferenceScore, S as pairRunRecords, T as EProcessState, a as evaluateHypothesis, b as comparePairedArms, c as signManifest, d as MatchedPair, g as PairedArmRow, i as SignedManifestAlgo, l as verifyManifest, m as PairArmsResult, n as HypothesisResult, o as hashJson, p as PairArmsOptions, r as SignedManifest, s as manifestContentDigest, t as HypothesisManifest, w as EProcessOptions, x as pairArms } from "../pre-registration-DHz6P_6f.js";
|
|
4
4
|
import { c as BOOTSTRAP_GATE_MIN_N, d as PairedBootstrapResult, h as pairedBootstrap, u as PairedBootstrapOptions } from "../paired-promotion-decision-CGzg0cI_.js";
|
|
5
|
-
import {
|
|
5
|
+
import { C as requiredSampleSize, S as requiredPairedSampleSize, _ as inMemoryExperimentStore, b as mcnemarRequiredN, c as ExperimentRep, d as ExperimentVerdict, h as fileExperimentStore, i as AttestedReport, l as ExperimentStats, n as AttestationProvenance, s as Experiment, u as ExperimentTracker, v as mulberry32, x as pairedMde, y as mcnemarPower } from "../attestation-CJBGmMVh.js";
|
|
6
6
|
import { n as PowerPreflightOptions, r as powerPreflight, t as PowerPreflight } from "../power-preflight-Ptse_Kq7.js";
|
|
7
|
-
import { c as PromotionPolicy, d as paretoSignificanceGate, i as EvidenceVector, l as buildEvidenceVector, u as paretoPolicy } from "../promotion-policy-
|
|
8
|
-
import { _ as sequentialPairedGate, c as pairHoldout, d as SequentialDecision, f as SequentialObservation, g as sequentialDecide, i as PairedHoldout, m as SequentialPairedGateOptions, n as HeldoutSignificance, p as SequentialPairedGate, r as HeldoutSignificanceOptions, s as heldoutSignificance } from "../statistical-heldout-
|
|
7
|
+
import { c as PromotionPolicy, d as paretoSignificanceGate, i as EvidenceVector, l as buildEvidenceVector, u as paretoPolicy } from "../promotion-policy-BBBcz5_3.js";
|
|
8
|
+
import { _ as sequentialPairedGate, c as pairHoldout, d as SequentialDecision, f as SequentialObservation, g as sequentialDecide, i as PairedHoldout, m as SequentialPairedGateOptions, n as HeldoutSignificance, p as SequentialPairedGate, r as HeldoutSignificanceOptions, s as heldoutSignificance } from "../statistical-heldout-Yldkntvy.js";
|
|
9
9
|
import { a as PairedEvalueStep, d as sequentialCrossingHorizon, i as PairedEvalueSequence, o as SequentialCrossingHorizon, r as PairedEvalueOptions, s as SequentialCrossingHorizonOptions, u as pairedEvalueSequence } from "../sequential-BhsrMupG.js";
|
|
10
10
|
import { z } from "zod";
|
|
11
11
|
//#region src/experiment/ast.d.ts
|
|
@@ -761,57 +761,6 @@ interface RegisteredExperiment {
|
|
|
761
761
|
*/
|
|
762
762
|
declare function openSealedExperiment(sealed: SealedExperiment): Promise<RegisteredExperiment>;
|
|
763
763
|
//#endregion
|
|
764
|
-
//#region src/attestation.d.ts
|
|
765
|
-
/**
|
|
766
|
-
* Reproducibility attestation for any serializable report object.
|
|
767
|
-
*
|
|
768
|
-
* `attest()` binds a report to its content address (sha-256 over canonical
|
|
769
|
-
* JSON) AND binds that address to the provenance needed to reproduce it:
|
|
770
|
-
* model versions, seeds, price-table hash, code SHA, inputs hash. The outer
|
|
771
|
-
* `envelopeHash` prevents provenance from being rewritten while leaving the
|
|
772
|
-
* report hash valid.
|
|
773
|
-
*
|
|
774
|
-
* Layering: content-addressing is the substrate's job; cryptographic SIGNING
|
|
775
|
-
* (who vouches for the attestation, key management, transparency logs) is the
|
|
776
|
-
* consumer's layer on top. An `AttestedReport` is a stable byte-identical
|
|
777
|
-
* payload a consumer can sign — the substrate never holds keys.
|
|
778
|
-
*
|
|
779
|
-
* Generic by design: the report parameter is ANY value `canonicalJson`
|
|
780
|
-
* accepts (campaign results, fuzz capsules, scorecards, cost ledgers). Do not
|
|
781
|
-
* couple this module to a specific report schema.
|
|
782
|
-
*/
|
|
783
|
-
/** Hash scheme identifier carried by every attestation. A verifier rejects
|
|
784
|
-
* unknown algorithms instead of guessing. */
|
|
785
|
-
declare const ATTESTATION_ALGORITHM: 'sha256/canonical-json';
|
|
786
|
-
interface AttestationProvenance {
|
|
787
|
-
/** Every model involved in producing the report, name → version/id. */
|
|
788
|
-
modelVersions: Record<string, string>;
|
|
789
|
-
/** RNG seeds the run was driven by, when seeded. */
|
|
790
|
-
seeds?: number[];
|
|
791
|
-
/** Content hash of the price table used for cost figures — cost numbers
|
|
792
|
-
* are only reproducible against the same prices. */
|
|
793
|
-
priceTableHash?: string;
|
|
794
|
-
/** Git SHA of the code that produced the report. */
|
|
795
|
-
codeSha: string;
|
|
796
|
-
/** Content hash of the input set (scenarios, dataset manifest, ...). */
|
|
797
|
-
inputsHash?: string;
|
|
798
|
-
/** ISO-8601 timestamp, caller-supplied — the substrate stays clock-free
|
|
799
|
-
* so attestation is deterministic and testable. */
|
|
800
|
-
createdAt: string;
|
|
801
|
-
}
|
|
802
|
-
interface AttestedReport {
|
|
803
|
-
/** Hex sha-256 over the canonical JSON of the report. */
|
|
804
|
-
reportHash: string;
|
|
805
|
-
provenance: AttestationProvenance;
|
|
806
|
-
algorithm: typeof ATTESTATION_ALGORITHM;
|
|
807
|
-
/**
|
|
808
|
-
* Hex sha-256 over `{ reportHash, provenance, algorithm }`. New attestations
|
|
809
|
-
* always carry it. Optional only so persisted pre-envelope attestations can
|
|
810
|
-
* still be read and explicitly recognized as legacy by callers.
|
|
811
|
-
*/
|
|
812
|
-
envelopeHash?: string;
|
|
813
|
-
}
|
|
814
|
-
//#endregion
|
|
815
764
|
//#region src/experiment/evidence-receipt.d.ts
|
|
816
765
|
declare const EVIDENCE_RECEIPT_VERSION: '1.0.0';
|
|
817
766
|
/** Closed promotion vocabulary. A typo or unknown future kind is never independent by default. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/experiment/ast.ts","../../src/experiment/budget.ts","../../src/experiment/funnel.ts","../../src/experiment/define.ts","../../src/
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/experiment/ast.ts","../../src/experiment/budget.ts","../../src/experiment/funnel.ts","../../src/experiment/define.ts","../../src/experiment/evidence-receipt.ts","../../src/experiment/evidence-record.ts","../../src/experiment/power.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA8BY,+CAA+C;GAAiB,YAAY;;;KAG5E,iBAAiB;;;;;KAWjB;EAEN;EACA;EACA;EACA,OAAO;;EAEP;EAAY;EAAe,QAAQ;;EACnC;EAAa,IAAI;;EACjB;EAAa,IAAI;;EACjB;EAAa,IAAI;;;iBAGP,UAAU,QAAQ,gBAAgB;;iBAUlC,kBAAkB,WAAW,WAAW,QAAQ;;;;;UA4C/C;EACf;EACA,MAAM;EACN;;;UAIe;EACf;;EAEA;EACA,MAAM;EACN;;;UAIe;EACf;EACA,QAAQ;EACR,aAAa;;;;;;;KAUH;EAEN;EACA;EACA;EACA;IAAe;IAAe;;EAC9B;EACA;;EAGA;;EAEA;EACA,MAAM;EACN;IAAS;IAAe;;;;;;;;;;;;;iBAad,iBACd,MAAM,eACN,kBAAkB,kBAClB;;EAEE;;EAEA,QAAQ;;EAER,YAAY,eAAe;;;KA0DnB;EACN;EAAoB;EAAa,OAAO;;EACxC;EAAmB,IAAI;;;;;;KAMjB;EACN;EAAc,OAAO;EAAW;;EAChC;EAA4B,OAAO;EAAW;;EAE9C;EACA;EACA;EACA;EACA;;;;;EAKA;;EAEA;;EAGA;EACA;EACA;EACA,WAAW;EACX,aAAa;;UAGF;EACf;EACA;EACA;;;iBA8Cc,gBACd,UAAU,UACV,eAAe,mBACd;;KA2ES;EAEN;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAyB;;UAEd;EACf;EACA;EACA;;;;;;;;;;iBAWc,gBACd,MAAM,cACN;EACM;EAAc,eAAe;EAAkB;;EAC/C;EAAkB;EAAmB;IAC1C;;KA8HS;EACN;EAAgC;EAAkB;;EAClD;EAAgC;;EAChC;EAA4B;EAAkB;EAAiC;;EAC/E;EAAwB;;EACxB;EAAa,IAAI;;EACjB;EAAa,IAAI;;EACjB;EAAa,IAAI;;;UAGN;EACf,WAAW;IAAiB;IAAe;;EAC3C,YAAY;EACZ,gBAAgB;;iBAGF,kBAAkB,WAAW,WAAW,UAAU;UAyCjD;EACf,MAAM;EACN;EACA;;;;;;;;;KAUU;EACN;EAAe,UAAU;;EAEzB;EACA;EACA;EACA;EACA;IAAmB;IAAuB;;;UAG/B;EACf;EACA;;iBAGc,oBACd,MAAM,cACN,UAAU,oBACT;;UAec;EACf;EACA;EACA;;;KAMU;EAEN;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAEA;EAA8B,OAAO;;EAErC;EACA;EACA;EACA;IAAO;IAAgB;IAAmB;;;EAE1C;EAAkB;EAAuB;EAAmB;;UAEjD;EACf;EACA;EACA,UAAU;;;;;;iBAOI,8BACd,YACA,MAAM,QAAQ;EAAgB;IAC9B,aAAa,qCACZ;;;;;;iBAiBa,sCACd,YACA,MAAM,QAAQ;EAAgB;IAC9B;EAAe,eAAe;EAAkB,gBAAgB;IAC/D;;iBAkBa,uBACd,YACA,MAAM,QAAQ;EAAgB;IAC9B,YAAY,iBACX;;iBAMa,qBACd,YACA,OAAO,QAAQ;EAAgB;IAC/B;EAAc;EAAgB;IAC7B;;;;;;iBAea,uBACd,YACA,MAAM,QAAQ;EAAgB;IAC9B;EAAkB;EAAgB;MACjC;;UAsBc;EACf;IAAQ;IAAyB;;EACjC;EACA;;UAGe;EACf;EACA;EACA;;iBAGc,iBAAiB,MAAM,UAAU,gBAAgB,eAAe;;;;;KAoBpE;EAEN;EACA;EACA;EACA;EACA;EACA;IAAU;IAAY;;EACtB;;EAGA;EACA;EACA;EACA;EACA;;UAGW;EACf;EACA;EACA;EACA;;UAGe;EACf,WAAW;EACX;;;;;;;;iBASc,qBACd,MAAM,QAAQ;EAAc;IAC5B,uCACC;UAuBc;EACf;EACA;IAAe;IAAW;IAAsB;;EAChD;IAAkB;IAAuC;;;;;;;;iBAQ3C,qBACd,MAAM,QAAQ;EAAc;IAC5B;EAAY;EAAqB;IAChC;;UAoBc;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;;KAGU;;;;;;iBAOI,gBACd,QAAQ,eACR,eACA,sBACC;;;;cCn6BU,2BAA2B;UAEvB;EACf;;EAEA;;UAGe;EACf,MAAM;EACN,MAAM;;EAEN;;EAEA;EACA;;EAEA;IAAkB;IAA2B;;;;;;;;iBAQ/B,qBACd,MAAM,mBACN,eAAe,sBACd;;iBAsCa,qBACd,MAAM,mBACN,eAAe,sBACd;;;;cC/DU,6BAA6B;UAEzB;EACf;EACA;EACA;EACA;;EAEA,aAAa;;EAEb;;UAGe;EACf;;EAEA;EACA;;EAEA;;UAGe;EACf;EACA;EACA,QAAQ;EACR;EACA,YAAY;;UAGG;EACf;;EAEA;EACA,aAAa;EACb;;;;;;;;;;;iBAYc,YAAY;EAC1B;EACA;EACA,QAAQ;EACR;IAAe;IAAY;IAAc;;IACvC;;iBAsEY,uBAAuB,QAAQ;UAwB9B;EACf,QAAQ;EACR,WAAW;;EAEX,eAAe,eAAe;;;;;;;;iBAShB,qBACd,MAAM,eACN,kBAAkB,mBACjB;;;;;;iBA8Ca,eACd,OAAO,kBACP,QAAQ,mBACP;;;;;;iBA4Ba,kBAAkB,QAAQ;;;;cCtM7B,2BAA2B;UAIvB;EACf;EACA;;EAEA;;EAEA;;EAEA,OAAO,eAAe;;KAGZ;EAEN;;EAEA;EACA;;EAEA;;EAEA;;EAGA;EACA;EACA;EACA;;;;;;UAOW;EACf;;UAGe;EACf;;EAEA;EACA,MAAM;EACN,SAAS;EACT,YAAY;;EAEZ,aAAa,eAAe;;;;;EAK5B,gBAAgB;EAChB,YAAY,eAAe;EAC3B,YAAY,eAAe;EAC3B,UAAU;EACV,cAAc;;EAEd,QAAQ,eAAe;EACvB,OAAO;EACP,SAAS;EACT,gBAAgB;EAChB,UAAU;EACV,iBAAiB;EACjB;;;;;;;;;;iBAwCc,iBAAiB,MAAM,iBAAiB;UAkHvC;;EAEf;EACA;;EAEA;;EAEA;;;;;;;KAQU;UAEK;EACf,MAAM;;EAEN;;;;;;EAMA,MAAM;EACN;;EAEA;EACA,YAAY;;;iBAIQ,eACpB,MAAM,gBACN;EAAW;IACV,QAAQ;;;;;;iBAkBW,gBACpB,QAAQ,kBACR;EAAa,MAAM;EAAgB;EAAgB;EAAiB;IACnE,QAAQ;;;iBAwBW,uBAAuB,QAAQ,mBAAmB;;KAoC5D;EACN;EAA4B,aAAa;;EAEzC;EACA,eAAe;EACf,gBAAgB;;EAEhB;EAA8B,YAAY;;EAC1C;EAAkB;EAAgB;;EAClC;EAAqB;IAAkB;IAAgB;;;;;;;UAM5C;WACN,QAAQ;;EAEjB,OAAO,UAAU,oBAAoB;;EAErC,MAAM,kBAAkB,mBAAmB;;EAE3C,OAAO,cAAc,kBAAkB,kBAAkB;IAAW;;;EAEpE,KAAK,cAAc,UAAU,eAAe;;EAE5C,KAAK,gBAAgB,eAAe;;EAEpC,qBAAqB,uCAAuC;;EAE5D,qBAAqB;IAAY;IAAqB;MAAiB;;EAEvE,eAAe,eAAe,sBAAsB;;EAEpD,SAAS,cAAc,eAAe,mBAAmB;;EAEzD,SACE,cACA;IACM;IAAc,eAAe;IAAkB;;IAC/C;IAAkB;IAAmB;MAC1C;;;;;;;iBAQiB,qBACpB,QAAQ,mBACP,QAAQ;;;cC7aE;;cAGA;KAQD,gCAAgC;cAE/B;UAOI;WACN,MAAM;;WAEN;;UAGM;WACN,sBAAsB;;WAEtB;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA,WAAW;;WAEX;;WAEA;;UAGM;WACN,SAAS;WACT,aAAa;;UAGP;WACN;WACA;;UAGM,mCAAmC,KAAK;;;;;;;iBAQzC,sBACd,OAAO,4BACP,YAAY,wBACX;;;;;;;iBA8Ba,sBAAsB,SAAS,kBAAkB;;;;;;iBA6CjD,sBAAsB,SAAS;;;;;;;;;;;;;;;cClIlC;KAQD,wBAAwB;;cAO9B,2BAAyB,EAAA;;;;GAM7B,EAAA,KAAA;KAEU,sBAAsB,EAAE,aAAa;cAEpC,8BAA4B,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCvC,EAAA,KAAA;KAEU,yBAAyB,EAAE,aAAa;;iBAGpC,4BAA4B,eAAe;;;;;;iBAW3C,yBAAyB,2BAA2B;;;;;;iBA4CpD,oBAAoB;;;;cCpIvB,2BAA2B;UAEvB;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA;IAAkB;IAAe;;;UAGlB;EACf;EACA;EACA;;UAGe;;EAEf;;EAEA;EACA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;;EAEP;EACA,eAAe;;EAEf;;EAEA,SAAS;;;;;;iBAOK,eAAe,SAAS,wBAAwB;;iBAsGhD,qBAAqB,QAAQ"}
|
package/dist/experiment/index.js
CHANGED
|
@@ -6,9 +6,8 @@ import { a as requiredSampleSize, c as holm, i as requiredPairedSampleSize, n as
|
|
|
6
6
|
import { f as mcnemar, g as pairedRiskDifferenceScore, h as pairedRiskDifferenceExact, m as pairedRiskDifference, n as pairArms, r as pairRunRecords, t as comparePairedArms, v as wilson } from "../paired-arms-D4aeIHUy.js";
|
|
7
7
|
import { r as pairedBootstrap, t as BOOTSTRAP_GATE_MIN_N } from "../paired-tests-C8iCsioC.js";
|
|
8
8
|
import { t as eProcess } from "../sequential-eprocess-D1jKoihe.js";
|
|
9
|
-
import { n as contentHash } from "../verdict-cache-B3eCVQtY.js";
|
|
10
9
|
import { a as heldoutSignificance, o as pairHoldout, t as powerPreflight } from "../power-preflight-CFXm0Vjo.js";
|
|
11
|
-
import {
|
|
10
|
+
import { c as inMemoryExperimentStore, i as ExperimentTracker, n as attest, o as fileExperimentStore, r as verifyAttestation } from "../attestation-XSUpbc4o.js";
|
|
12
11
|
import { n as pairedEvalueSequence, r as sequentialCrossingHorizon } from "../sequential-CzK5DarL.js";
|
|
13
12
|
import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-LY9mVQ7W.js";
|
|
14
13
|
import { n as sequentialPairedGate, t as sequentialDecide } from "../sequential-B51qAYE4.js";
|
|
@@ -973,99 +972,6 @@ async function openSealedExperiment(sealed) {
|
|
|
973
972
|
};
|
|
974
973
|
}
|
|
975
974
|
//#endregion
|
|
976
|
-
//#region src/attestation.ts
|
|
977
|
-
/**
|
|
978
|
-
* Reproducibility attestation for any serializable report object.
|
|
979
|
-
*
|
|
980
|
-
* `attest()` binds a report to its content address (sha-256 over canonical
|
|
981
|
-
* JSON) AND binds that address to the provenance needed to reproduce it:
|
|
982
|
-
* model versions, seeds, price-table hash, code SHA, inputs hash. The outer
|
|
983
|
-
* `envelopeHash` prevents provenance from being rewritten while leaving the
|
|
984
|
-
* report hash valid.
|
|
985
|
-
*
|
|
986
|
-
* Layering: content-addressing is the substrate's job; cryptographic SIGNING
|
|
987
|
-
* (who vouches for the attestation, key management, transparency logs) is the
|
|
988
|
-
* consumer's layer on top. An `AttestedReport` is a stable byte-identical
|
|
989
|
-
* payload a consumer can sign — the substrate never holds keys.
|
|
990
|
-
*
|
|
991
|
-
* Generic by design: the report parameter is ANY value `canonicalJson`
|
|
992
|
-
* accepts (campaign results, fuzz capsules, scorecards, cost ledgers). Do not
|
|
993
|
-
* couple this module to a specific report schema.
|
|
994
|
-
*/
|
|
995
|
-
/** Hash scheme identifier carried by every attestation. A verifier rejects
|
|
996
|
-
* unknown algorithms instead of guessing. */
|
|
997
|
-
const ATTESTATION_ALGORITHM = "sha256/canonical-json";
|
|
998
|
-
function envelopeMaterial(reportHash, provenance, algorithm) {
|
|
999
|
-
return {
|
|
1000
|
-
reportHash,
|
|
1001
|
-
provenance,
|
|
1002
|
-
algorithm
|
|
1003
|
-
};
|
|
1004
|
-
}
|
|
1005
|
-
/**
|
|
1006
|
-
* Content-address a report and bind it to its provenance. Throws (via
|
|
1007
|
-
* `canonicalJson`) if the report or provenance contains undefined / function /
|
|
1008
|
-
* symbol / non-finite numbers — an attestation that cannot be unambiguously
|
|
1009
|
-
* serialized cannot be trusted.
|
|
1010
|
-
*/
|
|
1011
|
-
function attest(report, provenance) {
|
|
1012
|
-
const reportHash = contentHash(report);
|
|
1013
|
-
const algorithm = ATTESTATION_ALGORITHM;
|
|
1014
|
-
return {
|
|
1015
|
-
reportHash,
|
|
1016
|
-
provenance,
|
|
1017
|
-
algorithm,
|
|
1018
|
-
envelopeHash: contentHash(envelopeMaterial(reportHash, provenance, algorithm))
|
|
1019
|
-
};
|
|
1020
|
-
}
|
|
1021
|
-
/**
|
|
1022
|
-
* Verify a report against its attestation. Returns a typed outcome rather
|
|
1023
|
-
* than throwing: an unverifiable report (e.g. one that no longer
|
|
1024
|
-
* canonicalizes) is a verification failure with the cause in `reason`, not a
|
|
1025
|
-
* crash — verifiers run in pipelines that must record WHY, not die.
|
|
1026
|
-
*
|
|
1027
|
-
* Legacy attestations without `envelopeHash` remain readable, but verification
|
|
1028
|
-
* explicitly marks their provenance as unbound so a promotion path can refuse
|
|
1029
|
-
* them instead of accidentally treating old metadata as cryptographic proof.
|
|
1030
|
-
*/
|
|
1031
|
-
function verifyAttestation(report, attested) {
|
|
1032
|
-
if (attested.algorithm !== "sha256/canonical-json") return {
|
|
1033
|
-
valid: false,
|
|
1034
|
-
reason: `unknown algorithm '${attested.algorithm}' — this verifier only checks '${ATTESTATION_ALGORITHM}'`
|
|
1035
|
-
};
|
|
1036
|
-
let recomputed;
|
|
1037
|
-
try {
|
|
1038
|
-
recomputed = contentHash(report);
|
|
1039
|
-
} catch (err) {
|
|
1040
|
-
return {
|
|
1041
|
-
valid: false,
|
|
1042
|
-
reason: `report is not canonicalizable: ${err instanceof Error ? err.message : String(err)}`
|
|
1043
|
-
};
|
|
1044
|
-
}
|
|
1045
|
-
if (recomputed !== attested.reportHash) return {
|
|
1046
|
-
valid: false,
|
|
1047
|
-
reason: `report hash mismatch: attested ${attested.reportHash}, recomputed ${recomputed}`
|
|
1048
|
-
};
|
|
1049
|
-
if (attested.envelopeHash === void 0) return {
|
|
1050
|
-
valid: true,
|
|
1051
|
-
legacyUnboundProvenance: true
|
|
1052
|
-
};
|
|
1053
|
-
let envelopeHash;
|
|
1054
|
-
try {
|
|
1055
|
-
envelopeHash = contentHash(envelopeMaterial(attested.reportHash, attested.provenance, attested.algorithm));
|
|
1056
|
-
} catch (err) {
|
|
1057
|
-
return {
|
|
1058
|
-
valid: false,
|
|
1059
|
-
reason: `attestation provenance is not canonicalizable: ${err instanceof Error ? err.message : String(err)}`
|
|
1060
|
-
};
|
|
1061
|
-
}
|
|
1062
|
-
if (envelopeHash !== attested.envelopeHash) return {
|
|
1063
|
-
valid: false,
|
|
1064
|
-
reason: `attestation envelope hash mismatch: attested ${attested.envelopeHash}, recomputed ${envelopeHash}`
|
|
1065
|
-
};
|
|
1066
|
-
return { valid: true };
|
|
1067
|
-
}
|
|
1068
|
-
//#endregion
|
|
1069
975
|
//#region src/experiment/evidence-receipt.ts
|
|
1070
976
|
/**
|
|
1071
977
|
* Evidence receipts are the join between Runtime execution and Eval promotion.
|