@tangle-network/agent-eval 0.147.0 → 0.149.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +7 -0
- package/dist/analyst/index.d.ts +3 -3
- package/dist/analyst/index.js +1 -1
- package/dist/{benchmark-command-BTiysiYU.js → benchmark-command-gufocQ-N.js} +2 -2
- package/dist/{benchmark-command-BTiysiYU.js.map → benchmark-command-gufocQ-N.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +2 -2
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.d.ts +4 -4
- package/dist/campaign/index.js +4 -4
- package/dist/{campaign-DCDdhuv2.js → campaign-BcfXzmPM.js} +4 -4
- package/dist/{campaign-DCDdhuv2.js.map → campaign-BcfXzmPM.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/contract/index.d.ts +3 -3
- package/dist/contract/index.js +3 -3
- package/dist/{define-agent-eval-D52ClbX2.d.ts → define-agent-eval-BWuk2O_N.d.ts} +3 -4
- package/dist/define-agent-eval-BWuk2O_N.d.ts.map +1 -0
- package/dist/{define-agent-eval-BqWFz3sK.js → define-agent-eval-CvZQW4u9.js} +2 -2
- package/dist/define-agent-eval-CvZQW4u9.js.map +1 -0
- package/dist/experiment/index.d.ts +117 -1
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +217 -1
- package/dist/experiment/index.js.map +1 -1
- package/dist/fuzz.d.ts +1 -1
- package/dist/{index-BjBjxiVv.d.ts → index-BCDyf_kT.d.ts} +12 -358
- package/dist/index-BCDyf_kT.d.ts.map +1 -0
- package/dist/{index-gtjtfLEJ.d.ts → index-aHDnhBC7.d.ts} +2 -2
- package/dist/{index-gtjtfLEJ.d.ts.map → index-aHDnhBC7.d.ts.map} +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +8 -15
- package/dist/index.js.map +1 -1
- package/dist/{llm-judge-DrzsVS5k.js → llm-judge-DWq1Ptco.js} +259 -5
- package/dist/llm-judge-DWq1Ptco.js.map +1 -0
- package/dist/{matrix-C-2Qx1Zr.d.ts → matrix-Bc111FKv.d.ts} +17 -1
- package/dist/{matrix-C-2Qx1Zr.d.ts.map → matrix-Bc111FKv.d.ts.map} +1 -1
- package/dist/{multi-layer-verifier-DIguZc8Z.d.ts → multi-layer-verifier-BUaQ4C17.d.ts} +3 -3
- package/dist/{multi-layer-verifier-DIguZc8Z.d.ts.map → multi-layer-verifier-BUaQ4C17.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +1 -1
- package/dist/multishot/index.js +16 -0
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-Dtx60bUQ.js → produced-state-C0oJ4vr-.js} +2 -2
- package/dist/{produced-state-Dtx60bUQ.js.map → produced-state-C0oJ4vr-.js.map} +1 -1
- package/dist/{provenance-LrOEOHQb.d.ts → provenance-DA-Pmyfv.d.ts} +441 -2
- package/dist/provenance-DA-Pmyfv.d.ts.map +1 -0
- package/dist/{reward-hacking-Bu8ev6PR.d.ts → reward-hacking-62tojkQd.d.ts} +2 -2
- package/dist/{reward-hacking-Bu8ev6PR.d.ts.map → reward-hacking-62tojkQd.d.ts.map} +1 -1
- package/dist/rl.d.ts +2 -2
- package/dist/{skillopt-optimization-method-BoC1Qccx.d.ts → skillopt-optimization-method-ChCTN4FD.d.ts} +3 -3
- package/dist/{skillopt-optimization-method-BoC1Qccx.d.ts.map → skillopt-optimization-method-ChCTN4FD.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-C_UrqZs2.js → skillopt-optimization-method-Donmq4sq.js} +2 -2
- package/dist/{skillopt-optimization-method-C_UrqZs2.js.map → skillopt-optimization-method-Donmq4sq.js.map} +1 -1
- package/docs/feature-guide.md +1 -1
- package/docs/multishot-golden-records.md +12 -1
- package/docs/search-history-receipts.md +139 -0
- package/package.json +2 -2
- package/dist/define-agent-eval-BqWFz3sK.js.map +0 -1
- package/dist/define-agent-eval-D52ClbX2.d.ts.map +0 -1
- package/dist/index-BjBjxiVv.d.ts.map +0 -1
- package/dist/llm-judge-DrzsVS5k.js.map +0 -1
- package/dist/provenance-LrOEOHQb.d.ts.map +0 -1
|
@@ -725,6 +725,122 @@ interface RegisteredExperiment {
|
|
|
725
725
|
*/
|
|
726
726
|
declare function openSealedExperiment(sealed: SealedExperiment): Promise<RegisteredExperiment>;
|
|
727
727
|
//#endregion
|
|
728
|
+
//#region src/attestation.d.ts
|
|
729
|
+
/**
|
|
730
|
+
* Reproducibility attestation for any serializable report object.
|
|
731
|
+
*
|
|
732
|
+
* `attest()` binds a report to its content address (sha-256 over canonical
|
|
733
|
+
* JSON) AND binds that address to the provenance needed to reproduce it:
|
|
734
|
+
* model versions, seeds, price-table hash, code SHA, inputs hash. The outer
|
|
735
|
+
* `envelopeHash` prevents provenance from being rewritten while leaving the
|
|
736
|
+
* report hash valid.
|
|
737
|
+
*
|
|
738
|
+
* Layering: content-addressing is the substrate's job; cryptographic SIGNING
|
|
739
|
+
* (who vouches for the attestation, key management, transparency logs) is the
|
|
740
|
+
* consumer's layer on top. An `AttestedReport` is a stable byte-identical
|
|
741
|
+
* payload a consumer can sign — the substrate never holds keys.
|
|
742
|
+
*
|
|
743
|
+
* Generic by design: the report parameter is ANY value `canonicalJson`
|
|
744
|
+
* accepts (campaign results, fuzz capsules, scorecards, cost ledgers). Do not
|
|
745
|
+
* couple this module to a specific report schema.
|
|
746
|
+
*/
|
|
747
|
+
/** Hash scheme identifier carried by every attestation. A verifier rejects
|
|
748
|
+
* unknown algorithms instead of guessing. */
|
|
749
|
+
declare const ATTESTATION_ALGORITHM: 'sha256/canonical-json';
|
|
750
|
+
interface AttestationProvenance {
|
|
751
|
+
/** Every model involved in producing the report, name → version/id. */
|
|
752
|
+
modelVersions: Record<string, string>;
|
|
753
|
+
/** RNG seeds the run was driven by, when seeded. */
|
|
754
|
+
seeds?: number[];
|
|
755
|
+
/** Content hash of the price table used for cost figures — cost numbers
|
|
756
|
+
* are only reproducible against the same prices. */
|
|
757
|
+
priceTableHash?: string;
|
|
758
|
+
/** Git SHA of the code that produced the report. */
|
|
759
|
+
codeSha: string;
|
|
760
|
+
/** Content hash of the input set (scenarios, dataset manifest, ...). */
|
|
761
|
+
inputsHash?: string;
|
|
762
|
+
/** ISO-8601 timestamp, caller-supplied — the substrate stays clock-free
|
|
763
|
+
* so attestation is deterministic and testable. */
|
|
764
|
+
createdAt: string;
|
|
765
|
+
}
|
|
766
|
+
interface AttestedReport {
|
|
767
|
+
/** Hex sha-256 over the canonical JSON of the report. */
|
|
768
|
+
reportHash: string;
|
|
769
|
+
provenance: AttestationProvenance;
|
|
770
|
+
algorithm: typeof ATTESTATION_ALGORITHM;
|
|
771
|
+
/**
|
|
772
|
+
* Hex sha-256 over `{ reportHash, provenance, algorithm }`. New attestations
|
|
773
|
+
* always carry it. Optional only so persisted pre-envelope attestations can
|
|
774
|
+
* still be read and explicitly recognized as legacy by callers.
|
|
775
|
+
*/
|
|
776
|
+
envelopeHash?: string;
|
|
777
|
+
}
|
|
778
|
+
//#endregion
|
|
779
|
+
//#region src/experiment/evidence-receipt.d.ts
|
|
780
|
+
declare const EVIDENCE_RECEIPT_VERSION: '1.0.0';
|
|
781
|
+
/** Closed promotion vocabulary. A typo or unknown future kind is never independent by default. */
|
|
782
|
+
declare const EVIDENCE_AUTHORITY_KINDS: readonly ['candidate-self-report', 'independent-evaluator', 'independent-replication', 'human-review', 'production-canary'];
|
|
783
|
+
type EvidenceAuthorityKind = (typeof EVIDENCE_AUTHORITY_KINDS)[number];
|
|
784
|
+
declare const INDEPENDENT_EVIDENCE_AUTHORITY_KINDS: readonly ["independent-evaluator", "independent-replication", "human-review", "production-canary"];
|
|
785
|
+
interface EvidenceAuthority {
|
|
786
|
+
readonly kind: EvidenceAuthorityKind;
|
|
787
|
+
/** Stable authority/service/person identifier; never inferred from model prose. */
|
|
788
|
+
readonly id: string;
|
|
789
|
+
}
|
|
790
|
+
interface EvidenceBinding {
|
|
791
|
+
readonly schemaVersion: typeof EVIDENCE_RECEIPT_VERSION;
|
|
792
|
+
/** Stable objective identity spanning retries, forks, resumes, and multiple runs. */
|
|
793
|
+
readonly pursuitId: string;
|
|
794
|
+
/** Concrete execution/evaluation run this evidence was produced from. */
|
|
795
|
+
readonly runId: string;
|
|
796
|
+
/** Exact candidate/program/profile bundle under evaluation. */
|
|
797
|
+
readonly candidateDigest: string;
|
|
798
|
+
/** Exact evaluator/checker program or registered experiment definition. */
|
|
799
|
+
readonly evaluatorDigest: string;
|
|
800
|
+
/** Exact sandbox/container/dependency/environment identity. */
|
|
801
|
+
readonly environmentDigest: string;
|
|
802
|
+
/** Commitment to the input set. The hidden inputs themselves need not be exposed. */
|
|
803
|
+
readonly inputSetCommitment: string;
|
|
804
|
+
/** Content identity of the raw evaluated output/artifact set. */
|
|
805
|
+
readonly outputDigest: string;
|
|
806
|
+
/** Content identity of the evaluation result/report/verdict body. */
|
|
807
|
+
readonly resultDigest: string;
|
|
808
|
+
readonly authority: EvidenceAuthority;
|
|
809
|
+
/** Optional experiment seal that governed the evaluation. */
|
|
810
|
+
readonly experimentDigest?: string;
|
|
811
|
+
/** Optional observer-journal chain tip covering the execution being evaluated. */
|
|
812
|
+
readonly observerDigest?: string;
|
|
813
|
+
}
|
|
814
|
+
interface EvidenceReceipt {
|
|
815
|
+
readonly binding: EvidenceBinding;
|
|
816
|
+
readonly attestation: AttestedReport;
|
|
817
|
+
}
|
|
818
|
+
interface EvidenceReceiptVerification {
|
|
819
|
+
readonly valid: boolean;
|
|
820
|
+
readonly reason?: string;
|
|
821
|
+
}
|
|
822
|
+
interface CreateEvidenceReceiptInput extends Omit<EvidenceBinding, 'schemaVersion'> {}
|
|
823
|
+
/**
|
|
824
|
+
* Mint a content-attested evidence receipt. Required identity fields are deliberately
|
|
825
|
+
* non-optional: unknown evidence stays unknown and cannot accidentally look certified.
|
|
826
|
+
* The attestation's input provenance must equal the receipt commitment so two competing
|
|
827
|
+
* descriptions of the evaluated population cannot coexist inside one valid receipt.
|
|
828
|
+
*/
|
|
829
|
+
declare function createEvidenceReceipt(input: CreateEvidenceReceiptInput, provenance: AttestationProvenance): EvidenceReceipt;
|
|
830
|
+
/**
|
|
831
|
+
* Verify promotion-grade evidence. Generic report attestation keeps a legacy read path,
|
|
832
|
+
* but an EvidenceReceipt never accepts unbound provenance: changing the evaluator code,
|
|
833
|
+
* model versions, input commitment provenance, or creation record must invalidate the
|
|
834
|
+
* evidence rather than merely annotating it as legacy.
|
|
835
|
+
*/
|
|
836
|
+
declare function verifyEvidenceReceipt(receipt: EvidenceReceipt): EvidenceReceiptVerification;
|
|
837
|
+
/**
|
|
838
|
+
* Promotion may choose a stricter policy, but this primitive makes the basic separation
|
|
839
|
+
* explicit: only a recognized independent authority is independent. Unknown/forged kinds
|
|
840
|
+
* and candidate self-reports both return false.
|
|
841
|
+
*/
|
|
842
|
+
declare function isIndependentEvidence(receipt: EvidenceReceipt): boolean;
|
|
843
|
+
//#endregion
|
|
728
844
|
//#region src/experiment/power.d.ts
|
|
729
845
|
/** A design refused at configuration time, before any spend. */
|
|
730
846
|
declare class DesignRefusalError extends ValidationError {}
|
|
@@ -805,5 +921,5 @@ declare function clusteredPower(options: ClusteredPowerOptions): ClusteredPowerR
|
|
|
805
921
|
/** Throw the refusal for callers that want configuration-time failure. */
|
|
806
922
|
declare function assertDesignAdequate(result: ClusteredPowerResult): void;
|
|
807
923
|
//#endregion
|
|
808
|
-
export { type AdmissionExecution, type AdmissionPartition, type AdmissionRule, type AdmissionStage, type ArmRealizedBudget, type ArmSpec, BOOTSTRAP_GATE_MIN_N, type BudgetRule, type ClusteredPowerOptions, type ClusteredPowerPoint, type ClusteredPowerRefusal, type ClusteredPowerResult, type ComputedInterval, type Condition, type DecisionBranch, type DecisionOutcome, type DecisionRule, DecisionTableNotTotalError, type DerivedQuantities, DesignRefusalError, type EProcess, type EProcessOptions, type EProcessState, type EProcessStep, type Estimand, type EstimandResult, type EvidenceRecord, type EvidenceVector, type Experiment, type ExperimentFunnel, type ExperimentRep, type ExperimentSpec, type ExperimentStats, ExperimentTracker, type ExperimentVerdict, FunnelIntegrityError, type FunnelPartitionCount, type FunnelStageCount, type FunnelStageInput, type GateEvidence, type GateResult, type HaltOutcome, type HaltRule, type HeldoutSignificance, type HeldoutSignificanceOptions, type HypothesisManifest, type HypothesisResult, type IntervalSpec, type JsonValue, MatchedBudgetError, type MatchedBudgetRule, type MatchedBudgetVerdict, type MatchedPair, type NLadderProjection, type Obligation, type OutcomeSpec, type PairArmsOptions, type PairArmsResult, type PairedArmRow, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type PairedHoldout, type PowerPreflight, type PowerPreflightOptions, type Predicate, type PromotionPolicy, type ProportionInterval, type RegisteredExperiment, type ReissuePolicy, type ReissueVerdict, type SealAmendment, SealIntegrityError, type SealedExperiment, type SeedDerivation, type SelectionRule, type SequentialCrossingHorizon, type SequentialCrossingHorizonOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SetExpr, type SignFlipFloor, type SignedManifest, type UniformPassDecision, type UniformPassSchedule, type ValidityGate, amendExperiment, assertDesignAdequate, assertFunnelReconciles, assertMatchedBudgets, benjaminiHochberg, bonferroni, buildEvidenceVector, buildFunnel, canonicalize, classifyReissue, clusteredPower, comparePairedArms, composeFunnels, computeEstimand, computeInterval, defineExperiment, eProcess, evaluateCondition, evaluateHaltRule, evaluateHypothesis, evaluateIdentityGate, evaluateOracleDeterminismGate, evaluatePopulationReproducibilityGate, evaluatePowerFloorGate, evaluatePredicate, evaluateProvenanceGate, executeAdmissionRule, executeDecisionRule, fileExperimentStore, hashJson, heldoutSignificance, holm, inMemoryExperimentStore, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openSealedExperiment, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, powerPreflight, projectNLadderBudget, readField, renderFunnelTable, requiredPairedSampleSize, requiredSampleSize, runSelectionRule, runUniformPassBudget, sealExperiment, sequentialCrossingHorizon, sequentialDecide, sequentialPairedGate, signManifest, verifyManifest, verifyMatchedBudgets, verifySealedExperiment, wilson };
|
|
924
|
+
export { type AdmissionExecution, type AdmissionPartition, type AdmissionRule, type AdmissionStage, type ArmRealizedBudget, type ArmSpec, BOOTSTRAP_GATE_MIN_N, type BudgetRule, type ClusteredPowerOptions, type ClusteredPowerPoint, type ClusteredPowerRefusal, type ClusteredPowerResult, type ComputedInterval, type Condition, type CreateEvidenceReceiptInput, type DecisionBranch, type DecisionOutcome, type DecisionRule, DecisionTableNotTotalError, type DerivedQuantities, DesignRefusalError, type EProcess, type EProcessOptions, type EProcessState, type EProcessStep, EVIDENCE_AUTHORITY_KINDS, EVIDENCE_RECEIPT_VERSION, type Estimand, type EstimandResult, type EvidenceAuthority, type EvidenceAuthorityKind, type EvidenceBinding, type EvidenceReceipt, type EvidenceReceiptVerification, type EvidenceRecord, type EvidenceVector, type Experiment, type ExperimentFunnel, type ExperimentRep, type ExperimentSpec, type ExperimentStats, ExperimentTracker, type ExperimentVerdict, FunnelIntegrityError, type FunnelPartitionCount, type FunnelStageCount, type FunnelStageInput, type GateEvidence, type GateResult, type HaltOutcome, type HaltRule, type HeldoutSignificance, type HeldoutSignificanceOptions, type HypothesisManifest, type HypothesisResult, INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, type IntervalSpec, type JsonValue, MatchedBudgetError, type MatchedBudgetRule, type MatchedBudgetVerdict, type MatchedPair, type NLadderProjection, type Obligation, type OutcomeSpec, type PairArmsOptions, type PairArmsResult, type PairedArmRow, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type PairedHoldout, type PowerPreflight, type PowerPreflightOptions, type Predicate, type PromotionPolicy, type ProportionInterval, type RegisteredExperiment, type ReissuePolicy, type ReissueVerdict, type SealAmendment, SealIntegrityError, type SealedExperiment, type SeedDerivation, type SelectionRule, type SequentialCrossingHorizon, type SequentialCrossingHorizonOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SetExpr, type SignFlipFloor, type SignedManifest, type UniformPassDecision, type UniformPassSchedule, type ValidityGate, amendExperiment, assertDesignAdequate, assertFunnelReconciles, assertMatchedBudgets, benjaminiHochberg, bonferroni, buildEvidenceVector, buildFunnel, canonicalize, classifyReissue, clusteredPower, comparePairedArms, composeFunnels, computeEstimand, computeInterval, createEvidenceReceipt, defineExperiment, eProcess, evaluateCondition, evaluateHaltRule, evaluateHypothesis, evaluateIdentityGate, evaluateOracleDeterminismGate, evaluatePopulationReproducibilityGate, evaluatePowerFloorGate, evaluatePredicate, evaluateProvenanceGate, executeAdmissionRule, executeDecisionRule, fileExperimentStore, hashJson, heldoutSignificance, holm, inMemoryExperimentStore, isIndependentEvidence, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openSealedExperiment, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, powerPreflight, projectNLadderBudget, readField, renderFunnelTable, requiredPairedSampleSize, requiredSampleSize, runSelectionRule, runUniformPassBudget, sealExperiment, sequentialCrossingHorizon, sequentialDecide, sequentialPairedGate, signManifest, verifyEvidenceReceipt, verifyManifest, verifyMatchedBudgets, verifySealedExperiment, wilson };
|
|
809
925
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/experiment/ast.ts","../../src/experiment/budget.ts","../../src/experiment/funnel.ts","../../src/experiment/define.ts","../../src/experiment/power.ts"],"mappings":";;;;;;;;;;;KA8BY,+CAA+C;GAAiB,YAAY;;;KAG5E,iBAAiB;;cAGhB,mCAAmC;;;;;KAQpC;EAEN;EACA;EACA;EACA,OAAO;;EAEP;EAAY;EAAe,QAAQ;;EACnC;EAAa,IAAI;;EACjB;EAAa,IAAI;;EACjB;EAAa,IAAI;;;iBAGP,UAAU,QAAQ,gBAAgB;;iBAUlC,kBAAkB,WAAW,WAAW,QAAQ;;;;;UA4C/C;EACf;EACA,MAAM;EACN;;;UAIe;EACf;;EAEA;EACA,MAAM;EACN;;;UAIe;EACf;EACA,QAAQ;EACR,aAAa;;;;;;;KAUH;EAEN;EACA;EACA;EACA;IAAe;IAAe;;EAC9B;EACA;;EAGA;;EAEA;EACA,MAAM;EACN;IAAS;IAAe;;;;;;;;;;;;;iBAad,iBACd,MAAM,eACN,kBAAkB,kBAClB;;EAEE;;EAEA,QAAQ;;EAER,YAAY,eAAe;;;KA0DnB;EACN;EAAoB;EAAa,OAAO;;EACxC;EAAmB,IAAI;;;;;;KAMjB;EACN;EAAc,OAAO;EAAW;;EAChC;EAA4B,OAAO;EAAW;;EAE9C;EACA;EACA;EACA;EACA;;;;;EAKA;;EAEA;;EAGA;EACA;EACA;EACA,WAAW;EACX,aAAa;;UAGF;EACf;EACA;EACA;;;iBA8Cc,gBACd,UAAU,UACV,eAAe,mBACd;;KA2ES;EAEN;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAyB;;UAEd;EACf;EACA;EACA;;;;;;;;;;iBAWc,gBACd,MAAM,cACN;EACM;EAAc,eAAe;EAAkB;;EAC/C;EAAkB;EAAmB;IAC1C;;KA8HS;EACN;EAAgC;EAAkB;;EAClD;EAAgC;;EAChC;EAA4B;EAAkB;EAAiC;;EAC/E;EAAwB;;EACxB;EAAa,IAAI;;EACjB;EAAa,IAAI;;EACjB;EAAa,IAAI;;;UAGN;EACf,WAAW;IAAiB;IAAe;;EAC3C,YAAY;EACZ,gBAAgB;;iBAGF,kBAAkB,WAAW,WAAW,UAAU;UAyCjD;EACf,MAAM;EACN;EACA;;;;;;;;;KAUU;EACN;EAAe,UAAU;;EAEzB;EACA;EACA;EACA;EACA;IAAmB;IAAuB;;;UAG/B;EACf;EACA;;iBAGc,oBACd,MAAM,cACN,UAAU,oBACT;;UAec;EACf;EACA;EACA;;;KAMU;EAEN;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAEA;EAA8B,OAAO;;EAErC;EACA;EACA;EACA;IAAO;IAAgB;IAAmB;;;EAE1C;EAAkB;EAAuB;EAAmB;;UAEjD;EACf;EACA;EACA,UAAU;;;;;;iBAOI,8BACd,YACA,MAAM,QAAQ;EAAgB;IAC9B,aAAa,qCACZ;;;;;;iBAiBa,sCACd,YACA,MAAM,QAAQ;EAAgB;IAC9B;EAAe,eAAe;EAAkB,gBAAgB;IAC/D;;iBAkBa,uBACd,YACA,MAAM,QAAQ;EAAgB;IAC9B,YAAY,iBACX;;iBAMa,qBACd,YACA,OAAO,QAAQ;EAAgB;IAC/B;EAAc;EAAgB;IAC7B;;;;;;iBAea,uBACd,YACA,MAAM,QAAQ;EAAgB;IAC9B;EAAkB;EAAgB;MACjC;;UAsBc;EACf;IAAQ;IAAyB;;EACjC;EACA;;UAGe;EACf;EACA;EACA;;iBAGc,iBAAiB,MAAM,UAAU,gBAAgB,eAAe;;;;;KAoBpE;EAEN;EACA;EACA;EACA;EACA;EACA;IAAU;IAAY;;EACtB;;EAGA;EACA;EACA;EACA;EACA;;UAGW;EACf;EACA;EACA;EACA;;UAGe;EACf,WAAW;EACX;;;;;;;;iBASc,qBACd,MAAM,QAAQ;EAAc;IAC5B,uCACC;UAuBc;EACf;EACA;IAAe;IAAW;IAAsB;;EAChD;IAAkB;IAAuC;;;;;;;;iBAQ3C,qBACd,MAAM,QAAQ;EAAc;IAC5B;EAAY;EAAqB;IAChC;;UAoBc;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;;KAGU;;;;;;iBAOI,gBACd,QAAQ,eACR,eACA,sBACC;;;;cCn6BU,2BAA2B;UAEvB;EACf;;EAEA;;UAGe;EACf,MAAM;EACN,MAAM;;EAEN;;EAEA;EACA;;EAEA;IAAkB;IAA2B;;;;;;;;iBAQ/B,qBACd,MAAM,mBACN,eAAe,sBACd;;iBAsCa,qBACd,MAAM,mBACN,eAAe,sBACd;;;;cC/DU,6BAA6B;UAEzB;EACf;EACA;EACA;EACA;;EAEA,aAAa;;EAEb;;UAGe;EACf;;EAEA;EACA;;EAEA;;UAGe;EACf;EACA;EACA,QAAQ;EACR;EACA,YAAY;;UAGG;EACf;;EAEA;EACA,aAAa;EACb;;;;;;;;;;;iBAYc,YAAY;EAC1B;EACA;EACA,QAAQ;EACR;IAAe;IAAY;IAAc;;IACvC;;iBAsEY,uBAAuB,QAAQ;UAwB9B;EACf,QAAQ;EACR,WAAW;;EAEX,eAAe,eAAe;;;;;;;;iBAShB,qBACd,MAAM,eACN,kBAAkB,mBACjB;;;;;;iBA8Ca,eACd,OAAO,kBACP,QAAQ,mBACP;;;;;;iBA4Ba,kBAAkB,QAAQ;;;;cCvM7B,2BAA2B;UAIvB;EACf;EACA;;EAEA;;EAEA;;EAEA,OAAO,eAAe;;KAGZ;EAEN;;EAEA;EACA;;EAEA;;EAEA;;EAGA;EACA;EACA;EACA;;;;;;UAOW;EACf;;UAGe;EACf;;EAEA;EACA,MAAM;EACN,SAAS;EACT,YAAY;;EAEZ,aAAa,eAAe;;;;;EAK5B,gBAAgB;EAChB,YAAY,eAAe;EAC3B,YAAY,eAAe;EAC3B,UAAU;EACV,cAAc;;EAEd,QAAQ,eAAe;EACvB,OAAO;EACP,SAAS;EACT,gBAAgB;EAChB,UAAU;EACV,iBAAiB;EACjB;;;;;;;;;;iBAwCc,iBAAiB,MAAM,iBAAiB;UAkHvC;;EAEf;EACA;;EAEA;;EAEA;;UAGe;EACf,MAAM;;EAEN;EACA;EACA;;EAEA;EACA,YAAY;;;iBAIQ,eACpB,MAAM,gBACN;EAAW;IACV,QAAQ;;;;;;iBAkBW,gBACpB,QAAQ,kBACR;EAAa,MAAM;EAAgB;EAAgB;EAAiB;IACnE,QAAQ;;iBAuBW,uBAAuB,QAAQ,mBAAmB;;KAe5D;EACN;EAA4B,aAAa;;EAEzC;EACA,eAAe;EACf,gBAAgB;;EAEhB;EAA8B,YAAY;;EAC1C;EAAkB;EAAgB;;EAClC;EAAqB;IAAkB;IAAgB;;;;;;;UAM5C;WACN,QAAQ;;EAEjB,OAAO,UAAU,oBAAoB;;EAErC,MAAM,kBAAkB,mBAAmB;;EAE3C,OAAO,cAAc,kBAAkB,kBAAkB;IAAW;;;EAEpE,KAAK,cAAc,UAAU,eAAe;;EAE5C,KAAK,gBAAgB,eAAe;;EAEpC,qBAAqB,uCAAuC;;EAE5D,qBAAqB;IAAY;IAAqB;MAAiB;;EAEvE,eAAe,eAAe,sBAAsB;;EAEpD,SAAS,cAAc,eAAe,mBAAmB;;EAEzD,SACE,cACA;IACM;IAAc,eAAe;IAAkB;;IAC/C;IAAkB;IAAmB;MAC1C;;;;;;;iBAQiB,qBACpB,QAAQ,mBACP,QAAQ;;;;
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/experiment/ast.ts","../../src/experiment/budget.ts","../../src/experiment/funnel.ts","../../src/experiment/define.ts","../../src/attestation.ts","../../src/experiment/evidence-receipt.ts","../../src/experiment/power.ts"],"mappings":";;;;;;;;;;;KA8BY,+CAA+C;GAAiB,YAAY;;;KAG5E,iBAAiB;;cAGhB,mCAAmC;;;;;KAQpC;EAEN;EACA;EACA;EACA,OAAO;;EAEP;EAAY;EAAe,QAAQ;;EACnC;EAAa,IAAI;;EACjB;EAAa,IAAI;;EACjB;EAAa,IAAI;;;iBAGP,UAAU,QAAQ,gBAAgB;;iBAUlC,kBAAkB,WAAW,WAAW,QAAQ;;;;;UA4C/C;EACf;EACA,MAAM;EACN;;;UAIe;EACf;;EAEA;EACA,MAAM;EACN;;;UAIe;EACf;EACA,QAAQ;EACR,aAAa;;;;;;;KAUH;EAEN;EACA;EACA;EACA;IAAe;IAAe;;EAC9B;EACA;;EAGA;;EAEA;EACA,MAAM;EACN;IAAS;IAAe;;;;;;;;;;;;;iBAad,iBACd,MAAM,eACN,kBAAkB,kBAClB;;EAEE;;EAEA,QAAQ;;EAER,YAAY,eAAe;;;KA0DnB;EACN;EAAoB;EAAa,OAAO;;EACxC;EAAmB,IAAI;;;;;;KAMjB;EACN;EAAc,OAAO;EAAW;;EAChC;EAA4B,OAAO;EAAW;;EAE9C;EACA;EACA;EACA;EACA;;;;;EAKA;;EAEA;;EAGA;EACA;EACA;EACA,WAAW;EACX,aAAa;;UAGF;EACf;EACA;EACA;;;iBA8Cc,gBACd,UAAU,UACV,eAAe,mBACd;;KA2ES;EAEN;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAyB;;UAEd;EACf;EACA;EACA;;;;;;;;;;iBAWc,gBACd,MAAM,cACN;EACM;EAAc,eAAe;EAAkB;;EAC/C;EAAkB;EAAmB;IAC1C;;KA8HS;EACN;EAAgC;EAAkB;;EAClD;EAAgC;;EAChC;EAA4B;EAAkB;EAAiC;;EAC/E;EAAwB;;EACxB;EAAa,IAAI;;EACjB;EAAa,IAAI;;EACjB;EAAa,IAAI;;;UAGN;EACf,WAAW;IAAiB;IAAe;;EAC3C,YAAY;EACZ,gBAAgB;;iBAGF,kBAAkB,WAAW,WAAW,UAAU;UAyCjD;EACf,MAAM;EACN;EACA;;;;;;;;;KAUU;EACN;EAAe,UAAU;;EAEzB;EACA;EACA;EACA;EACA;IAAmB;IAAuB;;;UAG/B;EACf;EACA;;iBAGc,oBACd,MAAM,cACN,UAAU,oBACT;;UAec;EACf;EACA;EACA;;;KAMU;EAEN;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAEA;EAA8B,OAAO;;EAErC;EACA;EACA;EACA;IAAO;IAAgB;IAAmB;;;EAE1C;EAAkB;EAAuB;EAAmB;;UAEjD;EACf;EACA;EACA,UAAU;;;;;;iBAOI,8BACd,YACA,MAAM,QAAQ;EAAgB;IAC9B,aAAa,qCACZ;;;;;;iBAiBa,sCACd,YACA,MAAM,QAAQ;EAAgB;IAC9B;EAAe,eAAe;EAAkB,gBAAgB;IAC/D;;iBAkBa,uBACd,YACA,MAAM,QAAQ;EAAgB;IAC9B,YAAY,iBACX;;iBAMa,qBACd,YACA,OAAO,QAAQ;EAAgB;IAC/B;EAAc;EAAgB;IAC7B;;;;;;iBAea,uBACd,YACA,MAAM,QAAQ;EAAgB;IAC9B;EAAkB;EAAgB;MACjC;;UAsBc;EACf;IAAQ;IAAyB;;EACjC;EACA;;UAGe;EACf;EACA;EACA;;iBAGc,iBAAiB,MAAM,UAAU,gBAAgB,eAAe;;;;;KAoBpE;EAEN;EACA;EACA;EACA;EACA;EACA;IAAU;IAAY;;EACtB;;EAGA;EACA;EACA;EACA;EACA;;UAGW;EACf;EACA;EACA;EACA;;UAGe;EACf,WAAW;EACX;;;;;;;;iBASc,qBACd,MAAM,QAAQ;EAAc;IAC5B,uCACC;UAuBc;EACf;EACA;IAAe;IAAW;IAAsB;;EAChD;IAAkB;IAAuC;;;;;;;;iBAQ3C,qBACd,MAAM,QAAQ;EAAc;IAC5B;EAAY;EAAqB;IAChC;;UAoBc;EACf;EACA;EACA;;;UAIe;EACf;EACA;EACA;;KAGU;;;;;;iBAOI,gBACd,QAAQ,eACR,eACA,sBACC;;;;cCn6BU,2BAA2B;UAEvB;EACf;;EAEA;;UAGe;EACf,MAAM;EACN,MAAM;;EAEN;;EAEA;EACA;;EAEA;IAAkB;IAA2B;;;;;;;;iBAQ/B,qBACd,MAAM,mBACN,eAAe,sBACd;;iBAsCa,qBACd,MAAM,mBACN,eAAe,sBACd;;;;cC/DU,6BAA6B;UAEzB;EACf;EACA;EACA;EACA;;EAEA,aAAa;;EAEb;;UAGe;EACf;;EAEA;EACA;;EAEA;;UAGe;EACf;EACA;EACA,QAAQ;EACR;EACA,YAAY;;UAGG;EACf;;EAEA;EACA,aAAa;EACb;;;;;;;;;;;iBAYc,YAAY;EAC1B;EACA;EACA,QAAQ;EACR;IAAe;IAAY;IAAc;;IACvC;;iBAsEY,uBAAuB,QAAQ;UAwB9B;EACf,QAAQ;EACR,WAAW;;EAEX,eAAe,eAAe;;;;;;;;iBAShB,qBACd,MAAM,eACN,kBAAkB,mBACjB;;;;;;iBA8Ca,eACd,OAAO,kBACP,QAAQ,mBACP;;;;;;iBA4Ba,kBAAkB,QAAQ;;;;cCvM7B,2BAA2B;UAIvB;EACf;EACA;;EAEA;;EAEA;;EAEA,OAAO,eAAe;;KAGZ;EAEN;;EAEA;EACA;;EAEA;;EAEA;;EAGA;EACA;EACA;EACA;;;;;;UAOW;EACf;;UAGe;EACf;;EAEA;EACA,MAAM;EACN,SAAS;EACT,YAAY;;EAEZ,aAAa,eAAe;;;;;EAK5B,gBAAgB;EAChB,YAAY,eAAe;EAC3B,YAAY,eAAe;EAC3B,UAAU;EACV,cAAc;;EAEd,QAAQ,eAAe;EACvB,OAAO;EACP,SAAS;EACT,gBAAgB;EAChB,UAAU;EACV,iBAAiB;EACjB;;;;;;;;;;iBAwCc,iBAAiB,MAAM,iBAAiB;UAkHvC;;EAEf;EACA;;EAEA;;EAEA;;UAGe;EACf,MAAM;;EAEN;EACA;EACA;;EAEA;EACA,YAAY;;;iBAIQ,eACpB,MAAM,gBACN;EAAW;IACV,QAAQ;;;;;;iBAkBW,gBACpB,QAAQ,kBACR;EAAa,MAAM;EAAgB;EAAgB;EAAiB;IACnE,QAAQ;;iBAuBW,uBAAuB,QAAQ,mBAAmB;;KAe5D;EACN;EAA4B,aAAa;;EAEzC;EACA,eAAe;EACf,gBAAgB;;EAEhB;EAA8B,YAAY;;EAC1C;EAAkB;EAAgB;;EAClC;EAAqB;IAAkB;IAAgB;;;;;;;UAM5C;WACN,QAAQ;;EAEjB,OAAO,UAAU,oBAAoB;;EAErC,MAAM,kBAAkB,mBAAmB;;EAE3C,OAAO,cAAc,kBAAkB,kBAAkB;IAAW;;;EAEpE,KAAK,cAAc,UAAU,eAAe;;EAE5C,KAAK,gBAAgB,eAAe;;EAEpC,qBAAqB,uCAAuC;;EAE5D,qBAAqB;IAAY;IAAqB;MAAiB;;EAEvE,eAAe,eAAe,sBAAsB;;EAEpD,SAAS,cAAc,eAAe,mBAAmB;;EAEzD,SACE,cACA;IACM;IAAc,eAAe;IAAkB;;IAC/C;IAAkB;IAAmB;MAC1C;;;;;;;iBAQiB,qBACpB,QAAQ,mBACP,QAAQ;;;;;;;;;;;;;;;;;;;;;;;cCvYE;UAEI;;EAEf,eAAe;;EAEf;;;EAGA;;EAEA;;EAEA;;;EAGA;;UAGe;;EAEf;EACA,YAAY;EACZ,kBAAkB;;;;;;EAMlB;;;;cChCW;;cAGA;KAQD,gCAAgC;cAE/B;UAOI;WACN,MAAM;;WAEN;;UAGM;WACN,sBAAsB;;WAEtB;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;WACA,WAAW;;WAEX;;WAEA;;UAGM;WACN,SAAS;WACT,aAAa;;UAGP;WACN;WACA;;UAGM,mCAAmC,KAAK;;;;;;;iBAQzC,sBACd,OAAO,4BACP,YAAY,wBACX;;;;;;;iBA8Ba,sBAAsB,SAAS,kBAAkB;;;;;;iBA6CjD,sBAAsB,SAAS;;;;cC3IlC,2BAA2B;UAEvB;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA;IAAkB;IAAe;;;UAGlB;EACf;EACA;EACA;;UAGe;;EAEf;;EAEA;EACA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;;EAEP;EACA,eAAe;;EAEf;;EAEA,SAAS;;;;;;iBAOK,eAAe,SAAS,wBAAwB;;iBAsGhD,qBAAqB,QAAQ"}
|
package/dist/experiment/index.js
CHANGED
|
@@ -5,6 +5,7 @@ import { a as requiredSampleSize, c as holm, i as requiredPairedSampleSize, n as
|
|
|
5
5
|
import { f as mcnemar, g as pairedRiskDifferenceScore, h as pairedRiskDifferenceExact, m as pairedRiskDifference, n as pairArms, r as pairRunRecords, t as comparePairedArms, v as wilson } from "../paired-arms-D-XRF_fy.js";
|
|
6
6
|
import { r as pairedBootstrap, t as BOOTSTRAP_GATE_MIN_N } from "../paired-tests-BHIhYVdu.js";
|
|
7
7
|
import { t as eProcess } from "../sequential-eprocess-CbUt2htw.js";
|
|
8
|
+
import { n as contentHash } from "../verdict-cache-mZf5FEiY.js";
|
|
8
9
|
import { a as heldoutSignificance, o as pairHoldout, t as powerPreflight } from "../power-preflight-DEw-uC7q.js";
|
|
9
10
|
import { a as inMemoryExperimentStore, r as fileExperimentStore, t as ExperimentTracker } from "../experiment-tracker-Ym6rEQT1.js";
|
|
10
11
|
import { n as pairedEvalueSequence, r as sequentialCrossingHorizon } from "../sequential-CzK5DarL.js";
|
|
@@ -952,6 +953,221 @@ async function openSealedExperiment(sealed) {
|
|
|
952
953
|
};
|
|
953
954
|
}
|
|
954
955
|
//#endregion
|
|
956
|
+
//#region src/attestation.ts
|
|
957
|
+
/**
|
|
958
|
+
* Reproducibility attestation for any serializable report object.
|
|
959
|
+
*
|
|
960
|
+
* `attest()` binds a report to its content address (sha-256 over canonical
|
|
961
|
+
* JSON) AND binds that address to the provenance needed to reproduce it:
|
|
962
|
+
* model versions, seeds, price-table hash, code SHA, inputs hash. The outer
|
|
963
|
+
* `envelopeHash` prevents provenance from being rewritten while leaving the
|
|
964
|
+
* report hash valid.
|
|
965
|
+
*
|
|
966
|
+
* Layering: content-addressing is the substrate's job; cryptographic SIGNING
|
|
967
|
+
* (who vouches for the attestation, key management, transparency logs) is the
|
|
968
|
+
* consumer's layer on top. An `AttestedReport` is a stable byte-identical
|
|
969
|
+
* payload a consumer can sign — the substrate never holds keys.
|
|
970
|
+
*
|
|
971
|
+
* Generic by design: the report parameter is ANY value `canonicalJson`
|
|
972
|
+
* accepts (campaign results, fuzz capsules, scorecards, cost ledgers). Do not
|
|
973
|
+
* couple this module to a specific report schema.
|
|
974
|
+
*/
|
|
975
|
+
/** Hash scheme identifier carried by every attestation. A verifier rejects
|
|
976
|
+
* unknown algorithms instead of guessing. */
|
|
977
|
+
const ATTESTATION_ALGORITHM = "sha256/canonical-json";
|
|
978
|
+
function envelopeMaterial(reportHash, provenance, algorithm) {
|
|
979
|
+
return {
|
|
980
|
+
reportHash,
|
|
981
|
+
provenance,
|
|
982
|
+
algorithm
|
|
983
|
+
};
|
|
984
|
+
}
|
|
985
|
+
/**
|
|
986
|
+
* Content-address a report and bind it to its provenance. Throws (via
|
|
987
|
+
* `canonicalJson`) if the report or provenance contains undefined / function /
|
|
988
|
+
* symbol / non-finite numbers — an attestation that cannot be unambiguously
|
|
989
|
+
* serialized cannot be trusted.
|
|
990
|
+
*/
|
|
991
|
+
function attest(report, provenance) {
|
|
992
|
+
const reportHash = contentHash(report);
|
|
993
|
+
const algorithm = ATTESTATION_ALGORITHM;
|
|
994
|
+
return {
|
|
995
|
+
reportHash,
|
|
996
|
+
provenance,
|
|
997
|
+
algorithm,
|
|
998
|
+
envelopeHash: contentHash(envelopeMaterial(reportHash, provenance, algorithm))
|
|
999
|
+
};
|
|
1000
|
+
}
|
|
1001
|
+
/**
|
|
1002
|
+
* Verify a report against its attestation. Returns a typed outcome rather
|
|
1003
|
+
* than throwing: an unverifiable report (e.g. one that no longer
|
|
1004
|
+
* canonicalizes) is a verification failure with the cause in `reason`, not a
|
|
1005
|
+
* crash — verifiers run in pipelines that must record WHY, not die.
|
|
1006
|
+
*
|
|
1007
|
+
* Legacy attestations without `envelopeHash` remain readable, but verification
|
|
1008
|
+
* explicitly marks their provenance as unbound so a promotion path can refuse
|
|
1009
|
+
* them instead of accidentally treating old metadata as cryptographic proof.
|
|
1010
|
+
*/
|
|
1011
|
+
function verifyAttestation(report, attested) {
|
|
1012
|
+
if (attested.algorithm !== "sha256/canonical-json") return {
|
|
1013
|
+
valid: false,
|
|
1014
|
+
reason: `unknown algorithm '${attested.algorithm}' — this verifier only checks '${ATTESTATION_ALGORITHM}'`
|
|
1015
|
+
};
|
|
1016
|
+
let recomputed;
|
|
1017
|
+
try {
|
|
1018
|
+
recomputed = contentHash(report);
|
|
1019
|
+
} catch (err) {
|
|
1020
|
+
return {
|
|
1021
|
+
valid: false,
|
|
1022
|
+
reason: `report is not canonicalizable: ${err instanceof Error ? err.message : String(err)}`
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
if (recomputed !== attested.reportHash) return {
|
|
1026
|
+
valid: false,
|
|
1027
|
+
reason: `report hash mismatch: attested ${attested.reportHash}, recomputed ${recomputed}`
|
|
1028
|
+
};
|
|
1029
|
+
if (attested.envelopeHash === void 0) return {
|
|
1030
|
+
valid: true,
|
|
1031
|
+
legacyUnboundProvenance: true
|
|
1032
|
+
};
|
|
1033
|
+
let envelopeHash;
|
|
1034
|
+
try {
|
|
1035
|
+
envelopeHash = contentHash(envelopeMaterial(attested.reportHash, attested.provenance, attested.algorithm));
|
|
1036
|
+
} catch (err) {
|
|
1037
|
+
return {
|
|
1038
|
+
valid: false,
|
|
1039
|
+
reason: `attestation provenance is not canonicalizable: ${err instanceof Error ? err.message : String(err)}`
|
|
1040
|
+
};
|
|
1041
|
+
}
|
|
1042
|
+
if (envelopeHash !== attested.envelopeHash) return {
|
|
1043
|
+
valid: false,
|
|
1044
|
+
reason: `attestation envelope hash mismatch: attested ${attested.envelopeHash}, recomputed ${envelopeHash}`
|
|
1045
|
+
};
|
|
1046
|
+
return { valid: true };
|
|
1047
|
+
}
|
|
1048
|
+
//#endregion
|
|
1049
|
+
//#region src/experiment/evidence-receipt.ts
|
|
1050
|
+
/**
|
|
1051
|
+
* Evidence receipts are the join between Runtime execution and Eval promotion.
|
|
1052
|
+
*
|
|
1053
|
+
* Runtime says what actually ran. Eval says what independently measured it.
|
|
1054
|
+
* This receipt binds those worlds without making either package import the other:
|
|
1055
|
+
* stable pursuit/run identity, exact candidate/evaluator/environment/input/output
|
|
1056
|
+
* content identities, and the authority class that produced the observation.
|
|
1057
|
+
*
|
|
1058
|
+
* The payload is attested with agent-eval's existing canonical report attestation.
|
|
1059
|
+
* Signing/key management deliberately remains outside this substrate; consumers may
|
|
1060
|
+
* sign the byte-stable receipt or anchor its attestation in a transparency log.
|
|
1061
|
+
*/
|
|
1062
|
+
const EVIDENCE_RECEIPT_VERSION = "1.0.0";
|
|
1063
|
+
/** Closed promotion vocabulary. A typo or unknown future kind is never independent by default. */
|
|
1064
|
+
const EVIDENCE_AUTHORITY_KINDS = [
|
|
1065
|
+
"candidate-self-report",
|
|
1066
|
+
"independent-evaluator",
|
|
1067
|
+
"independent-replication",
|
|
1068
|
+
"human-review",
|
|
1069
|
+
"production-canary"
|
|
1070
|
+
];
|
|
1071
|
+
const INDEPENDENT_EVIDENCE_AUTHORITY_KINDS = [
|
|
1072
|
+
"independent-evaluator",
|
|
1073
|
+
"independent-replication",
|
|
1074
|
+
"human-review",
|
|
1075
|
+
"production-canary"
|
|
1076
|
+
];
|
|
1077
|
+
/**
|
|
1078
|
+
* Mint a content-attested evidence receipt. Required identity fields are deliberately
|
|
1079
|
+
* non-optional: unknown evidence stays unknown and cannot accidentally look certified.
|
|
1080
|
+
* The attestation's input provenance must equal the receipt commitment so two competing
|
|
1081
|
+
* descriptions of the evaluated population cannot coexist inside one valid receipt.
|
|
1082
|
+
*/
|
|
1083
|
+
function createEvidenceReceipt(input, provenance) {
|
|
1084
|
+
const inputSetCommitment = requiredIdentity(input.inputSetCommitment, "inputSetCommitment");
|
|
1085
|
+
assertInputCommitment(provenance, inputSetCommitment);
|
|
1086
|
+
const binding = Object.freeze({
|
|
1087
|
+
schemaVersion: EVIDENCE_RECEIPT_VERSION,
|
|
1088
|
+
pursuitId: requiredIdentity(input.pursuitId, "pursuitId"),
|
|
1089
|
+
runId: requiredIdentity(input.runId, "runId"),
|
|
1090
|
+
candidateDigest: requiredIdentity(input.candidateDigest, "candidateDigest"),
|
|
1091
|
+
evaluatorDigest: requiredIdentity(input.evaluatorDigest, "evaluatorDigest"),
|
|
1092
|
+
environmentDigest: requiredIdentity(input.environmentDigest, "environmentDigest"),
|
|
1093
|
+
inputSetCommitment,
|
|
1094
|
+
outputDigest: requiredIdentity(input.outputDigest, "outputDigest"),
|
|
1095
|
+
resultDigest: requiredIdentity(input.resultDigest, "resultDigest"),
|
|
1096
|
+
authority: requiredAuthority(input.authority),
|
|
1097
|
+
...input.experimentDigest === void 0 ? {} : { experimentDigest: requiredIdentity(input.experimentDigest, "experimentDigest") },
|
|
1098
|
+
...input.observerDigest === void 0 ? {} : { observerDigest: requiredIdentity(input.observerDigest, "observerDigest") }
|
|
1099
|
+
});
|
|
1100
|
+
return Object.freeze({
|
|
1101
|
+
binding,
|
|
1102
|
+
attestation: attest(binding, provenance)
|
|
1103
|
+
});
|
|
1104
|
+
}
|
|
1105
|
+
/**
|
|
1106
|
+
* Verify promotion-grade evidence. Generic report attestation keeps a legacy read path,
|
|
1107
|
+
* but an EvidenceReceipt never accepts unbound provenance: changing the evaluator code,
|
|
1108
|
+
* model versions, input commitment provenance, or creation record must invalidate the
|
|
1109
|
+
* evidence rather than merely annotating it as legacy.
|
|
1110
|
+
*/
|
|
1111
|
+
function verifyEvidenceReceipt(receipt) {
|
|
1112
|
+
if (receipt.binding.schemaVersion !== "1.0.0") return {
|
|
1113
|
+
valid: false,
|
|
1114
|
+
reason: `unsupported evidence receipt version '${receipt.binding.schemaVersion}'`
|
|
1115
|
+
};
|
|
1116
|
+
try {
|
|
1117
|
+
for (const [field, value] of Object.entries({
|
|
1118
|
+
pursuitId: receipt.binding.pursuitId,
|
|
1119
|
+
runId: receipt.binding.runId,
|
|
1120
|
+
candidateDigest: receipt.binding.candidateDigest,
|
|
1121
|
+
evaluatorDigest: receipt.binding.evaluatorDigest,
|
|
1122
|
+
environmentDigest: receipt.binding.environmentDigest,
|
|
1123
|
+
inputSetCommitment: receipt.binding.inputSetCommitment,
|
|
1124
|
+
outputDigest: receipt.binding.outputDigest,
|
|
1125
|
+
resultDigest: receipt.binding.resultDigest
|
|
1126
|
+
})) requiredIdentity(value, field);
|
|
1127
|
+
requiredAuthority(receipt.binding.authority);
|
|
1128
|
+
assertInputCommitment(receipt.attestation.provenance, receipt.binding.inputSetCommitment);
|
|
1129
|
+
} catch (error) {
|
|
1130
|
+
return {
|
|
1131
|
+
valid: false,
|
|
1132
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
1133
|
+
};
|
|
1134
|
+
}
|
|
1135
|
+
const verification = verifyAttestation(receipt.binding, receipt.attestation);
|
|
1136
|
+
if (!verification.valid) return verification;
|
|
1137
|
+
if (verification.legacyUnboundProvenance === true || receipt.attestation.envelopeHash === void 0) return {
|
|
1138
|
+
valid: false,
|
|
1139
|
+
reason: "evidence receipt provenance is not bound by an attestation envelope"
|
|
1140
|
+
};
|
|
1141
|
+
return { valid: true };
|
|
1142
|
+
}
|
|
1143
|
+
/**
|
|
1144
|
+
* Promotion may choose a stricter policy, but this primitive makes the basic separation
|
|
1145
|
+
* explicit: only a recognized independent authority is independent. Unknown/forged kinds
|
|
1146
|
+
* and candidate self-reports both return false.
|
|
1147
|
+
*/
|
|
1148
|
+
function isIndependentEvidence(receipt) {
|
|
1149
|
+
return INDEPENDENT_EVIDENCE_AUTHORITY_KINDS.includes(receipt.binding.authority?.kind);
|
|
1150
|
+
}
|
|
1151
|
+
function requiredAuthority(value) {
|
|
1152
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new TypeError("evidence receipt: authority must be an object");
|
|
1153
|
+
const authority = value;
|
|
1154
|
+
if (typeof authority.kind !== "string" || !EVIDENCE_AUTHORITY_KINDS.includes(authority.kind)) throw new TypeError(`evidence receipt: unknown authority kind '${String(authority.kind)}'`);
|
|
1155
|
+
if (typeof authority.id !== "string") throw new TypeError("evidence receipt: authority.id must be a string");
|
|
1156
|
+
return Object.freeze({
|
|
1157
|
+
kind: authority.kind,
|
|
1158
|
+
id: requiredIdentity(authority.id, "authority.id")
|
|
1159
|
+
});
|
|
1160
|
+
}
|
|
1161
|
+
function assertInputCommitment(provenance, inputSetCommitment) {
|
|
1162
|
+
if (typeof provenance?.inputsHash !== "string" || provenance.inputsHash.trim().length === 0) throw new TypeError("evidence receipt: provenance.inputsHash is required");
|
|
1163
|
+
if (provenance.inputsHash.trim() !== inputSetCommitment) throw new TypeError("evidence receipt: provenance.inputsHash must equal binding.inputSetCommitment");
|
|
1164
|
+
}
|
|
1165
|
+
function requiredIdentity(value, field) {
|
|
1166
|
+
const normalized = value.trim();
|
|
1167
|
+
if (normalized.length === 0) throw new TypeError(`evidence receipt: ${field} must be non-empty`);
|
|
1168
|
+
return normalized;
|
|
1169
|
+
}
|
|
1170
|
+
//#endregion
|
|
955
1171
|
//#region src/experiment/power.ts
|
|
956
1172
|
/**
|
|
957
1173
|
* Design-time power for task-clustered paired designs, with a refusal verdict.
|
|
@@ -1120,6 +1336,6 @@ function mixSeed(seed, effect) {
|
|
|
1120
1336
|
return (seed ^ Math.round(effect * 1000003) * 2654435769) >>> 0 | 0;
|
|
1121
1337
|
}
|
|
1122
1338
|
//#endregion
|
|
1123
|
-
export { BOOTSTRAP_GATE_MIN_N, DecisionTableNotTotalError, DesignRefusalError, ExperimentTracker, FunnelIntegrityError, MatchedBudgetError, SealIntegrityError, amendExperiment, assertDesignAdequate, assertFunnelReconciles, assertMatchedBudgets, benjaminiHochberg, bonferroni, buildEvidenceVector, buildFunnel, canonicalize, classifyReissue, clusteredPower, comparePairedArms, composeFunnels, computeEstimand, computeInterval, defineExperiment, eProcess, evaluateCondition, evaluateHaltRule, evaluateHypothesis, evaluateIdentityGate, evaluateOracleDeterminismGate, evaluatePopulationReproducibilityGate, evaluatePowerFloorGate, evaluatePredicate, evaluateProvenanceGate, executeAdmissionRule, executeDecisionRule, fileExperimentStore, hashJson, heldoutSignificance, holm, inMemoryExperimentStore, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openSealedExperiment, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, powerPreflight, projectNLadderBudget, readField, renderFunnelTable, requiredPairedSampleSize, requiredSampleSize, runSelectionRule, runUniformPassBudget, sealExperiment, sequentialCrossingHorizon, sequentialDecide, sequentialPairedGate, signManifest, verifyManifest, verifyMatchedBudgets, verifySealedExperiment, wilson };
|
|
1339
|
+
export { BOOTSTRAP_GATE_MIN_N, DecisionTableNotTotalError, DesignRefusalError, EVIDENCE_AUTHORITY_KINDS, EVIDENCE_RECEIPT_VERSION, ExperimentTracker, FunnelIntegrityError, INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, MatchedBudgetError, SealIntegrityError, amendExperiment, assertDesignAdequate, assertFunnelReconciles, assertMatchedBudgets, benjaminiHochberg, bonferroni, buildEvidenceVector, buildFunnel, canonicalize, classifyReissue, clusteredPower, comparePairedArms, composeFunnels, computeEstimand, computeInterval, createEvidenceReceipt, defineExperiment, eProcess, evaluateCondition, evaluateHaltRule, evaluateHypothesis, evaluateIdentityGate, evaluateOracleDeterminismGate, evaluatePopulationReproducibilityGate, evaluatePowerFloorGate, evaluatePredicate, evaluateProvenanceGate, executeAdmissionRule, executeDecisionRule, fileExperimentStore, hashJson, heldoutSignificance, holm, inMemoryExperimentStore, isIndependentEvidence, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openSealedExperiment, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, powerPreflight, projectNLadderBudget, readField, renderFunnelTable, requiredPairedSampleSize, requiredSampleSize, runSelectionRule, runUniformPassBudget, sealExperiment, sequentialCrossingHorizon, sequentialDecide, sequentialPairedGate, signManifest, verifyEvidenceReceipt, verifyManifest, verifyMatchedBudgets, verifySealedExperiment, wilson };
|
|
1124
1340
|
|
|
1125
1341
|
//# sourceMappingURL=index.js.map
|