@tangle-network/agent-eval 0.187.1 → 0.187.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/analyst/index.d.ts +1 -1
- package/dist/analyst/index.js +1 -1
- package/dist/{benchmark-command-CERRH0E6.js → benchmark-command-DGokT8zd.js} +2 -2
- package/dist/{benchmark-command-CERRH0E6.js.map → benchmark-command-DGokT8zd.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/experiment/index.d.ts +55 -1
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +119 -1
- package/dist/experiment/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/docs/experiment.md +8 -0
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { i as runRolloutReleaseCli } from "./hf-dataset-CRqtSjnR.js";
|
|
3
3
|
import { t as createChatClient } from "./chat-client-DqPqWzVV.js";
|
|
4
|
-
import { t as runAnalystBenchmarkCommand } from "./benchmark-command-
|
|
4
|
+
import { t as runAnalystBenchmarkCommand } from "./benchmark-command-DGokT8zd.js";
|
|
5
5
|
import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-DmpcFiVK.js";
|
|
6
6
|
import { t as runSupervisorRunCommand } from "./report-command-CSTjzFGY.js";
|
|
7
7
|
import { writeFileSync } from "node:fs";
|
|
@@ -609,5 +609,59 @@ declare function sealRandomizedCanaryRule(rule: RandomizedCanaryRule): SealedRan
|
|
|
609
609
|
/** Apply one registered analysis to the entire frozen assignment roster. */
|
|
610
610
|
declare function decideRandomizedCanary(sealed: SealedRandomizedCanaryRule, cohort: CanaryCohortReceipt, rows: readonly CanaryObservation[], verifyCohort: CanaryCohortVerifier): RandomizedCanaryDecision;
|
|
611
611
|
//#endregion
|
|
612
|
-
|
|
612
|
+
//#region src/experiment/prospective-canary-join.d.ts
|
|
613
|
+
/** Host-ledger assignment written when an eligible request is routed. */
|
|
614
|
+
interface ProspectiveCanaryAssignment extends CanaryAssignmentReceipt {
|
|
615
|
+
sourceUnitId: string;
|
|
616
|
+
assignedAt: string;
|
|
617
|
+
}
|
|
618
|
+
/** Independent checker result, including a null when the outcome is unavailable. */
|
|
619
|
+
interface ProspectiveCanaryOutcome {
|
|
620
|
+
assignmentId: string;
|
|
621
|
+
outcomeSourceId: string;
|
|
622
|
+
checkedOutcome: 0 | 1 | null;
|
|
623
|
+
outcomeCheckerDigest: string;
|
|
624
|
+
checkedAt: string;
|
|
625
|
+
}
|
|
626
|
+
/** Served trace, or an independent receipt that the assigned unit never executed. */
|
|
627
|
+
interface ProspectiveCanaryExecution {
|
|
628
|
+
assignmentId: string;
|
|
629
|
+
traceSourceId: string | null;
|
|
630
|
+
noExecutionSourceId: string | null;
|
|
631
|
+
servedProfileDigest: string | null;
|
|
632
|
+
servedCodeRevisionDigest: string | null;
|
|
633
|
+
turns: number | null;
|
|
634
|
+
providerCallIds: readonly string[];
|
|
635
|
+
recordedAt: string;
|
|
636
|
+
}
|
|
637
|
+
/** Billing authority's settled snapshot for every provider call in one assignment. */
|
|
638
|
+
interface ProspectiveCanaryBilling {
|
|
639
|
+
assignmentId: string;
|
|
640
|
+
billingSourceId: string;
|
|
641
|
+
settlement: 'settled' | 'unknown';
|
|
642
|
+
billedCostUsd: number | null;
|
|
643
|
+
providerCallIds: readonly string[];
|
|
644
|
+
recordedAt: string;
|
|
645
|
+
}
|
|
646
|
+
interface ProspectiveCanaryJoinInput {
|
|
647
|
+
/** An opt-in host flag. Calling this function never routes product traffic. */
|
|
648
|
+
enabled?: boolean;
|
|
649
|
+
sealed: SealedRandomizedCanaryRule;
|
|
650
|
+
/** The host must independently verify this witness predates every assignment. */
|
|
651
|
+
protocolWitnessedAt: string;
|
|
652
|
+
/** The complete prior-source roster must be attested by the host verifier. */
|
|
653
|
+
historicalSourceUnitIds: readonly string[];
|
|
654
|
+
assignments: readonly ProspectiveCanaryAssignment[];
|
|
655
|
+
outcomes: readonly ProspectiveCanaryOutcome[];
|
|
656
|
+
executions: readonly ProspectiveCanaryExecution[];
|
|
657
|
+
billing: readonly ProspectiveCanaryBilling[];
|
|
658
|
+
}
|
|
659
|
+
/**
|
|
660
|
+
* Join independently sourced served receipts without inventing missing costs.
|
|
661
|
+
* The host still verifies ledger completeness, source authority, isolation,
|
|
662
|
+
* randomization and the pre-traffic witness in `decideRandomizedCanary`.
|
|
663
|
+
*/
|
|
664
|
+
declare function joinProspectiveCanaryReceipts(input: ProspectiveCanaryJoinInput): CanaryObservation[];
|
|
665
|
+
//#endregion
|
|
666
|
+
export { type AdmissionExecution, type AdmissionPartition, type AdmissionRule, type AdmissionStage, type ArmRealizedBudget, type ArmSpec, BOOTSTRAP_GATE_MIN_N, type BudgetRule, type CampaignEvidenceContext, type CanaryArm, type CanaryAssignmentReceipt, type CanaryCohortReceipt, type CanaryCohortVerifier, type CanaryObservation, type CanaryRefusal, type ClusteredPowerOptions, type ClusteredPowerPoint, type ClusteredPowerRefusal, type ClusteredPowerResult, type ComputedInterval, type Condition, type CreateEvidenceReceiptInput, type DecisionBranch, type DecisionOutcome, type DecisionRule, type DerivedQuantities, DesignRefusalError, type EProcess, type EProcessOptions, type EProcessState, type EProcessStep, EVIDENCE_AUTHORITY_KINDS, EVIDENCE_RECEIPT_VERSION, EVIDENCE_STATES, type Estimand, type EstimandResult, type EvaluationClaim, type EvaluationUnitSummary, type EvidenceAuthority, type EvidenceAuthorityKind, type EvidenceBinding, type EvidenceDenominator, type EvidenceReceipt, type EvidenceReceiptVerification, type EvidenceRecord, type EvidenceRegistryRecord, type EvidenceState, type EvidenceVector, type Experiment, type ExperimentFunnel, type ExperimentRep, type ExperimentSpec, type ExperimentStats, ExperimentTracker, type ExperimentVerdict, FinalEvidenceConflictError, FinalEvidenceError, type FinalEvidenceLedger, type FinalEvidenceMeasurement, type FinalEvidenceOutcome, type FinalEvidenceRecord, type FinalEvidenceReservation, FunnelIntegrityError, type FunnelPartitionCount, type FunnelStageCount, type FunnelStageInput, type GateEvidence, type GateResult, type HaltOutcome, type HaltRule, type HeldoutSignificance, type HeldoutSignificanceOptions, type HypothesisManifest, type HypothesisResult, INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, type IntervalSpec, type JsonValue, MatchedBudgetError, type MatchedBudgetRule, type MatchedBudgetVerdict, type MatchedPair, type NLadderProjection, type Obligation, type OutcomeSpec, type PairArmsOptions, type PairArmsResult, type PairedArmRow, type PairedBootstrapOptions, type PairedBootstrapResult, type PairedEvalueOptions, type PairedEvalueSequence, type PairedEvalueStep, type PairedHoldout, type PowerPreflight, type PowerPreflightOptions, type Predicate, type PromotionPolicy, type ProportionInterval, type ProspectiveCanaryAssignment, type ProspectiveCanaryBilling, type ProspectiveCanaryExecution, type ProspectiveCanaryJoinInput, type ProspectiveCanaryOutcome, type RandomizedCanaryDecision, type RandomizedCanaryRule, type RegisteredExperiment, type ReissuePolicy, type ReissueVerdict, type SealAmendment, SealIntegrityError, type SealedExperiment, type SealedRandomizedCanaryRule, type SeedDerivation, type SelectionRule, type SequentialCrossingHorizon, type SequentialCrossingHorizonOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SetExpr, type SignFlipFloor, type SignedManifest, type SignedManifestAlgo, type UniformPassDecision, type UniformPassSchedule, type ValidityGate, amendExperiment, assertDesignAdequate, assertFunnelReconciles, assertMatchedBudgets, benjaminiHochberg, bonferroni, buildEvidenceVector, buildFunnel, classifyReissue, clusteredPower, comparePairedArms, composeFunnels, computeEstimand, computeInterval, createCampaignEvidenceReceipt, createEvidenceReceipt, decideRandomizedCanary, defineEvaluationClaim, defineExperiment, eProcess, evaluateCondition, evaluateHaltRule, evaluateHypothesis, evaluateIdentityGate, evaluateOracleDeterminismGate, evaluatePopulationReproducibilityGate, evaluatePowerFloorGate, evaluatePredicate, evaluateProvenanceGate, evidenceRegistryRecordSchema, executeAdmissionRule, executeDecisionRule, fileExperimentStore, hashJson, heldoutSignificance, holm, inMemoryExperimentStore, isIndependentEvidence, joinProspectiveCanaryReceipts, manifestContentDigest, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openFinalEvidenceLedger, openSealedExperiment, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, parseEvidenceRegistryRecord, powerPreflight, projectNLadderBudget, randomizedCanaryObservationDigest, randomizedCanaryRosterDigest, readField, renderEvidenceIndex, renderFunnelTable, requiredPairedSampleSize, requiredSampleSize, runSelectionRule, runUniformPassBudget, sealExperiment, sealRandomizedCanaryRule, sequentialCrossingHorizon, sequentialDecide, sequentialPairedGate, signManifest, summarizeEvaluationUnits, validateEvidenceRegistry, verifyEvidenceReceipt, verifyManifest, verifyMatchedBudgets, verifySealedExperiment, wilson };
|
|
613
667
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/experiment/budget.ts","../../src/experiment/funnel.ts","../../src/experiment/define.ts","../../src/experiment/evidence-record.ts","../../src/experiment/power.ts","../../src/experiment/randomized-canary.ts"],"mappings":";;;;;;;;;;cAaa,2BAA2B;UAEvB;EACf;;EAEA;;UAGe;EACf,MAAM;EACN,MAAM;;EAEN;;EAEA;EACA;;EAEA;IAAkB;IAA2B;;;;;;;;iBAQ/B,qBACd,MAAM,mBACN,eAAe,sBACd;;iBAsCa,qBACd,MAAM,mBACN,eAAe,sBACd;;;;cC/DU,6BAA6B;UAEzB;EACf;EACA;EACA;EACA;;EAEA,aAAa;;EAEb;;UAGe;EACf;;EAEA;EACA;;EAEA;;UAGe;EACf;EACA;EACA,QAAQ;EACR;EACA,YAAY;;UAGG;EACf;;EAEA;EACA,aAAa;EACb;;;;;;;;;;;iBAYc,YAAY;EAC1B;EACA;EACA,QAAQ;EACR;IAAe;IAAY;IAAc;;IACvC;;iBAsEY,uBAAuB,QAAQ;UAwB9B;EACf,QAAQ;EACR,WAAW;;EAEX,eAAe,eAAe;;;;;;;;iBAShB,qBACd,MAAM,eACN,kBAAkB,mBACjB;;;;;;iBA8Ca,eACd,OAAO,kBACP,QAAQ,mBACP;;;;;;iBA4Ba,kBAAkB,QAAQ;;;;cC/L7B,2BAA2B;UAIvB;EACf;EACA;;EAEA;;EAEA;;EAEA,OAAO,eAAe;;KAGZ;EAEN;;EAEA;EACA;;EAEA;;EAEA;;EAGA;EACA;EACA;EACA;;;;;;UAOW;EACf;;UAGe;EACf;;EAEA,QAAQ;;EAER;EACA,MAAM;EACN,SAAS;EACT,YAAY;;EAEZ,aAAa,eAAe;;;;;EAK5B,gBAAgB;EAChB,YAAY,eAAe;EAC3B,YAAY,eAAe;EAC3B,UAAU;EACV,cAAc;;EAEd,QAAQ,eAAe;EACvB,OAAO;EACP,SAAS;EACT,gBAAgB;EAChB,UAAU;EACV,iBAAiB;EACjB;;;;;;;;;;iBAwCc,iBAAiB,MAAM,iBAAiB;UA0IvC;;EAEf;EACA;;EAEA;;EAEA;;;KAIU;UAEK;EACf,MAAM;;EAEN;;EAEA,MAAM;EACN;;EAEA;EACA,YAAY;;;iBAIQ,eACpB,MAAM,gBACN;EAAW;IACV,QAAQ;;;;;;iBAkBW,gBACpB,QAAQ,kBACR;EAAa,MAAM;EAAgB;EAAgB;EAAiB;IACnE,QAAQ;;iBAyBW,uBAAuB,QAAQ,mBAAmB;;KAwB5D;EACN;EAA4B,aAAa;;EAEzC;EACA,eAAe;EACf,gBAAgB;;EAEhB;EAA8B,YAAY;;EAC1C;EAAkB;EAAgB;;EAClC;EAAqB;IAAkB;IAAgB;;;;;;;UAM5C;WACN,QAAQ;;EAEjB,MAAM,kBAAkB,mBAAmB;;EAE3C,OAAO,UAAU,oBAAoB;;EAErC,MAAM,kBAAkB,mBAAmB;;EAE3C,OAAO,cAAc,kBAAkB,kBAAkB;IAAW;;;EAEpE,KAAK,cAAc,UAAU,eAAe;;EAE5C,KAAK,gBAAgB,eAAe;;EAEpC,qBAAqB,uCAAuC;;EAE5D,qBAAqB;IAAY;IAAqB;MAAiB;;EAEvE,eAAe,eAAe,sBAAsB;;EAEpD,SAAS,cAAc,eAAe,mBAAmB;;EAEzD,SACE,cACA;IACM;IAAc,eAAe;;IAC7B;IAAkB;IAAmB;IAAgB;MAC1D;IAAqB,QAAQ;;;;;;;;iBAQZ,qBACpB,QAAQ,mBACP,QAAQ;;;;;;;;;;;;;;;cC5aE;KAQD,wBAAwB;;cAO9B,2BAAyB,EAAA;;;;GAM7B,EAAA,KAAA;KAEU,sBAAsB,EAAE,aAAa;cAEpC,8BAA4B,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCvC,EAAA,KAAA;KAEU,yBAAyB,EAAE,aAAa;;iBAGpC,4BAA4B,eAAe;;;;;;iBAW3C,yBAAyB,2BAA2B;;;;;;iBA4CpD,oBAAoB;;;;cCpIvB,2BAA2B;UAEvB;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA;IAAkB;IAAe;;;UAGlB;EACf;EACA;EACA;;UAGe;;EAEf;;EAEA;EACA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;;EAEP;EACA,eAAe;;EAEf;;EAEA,SAAS;;;;;;iBAOK,eAAe,SAAS,wBAAwB;;iBAgKhD,qBAAqB,QAAQ;;;;KCvQjC;;UAGK;EACf;EACA;EACA;EACA,KAAK;;UAGU,0BAA0B;;EAEzC;EACA;;EAEA;EACA;;EAEA;;EAEA;;EAEA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;EAEA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;;KAIU,wBACV,SAAS,qBACT,UAAU;EAGN;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAiB;;UAEN;EACf,MAAM;EACN;EACA;;KAGU;UASK;EACf;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU,OACR;IAEE;IACA;IACA;IACA;;EAGJ;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IAAqB;IAAa;;EAClC;EACA;IAA0B;IAAa;;EACvC;;EAEA;EACA;EACA;EACA;EACA;EACA,SAAS;;EAET;;;iBAwBc,6BAA6B,eAAe;;iBA6B5C,kCAAkC,eAAe;;iBAiFjD,yBAAyB,MAAM,uBAAuB;;iBAOtD,uBACd,QAAQ,4BACR,QAAQ,qBACR,eAAe,qBACf,cAAc,uBACb"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/experiment/budget.ts","../../src/experiment/funnel.ts","../../src/experiment/define.ts","../../src/experiment/evidence-record.ts","../../src/experiment/power.ts","../../src/experiment/randomized-canary.ts","../../src/experiment/prospective-canary-join.ts"],"mappings":";;;;;;;;;;cAaa,2BAA2B;UAEvB;EACf;;EAEA;;UAGe;EACf,MAAM;EACN,MAAM;;EAEN;;EAEA;EACA;;EAEA;IAAkB;IAA2B;;;;;;;;iBAQ/B,qBACd,MAAM,mBACN,eAAe,sBACd;;iBAsCa,qBACd,MAAM,mBACN,eAAe,sBACd;;;;cC/DU,6BAA6B;UAEzB;EACf;EACA;EACA;EACA;;EAEA,aAAa;;EAEb;;UAGe;EACf;;EAEA;EACA;;EAEA;;UAGe;EACf;EACA;EACA,QAAQ;EACR;EACA,YAAY;;UAGG;EACf;;EAEA;EACA,aAAa;EACb;;;;;;;;;;;iBAYc,YAAY;EAC1B;EACA;EACA,QAAQ;EACR;IAAe;IAAY;IAAc;;IACvC;;iBAsEY,uBAAuB,QAAQ;UAwB9B;EACf,QAAQ;EACR,WAAW;;EAEX,eAAe,eAAe;;;;;;;;iBAShB,qBACd,MAAM,eACN,kBAAkB,mBACjB;;;;;;iBA8Ca,eACd,OAAO,kBACP,QAAQ,mBACP;;;;;;iBA4Ba,kBAAkB,QAAQ;;;;cC/L7B,2BAA2B;UAIvB;EACf;EACA;;EAEA;;EAEA;;EAEA,OAAO,eAAe;;KAGZ;EAEN;;EAEA;EACA;;EAEA;;EAEA;;EAGA;EACA;EACA;EACA;;;;;;UAOW;EACf;;UAGe;EACf;;EAEA,QAAQ;;EAER;EACA,MAAM;EACN,SAAS;EACT,YAAY;;EAEZ,aAAa,eAAe;;;;;EAK5B,gBAAgB;EAChB,YAAY,eAAe;EAC3B,YAAY,eAAe;EAC3B,UAAU;EACV,cAAc;;EAEd,QAAQ,eAAe;EACvB,OAAO;EACP,SAAS;EACT,gBAAgB;EAChB,UAAU;EACV,iBAAiB;EACjB;;;;;;;;;;iBAwCc,iBAAiB,MAAM,iBAAiB;UA0IvC;;EAEf;EACA;;EAEA;;EAEA;;;KAIU;UAEK;EACf,MAAM;;EAEN;;EAEA,MAAM;EACN;;EAEA;EACA,YAAY;;;iBAIQ,eACpB,MAAM,gBACN;EAAW;IACV,QAAQ;;;;;;iBAkBW,gBACpB,QAAQ,kBACR;EAAa,MAAM;EAAgB;EAAgB;EAAiB;IACnE,QAAQ;;iBAyBW,uBAAuB,QAAQ,mBAAmB;;KAwB5D;EACN;EAA4B,aAAa;;EAEzC;EACA,eAAe;EACf,gBAAgB;;EAEhB;EAA8B,YAAY;;EAC1C;EAAkB;EAAgB;;EAClC;EAAqB;IAAkB;IAAgB;;;;;;;UAM5C;WACN,QAAQ;;EAEjB,MAAM,kBAAkB,mBAAmB;;EAE3C,OAAO,UAAU,oBAAoB;;EAErC,MAAM,kBAAkB,mBAAmB;;EAE3C,OAAO,cAAc,kBAAkB,kBAAkB;IAAW;;;EAEpE,KAAK,cAAc,UAAU,eAAe;;EAE5C,KAAK,gBAAgB,eAAe;;EAEpC,qBAAqB,uCAAuC;;EAE5D,qBAAqB;IAAY;IAAqB;MAAiB;;EAEvE,eAAe,eAAe,sBAAsB;;EAEpD,SAAS,cAAc,eAAe,mBAAmB;;EAEzD,SACE,cACA;IACM;IAAc,eAAe;;IAC7B;IAAkB;IAAmB;IAAgB;MAC1D;IAAqB,QAAQ;;;;;;;;iBAQZ,qBACpB,QAAQ,mBACP,QAAQ;;;;;;;;;;;;;;;cC5aE;KAQD,wBAAwB;;cAO9B,2BAAyB,EAAA;;;;GAM7B,EAAA,KAAA;KAEU,sBAAsB,EAAE,aAAa;cAEpC,8BAA4B,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCvC,EAAA,KAAA;KAEU,yBAAyB,EAAE,aAAa;;iBAGpC,4BAA4B,eAAe;;;;;;iBAW3C,yBAAyB,2BAA2B;;;;;;iBA4CpD,oBAAoB;;;;cCpIvB,2BAA2B;UAEvB;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;EAMA;IAAkB;IAAe;;;UAGlB;EACf;EACA;EACA;;UAGe;;EAEf;;EAEA;EACA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,OAAO;;EAEP;EACA,eAAe;;EAEf;;EAEA,SAAS;;;;;;iBAOK,eAAe,SAAS,wBAAwB;;iBAgKhD,qBAAqB,QAAQ;;;;KCvQjC;;UAGK;EACf;EACA;EACA;EACA,KAAK;;UAGU,0BAA0B;;EAEzC;EACA;;EAEA;EACA;;EAEA;;EAEA;;EAEA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;EAEA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;;KAIU,wBACV,SAAS,qBACT,UAAU;EAGN;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;EAEA;EAAiB;;UAEN;EACf,MAAM;EACN;EACA;;KAGU;UASK;EACf;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU,OACR;IAEE;IACA;IACA;IACA;;EAGJ;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IAAqB;IAAa;;EAClC;EACA;IAA0B;IAAa;;EACvC;;EAEA;EACA;EACA;EACA;EACA;EACA,SAAS;;EAET;;;iBAwBc,6BAA6B,eAAe;;iBA6B5C,kCAAkC,eAAe;;iBAiFjD,yBAAyB,MAAM,uBAAuB;;iBAOtD,uBACd,QAAQ,4BACR,QAAQ,qBACR,eAAe,qBACf,cAAc,uBACb;;;;UC3Sc,oCAAoC;EACnD;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;EACA,QAAQ;;EAER;;EAEA;EACA,sBAAsB;EACtB,mBAAmB;EACnB,qBAAqB;EACrB,kBAAkB;;;;;;;iBA4DJ,8BACd,OAAO,6BACN"}
|
package/dist/experiment/index.js
CHANGED
|
@@ -851,6 +851,124 @@ function mixSeed(seed, effect) {
|
|
|
851
851
|
return (seed ^ Math.round(effect * 1000003) * 2654435769) >>> 0 | 0;
|
|
852
852
|
}
|
|
853
853
|
//#endregion
|
|
854
|
+
//#region src/experiment/prospective-canary-join.ts
|
|
855
|
+
function nonEmpty$1(value, label) {
|
|
856
|
+
if (typeof value !== "string" || value.length === 0 || value.trim() !== value) throw new ValidationError(`prospective canary: ${label} must be a nonempty trimmed string`);
|
|
857
|
+
}
|
|
858
|
+
function utc(value, label) {
|
|
859
|
+
nonEmpty$1(value, label);
|
|
860
|
+
const parsed = Date.parse(value);
|
|
861
|
+
const canonical = Number.isFinite(parsed) ? new Date(parsed).toISOString() : "";
|
|
862
|
+
if (!value.endsWith("Z") || !Number.isFinite(parsed) || value !== canonical && value !== canonical.replace(".000Z", "Z")) throw new ValidationError(`prospective canary: ${label} must be a canonical UTC timestamp`);
|
|
863
|
+
return parsed;
|
|
864
|
+
}
|
|
865
|
+
function indexRows(rows, label, assignments) {
|
|
866
|
+
const indexed = /* @__PURE__ */ new Map();
|
|
867
|
+
for (const row of rows) {
|
|
868
|
+
nonEmpty$1(row.assignmentId, `${label} assignmentId`);
|
|
869
|
+
if (!assignments.has(row.assignmentId)) throw new ValidationError(`prospective canary: orphan ${label} for '${row.assignmentId}'`);
|
|
870
|
+
if (indexed.has(row.assignmentId)) throw new ValidationError(`prospective canary: duplicate ${label} for '${row.assignmentId}'`);
|
|
871
|
+
indexed.set(row.assignmentId, row);
|
|
872
|
+
}
|
|
873
|
+
return indexed;
|
|
874
|
+
}
|
|
875
|
+
function uniqueIds(ids, label) {
|
|
876
|
+
if (!Array.isArray(ids)) throw new ValidationError(`prospective canary: ${label} must be an array`);
|
|
877
|
+
const seen = /* @__PURE__ */ new Set();
|
|
878
|
+
for (const id of ids) {
|
|
879
|
+
nonEmpty$1(id, label);
|
|
880
|
+
if (seen.has(id)) throw new ValidationError(`prospective canary: duplicate ${label} '${id}'`);
|
|
881
|
+
seen.add(id);
|
|
882
|
+
}
|
|
883
|
+
return [...seen].sort();
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* Join independently sourced served receipts without inventing missing costs.
|
|
887
|
+
* The host still verifies ledger completeness, source authority, isolation,
|
|
888
|
+
* randomization and the pre-traffic witness in `decideRandomizedCanary`.
|
|
889
|
+
*/
|
|
890
|
+
function joinProspectiveCanaryReceipts(input) {
|
|
891
|
+
if (input.enabled !== true) throw new ValidationError("prospective canary: pilot is disabled");
|
|
892
|
+
const witnessedAt = utc(input.protocolWitnessedAt, "protocolWitnessedAt");
|
|
893
|
+
const cutoff = utc(input.sealed.rule.analysisCutoff, "analysisCutoff");
|
|
894
|
+
if (witnessedAt >= cutoff) throw new ValidationError("prospective canary: protocol witness must precede cutoff");
|
|
895
|
+
if (!Array.isArray(input.assignments) || input.assignments.length === 0) throw new ValidationError("prospective canary: assignment ledger is empty");
|
|
896
|
+
const historical = new Set(uniqueIds(input.historicalSourceUnitIds, "historical source unit"));
|
|
897
|
+
const assignmentIds = /* @__PURE__ */ new Set();
|
|
898
|
+
const assignmentSources = /* @__PURE__ */ new Set();
|
|
899
|
+
const sourceUnits = /* @__PURE__ */ new Set();
|
|
900
|
+
for (const row of input.assignments) {
|
|
901
|
+
nonEmpty$1(row.assignmentId, "assignmentId");
|
|
902
|
+
nonEmpty$1(row.assignmentSourceId, "assignmentSourceId");
|
|
903
|
+
nonEmpty$1(row.clusterId, "clusterId");
|
|
904
|
+
nonEmpty$1(row.sourceUnitId, "sourceUnitId");
|
|
905
|
+
if (row.arm !== "control" && row.arm !== "candidate") throw new ValidationError("prospective canary: assignment arm is invalid");
|
|
906
|
+
if (assignmentIds.has(row.assignmentId) || assignmentSources.has(row.assignmentSourceId)) throw new ValidationError("prospective canary: duplicate assignment identity");
|
|
907
|
+
if (sourceUnits.has(row.sourceUnitId) || historical.has(row.sourceUnitId)) throw new ValidationError(`prospective canary: source unit '${row.sourceUnitId}' is reused`);
|
|
908
|
+
const assignedAt = utc(row.assignedAt, `assignment '${row.assignmentId}' assignedAt`);
|
|
909
|
+
if (assignedAt <= witnessedAt || assignedAt > cutoff) throw new ValidationError("prospective canary: assignment falls outside witnessed window");
|
|
910
|
+
assignmentIds.add(row.assignmentId);
|
|
911
|
+
assignmentSources.add(row.assignmentSourceId);
|
|
912
|
+
sourceUnits.add(row.sourceUnitId);
|
|
913
|
+
}
|
|
914
|
+
const outcomes = indexRows(input.outcomes, "outcome", assignmentIds);
|
|
915
|
+
const executions = indexRows(input.executions, "execution", assignmentIds);
|
|
916
|
+
const billing = indexRows(input.billing, "billing", assignmentIds);
|
|
917
|
+
const sourceIds = new Set(assignmentSources);
|
|
918
|
+
const claimSource = (id, label) => {
|
|
919
|
+
nonEmpty$1(id, label);
|
|
920
|
+
if (sourceIds.has(id)) throw new ValidationError(`prospective canary: reused independent source '${id}'`);
|
|
921
|
+
sourceIds.add(id);
|
|
922
|
+
};
|
|
923
|
+
return input.assignments.map((assignment) => {
|
|
924
|
+
const outcome = outcomes.get(assignment.assignmentId);
|
|
925
|
+
const execution = executions.get(assignment.assignmentId);
|
|
926
|
+
const bill = billing.get(assignment.assignmentId);
|
|
927
|
+
const assignedAt = Date.parse(assignment.assignedAt);
|
|
928
|
+
if (outcome) {
|
|
929
|
+
claimSource(outcome.outcomeSourceId, "outcomeSourceId");
|
|
930
|
+
nonEmpty$1(outcome.outcomeCheckerDigest, "outcomeCheckerDigest");
|
|
931
|
+
if (utc(outcome.checkedAt, "checkedAt") < assignedAt) throw new ValidationError("prospective canary: outcome precedes assignment");
|
|
932
|
+
if (![
|
|
933
|
+
0,
|
|
934
|
+
1,
|
|
935
|
+
null
|
|
936
|
+
].includes(outcome.checkedOutcome)) throw new ValidationError("prospective canary: checked outcome must be 0, 1 or null");
|
|
937
|
+
}
|
|
938
|
+
if (execution) {
|
|
939
|
+
if (execution.traceSourceId === null === (execution.noExecutionSourceId === null)) throw new ValidationError("prospective canary: execution needs one trace or no-execution source");
|
|
940
|
+
claimSource(execution.traceSourceId ?? execution.noExecutionSourceId, "execution source");
|
|
941
|
+
if (utc(execution.recordedAt, "execution recordedAt") < assignedAt) throw new ValidationError("prospective canary: execution precedes assignment");
|
|
942
|
+
if (execution.turns !== null && (!Number.isSafeInteger(execution.turns) || execution.turns < 0)) throw new ValidationError("prospective canary: turns must be a nonnegative integer");
|
|
943
|
+
if (execution.noExecutionSourceId !== null && (execution.turns !== 0 || execution.providerCallIds.length !== 0)) throw new ValidationError("prospective canary: no-execution receipt contains turns or calls");
|
|
944
|
+
}
|
|
945
|
+
if (bill) {
|
|
946
|
+
claimSource(bill.billingSourceId, "billingSourceId");
|
|
947
|
+
if (utc(bill.recordedAt, "billing recordedAt") < assignedAt) throw new ValidationError("prospective canary: billing precedes assignment");
|
|
948
|
+
if (bill.settlement === "settled") {
|
|
949
|
+
if (bill.billedCostUsd === null || !Number.isFinite(bill.billedCostUsd) || bill.billedCostUsd < 0) throw new ValidationError("prospective canary: settled billing needs nonnegative actual cost");
|
|
950
|
+
} else if (bill.settlement !== "unknown" || bill.billedCostUsd !== null) throw new ValidationError("prospective canary: unknown billing cost must stay null");
|
|
951
|
+
if (execution && uniqueIds(bill.providerCallIds, "billing call").join("\0") !== uniqueIds(execution.providerCallIds, "execution call").join("\0")) throw new ValidationError("prospective canary: provider-call billing join is incomplete");
|
|
952
|
+
}
|
|
953
|
+
return {
|
|
954
|
+
assignmentId: assignment.assignmentId,
|
|
955
|
+
assignmentSourceId: assignment.assignmentSourceId,
|
|
956
|
+
clusterId: assignment.clusterId,
|
|
957
|
+
arm: assignment.arm,
|
|
958
|
+
checkedOutcome: outcome?.checkedOutcome ?? null,
|
|
959
|
+
outcomeSourceId: outcome?.outcomeSourceId ?? null,
|
|
960
|
+
billedCostUsd: bill?.settlement === "settled" ? bill.billedCostUsd : null,
|
|
961
|
+
billingSourceId: bill?.billingSourceId ?? null,
|
|
962
|
+
traceSourceId: execution?.traceSourceId ?? null,
|
|
963
|
+
noExecutionSourceId: execution?.noExecutionSourceId ?? null,
|
|
964
|
+
servedProfileDigest: execution?.servedProfileDigest ?? null,
|
|
965
|
+
servedCodeRevisionDigest: execution?.servedCodeRevisionDigest ?? null,
|
|
966
|
+
outcomeCheckerDigest: outcome?.outcomeCheckerDigest ?? null,
|
|
967
|
+
turns: execution?.turns ?? null
|
|
968
|
+
};
|
|
969
|
+
});
|
|
970
|
+
}
|
|
971
|
+
//#endregion
|
|
854
972
|
//#region src/experiment/randomized-canary.ts
|
|
855
973
|
/** Fixed-horizon, assignment-level decision for an unpaired served canary. */
|
|
856
974
|
function nonEmpty(value, label) {
|
|
@@ -1116,6 +1234,6 @@ function decideRandomizedCanary(sealed, cohort, rows, verifyCohort) {
|
|
|
1116
1234
|
return result;
|
|
1117
1235
|
}
|
|
1118
1236
|
//#endregion
|
|
1119
|
-
export { BOOTSTRAP_GATE_MIN_N, DesignRefusalError, EVIDENCE_AUTHORITY_KINDS, EVIDENCE_RECEIPT_VERSION, EVIDENCE_STATES, ExperimentTracker, FinalEvidenceConflictError, FinalEvidenceError, FunnelIntegrityError, INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, MatchedBudgetError, SealIntegrityError, amendExperiment, assertDesignAdequate, assertFunnelReconciles, assertMatchedBudgets, benjaminiHochberg, bonferroni, buildEvidenceVector, buildFunnel, classifyReissue, clusteredPower, comparePairedArms, composeFunnels, computeEstimand, computeInterval, createCampaignEvidenceReceipt, createEvidenceReceipt, decideRandomizedCanary, defineEvaluationClaim, defineExperiment, eProcess, evaluateCondition, evaluateHaltRule, evaluateHypothesis, evaluateIdentityGate, evaluateOracleDeterminismGate, evaluatePopulationReproducibilityGate, evaluatePowerFloorGate, evaluatePredicate, evaluateProvenanceGate, evidenceRegistryRecordSchema, executeAdmissionRule, executeDecisionRule, fileExperimentStore, hashJson, heldoutSignificance, holm, inMemoryExperimentStore, isIndependentEvidence, manifestContentDigest, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openFinalEvidenceLedger, openSealedExperiment, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, parseEvidenceRegistryRecord, powerPreflight, projectNLadderBudget, randomizedCanaryObservationDigest, randomizedCanaryRosterDigest, readField, renderEvidenceIndex, renderFunnelTable, requiredPairedSampleSize, requiredSampleSize, runSelectionRule, runUniformPassBudget, sealExperiment, sealRandomizedCanaryRule, sequentialCrossingHorizon, sequentialDecide, sequentialPairedGate, signManifest, summarizeEvaluationUnits, validateEvidenceRegistry, verifyEvidenceReceipt, verifyManifest, verifyMatchedBudgets, verifySealedExperiment, wilson };
|
|
1237
|
+
export { BOOTSTRAP_GATE_MIN_N, DesignRefusalError, EVIDENCE_AUTHORITY_KINDS, EVIDENCE_RECEIPT_VERSION, EVIDENCE_STATES, ExperimentTracker, FinalEvidenceConflictError, FinalEvidenceError, FunnelIntegrityError, INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, MatchedBudgetError, SealIntegrityError, amendExperiment, assertDesignAdequate, assertFunnelReconciles, assertMatchedBudgets, benjaminiHochberg, bonferroni, buildEvidenceVector, buildFunnel, classifyReissue, clusteredPower, comparePairedArms, composeFunnels, computeEstimand, computeInterval, createCampaignEvidenceReceipt, createEvidenceReceipt, decideRandomizedCanary, defineEvaluationClaim, defineExperiment, eProcess, evaluateCondition, evaluateHaltRule, evaluateHypothesis, evaluateIdentityGate, evaluateOracleDeterminismGate, evaluatePopulationReproducibilityGate, evaluatePowerFloorGate, evaluatePredicate, evaluateProvenanceGate, evidenceRegistryRecordSchema, executeAdmissionRule, executeDecisionRule, fileExperimentStore, hashJson, heldoutSignificance, holm, inMemoryExperimentStore, isIndependentEvidence, joinProspectiveCanaryReceipts, manifestContentDigest, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openFinalEvidenceLedger, openSealedExperiment, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, parseEvidenceRegistryRecord, powerPreflight, projectNLadderBudget, randomizedCanaryObservationDigest, randomizedCanaryRosterDigest, readField, renderEvidenceIndex, renderFunnelTable, requiredPairedSampleSize, requiredSampleSize, runSelectionRule, runUniformPassBudget, sealExperiment, sealRandomizedCanaryRule, sequentialCrossingHorizon, sequentialDecide, sequentialPairedGate, signManifest, summarizeEvaluationUnits, validateEvidenceRegistry, verifyEvidenceReceipt, verifyManifest, verifyMatchedBudgets, verifySealedExperiment, wilson };
|
|
1120
1238
|
|
|
1121
1239
|
//# sourceMappingURL=index.js.map
|