@tangle-network/agent-eval 0.175.0 → 0.176.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/dist/adapters/http.d.ts +1 -1
  3. package/dist/agent-profile-cell-0gSi5ffD.js +374 -0
  4. package/dist/agent-profile-cell-0gSi5ffD.js.map +1 -0
  5. package/dist/analyst/index.d.ts +2 -2
  6. package/dist/analyst/index.js +3 -3
  7. package/dist/{benchmark-command-D_5xG9LG.js → benchmark-command-yPqjcZnC.js} +7 -7
  8. package/dist/{benchmark-command-D_5xG9LG.js.map → benchmark-command-yPqjcZnC.js.map} +1 -1
  9. package/dist/benchmarks/index.d.ts +1 -1
  10. package/dist/benchmarks/index.js +3 -3
  11. package/dist/campaign/index.d.ts +3 -3
  12. package/dist/campaign/index.js +9 -9
  13. package/dist/{campaign-BzMSCejE.js → campaign-85igdlgG.js} +12 -12
  14. package/dist/{campaign-BzMSCejE.js.map → campaign-85igdlgG.js.map} +1 -1
  15. package/dist/campaign-evidence-D8DBLqLI.js +2083 -0
  16. package/dist/campaign-evidence-D8DBLqLI.js.map +1 -0
  17. package/dist/cli.js +1 -1
  18. package/dist/contract/index.d.ts +4 -4
  19. package/dist/contract/index.js +8 -8
  20. package/dist/{define-agent-eval-V1jQyCDR.d.ts → define-agent-eval-CCbl8k2E.d.ts} +11 -4
  21. package/dist/define-agent-eval-CCbl8k2E.d.ts.map +1 -0
  22. package/dist/{define-agent-eval-ox5McL6e.js → define-agent-eval-DEMsu5eA.js} +54 -36
  23. package/dist/define-agent-eval-DEMsu5eA.js.map +1 -0
  24. package/dist/{dspy-rlm-engine-Caz2pl4L.js → dspy-rlm-engine-DqjER2sV.js} +2 -2
  25. package/dist/{dspy-rlm-engine-Caz2pl4L.js.map → dspy-rlm-engine-DqjER2sV.js.map} +1 -1
  26. package/dist/{eval-campaign-BeAjdhzC.js → eval-campaign-Cs-7MiCs.js} +4 -5
  27. package/dist/{eval-campaign-BeAjdhzC.js.map → eval-campaign-Cs-7MiCs.js.map} +1 -1
  28. package/dist/experiment/index.d.ts +3 -68
  29. package/dist/experiment/index.d.ts.map +1 -1
  30. package/dist/experiment/index.js +6 -128
  31. package/dist/experiment/index.js.map +1 -1
  32. package/dist/{attestation-XSUpbc4o.js → experiment-tracker-BKEumQug.js} +2 -96
  33. package/dist/experiment-tracker-BKEumQug.js.map +1 -0
  34. package/dist/{attestation-c1QvaBdX.d.ts → experiment-tracker-CNwqCZFD.d.ts} +2 -78
  35. package/dist/experiment-tracker-CNwqCZFD.d.ts.map +1 -0
  36. package/dist/{external-optimizer-process-CxnFL1hd.js → external-optimizer-process-Dlz8YxrT.js} +3 -3
  37. package/dist/{external-optimizer-process-CxnFL1hd.js.map → external-optimizer-process-Dlz8YxrT.js.map} +1 -1
  38. package/dist/{external-optimizer-subprocess-CQi27uEI.js → external-optimizer-subprocess-q3VzlGAO.js} +2 -2
  39. package/dist/{external-optimizer-subprocess-CQi27uEI.js.map → external-optimizer-subprocess-q3VzlGAO.js.map} +1 -1
  40. package/dist/{index-DKXuBPXf.d.ts → index-Bg6OT2Dd.d.ts} +23 -10
  41. package/dist/{index-DKXuBPXf.d.ts.map → index-Bg6OT2Dd.d.ts.map} +1 -1
  42. package/dist/{index-BTrx5s8m.d.ts → index-DBkcm_9H.d.ts} +4 -4
  43. package/dist/{index-BTrx5s8m.d.ts.map → index-DBkcm_9H.d.ts.map} +1 -1
  44. package/dist/{index-D-UdhAmg.d.ts → index-u0d1Jp4F.d.ts} +4 -2
  45. package/dist/{index-D-UdhAmg.d.ts.map → index-u0d1Jp4F.d.ts.map} +1 -1
  46. package/dist/index.d.ts +4 -4
  47. package/dist/index.js +14 -15
  48. package/dist/index.js.map +1 -1
  49. package/dist/ledger-core/index.d.ts +2 -2
  50. package/dist/ledger-core/index.js +2 -2
  51. package/dist/{ledger-core-PIfjCbKn.js → ledger-core-Cs9f7385.js} +60 -47
  52. package/dist/{ledger-core-PIfjCbKn.js.map → ledger-core-Cs9f7385.js.map} +1 -1
  53. package/dist/{llm-judge-DmNaBrXB.js → llm-judge-DliimmRb.js} +994 -1517
  54. package/dist/llm-judge-DliimmRb.js.map +1 -0
  55. package/dist/{mint-vWOdD8Ae.js → mint-Cc1_zwRQ.js} +2 -2
  56. package/dist/{mint-vWOdD8Ae.js.map → mint-Cc1_zwRQ.js.map} +1 -1
  57. package/dist/openapi.json +1 -1
  58. package/dist/{produced-state-B8mw6zj9.js → produced-state-DrMqa2HD.js} +3 -2
  59. package/dist/{produced-state-B8mw6zj9.js.map → produced-state-DrMqa2HD.js.map} +1 -1
  60. package/dist/profile-cell.js +1 -268
  61. package/dist/{promotion-policy-LY9mVQ7W.js → promotion-policy-DWOm70gx.js} +2 -2
  62. package/dist/{promotion-policy-LY9mVQ7W.js.map → promotion-policy-DWOm70gx.js.map} +1 -1
  63. package/dist/{release-confidence-BsGEg_xg.js → release-confidence-BcGCclTB.js} +2 -2
  64. package/dist/{release-confidence-BsGEg_xg.js.map → release-confidence-BcGCclTB.js.map} +1 -1
  65. package/dist/reporting.js +2 -2
  66. package/dist/{reward-hacking-CKW4teig.js → reward-hacking-D0XwhVWE.js} +2 -215
  67. package/dist/reward-hacking-D0XwhVWE.js.map +1 -0
  68. package/dist/rl.js +5 -4
  69. package/dist/rl.js.map +1 -1
  70. package/dist/rollout/index.js +2 -2
  71. package/dist/{rollout-CGlDq1GI.js → rollout-DmoJVqrF.js} +2 -2
  72. package/dist/{rollout-CGlDq1GI.js.map → rollout-DmoJVqrF.js.map} +1 -1
  73. package/dist/run-record-CR63CpHK.js +216 -0
  74. package/dist/run-record-CR63CpHK.js.map +1 -0
  75. package/dist/{run-record-ZIsR9Fif.js → run-record-DQpSf7t-.js} +2 -2
  76. package/dist/{run-record-ZIsR9Fif.js.map → run-record-DQpSf7t-.js.map} +1 -1
  77. package/dist/{semantic-concept-judge-E3s_fEjB.js → semantic-concept-judge-Dw-f7TEs.js} +3 -3
  78. package/dist/{semantic-concept-judge-E3s_fEjB.js.map → semantic-concept-judge-Dw-f7TEs.js.map} +1 -1
  79. package/dist/{sequential-B51qAYE4.js → sequential-B5gXgcyp.js} +3 -3
  80. package/dist/{sequential-B51qAYE4.js.map → sequential-B5gXgcyp.js.map} +1 -1
  81. package/dist/{skillopt-optimization-method-f7399oGb.js → skillopt-optimization-method-CV7go7ex.js} +6 -749
  82. package/dist/skillopt-optimization-method-CV7go7ex.js.map +1 -0
  83. package/dist/{statistical-heldout-Cqb73yE9.d.ts → statistical-heldout-Z9NROFFS.d.ts} +156 -3
  84. package/dist/statistical-heldout-Z9NROFFS.d.ts.map +1 -0
  85. package/dist/{summary-report-Bgh8CpNK.js → summary-report-B16xy9Kd.js} +2 -2
  86. package/dist/{summary-report-Bgh8CpNK.js.map → summary-report-B16xy9Kd.js.map} +1 -1
  87. package/dist/traces.js +1 -1
  88. package/docs/public-api.md +62 -39
  89. package/docs/search-history-receipts.md +48 -1
  90. package/package.json +1 -1
  91. package/dist/attestation-XSUpbc4o.js.map +0 -1
  92. package/dist/attestation-c1QvaBdX.d.ts.map +0 -1
  93. package/dist/define-agent-eval-V1jQyCDR.d.ts.map +0 -1
  94. package/dist/define-agent-eval-ox5McL6e.js.map +0 -1
  95. package/dist/llm-judge-DmNaBrXB.js.map +0 -1
  96. package/dist/power-preflight-CFXm0Vjo.js +0 -502
  97. package/dist/power-preflight-CFXm0Vjo.js.map +0 -1
  98. package/dist/pre-registration-D94b7Of5.js +0 -110
  99. package/dist/pre-registration-D94b7Of5.js.map +0 -1
  100. package/dist/profile-cell.js.map +0 -1
  101. package/dist/reward-hacking-CKW4teig.js.map +0 -1
  102. package/dist/skillopt-optimization-method-f7399oGb.js.map +0 -1
  103. package/dist/statistical-heldout-Cqb73yE9.d.ts.map +0 -1
@@ -1,16 +1,16 @@
1
1
  import { n as CaptureIntegrityError, s as ValidationError } from "../errors-Dngq5h35.js";
2
2
  import { r as canonicalString } from "../canonical-DPyQ_rpt.js";
3
- import { a as verifyManifest, i as signManifest, n as hashJson, r as manifestContentDigest, t as evaluateHypothesis } from "../pre-registration-D94b7Of5.js";
3
+ import { _ as verifyManifest, g as signManifest, h as manifestContentDigest, m as hashJson, p as evaluateHypothesis } from "../agent-profile-cell-0gSi5ffD.js";
4
4
  import { t as mulberry32 } from "../random-Dn5fPWkt.js";
5
5
  import { a as requiredSampleSize, c as holm, i as requiredPairedSampleSize, n as mcnemarRequiredN, o as benjaminiHochberg, r as pairedMde, s as bonferroni, t as mcnemarPower } from "../power-and-mde-B8F2RdcD.js";
6
6
  import { f as mcnemar, g as pairedRiskDifferenceScore, h as pairedRiskDifferenceExact, m as pairedRiskDifference, n as pairArms, r as pairRunRecords, t as comparePairedArms, v as wilson } from "../paired-arms-D4aeIHUy.js";
7
7
  import { r as pairedBootstrap, t as BOOTSTRAP_GATE_MIN_N } from "../paired-tests-C8iCsioC.js";
8
8
  import { t as eProcess } from "../sequential-eprocess-D1jKoihe.js";
9
- import { a as heldoutSignificance, o as pairHoldout, t as powerPreflight } from "../power-preflight-CFXm0Vjo.js";
10
- import { c as inMemoryExperimentStore, i as ExperimentTracker, n as attest, o as fileExperimentStore, r as verifyAttestation } from "../attestation-XSUpbc4o.js";
9
+ import { M as heldoutSignificance, N as pairHoldout, O as powerPreflight, a as createEvidenceReceipt, i as INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, n as EVIDENCE_AUTHORITY_KINDS, o as isIndependentEvidence, r as EVIDENCE_RECEIPT_VERSION, s as verifyEvidenceReceipt, t as createCampaignEvidenceReceipt } from "../campaign-evidence-D8DBLqLI.js";
10
+ import { a as inMemoryExperimentStore, r as fileExperimentStore, t as ExperimentTracker } from "../experiment-tracker-BKEumQug.js";
11
11
  import { n as pairedEvalueSequence, r as sequentialCrossingHorizon } from "../sequential-CzK5DarL.js";
12
- import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-LY9mVQ7W.js";
13
- import { n as sequentialPairedGate, t as sequentialDecide } from "../sequential-B51qAYE4.js";
12
+ import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-DWOm70gx.js";
13
+ import { n as sequentialPairedGate, t as sequentialDecide } from "../sequential-B5gXgcyp.js";
14
14
  import { createHash } from "node:crypto";
15
15
  import { z } from "zod";
16
16
  //#region src/experiment/ast.ts
@@ -972,128 +972,6 @@ async function openSealedExperiment(sealed) {
972
972
  };
973
973
  }
974
974
  //#endregion
975
- //#region src/experiment/evidence-receipt.ts
976
- /**
977
- * Evidence receipts are the join between Runtime execution and Eval promotion.
978
- *
979
- * Runtime says what actually ran. Eval says what independently measured it.
980
- * This receipt binds those worlds without making either package import the other:
981
- * stable pursuit/run identity, exact candidate/evaluator/environment/input/output
982
- * content identities, and the authority class that produced the observation.
983
- *
984
- * The payload is attested with agent-eval's existing canonical report attestation.
985
- * Signing/key management deliberately remains outside this substrate; consumers may
986
- * sign the byte-stable receipt or anchor its attestation in a transparency log.
987
- */
988
- const EVIDENCE_RECEIPT_VERSION = "1.0.0";
989
- /** Closed promotion vocabulary. A typo or unknown future kind is never independent by default. */
990
- const EVIDENCE_AUTHORITY_KINDS = [
991
- "candidate-self-report",
992
- "independent-evaluator",
993
- "independent-replication",
994
- "human-review",
995
- "production-canary"
996
- ];
997
- const INDEPENDENT_EVIDENCE_AUTHORITY_KINDS = [
998
- "independent-evaluator",
999
- "independent-replication",
1000
- "human-review",
1001
- "production-canary"
1002
- ];
1003
- /**
1004
- * Mint a content-attested evidence receipt. Required identity fields are deliberately
1005
- * non-optional: unknown evidence stays unknown and cannot accidentally look certified.
1006
- * The attestation's input provenance must equal the receipt commitment so two competing
1007
- * descriptions of the evaluated population cannot coexist inside one valid receipt.
1008
- */
1009
- function createEvidenceReceipt(input, provenance) {
1010
- const inputSetCommitment = requiredIdentity(input.inputSetCommitment, "inputSetCommitment");
1011
- assertInputCommitment(provenance, inputSetCommitment);
1012
- const binding = Object.freeze({
1013
- schemaVersion: EVIDENCE_RECEIPT_VERSION,
1014
- pursuitId: requiredIdentity(input.pursuitId, "pursuitId"),
1015
- runId: requiredIdentity(input.runId, "runId"),
1016
- candidateDigest: requiredIdentity(input.candidateDigest, "candidateDigest"),
1017
- evaluatorDigest: requiredIdentity(input.evaluatorDigest, "evaluatorDigest"),
1018
- environmentDigest: requiredIdentity(input.environmentDigest, "environmentDigest"),
1019
- inputSetCommitment,
1020
- outputDigest: requiredIdentity(input.outputDigest, "outputDigest"),
1021
- resultDigest: requiredIdentity(input.resultDigest, "resultDigest"),
1022
- authority: requiredAuthority(input.authority),
1023
- ...input.experimentDigest === void 0 ? {} : { experimentDigest: requiredIdentity(input.experimentDigest, "experimentDigest") },
1024
- ...input.observerDigest === void 0 ? {} : { observerDigest: requiredIdentity(input.observerDigest, "observerDigest") }
1025
- });
1026
- return Object.freeze({
1027
- binding,
1028
- attestation: attest(binding, provenance)
1029
- });
1030
- }
1031
- /**
1032
- * Verify promotion-grade evidence. Generic report attestation keeps a legacy read path,
1033
- * but an EvidenceReceipt never accepts unbound provenance: changing the evaluator code,
1034
- * model versions, input commitment provenance, or creation record must invalidate the
1035
- * evidence rather than merely annotating it as legacy.
1036
- */
1037
- function verifyEvidenceReceipt(receipt) {
1038
- if (receipt.binding.schemaVersion !== "1.0.0") return {
1039
- valid: false,
1040
- reason: `unsupported evidence receipt version '${receipt.binding.schemaVersion}'`
1041
- };
1042
- try {
1043
- for (const [field, value] of Object.entries({
1044
- pursuitId: receipt.binding.pursuitId,
1045
- runId: receipt.binding.runId,
1046
- candidateDigest: receipt.binding.candidateDigest,
1047
- evaluatorDigest: receipt.binding.evaluatorDigest,
1048
- environmentDigest: receipt.binding.environmentDigest,
1049
- inputSetCommitment: receipt.binding.inputSetCommitment,
1050
- outputDigest: receipt.binding.outputDigest,
1051
- resultDigest: receipt.binding.resultDigest
1052
- })) requiredIdentity(value, field);
1053
- requiredAuthority(receipt.binding.authority);
1054
- assertInputCommitment(receipt.attestation.provenance, receipt.binding.inputSetCommitment);
1055
- } catch (error) {
1056
- return {
1057
- valid: false,
1058
- reason: error instanceof Error ? error.message : String(error)
1059
- };
1060
- }
1061
- const verification = verifyAttestation(receipt.binding, receipt.attestation);
1062
- if (!verification.valid) return verification;
1063
- if (verification.legacyUnboundProvenance === true || receipt.attestation.envelopeHash === void 0) return {
1064
- valid: false,
1065
- reason: "evidence receipt provenance is not bound by an attestation envelope"
1066
- };
1067
- return { valid: true };
1068
- }
1069
- /**
1070
- * Promotion may choose a stricter policy, but this primitive makes the basic separation
1071
- * explicit: only a recognized independent authority is independent. Unknown/forged kinds
1072
- * and candidate self-reports both return false.
1073
- */
1074
- function isIndependentEvidence(receipt) {
1075
- return INDEPENDENT_EVIDENCE_AUTHORITY_KINDS.includes(receipt.binding.authority?.kind);
1076
- }
1077
- function requiredAuthority(value) {
1078
- if (typeof value !== "object" || value === null || Array.isArray(value)) throw new TypeError("evidence receipt: authority must be an object");
1079
- const authority = value;
1080
- if (typeof authority.kind !== "string" || !EVIDENCE_AUTHORITY_KINDS.includes(authority.kind)) throw new TypeError(`evidence receipt: unknown authority kind '${String(authority.kind)}'`);
1081
- if (typeof authority.id !== "string") throw new TypeError("evidence receipt: authority.id must be a string");
1082
- return Object.freeze({
1083
- kind: authority.kind,
1084
- id: requiredIdentity(authority.id, "authority.id")
1085
- });
1086
- }
1087
- function assertInputCommitment(provenance, inputSetCommitment) {
1088
- if (typeof provenance?.inputsHash !== "string" || provenance.inputsHash.trim().length === 0) throw new TypeError("evidence receipt: provenance.inputsHash is required");
1089
- if (provenance.inputsHash.trim() !== inputSetCommitment) throw new TypeError("evidence receipt: provenance.inputsHash must equal binding.inputSetCommitment");
1090
- }
1091
- function requiredIdentity(value, field) {
1092
- const normalized = value.trim();
1093
- if (normalized.length === 0) throw new TypeError(`evidence receipt: ${field} must be non-empty`);
1094
- return normalized;
1095
- }
1096
- //#endregion
1097
975
  //#region src/experiment/evidence-record.ts
1098
976
  /**
1099
977
  * The evidence registry record — the canonical, machine-readable unit for a
@@ -1435,6 +1313,6 @@ function mixSeed(seed, effect) {
1435
1313
  return (seed ^ Math.round(effect * 1000003) * 2654435769) >>> 0 | 0;
1436
1314
  }
1437
1315
  //#endregion
1438
- export { BOOTSTRAP_GATE_MIN_N, DesignRefusalError, EVIDENCE_AUTHORITY_KINDS, EVIDENCE_RECEIPT_VERSION, EVIDENCE_STATES, ExperimentTracker, FunnelIntegrityError, INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, MatchedBudgetError, SealIntegrityError, amendExperiment, assertDesignAdequate, assertFunnelReconciles, assertMatchedBudgets, benjaminiHochberg, bonferroni, buildEvidenceVector, buildFunnel, classifyReissue, clusteredPower, comparePairedArms, composeFunnels, computeEstimand, computeInterval, createEvidenceReceipt, defineExperiment, eProcess, evaluateCondition, evaluateHaltRule, evaluateHypothesis, evaluateIdentityGate, evaluateOracleDeterminismGate, evaluatePopulationReproducibilityGate, evaluatePowerFloorGate, evaluatePredicate, evaluateProvenanceGate, evidenceRegistryRecordSchema, executeAdmissionRule, executeDecisionRule, fileExperimentStore, hashJson, heldoutSignificance, holm, inMemoryExperimentStore, isIndependentEvidence, manifestContentDigest, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openSealedExperiment, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, parseEvidenceRegistryRecord, powerPreflight, projectNLadderBudget, readField, renderEvidenceIndex, renderFunnelTable, requiredPairedSampleSize, requiredSampleSize, runSelectionRule, runUniformPassBudget, sealExperiment, sequentialCrossingHorizon, sequentialDecide, sequentialPairedGate, signManifest, validateEvidenceRegistry, verifyEvidenceReceipt, verifyManifest, verifyMatchedBudgets, verifySealedExperiment, wilson };
1316
+ export { BOOTSTRAP_GATE_MIN_N, DesignRefusalError, EVIDENCE_AUTHORITY_KINDS, EVIDENCE_RECEIPT_VERSION, EVIDENCE_STATES, ExperimentTracker, FunnelIntegrityError, INDEPENDENT_EVIDENCE_AUTHORITY_KINDS, MatchedBudgetError, SealIntegrityError, amendExperiment, assertDesignAdequate, assertFunnelReconciles, assertMatchedBudgets, benjaminiHochberg, bonferroni, buildEvidenceVector, buildFunnel, classifyReissue, clusteredPower, comparePairedArms, composeFunnels, computeEstimand, computeInterval, createCampaignEvidenceReceipt, createEvidenceReceipt, defineExperiment, eProcess, evaluateCondition, evaluateHaltRule, evaluateHypothesis, evaluateIdentityGate, evaluateOracleDeterminismGate, evaluatePopulationReproducibilityGate, evaluatePowerFloorGate, evaluatePredicate, evaluateProvenanceGate, evidenceRegistryRecordSchema, executeAdmissionRule, executeDecisionRule, fileExperimentStore, hashJson, heldoutSignificance, holm, inMemoryExperimentStore, isIndependentEvidence, manifestContentDigest, mcnemar, mcnemarPower, mcnemarRequiredN, mulberry32, openSealedExperiment, pairArms, pairHoldout, pairRunRecords, pairedBootstrap, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, paretoPolicy, paretoSignificanceGate, parseEvidenceRegistryRecord, powerPreflight, projectNLadderBudget, readField, renderEvidenceIndex, renderFunnelTable, requiredPairedSampleSize, requiredSampleSize, runSelectionRule, runUniformPassBudget, sealExperiment, sequentialCrossingHorizon, sequentialDecide, sequentialPairedGate, signManifest, validateEvidenceRegistry, verifyEvidenceReceipt, verifyManifest, verifyMatchedBudgets, verifySealedExperiment, wilson };
1439
1317
 
1440
1318
  //# sourceMappingURL=index.js.map