@tangle-network/agent-eval 0.133.2 → 0.134.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/CHANGELOG.md +218 -0
  2. package/dist/analyst/index.d.ts +11 -35
  3. package/dist/analyst/index.d.ts.map +1 -1
  4. package/dist/analyst/index.js +4 -53
  5. package/dist/analyst/index.js.map +1 -1
  6. package/dist/{analyze-runs-BmX-h_yn.d.ts → analyze-runs-DMo3Lb_y.d.ts} +4 -4
  7. package/dist/{analyze-runs-BmX-h_yn.d.ts.map → analyze-runs-DMo3Lb_y.d.ts.map} +1 -1
  8. package/dist/{analyze-runs-B-afTpCv.js → analyze-runs-qk8op0tN.js} +63 -42
  9. package/dist/analyze-runs-qk8op0tN.js.map +1 -0
  10. package/dist/baseline-BaPxoROc.js +149 -0
  11. package/dist/baseline-BaPxoROc.js.map +1 -0
  12. package/dist/{baseline-hG3K85h4.d.ts → baseline-D_fT6277.d.ts} +43 -11
  13. package/dist/baseline-D_fT6277.d.ts.map +1 -0
  14. package/dist/benchmarks/index.d.ts +1 -1
  15. package/dist/benchmarks/index.js +1 -1
  16. package/dist/{benchmarks-CJr1H1_a.js → benchmarks-v5piCeDl.js} +3 -3
  17. package/dist/{benchmarks-CJr1H1_a.js.map → benchmarks-v5piCeDl.js.map} +1 -1
  18. package/dist/builder-eval/index.js +1 -1
  19. package/dist/campaign/index.d.ts +5 -4
  20. package/dist/campaign/index.js +4 -3
  21. package/dist/{campaign-BJjn1rhw.js → campaign-DEC_7DLn.js} +12 -6
  22. package/dist/{campaign-BJjn1rhw.js.map → campaign-DEC_7DLn.js.map} +1 -1
  23. package/dist/{client-COvaLoQG.d.ts → client-BIyh1RCr.d.ts} +29 -15
  24. package/dist/client-BIyh1RCr.d.ts.map +1 -0
  25. package/dist/{client-CYzbdJOZ.js → client-LIuo-KPv.js} +19 -7
  26. package/dist/client-LIuo-KPv.js.map +1 -0
  27. package/dist/contract/index.d.ts +12 -11
  28. package/dist/contract/index.d.ts.map +1 -1
  29. package/dist/contract/index.js +12 -18
  30. package/dist/contract/index.js.map +1 -1
  31. package/dist/{default-registry-Cl3pHo4n.d.ts → default-registry-Brxr728w.d.ts} +4 -268
  32. package/dist/default-registry-Brxr728w.d.ts.map +1 -0
  33. package/dist/{default-registry-D3T9XbuY.js → default-registry-IjYs7T8l.js} +4 -61
  34. package/dist/default-registry-IjYs7T8l.js.map +1 -0
  35. package/dist/{eval-campaign-DXhpZghy.js → eval-campaign-CvPcvqXC.js} +2 -2
  36. package/dist/{eval-campaign-DXhpZghy.js.map → eval-campaign-CvPcvqXC.js.map} +1 -1
  37. package/dist/hosted/index.d.ts +2 -2
  38. package/dist/hosted/index.d.ts.map +1 -1
  39. package/dist/hosted/index.js +1 -1
  40. package/dist/{index-C7Wue8R6.d.ts → index-BoJNQR6n.d.ts} +29 -11
  41. package/dist/index-BoJNQR6n.d.ts.map +1 -0
  42. package/dist/{index-BREtv3ZZ.d.ts → index-C21xKtxu.d.ts} +4 -4
  43. package/dist/{index-BREtv3ZZ.d.ts.map → index-C21xKtxu.d.ts.map} +1 -1
  44. package/dist/{index-DSC51roc.d.ts → index-DSC51roc2.d.ts} +1 -1
  45. package/dist/index-DSC51roc2.d.ts.map +1 -0
  46. package/dist/{index-nhIYz9hn.d.ts → index-DuhJaaiH.d.ts} +68 -7
  47. package/dist/index-DuhJaaiH.d.ts.map +1 -0
  48. package/dist/index.d.ts +60 -13
  49. package/dist/index.d.ts.map +1 -1
  50. package/dist/index.js +134 -27
  51. package/dist/index.js.map +1 -1
  52. package/dist/ledger-core/index.d.ts +2 -2
  53. package/dist/ledger-core/index.js +2 -2
  54. package/dist/{ledger-core-CPZfcrC2.js → ledger-core-DAKFKRzi.js} +136 -18
  55. package/dist/ledger-core-DAKFKRzi.js.map +1 -0
  56. package/dist/matrix/index.d.ts +1 -1
  57. package/dist/meta-eval/index.d.ts +1 -1
  58. package/dist/meta-eval/index.js +2 -2
  59. package/dist/multishot/index.d.ts +2 -2
  60. package/dist/openapi.json +1 -1
  61. package/dist/{paired-arms-6XItKzd1.js → paired-arms-CA_8pN01.js} +2 -2
  62. package/dist/{paired-arms-6XItKzd1.js.map → paired-arms-CA_8pN01.js.map} +1 -1
  63. package/dist/pipelines/index.d.ts +1 -1
  64. package/dist/pipelines/index.js +3 -2
  65. package/dist/pipelines/index.js.map +1 -1
  66. package/dist/proposal-findings-DCawte-y.js +164 -0
  67. package/dist/proposal-findings-DCawte-y.js.map +1 -0
  68. package/dist/{release-report-wuilQkvK.js → release-report-BVZBmRZp.js} +2 -2
  69. package/dist/{release-report-wuilQkvK.js.map → release-report-BVZBmRZp.js.map} +1 -1
  70. package/dist/{release-report-CjHWa8Ia.d.ts → release-report-CuULWKyk.d.ts} +2 -2
  71. package/dist/{release-report-CjHWa8Ia.d.ts.map → release-report-CuULWKyk.d.ts.map} +1 -1
  72. package/dist/reporting.d.ts +3 -3
  73. package/dist/reporting.js +4 -4
  74. package/dist/{researcher-CbSKhK8z.d.ts → researcher-DVtruQ9U.d.ts} +2 -2
  75. package/dist/{researcher-CbSKhK8z.d.ts.map → researcher-DVtruQ9U.d.ts.map} +1 -1
  76. package/dist/{reward-hacking-Dl2UBzej.js → reward-hacking-DCdRK9TY.js} +2 -2
  77. package/dist/{reward-hacking-Dl2UBzej.js.map → reward-hacking-DCdRK9TY.js.map} +1 -1
  78. package/dist/rl.d.ts +2 -2
  79. package/dist/rl.d.ts.map +1 -1
  80. package/dist/rl.js +18 -5
  81. package/dist/rl.js.map +1 -1
  82. package/dist/{rubric-predictive-validity-QG7ydk0s.js → rubric-predictive-validity-D6Q6n9oq.js} +2 -2
  83. package/dist/{rubric-predictive-validity-QG7ydk0s.js.map → rubric-predictive-validity-D6Q6n9oq.js.map} +1 -1
  84. package/dist/{semantic-concept-judge-BypLt6Fw.js → semantic-concept-judge-C0P1VTXD.js} +2 -3
  85. package/dist/{semantic-concept-judge-BypLt6Fw.js.map → semantic-concept-judge-C0P1VTXD.js.map} +1 -1
  86. package/dist/{skill-usage-BaaxFSJR.d.ts → skill-usage-BDQVPIG1.d.ts} +3 -2
  87. package/dist/skill-usage-BDQVPIG1.d.ts.map +1 -0
  88. package/dist/{skillopt-optimization-method-CF6a327Q.js → skillopt-optimization-method-BY6vKLJB.js} +169 -53
  89. package/dist/skillopt-optimization-method-BY6vKLJB.js.map +1 -0
  90. package/dist/{skillopt-optimization-method-wHF5xsUv.d.ts → skillopt-optimization-method-DJ3l4w8W.d.ts} +20 -18
  91. package/dist/skillopt-optimization-method-DJ3l4w8W.d.ts.map +1 -0
  92. package/dist/{statistics-DbvkkDPa.d.ts → statistics-D_4Snl-5.d.ts} +158 -30
  93. package/dist/statistics-D_4Snl-5.d.ts.map +1 -0
  94. package/dist/{statistics-DWM_AyLe.js → statistics-RwRNu2__.js} +546 -98
  95. package/dist/statistics-RwRNu2__.js.map +1 -0
  96. package/dist/{summary-report-Ci17nIdU.js → summary-report-BxtossFi.js} +3 -3
  97. package/dist/{summary-report-Ci17nIdU.js.map → summary-report-BxtossFi.js.map} +1 -1
  98. package/dist/{summary-report-CFnQgNfg.d.ts → summary-report-DGp0-_XO.d.ts} +61 -4
  99. package/dist/summary-report-DGp0-_XO.d.ts.map +1 -0
  100. package/dist/{baseline-DcX5hQDv.js → tool-use-metrics-DEGMKycK.js} +2 -114
  101. package/dist/tool-use-metrics-DEGMKycK.js.map +1 -0
  102. package/dist/types-DVjczBM9.d.ts +276 -0
  103. package/dist/types-DVjczBM9.d.ts.map +1 -0
  104. package/dist/{types-BokuXvOG.d.ts → types-DiWLru6Z.d.ts} +20 -37
  105. package/dist/types-DiWLru6Z.d.ts.map +1 -0
  106. package/docs/campaign-proposers.md +5 -0
  107. package/docs/design/statistics-decisions.md +271 -0
  108. package/docs/design.md +1 -0
  109. package/docs/insight-report.md +1 -1
  110. package/docs/research-report-methodology.md +4 -1
  111. package/package.json +2 -1
  112. package/dist/analyze-runs-B-afTpCv.js.map +0 -1
  113. package/dist/baseline-DcX5hQDv.js.map +0 -1
  114. package/dist/baseline-hG3K85h4.d.ts.map +0 -1
  115. package/dist/client-COvaLoQG.d.ts.map +0 -1
  116. package/dist/client-CYzbdJOZ.js.map +0 -1
  117. package/dist/default-registry-Cl3pHo4n.d.ts.map +0 -1
  118. package/dist/default-registry-D3T9XbuY.js.map +0 -1
  119. package/dist/index-C7Wue8R6.d.ts.map +0 -1
  120. package/dist/index-DSC51roc.d.ts.map +0 -1
  121. package/dist/index-nhIYz9hn.d.ts.map +0 -1
  122. package/dist/ledger-core-CPZfcrC2.js.map +0 -1
  123. package/dist/run-score-iEEAWiBY.js +0 -41
  124. package/dist/run-score-iEEAWiBY.js.map +0 -1
  125. package/dist/skill-usage-BaaxFSJR.d.ts.map +0 -1
  126. package/dist/skillopt-optimization-method-CF6a327Q.js.map +0 -1
  127. package/dist/skillopt-optimization-method-wHF5xsUv.d.ts.map +0 -1
  128. package/dist/statistics-DWM_AyLe.js.map +0 -1
  129. package/dist/statistics-DbvkkDPa.d.ts.map +0 -1
  130. package/dist/summary-report-CFnQgNfg.d.ts.map +0 -1
  131. package/dist/types-BokuXvOG.d.ts.map +0 -1
@@ -2,13 +2,13 @@ import { s as ValidationError, t as AgentEvalError } from "./errors-8YnH8WlF.js"
2
2
  import { p as canonicalize, s as buildAgentProfileCell } from "./agent-profile-cell-OhuTee9n.js";
3
3
  import { i as CostLedger } from "./cost-ledger-BrJxbrMy.js";
4
4
  import { f as maximumChargeForLlmRequest, l as costReceiptFromLlm, u as costReceiptFromLlmError } from "./llm-client-ClPW-dWB.js";
5
- import { $ as SEARCH_LEDGER_FILE_CONTEXT, B as surfaceContentHash, Bt as JudgeParseError, E as pairHoldout, Et as contentHash, F as assertCodeSurfaceIdentity, J as planCampaignRun, Lt as assertRealBackend, Pt as recoverTruncatedJson, Y as runCampaign, et as SearchLedgerConflictError, h as labelTrustRank, nt as SearchLedgerIntegrityError, tt as SearchLedgerError, zt as summarizeBackendIntegrity } from "./skillopt-optimization-method-CF6a327Q.js";
6
- import { c as eProcess, g as mulberry32, v as pairedBootstrap } from "./statistics-DWM_AyLe.js";
7
- import { t as comparePairedArms } from "./paired-arms-6XItKzd1.js";
5
+ import { $ as SEARCH_LEDGER_FILE_CONTEXT, B as surfaceContentHash, E as pairHoldout, Et as contentHash, F as assertCodeSurfaceIdentity, Ht as JudgeParseError, J as planCampaignRun, Pt as recoverTruncatedJson, Vt as summarizeBackendIntegrity, Y as runCampaign, et as SearchLedgerConflictError, h as labelTrustRank, nt as SearchLedgerIntegrityError, tt as SearchLedgerError, zt as assertRealBackend } from "./skillopt-optimization-method-BY6vKLJB.js";
6
+ import { C as pairedBootstrap, p as eProcess, x as mulberry32 } from "./statistics-RwRNu2__.js";
7
+ import { t as comparePairedArms } from "./paired-arms-CA_8pN01.js";
8
8
  import { o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord } from "./run-record-BIwU2wdV.js";
9
9
  import { t as analyzeTraces } from "./analyst-LsnNpSkm.js";
10
- import { c as campaignCellToRunRecord } from "./reward-hacking-Dl2UBzej.js";
11
- import { t as FileLedgerJournal, u as canonicalString } from "./ledger-core-CPZfcrC2.js";
10
+ import { c as campaignCellToRunRecord } from "./reward-hacking-DCdRK9TY.js";
11
+ import { d as canonicalString, t as FileLedgerJournal } from "./ledger-core-DAKFKRzi.js";
12
12
  import { z } from "zod";
13
13
  import { closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
14
14
  import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -3021,6 +3021,12 @@ var FileSearchLedger = class {
3021
3021
  async trustedHead() {
3022
3022
  return this.journal.trustedHead();
3023
3023
  }
3024
+ async pinTrustedHead() {
3025
+ return this.journal.pinTrustedHead();
3026
+ }
3027
+ async clearTrustedHead() {
3028
+ return this.journal.clearTrustedHead();
3029
+ }
3024
3030
  };
3025
3031
  function parseSearchLedgerEntry(raw, context) {
3026
3032
  const parsed = EntrySchema.safeParse(raw);
@@ -3890,4 +3896,4 @@ function resolveWorktreePath(surface, worktreeDir) {
3890
3896
  //#endregion
3891
3897
  export { planEvalFixtureRun as A, harnessAxisOf as B, rolloutArgumentDiff as C, discoverEvalFixtures as D, neutralizationGate as E, HARNESS_NATIVE_MODEL as F, parseCorrectnessResponse as G, completionVerdict as H, agentProfileHash as I, verifyCompletion as K, agentProfileId as L, buildAnalystSurfaceDispatch as M, failureModeRecallJudge as N, loadEvalFixture as O, CODING_HARNESSES as P, agentProfileModelId as R, classifyUngroundedLiterals as S, sequentialPairedGate as T, createLlmCorrectnessChecker as U, extractProducedState as V, createTokenRecallChecker as W, scoreboardSummary as _, isTransientTransportFailure as a, FsLabeledScenarioStore as b, openSearchLedger as c, selectDiscriminative as d, ProfileMatrixError as f, scoreUserStory as g, renderScoreboardMarkdown as h, verifyCodeSurface as i, analyzeCrossSurfaceInteractions as j, loadEvalFixtureScenarios as k, validateSearchLedgerEvent as l, makePlaybackDispatch as m, gitWorktreeAdapter as n, FileSearchLedger as o, runProfileMatrix as p, resolveWorktreePath as r, SEARCH_LEDGER_SCHEMA as s, WorktreeAdapterError as t, scoreDiscrimination as u, userStoryScoreboard as v, sequentialDecide as w, LabeledScenarioStoreError as x, neutralizeText as y, expandProfileAxes as z };
3892
3898
 
3893
- //# sourceMappingURL=campaign-BJjn1rhw.js.map
3899
+ //# sourceMappingURL=campaign-DEC_7DLn.js.map