@tangle-network/agent-eval 0.144.7 → 0.144.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/{agent-profile-CgDTo40f.d.ts → agent-profile-DPi7IZg7.d.ts} +3 -3
  3. package/dist/{agent-profile-CgDTo40f.d.ts.map → agent-profile-DPi7IZg7.d.ts.map} +1 -1
  4. package/dist/analyst/index.d.ts +9 -9
  5. package/dist/analyst/index.js +1 -1
  6. package/dist/{analyze-runs-BNNK7irB.js → analyze-runs-C30yljDJ.js} +2 -2
  7. package/dist/{analyze-runs-BNNK7irB.js.map → analyze-runs-C30yljDJ.js.map} +1 -1
  8. package/dist/{benchmark-command-BCafwNrf.js → benchmark-command-Dtym4pA-.js} +2 -2
  9. package/dist/{benchmark-command-BCafwNrf.js.map → benchmark-command-Dtym4pA-.js.map} +1 -1
  10. package/dist/benchmarks/index.d.ts +1 -1
  11. package/dist/benchmarks/index.js +1 -1
  12. package/dist/{benchmarks-CDSolHq7.js → benchmarks-Ce4Ae5fX.js} +3 -3
  13. package/dist/{benchmarks-CDSolHq7.js.map → benchmarks-Ce4Ae5fX.js.map} +1 -1
  14. package/dist/campaign/index.d.ts +6 -6
  15. package/dist/campaign/index.js +2 -2
  16. package/dist/{campaign-Tdy3h62h.js → campaign-FG3thH2m.js} +4 -4
  17. package/dist/{campaign-Tdy3h62h.js.map → campaign-FG3thH2m.js.map} +1 -1
  18. package/dist/cli.js +1 -1
  19. package/dist/{client-DjXROWpx.d.ts → client-C9gzZE59.d.ts} +4 -4
  20. package/dist/{client-DjXROWpx.d.ts.map → client-C9gzZE59.d.ts.map} +1 -1
  21. package/dist/{completion-verifier-foUCLif_.d.ts → completion-verifier-DJA5BhPb.d.ts} +3 -3
  22. package/dist/{completion-verifier-foUCLif_.d.ts.map → completion-verifier-DJA5BhPb.d.ts.map} +1 -1
  23. package/dist/contract/index.d.ts +9 -9
  24. package/dist/contract/index.js +4 -4
  25. package/dist/contract/index.js.map +1 -1
  26. package/dist/control.d.ts +2 -2
  27. package/dist/control.js +1 -1
  28. package/dist/{default-registry-BZhStdjl.d.ts → default-registry-Bf8Woqmq.d.ts} +6 -6
  29. package/dist/{default-registry-BZhStdjl.d.ts.map → default-registry-Bf8Woqmq.d.ts.map} +1 -1
  30. package/dist/{engine-nB64f48I.d.ts → engine-3hL-XqwJ.d.ts} +7 -7
  31. package/dist/{engine-nB64f48I.d.ts.map → engine-3hL-XqwJ.d.ts.map} +1 -1
  32. package/dist/{eval-campaign-DNjCvAm-.js → eval-campaign-B_7wcnav.js} +2 -2
  33. package/dist/{eval-campaign-DNjCvAm-.js.map → eval-campaign-B_7wcnav.js.map} +1 -1
  34. package/dist/{exact-types-Djvzosly.d.ts → exact-types-DSFFpLLI.d.ts} +2 -2
  35. package/dist/{exact-types-Djvzosly.d.ts.map → exact-types-DSFFpLLI.d.ts.map} +1 -1
  36. package/dist/experiment/index.d.ts +2 -2
  37. package/dist/{feedback-trajectory-Rh280oXo.d.ts → feedback-trajectory-CacHpxsp.d.ts} +3 -3
  38. package/dist/{feedback-trajectory-Rh280oXo.d.ts.map → feedback-trajectory-CacHpxsp.d.ts.map} +1 -1
  39. package/dist/hosted/index.d.ts +3 -3
  40. package/dist/{index-CvXXlyz7.d.ts → index-BnEuDAK2.d.ts} +2 -2
  41. package/dist/{index-CvXXlyz7.d.ts.map → index-BnEuDAK2.d.ts.map} +1 -1
  42. package/dist/{index-Sh2I0DRc.d.ts → index-CBEMKO8d.d.ts} +9 -9
  43. package/dist/{index-Sh2I0DRc.d.ts.map → index-CBEMKO8d.d.ts.map} +1 -1
  44. package/dist/{index-C5HOo4ZF2.d.ts → index-CLKMJTJF2.d.ts} +5 -5
  45. package/dist/{index-C5HOo4ZF2.d.ts.map → index-CLKMJTJF2.d.ts.map} +1 -1
  46. package/dist/{index-CwDrUMe0.d.ts → index-CQTZ-4XN.d.ts} +2 -2
  47. package/dist/{index-CwDrUMe0.d.ts.map → index-CQTZ-4XN.d.ts.map} +1 -1
  48. package/dist/index.d.ts +22 -22
  49. package/dist/index.js +11 -11
  50. package/dist/{insight-report-C6h6F_4L.d.ts → insight-report-CgX_s0Ez.d.ts} +3 -3
  51. package/dist/{insight-report-C6h6F_4L.d.ts.map → insight-report-CgX_s0Ez.d.ts.map} +1 -1
  52. package/dist/meta-eval/index.d.ts +1 -1
  53. package/dist/{mint-CGEkzPLf.js → mint-B2O60ACG.js} +2 -2
  54. package/dist/{mint-CGEkzPLf.js.map → mint-B2O60ACG.js.map} +1 -1
  55. package/dist/multishot/index.d.ts +1 -1
  56. package/dist/openapi.json +1 -1
  57. package/dist/{promotion-policy-ChWhTDBH.d.ts → promotion-policy-Ckjhzg_4.d.ts} +2 -2
  58. package/dist/{promotion-policy-ChWhTDBH.d.ts.map → promotion-policy-Ckjhzg_4.d.ts.map} +1 -1
  59. package/dist/{propose-review-control-qgWLA6E9.js → propose-review-control-dSNPjFUH.js} +2 -2
  60. package/dist/{propose-review-control-qgWLA6E9.js.map → propose-review-control-dSNPjFUH.js.map} +1 -1
  61. package/dist/{release-report-Dy39cbFF.js → release-report-BUYmoKo2.js} +2 -2
  62. package/dist/{release-report-Dy39cbFF.js.map → release-report-BUYmoKo2.js.map} +1 -1
  63. package/dist/{release-report-CI8uisI1.d.ts → release-report-DA2BCu5a.d.ts} +3 -3
  64. package/dist/{release-report-CI8uisI1.d.ts.map → release-report-DA2BCu5a.d.ts.map} +1 -1
  65. package/dist/{replay-DyBLaKFc.js → replay-B3gACG_H.js} +2 -2
  66. package/dist/{replay-DyBLaKFc.js.map → replay-B3gACG_H.js.map} +1 -1
  67. package/dist/{replay-DFf-teiC.d.ts → replay-DbhZ4Ked.d.ts} +5 -5
  68. package/dist/{replay-DFf-teiC.d.ts.map → replay-DbhZ4Ked.d.ts.map} +1 -1
  69. package/dist/reporting.d.ts +3 -3
  70. package/dist/reporting.js +1 -1
  71. package/dist/{researcher-BoaxeCzP.d.ts → researcher-BchpD55R.d.ts} +3 -3
  72. package/dist/{researcher-BoaxeCzP.d.ts.map → researcher-BchpD55R.d.ts.map} +1 -1
  73. package/dist/{reward-hacking-B2tPL9a3.js → reward-hacking-BDToousL.js} +4 -3
  74. package/dist/reward-hacking-BDToousL.js.map +1 -0
  75. package/dist/{reward-hacking-Cf1PtEOz.d.ts → reward-hacking-BI0OMAlo.d.ts} +2 -2
  76. package/dist/{reward-hacking-Cf1PtEOz.d.ts.map → reward-hacking-BI0OMAlo.d.ts.map} +1 -1
  77. package/dist/rl.d.ts +5 -5
  78. package/dist/rl.js +4 -4
  79. package/dist/rollout/index.d.ts +1 -1
  80. package/dist/rollout/index.js +2 -2
  81. package/dist/{rollout-2ECTXb2N.js → rollout-DRrksrcV.js} +2 -2
  82. package/dist/{rollout-2ECTXb2N.js.map → rollout-DRrksrcV.js.map} +1 -1
  83. package/dist/{rubric-predictive-validity-9qAwzkZm.d.ts → rubric-predictive-validity-DvkjPYCe.d.ts} +2 -2
  84. package/dist/{rubric-predictive-validity-9qAwzkZm.d.ts.map → rubric-predictive-validity-DvkjPYCe.d.ts.map} +1 -1
  85. package/dist/{run-evidence-BDFFai9R.d.ts → run-evidence-Dhi3C81V.d.ts} +3 -3
  86. package/dist/{run-evidence-BDFFai9R.d.ts.map → run-evidence-Dhi3C81V.d.ts.map} +1 -1
  87. package/dist/{run-record-DqOw5X6_.js → run-record-BmSPWXJR.js} +2 -1
  88. package/dist/run-record-BmSPWXJR.js.map +1 -0
  89. package/dist/{run-record-DdSa93_W.d.ts → run-record-CF4Dwpxr.d.ts} +4 -1
  90. package/dist/run-record-CF4Dwpxr.d.ts.map +1 -0
  91. package/dist/{skill-usage-CJlWEUFt.d.ts → skill-usage-CO9OLRBx.d.ts} +6 -6
  92. package/dist/{skill-usage-CJlWEUFt.d.ts.map → skill-usage-CO9OLRBx.d.ts.map} +1 -1
  93. package/dist/{skillopt-optimization-method-BO7NAl3b.d.ts → skillopt-optimization-method-D8wysvU1.d.ts} +6 -6
  94. package/dist/{skillopt-optimization-method-BO7NAl3b.d.ts.map → skillopt-optimization-method-D8wysvU1.d.ts.map} +1 -1
  95. package/dist/{skillopt-optimization-method-CQwZ-ZX8.js → skillopt-optimization-method-Dtzp7QpP.js} +14 -4
  96. package/dist/skillopt-optimization-method-Dtzp7QpP.js.map +1 -0
  97. package/dist/{statistical-heldout-Dn9ruizm.d.ts → statistical-heldout-TQ-4CYiN.d.ts} +3 -3
  98. package/dist/{statistical-heldout-Dn9ruizm.d.ts.map → statistical-heldout-TQ-4CYiN.d.ts.map} +1 -1
  99. package/dist/{summary-report-DuUS_i7W.d.ts → summary-report-BNR7DWTj.d.ts} +2 -2
  100. package/dist/{summary-report-DuUS_i7W.d.ts.map → summary-report-BNR7DWTj.d.ts.map} +1 -1
  101. package/dist/{tool-groups-DIVBnJyl.d.ts → tool-groups-CZPGGlHf.d.ts} +3 -3
  102. package/dist/tool-groups-CZPGGlHf.d.ts.map +1 -0
  103. package/dist/trace-repair/index.d.ts +2 -2
  104. package/dist/traces.d.ts +4 -4
  105. package/dist/traces.js +1 -1
  106. package/dist/{types-DYuNHo9R.d.ts → types-BnjdJ70P.d.ts} +3 -3
  107. package/dist/{types-DYuNHo9R.d.ts.map → types-BnjdJ70P.d.ts.map} +1 -1
  108. package/dist/{types-DF_Udrp-.d.ts → types-BvDKaULh.d.ts} +2 -2
  109. package/dist/{types-DF_Udrp-.d.ts.map → types-BvDKaULh.d.ts.map} +1 -1
  110. package/dist/wire/index.d.ts +1 -1
  111. package/docs/campaign-proposers.md +5 -0
  112. package/docs/concepts.md +1 -0
  113. package/package.json +1 -1
  114. package/dist/reward-hacking-B2tPL9a3.js.map +0 -1
  115. package/dist/run-record-DdSa93_W.d.ts.map +0 -1
  116. package/dist/run-record-DqOw5X6_.js.map +0 -1
  117. package/dist/skillopt-optimization-method-CQwZ-ZX8.js.map +0 -1
  118. package/dist/tool-groups-DIVBnJyl.d.ts.map +0 -1
@@ -8,7 +8,7 @@ import { p as mapConcurrent } from "./ledger-core-DXZIqu17.js";
8
8
  import { E as pairedBootstrap, H as weightedComposite, d as confidenceInterval } from "./statistics-ByxzSiOM.js";
9
9
  import { c as heldoutSignificance, l as pairHoldout, s as dimensionRegressions } from "./promotion-policy-CrLrmys8.js";
10
10
  import { t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
11
- import { a as campaignCellCostProvenance, o as campaignCellExecutionEvidence, t as detectRewardHacking, u as projectCampaignCellQuality } from "./reward-hacking-B2tPL9a3.js";
11
+ import { a as campaignCellCostProvenance, o as campaignCellExecutionEvidence, t as detectRewardHacking, u as projectCampaignCellQuality } from "./reward-hacking-BDToousL.js";
12
12
  import { z } from "zod";
13
13
  import { writeFileSync } from "node:fs";
14
14
  import { basename, isAbsolute, join } from "node:path";
@@ -1809,6 +1809,7 @@ async function executeCell(args) {
1809
1809
  const tokenUsage = {
1810
1810
  input: agentCost.inputTokens,
1811
1811
  output: agentCost.outputTokens,
1812
+ ...agentCost.usageComplete ? {} : { tokensKnown: false },
1812
1813
  ...agentCost.reasoningTokens !== void 0 && agentCost.reasoningTokens > 0 ? { reasoning: agentCost.reasoningTokens } : {},
1813
1814
  ...agentCost.cachedTokens > 0 ? { cached: agentCost.cachedTokens } : {},
1814
1815
  ...agentCost.cacheWriteTokens !== void 0 && agentCost.cacheWriteTokens > 0 ? { cacheWrite: agentCost.cacheWriteTokens } : {}
@@ -5207,6 +5208,7 @@ async function runOptimization(opts) {
5207
5208
  const selectionRankKey = opts.selectionRankKey ?? ((campaign) => [campaignMeanComposite(campaign)]);
5208
5209
  let winnerSurface = baselineSurface;
5209
5210
  let winnerSurfaceHash = surfaceHash(baselineSurface);
5211
+ const admittedCandidateHashes = /* @__PURE__ */ new Set([winnerSurfaceHash]);
5210
5212
  let winnerComposite = campaignMeanComposite(baselineCampaign);
5211
5213
  let winnerRankKey = selectionRankKey(baselineCampaign);
5212
5214
  assertFiniteRankKey(winnerRankKey, "selectionRankKey for baseline");
@@ -5255,11 +5257,19 @@ async function runOptimization(opts) {
5255
5257
  if (!Array.isArray(proposed)) throw new TypeError("runOptimization: proposer must return an array");
5256
5258
  const proposalSnapshot = immutableProposalSnapshot(proposed, "candidate outputs");
5257
5259
  if (proposalSnapshot.length === 0) break;
5258
- const surfaceResults = await mapConcurrent(proposalSnapshot.map((p) => isProposedCandidate(p) ? p : {
5260
+ const candidates = proposalSnapshot.map((p) => isProposedCandidate(p) ? p : {
5259
5261
  surface: p,
5260
5262
  label: "",
5261
5263
  rationale: ""
5262
- }), candidateConcurrency, async ({ surface, label, rationale }, i) => {
5264
+ });
5265
+ const generationHashes = /* @__PURE__ */ new Set();
5266
+ for (const { surface } of candidates) {
5267
+ const hash = surfaceHash(surface);
5268
+ if (admittedCandidateHashes.has(hash) || generationHashes.has(hash)) throw new Error(`runOptimization: duplicate candidate surface hash "${hash}" in generation ${gen}; candidate surfaces must be unique`);
5269
+ generationHashes.add(hash);
5270
+ }
5271
+ for (const hash of generationHashes) admittedCandidateHashes.add(hash);
5272
+ const surfaceResults = await mapConcurrent(candidates, candidateConcurrency, async ({ surface, label, rationale }, i) => {
5263
5273
  const hash = surfaceHash(surface);
5264
5274
  const campaign = await runCampaign({
5265
5275
  ...opts,
@@ -6520,4 +6530,4 @@ function skillOptOptimizationMethod(config) {
6520
6530
  //#endregion
6521
6531
  export { runReferenceEquivalenceJudge as $, componentSurfaceIdentityMaterial as A, planCampaignRun as B, combineComparisonCosts as C, assertCodeSurfaceIdentity as D, optimizationTokenUsageFromSummary as E, campaignMeanComposite as F, assertCampaignSplitIdentity as G, resolveRunDir as H, compareRankKeys as I, campaignSplitDigestFromIdentities as J, campaignScenarioIdentity as K, DEFAULT_MUTATION_PRIMITIVES as L, surfaceContentHash as M, surfaceHash as N, assertComponentSurface as O, campaignBreakdown as P, createReferenceEquivalenceJudge as Q, buildReflectionPrompt as R, assertOptimizationResult as S, costFromLedgerSummary as T, tangleTracesRoot as U, runCampaign as V, assertCampaignDesign as W, REFERENCE_EQUIVALENCE_INPUT_LIMITS as X, openAutoPr as Y, REFERENCE_EQUIVALENCE_JUDGE_VERSION as Z, heldOutGate as _, assertRealBackend as _t, emitLoopProvenance as a, Dataset as at, composeGate as b, JudgeParseError as bt, provenanceRecordPath as c, llmJudge as ct, runImprovementLoop as d, paretoFrontier as dt, DEFAULT_RED_TEAM_CORPUS as et, runOptimization as f, paretoFrontierWithCrowding as ft, gepaOptimizationMethod as g, assertRealAgentReceipts as gt, labelTrustRank as h, BackendIntegrityError as ht, canonicalDigest as i, toolNamesForRun as it, renderSurfaceDiff as j, codeSurfaceIdentityMaterial as k, provenanceSpansPath as l, crowdingDistance as lt, isProposedCandidate as m, recoverTruncatedJson as mt, buildLoopProvenanceRecord as n, redTeamReport as nt, loopProvenanceArgsFromResult as o, HoldoutLockedError as ot, runEval as p, scalarScore as pt, campaignSplitDigest as q, campaignMeasurementDigest as r, scoreRedTeamOutput as rt, loopProvenanceSpans as s, hashScenarios as st, skillOptOptimizationMethod as t, redTeamDataset as tt, verifyLoopProvenanceRecord as u, dominates as ut, defaultProductionGate as v, summarizeAgentReceiptIntegrity as vt, compareOptimizationMethods as w, externalTextOptimizationMethod as x, runCanaries as y, summarizeBackendIntegrity as yt, parseReflectionResponse as z };
6522
6532
 
6523
- //# sourceMappingURL=skillopt-optimization-method-CQwZ-ZX8.js.map
6533
+ //# sourceMappingURL=skillopt-optimization-method-Dtzp7QpP.js.map