@tangle-network/agent-eval 0.173.3 → 0.175.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/CHANGELOG.md +54 -0
  2. package/README.md +1 -1
  3. package/dist/{proposal-findings-bko3GGy-.js → abort-signal-CtzAM_sJ.js} +11 -11
  4. package/dist/abort-signal-CtzAM_sJ.js.map +1 -0
  5. package/dist/adapters/http.d.ts +2 -2
  6. package/dist/agent-profile-_xPxqVJt.d.ts +488 -0
  7. package/dist/agent-profile-_xPxqVJt.d.ts.map +1 -0
  8. package/dist/analyst/index.d.ts +7 -9
  9. package/dist/analyst/index.d.ts.map +1 -1
  10. package/dist/analyst/index.js +8 -8
  11. package/dist/{benchmark-C4wk_Sjr.js → benchmark-DQKzykkO.js} +2 -2
  12. package/dist/{benchmark-C4wk_Sjr.js.map → benchmark-DQKzykkO.js.map} +1 -1
  13. package/dist/{benchmark-command-BY9oscke.js → benchmark-command-D_5xG9LG.js} +13 -13
  14. package/dist/{benchmark-command-BY9oscke.js.map → benchmark-command-D_5xG9LG.js.map} +1 -1
  15. package/dist/benchmarks/index.d.ts +3 -4
  16. package/dist/benchmarks/index.d.ts.map +1 -1
  17. package/dist/benchmarks/index.js +3 -3
  18. package/dist/campaign/index.d.ts +5 -9
  19. package/dist/campaign/index.js +7 -7
  20. package/dist/{campaign-B3kPMU8S.js → campaign-BzMSCejE.js} +8 -8
  21. package/dist/{campaign-B3kPMU8S.js.map → campaign-BzMSCejE.js.map} +1 -1
  22. package/dist/{opencode-sqlite-eK6HW6dr.js → claude-jsonl-CxZZrDJ3.js} +9 -149
  23. package/dist/claude-jsonl-CxZZrDJ3.js.map +1 -0
  24. package/dist/cli.js +9 -2
  25. package/dist/cli.js.map +1 -1
  26. package/dist/{client-DlqdbM7n.d.ts → client-vyYQg3bm.d.ts} +2 -2
  27. package/dist/{client-DlqdbM7n.d.ts.map → client-vyYQg3bm.d.ts.map} +1 -1
  28. package/dist/contract/index.d.ts +9 -10
  29. package/dist/contract/index.js +8 -8
  30. package/dist/{default-registry-B0bKikCb.js → default-registry-DBqVI4pq.js} +5 -5
  31. package/dist/{default-registry-B0bKikCb.js.map → default-registry-DBqVI4pq.js.map} +1 -1
  32. package/dist/{default-registry-BKwc8bN5.d.ts → default-registry-FfNzaUHV.d.ts} +3 -3
  33. package/dist/{default-registry-BKwc8bN5.d.ts.map → default-registry-FfNzaUHV.d.ts.map} +1 -1
  34. package/dist/{define-agent-eval-CY6qdlGV.d.ts → define-agent-eval-V1jQyCDR.d.ts} +102 -11
  35. package/dist/define-agent-eval-V1jQyCDR.d.ts.map +1 -0
  36. package/dist/{define-agent-eval-8h3lXXee.js → define-agent-eval-ox5McL6e.js} +331 -144
  37. package/dist/define-agent-eval-ox5McL6e.js.map +1 -0
  38. package/dist/{dspy-rlm-engine-CF0t2ITD.js → dspy-rlm-engine-Caz2pl4L.js} +3 -3
  39. package/dist/{dspy-rlm-engine-CF0t2ITD.js.map → dspy-rlm-engine-Caz2pl4L.js.map} +1 -1
  40. package/dist/{engine-DhFir3Ys.d.ts → engine-CvW_I72-.d.ts} +2 -2
  41. package/dist/{engine-DhFir3Ys.d.ts.map → engine-CvW_I72-.d.ts.map} +1 -1
  42. package/dist/experiment/index.d.ts +1 -4
  43. package/dist/experiment/index.d.ts.map +1 -1
  44. package/dist/{external-optimizer-process-BwITA9Jp.js → external-optimizer-process-CxnFL1hd.js} +2 -2
  45. package/dist/{external-optimizer-process-BwITA9Jp.js.map → external-optimizer-process-CxnFL1hd.js.map} +1 -1
  46. package/dist/{external-optimizer-subprocess-wBWeoG6A.js → external-optimizer-subprocess-CQi27uEI.js} +2 -2
  47. package/dist/{external-optimizer-subprocess-wBWeoG6A.js.map → external-optimizer-subprocess-CQi27uEI.js.map} +1 -1
  48. package/dist/fuzz.js +1 -1
  49. package/dist/fuzz.js.map +1 -1
  50. package/dist/hosted/index.d.ts +1 -1
  51. package/dist/{index-D0Db5X-4.d.ts → index-BAAiSF3_.d.ts} +5 -5
  52. package/dist/{index-D0Db5X-4.d.ts.map → index-BAAiSF3_.d.ts.map} +1 -1
  53. package/dist/{index-BQqOjerE.d.ts → index-BTrx5s8m.d.ts} +8 -9
  54. package/dist/index-BTrx5s8m.d.ts.map +1 -0
  55. package/dist/index-DKXuBPXf.d.ts +3840 -0
  56. package/dist/index-DKXuBPXf.d.ts.map +1 -0
  57. package/dist/index.d.ts +11 -13
  58. package/dist/index.d.ts.map +1 -1
  59. package/dist/index.js +11 -11
  60. package/dist/{integrity-BWywb34E.js → integrity-DsHWCebQ.js} +11 -435
  61. package/dist/integrity-DsHWCebQ.js.map +1 -0
  62. package/dist/{kind-factory-gP6lDySe.js → kind-factory-BLvL-E44.js} +2 -2
  63. package/dist/{kind-factory-gP6lDySe.js.map → kind-factory-BLvL-E44.js.map} +1 -1
  64. package/dist/{llm-judge-BfqMFo4h.js → llm-judge-DmNaBrXB.js} +2541 -2435
  65. package/dist/llm-judge-DmNaBrXB.js.map +1 -0
  66. package/dist/{matrix-DGu8KhSs.d.ts → matrix-CJtXz1ky.d.ts} +2 -2
  67. package/dist/{matrix-DGu8KhSs.d.ts.map → matrix-CJtXz1ky.d.ts.map} +1 -1
  68. package/dist/multishot/golden/index.d.ts +1 -1
  69. package/dist/multishot/index.d.ts +2 -2
  70. package/dist/openapi.json +1 -1
  71. package/dist/opencode-sqlite-CNw3vubS.js +145 -0
  72. package/dist/opencode-sqlite-CNw3vubS.js.map +1 -0
  73. package/dist/{produced-state-D91uDvQw.js → produced-state-B8mw6zj9.js} +2 -2
  74. package/dist/{produced-state-D91uDvQw.js.map → produced-state-B8mw6zj9.js.map} +1 -1
  75. package/dist/report-command-DKlXfU5r.js +1528 -0
  76. package/dist/report-command-DKlXfU5r.js.map +1 -0
  77. package/dist/rl.d.ts +1 -1
  78. package/dist/rl.d.ts.map +1 -1
  79. package/dist/rl.js.map +1 -1
  80. package/dist/rollout/index.js +3 -2
  81. package/dist/{rollout-C-znbbYg.js → rollout-CGlDq1GI.js} +3 -2
  82. package/dist/{rollout-C-znbbYg.js.map → rollout-CGlDq1GI.js.map} +1 -1
  83. package/dist/{semantic-concept-judge-Dok7_35a.js → semantic-concept-judge-E3s_fEjB.js} +3 -3
  84. package/dist/{semantic-concept-judge-Dok7_35a.js.map → semantic-concept-judge-E3s_fEjB.js.map} +1 -1
  85. package/dist/{skillopt-optimization-method-DDw3v3gA.js → skillopt-optimization-method-f7399oGb.js} +5 -5
  86. package/dist/{skillopt-optimization-method-DDw3v3gA.js.map → skillopt-optimization-method-f7399oGb.js.map} +1 -1
  87. package/dist/statistical-heldout-Cqb73yE9.d.ts +1127 -0
  88. package/dist/statistical-heldout-Cqb73yE9.d.ts.map +1 -0
  89. package/dist/{store-otlp-Dow0pk_5.js → store-otlp-DV_H2HDu.js} +2 -2
  90. package/dist/{store-otlp-Dow0pk_5.js.map → store-otlp-DV_H2HDu.js.map} +1 -1
  91. package/dist/{store-tool-spans-CCZNsihA.d.ts → store-tool-spans-4o55ABER.d.ts} +3 -3
  92. package/dist/{store-tool-spans-CCZNsihA.d.ts.map → store-tool-spans-4o55ABER.d.ts.map} +1 -1
  93. package/dist/{store-tool-spans-CeNj_m2L.js → store-tool-spans-B9tjys_h.js} +3 -3
  94. package/dist/{store-tool-spans-CeNj_m2L.js.map → store-tool-spans-B9tjys_h.js.map} +1 -1
  95. package/dist/supervisor-run/index.d.ts +71 -6
  96. package/dist/supervisor-run/index.d.ts.map +1 -1
  97. package/dist/supervisor-run/index.js +6 -1357
  98. package/dist/supervisor-run/index.js.map +1 -1
  99. package/dist/{task-failure-attributes-CZjZeBsY.js → task-failure-attributes-CUy9mkIY.js} +2 -2
  100. package/dist/{task-failure-attributes-CZjZeBsY.js.map → task-failure-attributes-CUy9mkIY.js.map} +1 -1
  101. package/dist/terminal-record-Ce9_UjRz.js +539 -0
  102. package/dist/terminal-record-Ce9_UjRz.js.map +1 -0
  103. package/dist/{tool-groups-Cp4Xdzrp.d.ts → tool-groups-DAe1t6zb.d.ts} +2 -2
  104. package/dist/tool-groups-DAe1t6zb.d.ts.map +1 -0
  105. package/dist/trace-repair/index.d.ts +1 -1
  106. package/dist/traces.d.ts +2 -2
  107. package/dist/traces.js +4 -4
  108. package/dist/{types-Ba5UQyVD.d.ts → types-BJz2CPTM.d.ts} +2 -2
  109. package/dist/{types-Ba5UQyVD.d.ts.map → types-BJz2CPTM.d.ts.map} +1 -1
  110. package/dist/{types-CiWITkGo.js → types-DQ0e2E7y.js} +2 -2
  111. package/dist/types-DQ0e2E7y.js.map +1 -0
  112. package/dist/{types-BDV4PiMR.d.ts → types-Dd1ejaeI.d.ts} +2 -2
  113. package/dist/{types-BDV4PiMR.d.ts.map → types-Dd1ejaeI.d.ts.map} +1 -1
  114. package/dist/{types-CoPUTiXb.d.ts → types-vUdAx2Cj.d.ts} +65 -3
  115. package/dist/types-vUdAx2Cj.d.ts.map +1 -0
  116. package/docs/campaign-proposers.md +42 -0
  117. package/package.json +1 -1
  118. package/dist/agent-profile-B9_GGsG8.d.ts +0 -84
  119. package/dist/agent-profile-B9_GGsG8.d.ts.map +0 -1
  120. package/dist/backend-integrity-CeuTgqsd.d.ts +0 -280
  121. package/dist/backend-integrity-CeuTgqsd.d.ts.map +0 -1
  122. package/dist/benchmark-BjLGkfnN.d.ts +0 -236
  123. package/dist/benchmark-BjLGkfnN.d.ts.map +0 -1
  124. package/dist/define-agent-eval-8h3lXXee.js.map +0 -1
  125. package/dist/define-agent-eval-CY6qdlGV.d.ts.map +0 -1
  126. package/dist/external-optimizer-contracts-CQCpyrIL.d.ts +0 -172
  127. package/dist/external-optimizer-contracts-CQCpyrIL.d.ts.map +0 -1
  128. package/dist/heldout-gate-Df5hsqmm.d.ts +0 -453
  129. package/dist/heldout-gate-Df5hsqmm.d.ts.map +0 -1
  130. package/dist/index-BQqOjerE.d.ts.map +0 -1
  131. package/dist/index-CFDffsKz.d.ts +0 -1135
  132. package/dist/index-CFDffsKz.d.ts.map +0 -1
  133. package/dist/integrity-BWywb34E.js.map +0 -1
  134. package/dist/llm-judge-BfqMFo4h.js.map +0 -1
  135. package/dist/opencode-sqlite-eK6HW6dr.js.map +0 -1
  136. package/dist/power-preflight-Ptse_Kq7.d.ts +0 -117
  137. package/dist/power-preflight-Ptse_Kq7.d.ts.map +0 -1
  138. package/dist/pre-registration-BoI4ucR3.d.ts +0 -592
  139. package/dist/pre-registration-BoI4ucR3.d.ts.map +0 -1
  140. package/dist/promotion-policy-CvMda3kU.d.ts +0 -134
  141. package/dist/promotion-policy-CvMda3kU.d.ts.map +0 -1
  142. package/dist/proposal-findings-bko3GGy-.js.map +0 -1
  143. package/dist/provenance-CRY67X50.d.ts +0 -1995
  144. package/dist/provenance-CRY67X50.d.ts.map +0 -1
  145. package/dist/statistical-heldout-DTyB_6-1.d.ts +0 -295
  146. package/dist/statistical-heldout-DTyB_6-1.d.ts.map +0 -1
  147. package/dist/tool-groups-Cp4Xdzrp.d.ts.map +0 -1
  148. package/dist/types-CiWITkGo.js.map +0 -1
  149. package/dist/types-CoPUTiXb.d.ts.map +0 -1
@@ -1,8 +1,8 @@
1
1
  import { i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-DN2WdT5S.js";
2
2
  import { l as ExactCapableAnalyst } from "./exact-types-BKOEILRP.js";
3
3
  import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-7pOUBrtX.js";
4
- import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine } from "./engine-DhFir3Ys.js";
5
- import { t as TraceToolGroupName } from "./tool-groups-Cp4Xdzrp.js";
4
+ import { a as TraceAnalystLimits, i as TraceAnalysisEngineResult, l as RawAnalystFinding, n as TraceAnalysisEngine } from "./engine-CvW_I72-.js";
5
+ import { t as TraceToolGroupName } from "./tool-groups-DAe1t6zb.js";
6
6
  //#region src/trace-analyst/behavioral-metrics.d.ts
7
7
  type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
8
8
  interface SuboptimalSignal {
@@ -110,4 +110,4 @@ interface DefaultAnalystRegistryOptions {
110
110
  declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
111
111
  //#endregion
112
112
  export { createTraceAnalyst as a, BehavioralAnalystOptions as c, TraceAnalystDefinition as i, behavioralAnalyst as l, buildDefaultAnalystRegistry as n, renderPriorFindings as o, CreateTraceAnalystOptions as r, runTraceAnalyst as s, DefaultAnalystRegistryOptions as t, deriveEfficiencyFindings as u };
113
- //# sourceMappingURL=default-registry-BKwc8bN5.d.ts.map
113
+ //# sourceMappingURL=default-registry-FfNzaUHV.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"default-registry-BKwc8bN5.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;UC1BR;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;;;UCpZ1B;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
1
+ {"version":3,"file":"default-registry-FfNzaUHV.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;UC1BR;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;;;UCpZ1B;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
@@ -1,10 +1,85 @@
1
1
  import { c as CostLedgerHandle, f as CostLedgerSummary, m as CostReceipt } from "./cost-ledger-DbQdN3nO.js";
2
2
  import { _ as ProposalFinding } from "./types-DN2WdT5S.js";
3
- import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate } from "./types-Ba5UQyVD.js";
4
- import { An as RunEvalOptions, B as OptimizationMethodResult, F as OptimizationMethod, Kn as CampaignStorage, Mt as SearchHistoryReceipt, Nn as CampaignCellRetryPolicy, ct as RunImprovementLoopResult, dt as RunOptimizationOptions, l as LoopProvenanceRecord, ut as PremeasuredOptimizationBaseline, z as OptimizationMethodProvenance } from "./provenance-CRY67X50.js";
5
- import { t as PowerPreflight } from "./power-preflight-Ptse_Kq7.js";
3
+ import { B as PowerPreflight } from "./statistical-heldout-Cqb73yE9.js";
4
+ import { H as SurfaceProposer, R as Scenario, S as JudgeConfig, a as CampaignResult, d as DispatchContext, j as MutableSurface, k as LabeledScenarioStore, p as Gate, v as GateResult } from "./types-BJz2CPTM.js";
5
+ import { $r as OptimizationMethodProvenance, Ni as PremeasuredOptimizationBaseline, Pi as RunOptimizationOptions, Yr as OptimizationMethod, Zn as LoopProvenanceRecord, ao as CampaignCellRetryPolicy, ei as OptimizationMethodResult, ji as RunImprovementLoopResult, no as RunEvalOptions, oo as RunCampaignOptions, si as ComparisonCost, ta as SearchHistoryReceipt, to as openAutoPr, vo as CampaignStorage } from "./index-DKXuBPXf.js";
6
6
  import { o as InsightReport } from "./insight-report-DETqPc_A.js";
7
- import { n as HostedTenant } from "./client-DlqdbM7n.js";
7
+ import { n as HostedTenant } from "./client-vyYQg3bm.js";
8
+ //#region src/campaign/presets/run-final-comparison.d.ts
9
+ interface FinalComparisonOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'dispatch'> {
10
+ baselineSurface: MutableSurface;
11
+ winnerSurface: MutableSurface;
12
+ dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: Parameters<RunCampaignOptions<TScenario, TArtifact>['dispatch']>[1]) => Promise<TArtifact>;
13
+ gate: Gate<TArtifact, TScenario>;
14
+ holdout?: 'measured' | 'deferred';
15
+ label?: string;
16
+ neutralize?: (winner: MutableSurface, baseline: MutableSurface) => MutableSurface;
17
+ }
18
+ /** Compare a search-selected surface without changing the search's selection. */
19
+ declare function runFinalComparison<TScenario extends Scenario, TArtifact>(opts: FinalComparisonOptions<TScenario, TArtifact>): Promise<{
20
+ baselineOnHoldout: CampaignResult<TArtifact, TScenario>;
21
+ winnerOnHoldout: CampaignResult<TArtifact, TScenario>;
22
+ neutralizedOnHoldout?: CampaignResult<TArtifact, TScenario> | undefined;
23
+ neutralizedSurface?: MutableSurface | undefined;
24
+ holdout?: "deferred" | undefined;
25
+ gateResult: GateResult;
26
+ promotedDiff: string;
27
+ }>;
28
+ //#endregion
29
+ //#region src/contract/self-improve-method.d.ts
30
+ interface SelfImproveMethodProvenance {
31
+ schema: 'tangle.method-improvement';
32
+ recordDigest: `sha256:${string}`;
33
+ runId: string;
34
+ runDir: string;
35
+ timestamp: string;
36
+ baselineContentHash: string;
37
+ winnerContentHash: string;
38
+ diff: string;
39
+ optimizationMethod: NonNullable<SelfImproveProposerResult<Scenario, unknown>['optimization']>;
40
+ evidence: {
41
+ trainSplitDigest: `sha256:${string}`;
42
+ selectionSplitDigest: `sha256:${string}`;
43
+ holdoutSplitDigest: `sha256:${string}`;
44
+ baselineCampaignDigest: `sha256:${string}`;
45
+ winnerCampaignDigest: `sha256:${string}`;
46
+ costReceiptsDigest: `sha256:${string}`;
47
+ neutralizedCampaignDigest?: `sha256:${string}`;
48
+ };
49
+ gate: Awaited<ReturnType<typeof runFinalComparison>>['gateResult'];
50
+ holdout?: 'deferred';
51
+ baselineHoldoutComposite?: number;
52
+ winnerHoldoutComposite?: number;
53
+ heldOutLift?: number;
54
+ cost: ComparisonCost;
55
+ totalCostUsd: number;
56
+ totalDurationMs: number;
57
+ }
58
+ interface SelfImproveMethodResult<TScenario extends Scenario, TArtifact> extends Omit<SelfImproveProposerResult<TScenario, TArtifact>, 'mode' | 'baseline' | 'winner' | 'provenance' | 'generationsExplored' | 'cost' | 'raw' | 'optimization' | 'insight'> {
59
+ mode: 'method';
60
+ /** No final measurement exists when holdout is deferred. */
61
+ baseline: SelfImproveProposerResult<TScenario, TArtifact>['baseline'] | null;
62
+ winner: Omit<SelfImproveProposerResult<TScenario, TArtifact>['winner'], 'compositeMean'> & {
63
+ compositeMean: number | null;
64
+ };
65
+ provenance: SelfImproveMethodProvenance;
66
+ optimization: NonNullable<SelfImproveProposerResult<TScenario, TArtifact>['optimization']>;
67
+ /** Includes the method's reported cost and final measurements without double counting receipts. */
68
+ cost: ComparisonCost;
69
+ /** Only calls actually recorded by the shared ledger have token and channel breakdowns. */
70
+ ledgerCost: CostLedgerSummary;
71
+ insight?: SelfImproveProposerResult<TScenario, TArtifact>['insight'];
72
+ raw: Awaited<ReturnType<typeof runFinalComparison<TScenario, TArtifact>>> & {
73
+ kind: 'method';
74
+ baselineSurface: SelfImproveProposerResult<TScenario, TArtifact>['raw']['baselineSurface'];
75
+ winnerSurface: SelfImproveProposerResult<TScenario, TArtifact>['raw']['winnerSurface'];
76
+ winnerSurfaceHash: string;
77
+ method: OptimizationMethodResult;
78
+ cost: ComparisonCost;
79
+ prResult?: ReturnType<typeof openAutoPr>;
80
+ };
81
+ }
82
+ //#endregion
8
83
  //#region src/contract/self-improve.d.ts
9
84
  interface SelfImproveBudget {
10
85
  /** Hard spend cap across the full run. Each paid call reserves its enforced
@@ -96,6 +171,8 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
96
171
  * Omit this when every cell reports its concrete model in a receipt.
97
172
  */
98
173
  model?: string;
174
+ /** Version of execution behavior outside the candidate surface, used by measurement caches. */
175
+ dispatchRef?: string;
99
176
  /** Scenarios to evaluate against. Train/holdout split is computed from
100
177
  * these unless `budget.holdoutScenarios` is set explicitly. */
101
178
  scenarios: TScenario[];
@@ -109,13 +186,14 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
109
186
  /**
110
187
  * Complete prior measurement of `baselineSurface` over the TRAIN split.
111
188
  * Forwarded to the loop body, which validates its surface hash, scenario
112
- * split, seed (42), reps, and coverage, then skips the baseline search
189
+ * split, seed (42), reps, evaluator manifest, and coverage, then skips the baseline search
113
190
  * campaign entirely — no baseline dispatch, no resumability lookup. The
114
191
  * train split is `scenarios` minus the holdout split, so premeasure with
115
192
  * exactly that scenario set (explicit `budget.holdoutScenarios`, or
116
193
  * `budget.holdout: 'deferred'` with no reserved set, makes the train split
117
194
  * deterministic). Prior spend stays in the imported campaign aggregates and
118
- * is not re-added to this run's cost ledger.
195
+ * is not re-added to this run's cost ledger. Premeasure with
196
+ * `dispatchRef: surfaceDispatchRef(baselineSurface, dispatchRef)` and the same judge revision.
119
197
  */
120
198
  premeasuredBaseline?: PremeasuredOptimizationBaseline<TArtifact, TScenario>;
121
199
  /**
@@ -151,9 +229,9 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
151
229
  * `mem://selfImprove-<timestamp>`. External method mode defaults to
152
230
  * `.agent-eval/runs/self-improve-<timestamp>`. */
153
231
  runDir?: string;
154
- /** Fires once the durable provenance record + OTel spans are emitted.
232
+ /** Fires once the durable provenance record is written.
155
233
  * Receives the structured record for inline assertions / custom routing. */
156
- onProvenance?: (record: LoopProvenanceRecord) => void;
234
+ onProvenance?: (record: LoopProvenanceRecord | SelfImproveMethodProvenance) => void;
157
235
  /** Distributed execution seam — same as `RunCampaignOptions.cellPlacement`.
158
236
  * Returns an opaque placement key the substrate forwards to your agent
159
237
  * as `ctx.placement`. Combined with `httpDispatch` from
@@ -243,7 +321,8 @@ interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
243
321
  * `RunOptimizationOptions.searchLedger`. */
244
322
  searchLedger?: RunOptimizationOptions<TScenario, TArtifact>['searchLedger'];
245
323
  }
246
- interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
324
+ interface SelfImproveProposerResult<TScenario extends Scenario, TArtifact> {
325
+ mode: 'proposer';
247
326
  /** Composite mean across all scenarios, baseline run. When
248
327
  * `budget.holdout === 'deferred'` this is measured on the improvement
249
328
  * (search) split — no holdout campaign ran. */
@@ -321,6 +400,16 @@ interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
321
400
  */
322
401
  raw: RunImprovementLoopResult<TArtifact, TScenario>;
323
402
  }
403
+ type SelfImproveResult<TScenario extends Scenario, TArtifact> = SelfImproveProposerResult<TScenario, TArtifact> | SelfImproveMethodResult<TScenario, TArtifact>;
404
+ type SelfImproveMethodOptions<TScenario extends Scenario, TArtifact> = Omit<SelfImproveOptions<TScenario, TArtifact>, 'proposer' | 'onProvenance'> & {
405
+ method: OptimizationMethod<TScenario, TArtifact>;
406
+ proposer?: never;
407
+ onProvenance?: (record: SelfImproveMethodProvenance) => void;
408
+ };
409
+ type SelfImproveProposerOptions<TScenario extends Scenario, TArtifact> = Omit<SelfImproveOptions<TScenario, TArtifact>, 'method' | 'onProvenance'> & {
410
+ method?: never;
411
+ onProvenance?: (record: LoopProvenanceRecord) => void;
412
+ };
324
413
  /** Failed self-improvement run with an immutable receipt snapshot. */
325
414
  declare class SelfImproveRunError extends Error {
326
415
  readonly cost: CostLedgerSummary;
@@ -353,6 +442,8 @@ declare class SelfImproveRunError extends Error {
353
442
  * budget: { maxConcurrency: 12 },
354
443
  * })
355
444
  */
445
+ declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveMethodOptions<TScenario, TArtifact>): Promise<SelfImproveMethodResult<TScenario, TArtifact>>;
446
+ declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveProposerOptions<TScenario, TArtifact>): Promise<SelfImproveProposerResult<TScenario, TArtifact>>;
356
447
  declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveOptions<TScenario, TArtifact>): Promise<SelfImproveResult<TScenario, TArtifact>>;
357
448
  //#endregion
358
449
  //#region src/contract/define-agent-eval.d.ts
@@ -403,5 +494,5 @@ interface DefinedAgentEval<TScenario extends Scenario, TArtifact> {
403
494
  */
404
495
  declare function defineAgentEval<TScenario extends Scenario, TArtifact>(defaults: DefineAgentEvalOptions<TScenario, TArtifact>): DefinedAgentEval<TScenario, TArtifact>;
405
496
  //#endregion
406
- export { DefinedAgentEval as a, SelfImproveOptions as c, SelfImproveRunError as d, selfImprove as f, DefineAgentEvalOptions as i, SelfImproveProgressEvent as l, AgentEvalEvaluateOptions as n, defineAgentEval as o, AgentEvalImproveOptions as r, SelfImproveBudget as s, AgentEvalAgent as t, SelfImproveResult as u };
407
- //# sourceMappingURL=define-agent-eval-CY6qdlGV.d.ts.map
497
+ export { SelfImproveMethodResult as _, DefinedAgentEval as a, SelfImproveMethodOptions as c, SelfImproveProposerOptions as d, SelfImproveProposerResult as f, SelfImproveMethodProvenance as g, selfImprove as h, DefineAgentEvalOptions as i, SelfImproveOptions as l, SelfImproveRunError as m, AgentEvalEvaluateOptions as n, defineAgentEval as o, SelfImproveResult as p, AgentEvalImproveOptions as r, SelfImproveBudget as s, AgentEvalAgent as t, SelfImproveProgressEvent as u };
498
+ //# sourceMappingURL=define-agent-eval-V1jQyCDR.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"define-agent-eval-V1jQyCDR.d.ts","names":[],"sources":["../src/campaign/presets/run-final-comparison.ts","../src/contract/self-improve-method.ts","../src/contract/self-improve.ts","../src/contract/define-agent-eval.ts"],"mappings":";;;;;;;;UAMiB,uBAAuB,kBAAkB,UAAU,mBAC1D,KAAK,mBAAmB,WAAW;EAC3C,iBAAiB;EACjB,eAAe;EACf,sBACE,SAAS,gBACT,UAAU,WACV,KAAK,WAAW,mBAAmB,WAAW,+BAC3C,QAAQ;EACb,MAAM,KAAK,WAAW;EACtB;EACA;EACA,cAAc,QAAQ,gBAAgB,UAAU,mBAAmB;;;iBAI/C,mBAAmB,kBAAkB,UAAU,WACnE,MAAM,uBAAuB,WAAW,aAAU;;;;;;;;;;;UCSnC;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,oBAAoB,YAAY,0BAA0B;EAC1D;IACE;IACA;IACA;IACA;IACA;IACA;IACA;;EAEF,MAAM,QAAQ,kBAAkB;EAChC;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;;UAGe,wBAAwB,kBAAkB,UAAU,mBAC3D,KACN,0BAA0B,WAAW;EAWvC;;EAEA,UAAU,0BAA0B,WAAW;EAC/C,QAAQ,KAAK,0BAA0B,WAAW;IAChD;;EAEF,YAAY;EACZ,cAAc,YAAY,0BAA0B,WAAW;;EAE/D,MAAM;;EAEN,YAAY;EACZ,UAAU,0BAA0B,WAAW;EAC/C,KAAK,QAAQ,kBAAkB,mBAAmB,WAAW;IAC3D;IACA,iBAAiB,0BAA0B,WAAW;IACtD,eAAe,0BAA0B,WAAW;IACpD;IACA,QAAQ;IACR,MAAM;IACN,WAAW,kBAAkB;;;;;UCzBhB;;;EAGf;;;;EAIA;;EAEA;;EAEA;;;EAGA;;;EAGA;;;;EAIA;;EAEA,mBAAmB;;;;;;;;;EASnB;;EAEA;;;;;EAKA;;KAGU;EACN;EAA0B;;EAC1B;EAA4B;EAAuB;;EACnD;EAA4B;EAAe;;EAC3C;EAA8B;EAAe;EAAuB;;EAGpE;EAAsB;EAAkB;;EACxC;EAAyB;EAAW;EAAY;EAAa;;UAElD,mBAAmB,kBAAkB,UAAU;;;;;;;;;;;;;;EAc9D,QAAQ,SAAS,gBAAgB,UAAU,WAAW,KAAK,oBAAoB,QAAQ;;;;;;;EAQvF;;EAEA;;;EAIA,WAAW;;EAGX,OAAO,YAAY,WAAW;;;EAI9B,iBAAiB;;EAGjB,SAAS;;;;;;;;;;;;;EAcT,sBAAsB,gCAAgC,WAAW;;;;;EAMjE,WAAW,gBAAgB;;;;;;EAO3B,SAAS,mBAAmB,WAAW;;;EAIvC,qBAAqB;;;EAIrB,OAAO,KAAK,WAAW;;;;;;;;EASvB,cAAc,eAAe,gBAAgB,iBAAiB,mBAAmB;;;;EAKjF,UAAU;;;;EAKV;;;EAIA,gBAAgB,QAAQ,uBAAuB;;;;;EAM/C,iBAAiB;IACf,UAAU;IACV;IACA;;;;EAKF;;;;;;;EAQA,YAAY;;;EAIZ,cAAc,OAAO;;;;EAKrB;EACA;EACA;;;;;;;;;;;;;;EAeA,eAAe;;;EAIf,eAAe;;;;EAKf,eAAe;;EAGf;;;;;;;;EASA;;;;;;;;;EAUA,oBAAoB,uBAAuB,WAAW;;;EAItD,WAAW;;;;;;;EAQX,mBAAmB,uBAAuB,WAAW;;;;;;EAOrD,eAAe,uBAAuB,WAAW;;;;EAKjD,eAAe,uBAAuB,WAAW;;UAGlC,0BAA0B,kBAAkB,UAAU;EACrE;;;;EAIA;IACE;IACA,aAAa;;;;;EAKf;IACE;IACA,aAAa;IACb,SAAS;;;IAGT;;;;IAIA;;;;;;EAMF;;;EAGA;;;;EAIA,YAAY;;EAEZ;;;EAGA;;EAEA;;EAEA;;EAEA,MAAM;;;EAGN,UAAU;;;EAGV,gBAAgB;;EAEhB;IACE;IACA,MAAM;IACN;IACA,aAAa;;;;;;;;EAQf,SAAS;;;;;EAKT,QAAQ;;;;;;EAMR,KAAK,yBAAyB,WAAW;;KAG/B,kBAAkB,kBAAkB,UAAU,aACtD,0BAA0B,WAAW,aACrC,wBAAwB,WAAW;KAE3B,yBAAyB,kBAAkB,UAAU,aAAa,KAC5E,mBAAmB,WAAW;EAG9B,QAAQ,mBAAmB,WAAW;EACtC;EACA,gBAAgB,QAAQ;;KAGd,2BAA2B,kBAAkB,UAAU,aAAa,KAC9E,mBAAmB,WAAW;EAG9B;EACA,gBAAgB,QAAQ;;;cAIb,4BAA4B;WAC9B,MAAM;WACN,UAAU;EAEnB,YAAY,gBAAgB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAwLtB,YAAY,kBAAkB,UAAU,WACtD,MAAM,yBAAyB,WAAW,aACzC,QAAQ,wBAAwB,WAAW;iBAC9B,YAAY,kBAAkB,UAAU,WACtD,MAAM,2BAA2B,WAAW,aAC3C,QAAQ,0BAA0B,WAAW;iBAChC,YAAY,kBAAkB,UAAU,WACtD,MAAM,mBAAmB,WAAW,aACnC,QAAQ,kBAAkB,WAAW;;;KC1lB5B,eAAe,kBAAkB,UAAU,cACrD,SAAS,gBACT,UAAU,WACV,KAAK,oBACF,QAAQ;KAED,uBAAuB,kBAAkB,UAAU,aAAa,mBAC1E,WACA;UAGe,yBAAyB,kBAAkB,UAAU,mBAC5D,KACN,eAAe,WAAW;;EAI5B,YAAY;;EAEZ,UAAU;;EAEV,QAAQ,eAAe,WAAW;;EAElC,QAAQ,YAAY,WAAW;;EAE/B,SAAS,YAAY,WAAW;;EAEhC;;KAGU,wBAAwB,kBAAkB,UAAU,aAAa,KAC3E,QAAQ,mBAAmB,WAAW;EAGtC,SAAS,QAAQ;EACjB,eAAe,QAAQ;;UAGR,iBAAiB,kBAAkB,UAAU;;WAEnD,oBAAoB;;WAEpB,iBAAiB;;;;;EAK1B,SACE,OAAO,yBAAyB,WAAW,aAC1C,QAAQ,eAAe,WAAW;;;;;;;EAOrC,QACE,OAAO,wBAAwB,WAAW,aACzC,QAAQ,kBAAkB,WAAW;;;;;;;;;iBAU1B,gBAAgB,kBAAkB,UAAU,WAC1D,UAAU,uBAAuB,WAAW,aAC3C,iBAAiB,WAAW"}