@tangle-network/agent-eval 0.220.2 → 0.221.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/adapters/http.d.ts +2 -2
  3. package/dist/{agent-profile-D01Io2Xw.d.ts → agent-profile-Bc7ADJY4.d.ts} +2 -2
  4. package/dist/{agent-profile-D01Io2Xw.d.ts.map → agent-profile-Bc7ADJY4.d.ts.map} +1 -1
  5. package/dist/analyst/index.d.ts +77 -78
  6. package/dist/analyst/index.d.ts.map +1 -1
  7. package/dist/analyst/index.js +204 -25
  8. package/dist/analyst/index.js.map +1 -1
  9. package/dist/{analyze-runs-DqPJaJ4X.js → analyze-runs-CHmsykNJ.js} +4 -4
  10. package/dist/{analyze-runs-DqPJaJ4X.js.map → analyze-runs-CHmsykNJ.js.map} +1 -1
  11. package/dist/{analyze-runs-BWAHBDAw.d.ts → analyze-runs-D7In4c3z.d.ts} +2 -2
  12. package/dist/{analyze-runs-BWAHBDAw.d.ts.map → analyze-runs-D7In4c3z.d.ts.map} +1 -1
  13. package/dist/{benchmark-command-Dx0E4a6z.js → benchmark-command-8XQfpUK3.js} +13 -11
  14. package/dist/benchmark-command-8XQfpUK3.js.map +1 -0
  15. package/dist/benchmarks/index.d.ts +4 -4
  16. package/dist/benchmarks/index.js +3 -3
  17. package/dist/calibration-CTwxpJ-t.js +197 -0
  18. package/dist/calibration-CTwxpJ-t.js.map +1 -0
  19. package/dist/campaign/index.d.ts +6 -6
  20. package/dist/campaign/index.js +5 -5
  21. package/dist/{campaign-BVPC6ODQ.js → campaign-Bt6rbWIU.js} +9 -8
  22. package/dist/{campaign-BVPC6ODQ.js.map → campaign-Bt6rbWIU.js.map} +1 -1
  23. package/dist/cli.js +1 -1
  24. package/dist/{completion-verifier-Cigap8CG.js → completion-verifier-YwoeujPl.js} +2 -2
  25. package/dist/{completion-verifier-Cigap8CG.js.map → completion-verifier-YwoeujPl.js.map} +1 -1
  26. package/dist/contract/index.d.ts +10 -10
  27. package/dist/contract/index.js +7 -7
  28. package/dist/decision-point-Co3u3W8k.js +831 -0
  29. package/dist/decision-point-Co3u3W8k.js.map +1 -0
  30. package/dist/default-registry-BMHqNR6_.d.ts +257 -0
  31. package/dist/default-registry-BMHqNR6_.d.ts.map +1 -0
  32. package/dist/{default-registry-ovrIBGmH.js → default-registry-BbFCkqvB.js} +607 -14
  33. package/dist/default-registry-BbFCkqvB.js.map +1 -0
  34. package/dist/{define-agent-eval-C2nFhHPu.d.ts → define-agent-eval-Bsoi2GUV.d.ts} +7 -7
  35. package/dist/{define-agent-eval-C2nFhHPu.d.ts.map → define-agent-eval-Bsoi2GUV.d.ts.map} +1 -1
  36. package/dist/{define-agent-eval-UQY3VbJ2.js → define-agent-eval-CJhECJNL.js} +4 -4
  37. package/dist/{define-agent-eval-UQY3VbJ2.js.map → define-agent-eval-CJhECJNL.js.map} +1 -1
  38. package/dist/diagnosis/index.d.ts +2 -2
  39. package/dist/diagnosis/index.js +1 -1
  40. package/dist/{dspy-rlm-engine-Bn_Mt_LW.js → dspy-rlm-engine-BTy7u0jQ.js} +3 -3
  41. package/dist/{dspy-rlm-engine-Bn_Mt_LW.js.map → dspy-rlm-engine-BTy7u0jQ.js.map} +1 -1
  42. package/dist/{engine-BpGGw6cd.d.ts → engine-BF95CWzH.d.ts} +2 -2
  43. package/dist/{engine-BpGGw6cd.d.ts.map → engine-BF95CWzH.d.ts.map} +1 -1
  44. package/dist/{eval-campaign-BUMIxKTE.js → eval-campaign-DoX5vz3E.js} +2 -2
  45. package/dist/{eval-campaign-BUMIxKTE.js.map → eval-campaign-DoX5vz3E.js.map} +1 -1
  46. package/dist/{evaluation-CBdQt_kr.d.ts → evaluation-D1PGzkPR.d.ts} +3 -3
  47. package/dist/{evaluation-CBdQt_kr.d.ts.map → evaluation-D1PGzkPR.d.ts.map} +1 -1
  48. package/dist/evaluation-wdGyZ7jV.js +147 -0
  49. package/dist/evaluation-wdGyZ7jV.js.map +1 -0
  50. package/dist/evaluation.d.ts +1 -1
  51. package/dist/evaluation.js +1 -136
  52. package/dist/{exact-types-BYFmYq44.d.ts → exact-types-CVn1QxFO.d.ts} +2 -2
  53. package/dist/{exact-types-BYFmYq44.d.ts.map → exact-types-CVn1QxFO.d.ts.map} +1 -1
  54. package/dist/experiment/index.d.ts +3 -3
  55. package/dist/experiment/index.js +2 -2
  56. package/dist/{external-optimizer-process-wOLfzsCC.js → external-optimizer-process-E5yY2SYS.js} +2 -2
  57. package/dist/{external-optimizer-process-wOLfzsCC.js.map → external-optimizer-process-E5yY2SYS.js.map} +1 -1
  58. package/dist/{external-optimizer-subprocess-D670Oxf2.js → external-optimizer-subprocess-DA02c_L6.js} +12 -19
  59. package/dist/{external-optimizer-subprocess-D670Oxf2.js.map → external-optimizer-subprocess-DA02c_L6.js.map} +1 -1
  60. package/dist/{feedback-trajectory-DhB9WzYg.d.ts → feedback-trajectory-BZhVCWuz.d.ts} +3 -3
  61. package/dist/{feedback-trajectory-DhB9WzYg.d.ts.map → feedback-trajectory-BZhVCWuz.d.ts.map} +1 -1
  62. package/dist/finding-composition-8n0fFgVI.js +131 -0
  63. package/dist/finding-composition-8n0fFgVI.js.map +1 -0
  64. package/dist/{findings-store-CvNo712L.js → findings-store-DRiFdoid.js} +4 -4
  65. package/dist/{findings-store-CvNo712L.js.map → findings-store-DRiFdoid.js.map} +1 -1
  66. package/dist/{index-CVFStrQD.d.ts → index-BJZ7s-sv.d.ts} +7 -58
  67. package/dist/index-BJZ7s-sv.d.ts.map +1 -0
  68. package/dist/{index-Q6DvdcwI.d.ts → index-BvX3CFNS.d.ts} +6 -6
  69. package/dist/{index-Q6DvdcwI.d.ts.map → index-BvX3CFNS.d.ts.map} +1 -1
  70. package/dist/index-DWd1gUwR.d.ts +1086 -0
  71. package/dist/index-DWd1gUwR.d.ts.map +1 -0
  72. package/dist/{index-DAXfpVf7.d.ts → index-y7i8l6Y_.d.ts} +8 -8
  73. package/dist/{index-DAXfpVf7.d.ts.map → index-y7i8l6Y_.d.ts.map} +1 -1
  74. package/dist/index.d.ts +18 -17
  75. package/dist/index.d.ts.map +1 -1
  76. package/dist/index.js +15 -15
  77. package/dist/{kind-factory-BhxbtUsQ.js → kind-factory-Dl0PmNQL.js} +2 -2
  78. package/dist/{kind-factory-BhxbtUsQ.js.map → kind-factory-Dl0PmNQL.js.map} +1 -1
  79. package/dist/{llm-judge-Cc8vDJh_.js → llm-judge-3uAeRawE.js} +4 -4
  80. package/dist/{llm-judge-Cc8vDJh_.js.map → llm-judge-3uAeRawE.js.map} +1 -1
  81. package/dist/{matrix-Cf-AGxGh.d.ts → matrix-BXo3PSqM.d.ts} +2 -2
  82. package/dist/{matrix-Cf-AGxGh.d.ts.map → matrix-BXo3PSqM.d.ts.map} +1 -1
  83. package/dist/meta-eval/index.d.ts +3 -638
  84. package/dist/meta-eval/index.js +3 -193
  85. package/dist/meta-eval/index.js.map +1 -1
  86. package/dist/multishot/golden/index.d.ts +1 -1
  87. package/dist/multishot/index.d.ts +2 -2
  88. package/dist/openapi.json +1 -1
  89. package/dist/outcome-observations-FLmRpgYa.js +78 -0
  90. package/dist/outcome-observations-FLmRpgYa.js.map +1 -0
  91. package/dist/{paired-arms-Ct4LTKc1.js → paired-arms-B2ttLc0_.js} +108 -31
  92. package/dist/paired-arms-B2ttLc0_.js.map +1 -0
  93. package/dist/{power-and-mde-N5nU4_Ly.js → power-and-mde-CQX-0weG.js} +2 -2
  94. package/dist/{power-and-mde-N5nU4_Ly.js.map → power-and-mde-CQX-0weG.js.map} +1 -1
  95. package/dist/{prime-protocol-CckHBxGu.d.ts → prime-protocol-BEaqBoQE.d.ts} +2 -2
  96. package/dist/{prime-protocol-CckHBxGu.d.ts.map → prime-protocol-BEaqBoQE.d.ts.map} +1 -1
  97. package/dist/{proposal-findings-E1aSC_cU.js → proposal-findings-D0_zkQ2l.js} +2 -2
  98. package/dist/{proposal-findings-E1aSC_cU.js.map → proposal-findings-D0_zkQ2l.js.map} +1 -1
  99. package/dist/{registry-B_yG-1jh.d.ts → registry-DDKaUrmM.d.ts} +3 -3
  100. package/dist/{registry-B_yG-1jh.d.ts.map → registry-DDKaUrmM.d.ts.map} +1 -1
  101. package/dist/reporting.d.ts +1 -1
  102. package/dist/reporting.js +3 -3
  103. package/dist/rl.d.ts +4 -5
  104. package/dist/rl.d.ts.map +1 -1
  105. package/dist/rl.js +5 -4
  106. package/dist/rl.js.map +1 -1
  107. package/dist/{rubric-predictive-validity-noY7uE03.d.ts → rubric-predictive-validity-Cd5SCtAf.d.ts} +58 -3
  108. package/dist/rubric-predictive-validity-Cd5SCtAf.d.ts.map +1 -0
  109. package/dist/{rubric-predictive-validity-D3_oI72N.js → rubric-predictive-validity-CeKW2GQN.js} +3 -77
  110. package/dist/rubric-predictive-validity-CeKW2GQN.js.map +1 -0
  111. package/dist/search/index.d.ts +2 -2
  112. package/dist/{search-kernel-Dloxd6gs.d.ts → search-kernel-C6__V0IS.d.ts} +2 -2
  113. package/dist/{search-kernel-Dloxd6gs.d.ts.map → search-kernel-C6__V0IS.d.ts.map} +1 -1
  114. package/dist/{skillopt-optimization-method-DLKqdhcR.js → skillopt-optimization-method-D21dH_Bh.js} +4 -4
  115. package/dist/{skillopt-optimization-method-DLKqdhcR.js.map → skillopt-optimization-method-D21dH_Bh.js.map} +1 -1
  116. package/dist/{statistical-heldout-3z3I-kJN.d.ts → statistical-heldout-pD1Yah-0.d.ts} +58 -8
  117. package/dist/statistical-heldout-pD1Yah-0.d.ts.map +1 -0
  118. package/dist/{store-otlp-dyWE3wAG.js → store-otlp-BVZfR168.js} +2 -2
  119. package/dist/{store-otlp-dyWE3wAG.js.map → store-otlp-BVZfR168.js.map} +1 -1
  120. package/dist/{store-tool-spans-DIUgZbzu.js → store-tool-spans-CjYiF4fF.js} +3 -3
  121. package/dist/{store-tool-spans-DIUgZbzu.js.map → store-tool-spans-CjYiF4fF.js.map} +1 -1
  122. package/dist/{store-tool-spans-DPwXHQkH.d.ts → store-tool-spans-DfnwWMFR.d.ts} +4 -4
  123. package/dist/{store-tool-spans-DPwXHQkH.d.ts.map → store-tool-spans-DfnwWMFR.d.ts.map} +1 -1
  124. package/dist/{summary-report-BsXJoYX5.js → summary-report-DHDWxe6p.js} +3 -3
  125. package/dist/{summary-report-BsXJoYX5.js.map → summary-report-DHDWxe6p.js.map} +1 -1
  126. package/dist/systemone-D3MrW63C.d.ts +400 -0
  127. package/dist/systemone-D3MrW63C.d.ts.map +1 -0
  128. package/dist/{systemone-protocol-DKoi1NdK.d.ts → systemone-protocol-Bl2EI2KQ.d.ts} +4 -2
  129. package/dist/{systemone-protocol-DKoi1NdK.d.ts.map → systemone-protocol-Bl2EI2KQ.d.ts.map} +1 -1
  130. package/dist/systemone-protocol.d.ts +2 -2
  131. package/dist/systemone-protocol.js +7 -7
  132. package/dist/systemone-protocol.js.map +1 -1
  133. package/dist/{systemone-review-CUBMuTLi.d.ts → systemone-review-C8DRiT_-.d.ts} +3 -3
  134. package/dist/{systemone-review-CUBMuTLi.d.ts.map → systemone-review-C8DRiT_-.d.ts.map} +1 -1
  135. package/dist/{systemone-review-DzdOJ8I6.js → systemone-review-DFGtIUcf.js} +2 -2
  136. package/dist/{systemone-review-DzdOJ8I6.js.map → systemone-review-DFGtIUcf.js.map} +1 -1
  137. package/dist/systemone.d.ts +5 -5
  138. package/dist/systemone.js +5 -343
  139. package/dist/{tool-groups-LISbBWCg.d.ts → tool-groups-DK1bu5TD.d.ts} +3 -3
  140. package/dist/tool-groups-DK1bu5TD.d.ts.map +1 -0
  141. package/dist/{trace-questions-C42fYEE4.d.ts → trace-questions-D_OKYZ6u.d.ts} +3 -3
  142. package/dist/{trace-questions-C42fYEE4.d.ts.map → trace-questions-D_OKYZ6u.d.ts.map} +1 -1
  143. package/dist/trace-repair/index.d.ts +1 -1
  144. package/dist/traces.d.ts +3 -3
  145. package/dist/traces.js +3 -3
  146. package/dist/{types-DqFMNTgK.js → types-CnWmj8NW.js} +19 -6
  147. package/dist/types-CnWmj8NW.js.map +1 -0
  148. package/dist/{types-CHfn1zg5.d.ts → types-DDKfH-N6.d.ts} +2 -2
  149. package/dist/{types-CHfn1zg5.d.ts.map → types-DDKfH-N6.d.ts.map} +1 -1
  150. package/dist/{types-8yQzNm8u.d.ts → types-DGImvHIr.d.ts} +150 -4
  151. package/dist/types-DGImvHIr.d.ts.map +1 -0
  152. package/dist/{types-D8GXK9Z5.d.ts → types-IERw_HxR.d.ts} +2 -2
  153. package/dist/{types-D8GXK9Z5.d.ts.map → types-IERw_HxR.d.ts.map} +1 -1
  154. package/dist/wire/index.d.ts +1 -1
  155. package/docs/behavioral-trace-questions.md +99 -0
  156. package/docs/experiment.md +27 -0
  157. package/docs/finding-layers.md +54 -1
  158. package/docs/systemone.md +107 -0
  159. package/package.json +1 -1
  160. package/dist/benchmark-command-Dx0E4a6z.js.map +0 -1
  161. package/dist/default-registry-Cwg_QN3d.d.ts +0 -70
  162. package/dist/default-registry-Cwg_QN3d.d.ts.map +0 -1
  163. package/dist/default-registry-ovrIBGmH.js.map +0 -1
  164. package/dist/evaluation.js.map +0 -1
  165. package/dist/index-CVFStrQD.d.ts.map +0 -1
  166. package/dist/judge-gate-Dc_mXQsL.d.ts +0 -453
  167. package/dist/judge-gate-Dc_mXQsL.d.ts.map +0 -1
  168. package/dist/meta-eval/index.d.ts.map +0 -1
  169. package/dist/outcome-store-CNt4iZ67.d.ts +0 -58
  170. package/dist/outcome-store-CNt4iZ67.d.ts.map +0 -1
  171. package/dist/paired-arms-Ct4LTKc1.js.map +0 -1
  172. package/dist/rubric-predictive-validity-D3_oI72N.js.map +0 -1
  173. package/dist/rubric-predictive-validity-noY7uE03.d.ts.map +0 -1
  174. package/dist/statistical-heldout-3z3I-kJN.d.ts.map +0 -1
  175. package/dist/systemone-DVA7t76V.d.ts +0 -135
  176. package/dist/systemone-DVA7t76V.d.ts.map +0 -1
  177. package/dist/systemone.js.map +0 -1
  178. package/dist/tool-groups-LISbBWCg.d.ts.map +0 -1
  179. package/dist/types-8yQzNm8u.d.ts.map +0 -1
  180. package/dist/types-DqFMNTgK.js.map +0 -1
@@ -1,70 +0,0 @@
1
- import { W as TraceAnalysisStore, n as AnalystContext } from "./types-8yQzNm8u.js";
2
- import { l as ExactCapableAnalyst } from "./exact-types-BYFmYq44.js";
3
- import { i as TraceAnalystLimits, r as TraceAnalysisEngineResult, s as RawAnalystFinding, t as TraceAnalysisEngine } from "./engine-BpGGw6cd.js";
4
- import { t as TraceToolGroupName } from "./tool-groups-LISbBWCg.js";
5
- import { i as AnalystRegistryOptions, t as AnalystRegistry } from "./registry-B_yG-1jh.js";
6
- //#region src/analyst/behavioral-analyst.d.ts
7
- interface BehavioralAnalystOptions {
8
- /** Refuse larger unfiltered datasets instead of sampling them silently. Default 1,000. */
9
- maxTraces?: number;
10
- /** Evidence locations retained per finding. Counts still cover every analyzed trace. Default 20. */
11
- maxEvidenceRefsPerFinding?: number;
12
- }
13
- /** The deterministic behavioral/efficiency analyst (no LLM, any-model). */
14
- declare function behavioralAnalyst(options?: BehavioralAnalystOptions): ExactCapableAnalyst<TraceAnalysisStore>;
15
- //#endregion
16
- //#region src/analyst/kind-factory.d.ts
17
- /** A named research question independent of the recursive engine that runs it. */
18
- interface TraceAnalystDefinition {
19
- id: string;
20
- description: string;
21
- area: string;
22
- version: string;
23
- /** User-facing question. A function may derive it from run context. */
24
- question?: string | ((context: AnalystContext) => string);
25
- /** Research policy, evidence rules, and domain-specific instructions. */
26
- instructions: string;
27
- /** Smallest trace-tool set that can answer the question. */
28
- toolGroup: TraceToolGroupName;
29
- /** Optional bounded context assembled deterministically before the investigation. */
30
- prepareContext?: (store: TraceAnalysisStore, context: AnalystContext) => string | undefined | Promise<string | undefined>;
31
- limits?: Partial<TraceAnalystLimits>;
32
- postProcess?: (row: RawAnalystFinding, context: AnalystContext) => RawAnalystFinding | null;
33
- minimumEvidenceCitations?: number;
34
- requireStructuredFindings?: boolean;
35
- }
36
- interface CreateTraceAnalystOptions {
37
- engine: TraceAnalysisEngine;
38
- /** Override the definition's `version` (e.g. when an optimizer has fitted new instructions). */
39
- versionSuffix?: string;
40
- settlementTimeoutMs?: number;
41
- }
42
- /** Run one definition and retain its answer, findings, and full investigation record. */
43
- declare function runTraceAnalyst(args: {
44
- definition: TraceAnalystDefinition;
45
- engine: TraceAnalysisEngine;
46
- store: TraceAnalysisStore;
47
- context: AnalystContext;
48
- settlementTimeoutMs?: number;
49
- }): Promise<TraceAnalysisEngineResult>;
50
- /** Adapt a research definition to the common Analyst registry contract. */
51
- declare function createTraceAnalyst(definition: TraceAnalystDefinition, options: CreateTraceAnalystOptions): ExactCapableAnalyst<TraceAnalysisStore>;
52
- //#endregion
53
- //#region src/analyst/default-registry.d.ts
54
- interface DefaultAnalystRegistryOptions {
55
- /**
56
- * Recursive engine for model-backed trace analysts. When omitted, the
57
- * registry contains only deterministic analysts. The engine's id, version,
58
- * and model become the exact-run identity of every analyst it executes.
59
- */
60
- engine?: TraceAnalysisEngine;
61
- definitions?: readonly TraceAnalystDefinition[];
62
- /** Set false to omit the deterministic behavioral analyst (default: include). */
63
- includeBehavioral?: boolean;
64
- behavioral?: BehavioralAnalystOptions;
65
- registry?: AnalystRegistryOptions;
66
- }
67
- declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
68
- //#endregion
69
- export { createTraceAnalyst as a, behavioralAnalyst as c, TraceAnalystDefinition as i, buildDefaultAnalystRegistry as n, runTraceAnalyst as o, CreateTraceAnalystOptions as r, BehavioralAnalystOptions as s, DefaultAnalystRegistryOptions as t };
70
- //# sourceMappingURL=default-registry-Cwg_QN3d.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"default-registry-Cwg_QN3d.d.ts","names":[],"sources":["../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;UA+BiB;;EAEf;;EAEA;;;iBAoGc,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgHI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;;;UCrKN;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}