@tangle-network/agent-eval 0.136.0 → 0.137.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. package/CHANGELOG.md +38 -1
  2. package/README.md +4 -2
  3. package/dist/{agent-profile-cell-OhuTee9n.js → agent-profile-cell-CbfBm2g6.js} +2 -2
  4. package/dist/{agent-profile-cell-OhuTee9n.js.map → agent-profile-cell-CbfBm2g6.js.map} +1 -1
  5. package/dist/{agent-profile-cell-CCm3l2v2.d.ts → agent-profile-cell-Cw0PVwDr.d.ts} +2 -2
  6. package/dist/{agent-profile-cell-CCm3l2v2.d.ts.map → agent-profile-cell-Cw0PVwDr.d.ts.map} +1 -1
  7. package/dist/analyst/index.d.ts +139 -17
  8. package/dist/analyst/index.d.ts.map +1 -1
  9. package/dist/analyst/index.js +606 -4
  10. package/dist/analyst/index.js.map +1 -1
  11. package/dist/{analyze-runs-jjCmF8pU.js → analyze-runs-BScZvqMV.js} +6 -6
  12. package/dist/{analyze-runs-jjCmF8pU.js.map → analyze-runs-BScZvqMV.js.map} +1 -1
  13. package/dist/{analyze-runs-Cda5Xkj1.d.ts → analyze-runs-PVtnfjvA.d.ts} +6 -6
  14. package/dist/{analyze-runs-Cda5Xkj1.d.ts.map → analyze-runs-PVtnfjvA.d.ts.map} +1 -1
  15. package/dist/{baseline-BUeFcgrn.js → baseline-C-GocmIW.js} +2 -2
  16. package/dist/{baseline-BUeFcgrn.js.map → baseline-C-GocmIW.js.map} +1 -1
  17. package/dist/benchmark-CHX4orG7.d.ts +184 -0
  18. package/dist/benchmark-CHX4orG7.d.ts.map +1 -0
  19. package/dist/benchmark-YDrpumqB.js +414 -0
  20. package/dist/benchmark-YDrpumqB.js.map +1 -0
  21. package/dist/benchmarks/index.d.ts +1 -1
  22. package/dist/benchmarks/index.js +1 -1
  23. package/dist/{benchmarks-Dfgm9ts5.js → benchmarks-DCLkQOmc.js} +3 -3
  24. package/dist/{benchmarks-Dfgm9ts5.js.map → benchmarks-DCLkQOmc.js.map} +1 -1
  25. package/dist/builder-eval/index.js +2 -2
  26. package/dist/campaign/index.d.ts +6 -6
  27. package/dist/campaign/index.js +3 -3
  28. package/dist/{campaign-Dz8uQnhC.js → campaign-lgObcHFC.js} +212 -78
  29. package/dist/campaign-lgObcHFC.js.map +1 -0
  30. package/dist/cli.js +1 -1
  31. package/dist/client-C8L6h6Wf.d.ts +202 -0
  32. package/dist/client-C8L6h6Wf.d.ts.map +1 -0
  33. package/dist/completion-verifier-DSyRNVzU.d.ts +240 -0
  34. package/dist/completion-verifier-DSyRNVzU.d.ts.map +1 -0
  35. package/dist/contract/index.d.ts +10 -9
  36. package/dist/contract/index.d.ts.map +1 -1
  37. package/dist/contract/index.js +10 -10
  38. package/dist/control.d.ts +2 -2
  39. package/dist/control.js +1 -1
  40. package/dist/{cost-ledger-DHAjwNj7.js → cost-ledger-D-5_-dhi.js} +2 -2
  41. package/dist/{cost-ledger-DHAjwNj7.js.map → cost-ledger-D-5_-dhi.js.map} +1 -1
  42. package/dist/{cost-ledger-fGS_u_O1.d.ts → cost-ledger-D2o6JOrL.d.ts} +2 -2
  43. package/dist/{cost-ledger-fGS_u_O1.d.ts.map → cost-ledger-D2o6JOrL.d.ts.map} +1 -1
  44. package/dist/{dataset-BvtnC8Dc.d.ts → dataset-v_Y5902-.d.ts} +2 -2
  45. package/dist/{dataset-BvtnC8Dc.d.ts.map → dataset-v_Y5902-.d.ts.map} +1 -1
  46. package/dist/{default-registry-CHmdy2An.js → default-registry-CLXbRt0f.js} +119 -38
  47. package/dist/default-registry-CLXbRt0f.js.map +1 -0
  48. package/dist/{default-registry-Brxr728w.d.ts → default-registry-Dc5D_Loc.d.ts} +63 -137
  49. package/dist/default-registry-Dc5D_Loc.d.ts.map +1 -0
  50. package/dist/{errors-8YnH8WlF.js → errors-D-LKuDhb.js} +8 -2
  51. package/dist/errors-D-LKuDhb.js.map +1 -0
  52. package/dist/{errors-CEk209JS.d.ts → errors-DkfjIDvD.d.ts} +9 -3
  53. package/dist/errors-DkfjIDvD.d.ts.map +1 -0
  54. package/dist/{eval-campaign-Cc8WZJ6b.js → eval-campaign-CHqfLnff.js} +6 -6
  55. package/dist/{eval-campaign-Cc8WZJ6b.js.map → eval-campaign-CHqfLnff.js.map} +1 -1
  56. package/dist/{extract-usage-DIQpN-ww.js → extract-usage-p-56bh8q.js} +3 -3
  57. package/dist/{extract-usage-DIQpN-ww.js.map → extract-usage-p-56bh8q.js.map} +1 -1
  58. package/dist/{feedback-trajectory-CVaeREXV.d.ts → feedback-trajectory-N_F0PwHz.d.ts} +90 -3
  59. package/dist/feedback-trajectory-N_F0PwHz.d.ts.map +1 -0
  60. package/dist/fuzz.d.ts +1 -1
  61. package/dist/fuzz.js +2 -2
  62. package/dist/hosted/index.d.ts +3 -2
  63. package/dist/hosted/index.d.ts.map +1 -1
  64. package/dist/index-BipJlj-C.d.ts +316 -0
  65. package/dist/index-BipJlj-C.d.ts.map +1 -0
  66. package/dist/{index-CQsJcqch.d.ts → index-BnP1QJUv.d.ts} +5 -5
  67. package/dist/{index-CQsJcqch.d.ts.map → index-BnP1QJUv.d.ts.map} +1 -1
  68. package/dist/{index-B4Fjfo5U.d.ts → index-C-Pr4OWg.d.ts} +88 -317
  69. package/dist/index-C-Pr4OWg.d.ts.map +1 -0
  70. package/dist/{index-DuhJaaiH.d.ts → index-DEb46kc6.d.ts} +2 -2
  71. package/dist/{index-DuhJaaiH.d.ts.map → index-DEb46kc6.d.ts.map} +1 -1
  72. package/dist/{index-C2fkZhv_.d.ts → index-DRNl6g_N.d.ts} +3 -3
  73. package/dist/{index-C2fkZhv_.d.ts.map → index-DRNl6g_N.d.ts.map} +1 -1
  74. package/dist/{index-AbhwHp0V.d.ts → index-U3RHOShi.d.ts} +2 -2
  75. package/dist/{index-AbhwHp0V.d.ts.map → index-U3RHOShi.d.ts.map} +1 -1
  76. package/dist/index.d.ts +29 -70
  77. package/dist/index.d.ts.map +1 -1
  78. package/dist/index.js +507 -33
  79. package/dist/index.js.map +1 -1
  80. package/dist/{client-DcvgkaZi.d.ts → insight-report-B9ooYH_g.d.ts} +5 -203
  81. package/dist/insight-report-B9ooYH_g.d.ts.map +1 -0
  82. package/dist/integrity-CCXTftiL.js +1360 -0
  83. package/dist/integrity-CCXTftiL.js.map +1 -0
  84. package/dist/{integrity-rmVhXWA7.d.ts → integrity-CKxosZ5Z.d.ts} +3 -3
  85. package/dist/{integrity-rmVhXWA7.d.ts.map → integrity-CKxosZ5Z.d.ts.map} +1 -1
  86. package/dist/{integrity-BzRbCHzi.js → integrity-fdt8XPAv.js} +2 -2
  87. package/dist/{integrity-BzRbCHzi.js.map → integrity-fdt8XPAv.js.map} +1 -1
  88. package/dist/ledger-core/index.d.ts +1 -1
  89. package/dist/ledger-core/index.js +1 -1
  90. package/dist/{ledger-core-DAKFKRzi.js → ledger-core-t6sItivm.js} +85 -85
  91. package/dist/{ledger-core-DAKFKRzi.js.map → ledger-core-t6sItivm.js.map} +1 -1
  92. package/dist/{llm-client-DHx8pzyJ.js → llm-client-DKB25jV8.js} +3 -3
  93. package/dist/{llm-client-DHx8pzyJ.js.map → llm-client-DKB25jV8.js.map} +1 -1
  94. package/dist/meta-eval/index.d.ts +2 -2
  95. package/dist/meta-eval/index.js +3 -3
  96. package/dist/{mint-DyRUc9k6.js → mint-Ctwk079K.js} +4 -4
  97. package/dist/{mint-DyRUc9k6.js.map → mint-Ctwk079K.js.map} +1 -1
  98. package/dist/multishot/index.d.ts +2 -2
  99. package/dist/openapi.json +1 -1
  100. package/dist/{paired-arms-BbFKrAU-.js → paired-arms-iZ08VFMN.js} +3 -3
  101. package/dist/{paired-arms-BbFKrAU-.js.map → paired-arms-iZ08VFMN.js.map} +1 -1
  102. package/dist/pipelines/index.js +2 -2
  103. package/dist/profile-cell.d.ts +1 -1
  104. package/dist/profile-cell.js +1 -1
  105. package/dist/{propose-review-control-SQ-n9-We.js → propose-review-control-DLXz4FCX.js} +2 -2
  106. package/dist/{propose-review-control-SQ-n9-We.js.map → propose-review-control-DLXz4FCX.js.map} +1 -1
  107. package/dist/registry-BdM7SuTr.d.ts +124 -0
  108. package/dist/registry-BdM7SuTr.d.ts.map +1 -0
  109. package/dist/{release-report-DooPguBc.js → release-report-B5XPBvAU.js} +4 -4
  110. package/dist/{release-report-DooPguBc.js.map → release-report-B5XPBvAU.js.map} +1 -1
  111. package/dist/{release-report-DpBxGGI1.d.ts → release-report-CofgVNZt.d.ts} +4 -4
  112. package/dist/{release-report-DpBxGGI1.d.ts.map → release-report-CofgVNZt.d.ts.map} +1 -1
  113. package/dist/{replay-C6wRg47C.js → replay-Bju0T8Ls.js} +248 -8
  114. package/dist/replay-Bju0T8Ls.js.map +1 -0
  115. package/dist/{replay-BRfMIs81.d.ts → replay-K8FaC0CB.d.ts} +227 -52
  116. package/dist/replay-K8FaC0CB.d.ts.map +1 -0
  117. package/dist/reporting.d.ts +4 -4
  118. package/dist/reporting.js +4 -4
  119. package/dist/{researcher-Doo95b50.d.ts → researcher-Da0Wj-bt.d.ts} +6 -7
  120. package/dist/researcher-Da0Wj-bt.d.ts.map +1 -0
  121. package/dist/{reward-hacking-D-QqXvg-.d.ts → reward-hacking-CQ3hTCO3.d.ts} +2 -2
  122. package/dist/{reward-hacking-D-QqXvg-.d.ts.map → reward-hacking-CQ3hTCO3.d.ts.map} +1 -1
  123. package/dist/{reward-hacking-a-kYs0-i.js → reward-hacking-GyN0kMd8.js} +3 -3
  124. package/dist/{reward-hacking-a-kYs0-i.js.map → reward-hacking-GyN0kMd8.js.map} +1 -1
  125. package/dist/rl.d.ts +6 -6
  126. package/dist/rl.js +9 -9
  127. package/dist/rollout/index.d.ts +1 -1
  128. package/dist/rollout/index.js +3 -3
  129. package/dist/{rollout-DLSUIWLu.js → rollout-DQFl0UXA.js} +2 -2
  130. package/dist/{rollout-DLSUIWLu.js.map → rollout-DQFl0UXA.js.map} +1 -1
  131. package/dist/{rubric-predictive-validity-BJf-8ejY.js → rubric-predictive-validity-BRR632r1.js} +2 -2
  132. package/dist/{rubric-predictive-validity-BJf-8ejY.js.map → rubric-predictive-validity-BRR632r1.js.map} +1 -1
  133. package/dist/{rubric-predictive-validity-C1dCLcvb.d.ts → rubric-predictive-validity-C4sztLR3.d.ts} +2 -2
  134. package/dist/{rubric-predictive-validity-C1dCLcvb.d.ts.map → rubric-predictive-validity-C4sztLR3.d.ts.map} +1 -1
  135. package/dist/{run-evidence-DokQtX0-.d.ts → run-evidence-BDIircdA.d.ts} +3 -3
  136. package/dist/{run-evidence-DokQtX0-.d.ts.map → run-evidence-BDIircdA.d.ts.map} +1 -1
  137. package/dist/{run-record-DcObtIGh.d.ts → run-record-BPCa2rQ8.d.ts} +4 -4
  138. package/dist/{run-record-DcObtIGh.d.ts.map → run-record-BPCa2rQ8.d.ts.map} +1 -1
  139. package/dist/{run-record-BIwU2wdV.js → run-record-vRgqWmJw.js} +3 -3
  140. package/dist/{run-record-BIwU2wdV.js.map → run-record-vRgqWmJw.js.map} +1 -1
  141. package/dist/{semantic-concept-judge-Btozx3Vc.js → semantic-concept-judge-Bz64IckK.js} +4 -4
  142. package/dist/{semantic-concept-judge-Btozx3Vc.js.map → semantic-concept-judge-Bz64IckK.js.map} +1 -1
  143. package/dist/{server-Bz3WQJs6.js → server-KjXZZUDX.js} +3 -3
  144. package/dist/{server-Bz3WQJs6.js.map → server-KjXZZUDX.js.map} +1 -1
  145. package/dist/{skill-usage-BDQVPIG1.d.ts → skill-usage-CFDLLlhF.d.ts} +25 -47
  146. package/dist/skill-usage-CFDLLlhF.d.ts.map +1 -0
  147. package/dist/{skillopt-optimization-method-CwSYkv35.d.ts → skillopt-optimization-method-BpbnlvAZ.d.ts} +11 -12
  148. package/dist/skillopt-optimization-method-BpbnlvAZ.d.ts.map +1 -0
  149. package/dist/{skillopt-optimization-method-0UmPD6aP.js → skillopt-optimization-method-f4o9sUT4.js} +7 -7
  150. package/dist/{skillopt-optimization-method-0UmPD6aP.js.map → skillopt-optimization-method-f4o9sUT4.js.map} +1 -1
  151. package/dist/{statistics-CnGCLLqc.js → statistics-ByxzSiOM.js} +2 -2
  152. package/dist/{statistics-CnGCLLqc.js.map → statistics-ByxzSiOM.js.map} +1 -1
  153. package/dist/{statistics-CKOqre5S.d.ts → statistics-_7P642CN.d.ts} +2 -2
  154. package/dist/{statistics-CKOqre5S.d.ts.map → statistics-_7P642CN.d.ts.map} +1 -1
  155. package/dist/{summary-report-BEk8OFLs.js → summary-report-9A5y7EsK.js} +4 -4
  156. package/dist/{summary-report-BEk8OFLs.js.map → summary-report-9A5y7EsK.js.map} +1 -1
  157. package/dist/{summary-report-CPMINBqs.d.ts → summary-report-DHipz9Kx.d.ts} +3 -3
  158. package/dist/{summary-report-CPMINBqs.d.ts.map → summary-report-DHipz9Kx.d.ts.map} +1 -1
  159. package/dist/supervisor-run/index.d.ts +3 -2
  160. package/dist/supervisor-run/index.js +3 -2
  161. package/dist/{supervisor-run-Dr5HnTup.js → supervisor-run-B2EWUmQY.js} +28 -454
  162. package/dist/supervisor-run-B2EWUmQY.js.map +1 -0
  163. package/dist/{test-graded-scenario-BsqWLmPt.js → test-graded-scenario-JHcKQNpq.js} +2 -2
  164. package/dist/{test-graded-scenario-BsqWLmPt.js.map → test-graded-scenario-JHcKQNpq.js.map} +1 -1
  165. package/dist/tools-DZk2Jn64.js +1876 -0
  166. package/dist/tools-DZk2Jn64.js.map +1 -0
  167. package/dist/traces.d.ts +6 -7
  168. package/dist/traces.js +5 -6
  169. package/dist/{types-DVjczBM9.d.ts → types-CKswbJGO.d.ts} +260 -6
  170. package/dist/types-CKswbJGO.d.ts.map +1 -0
  171. package/dist/{types-DiWLru6Z.d.ts → types-CTGbIm57.d.ts} +5 -5
  172. package/dist/{types-DiWLru6Z.d.ts.map → types-CTGbIm57.d.ts.map} +1 -1
  173. package/dist/types-CTvKfr5F.d.ts +804 -0
  174. package/dist/types-CTvKfr5F.d.ts.map +1 -0
  175. package/dist/{index-CyC1BTmn.d.ts → types-Dea6tiVI.d.ts} +16 -238
  176. package/dist/types-Dea6tiVI.d.ts.map +1 -0
  177. package/dist/wire/index.d.ts +3 -3
  178. package/dist/wire/index.js +1 -1
  179. package/docs/feedback-trajectories.md +100 -1
  180. package/docs/trace-analysis.md +374 -58
  181. package/package.json +5 -1
  182. package/dist/analyst-BkTS3C58.d.ts +0 -89
  183. package/dist/analyst-BkTS3C58.d.ts.map +0 -1
  184. package/dist/analyst-j5je5J7c.js +0 -152
  185. package/dist/analyst-j5je5J7c.js.map +0 -1
  186. package/dist/campaign-Dz8uQnhC.js.map +0 -1
  187. package/dist/client-DcvgkaZi.d.ts.map +0 -1
  188. package/dist/default-registry-Brxr728w.d.ts.map +0 -1
  189. package/dist/default-registry-CHmdy2An.js.map +0 -1
  190. package/dist/errors-8YnH8WlF.js.map +0 -1
  191. package/dist/errors-CEk209JS.d.ts.map +0 -1
  192. package/dist/feedback-trajectory-CVaeREXV.d.ts.map +0 -1
  193. package/dist/index-B4Fjfo5U.d.ts.map +0 -1
  194. package/dist/index-CyC1BTmn.d.ts.map +0 -1
  195. package/dist/llm-client-BiK4HW0u.d.ts +0 -290
  196. package/dist/llm-client-BiK4HW0u.d.ts.map +0 -1
  197. package/dist/raw-provider-sink-BU29Sh8h.d.ts +0 -134
  198. package/dist/raw-provider-sink-BU29Sh8h.d.ts.map +0 -1
  199. package/dist/replay-BRfMIs81.d.ts.map +0 -1
  200. package/dist/replay-C6wRg47C.js.map +0 -1
  201. package/dist/researcher-Doo95b50.d.ts.map +0 -1
  202. package/dist/skill-usage-BDQVPIG1.d.ts.map +0 -1
  203. package/dist/skillopt-optimization-method-CwSYkv35.d.ts.map +0 -1
  204. package/dist/store-CxJry_cs.d.ts +0 -229
  205. package/dist/store-CxJry_cs.d.ts.map +0 -1
  206. package/dist/supervisor-run-Dr5HnTup.js.map +0 -1
  207. package/dist/tools-D8yTtNSN.js +0 -1190
  208. package/dist/tools-D8yTtNSN.js.map +0 -1
  209. package/dist/types-Cc3qbqzj.d.ts +0 -387
  210. package/dist/types-Cc3qbqzj.d.ts.map +0 -1
  211. package/dist/types-DVjczBM9.d.ts.map +0 -1
@@ -1,152 +0,0 @@
1
- import { i as OtlpFileTraceStore, r as runTraceAnalysisLoop, t as buildTraceAnalystTools } from "./tools-D8yTtNSN.js";
2
- //#region src/trace-analyst/prompts.ts
3
- /** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */
4
- const TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \`traces\` namespace.
5
-
6
- DISCOVERY → NARROW → DEEP-READ protocol — follow exactly:
7
-
8
- 1. ALWAYS call \`traces.getDatasetOverview({})\` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids — never fabricate one).
9
-
10
- 2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. \`filters.regex_pattern\` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces — narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.
11
-
12
- 3. To list more traces than the sample, call \`traces.queryTraces({ filters?, limit, offset? })\`. Each summary carries raw_jsonl_bytes — use it to choose between viewTrace and searchTrace BEFORE calling either.
13
-
14
- 4. Per-trace inspection:
15
- - SMALL trace (raw_jsonl_bytes well under 150_000): call \`traces.viewTrace({ trace_id })\`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large \`input.value\` / \`output.value\` / \`llm.input_messages\` will show a \`[trace-analyst truncated: N bytes]\` marker.
16
- - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an \`oversized\` response): use \`traces.searchTrace({ trace_id, regex_pattern })\` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call \`traces.viewSpans({ trace_id, span_ids: [...] })\` for surgical reads (~16KB cap, 4× higher than discovery), or \`traces.searchSpan({ trace_id, span_id, regex_pattern })\` for one large span. Stays bounded regardless of trace size.
17
- - Useful regex patterns: \`STATUS_CODE_ERROR\` (failures), tool names like \`grep\` or \`view_trace\`, error strings like \`MaxTurnsExceeded\`, model names, attribute keys.
18
-
19
- 5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.
20
-
21
- 5a. **Result-shape contract** — searchTrace and searchSpan return \`{ trace_id, hits, total_matches, has_more }\`. Iterate \`result.hits\` (NOT result.matches). Each hit has \`{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }\`. viewTrace returns \`{ trace_id, spans }\` (or \`oversized\`). viewSpans returns \`{ trace_id, spans, missing_span_ids, truncated_attribute_count }\`. Never assume a field name — log the result shape first if unsure.
22
-
23
- 6. If viewTrace returns an \`oversized\` summary instead of \`spans\`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.
24
-
25
- 7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.
26
-
27
- 8. If a tool errors (invalid regex, range error), STOP and reconsider — don't retry with a guessed id or argument. Use the discovery tools above to recover.
28
-
29
- 9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.
30
-
31
- 10. If the question splits into independent reasoning branches, use bounded \`llmQuery(...)\` calls over evidence you already loaded. Subqueries cannot inspect the trace store, so pass the exact trace excerpts they need. Example:
32
-
33
- const reviews = await llmQuery([
34
- { query: 'Classify the failure mechanism in this excerpt.', context: traceAbcExcerpt },
35
- { query: 'Classify the failure mechanism in this excerpt.', context: traceDefExcerpt },
36
- ]);
37
-
38
- OBSERVABILITY rules:
39
- - Each discovery turn must emit at least one concise \`console.log(...)\` showing what evidence was learned.
40
- - Finish gathering evidence before submitting the analysis.
41
- - Reuse runtime variables across turns; don't recompute.
42
-
43
- OUTPUT contract — your final answer must include:
44
- - A clear prose conclusion answering the user's question.
45
- - Trace ids and span ids cited as evidence for each claim.
46
- - Failure modes named in the user's domain language, with frequency and concrete examples.
47
- - A concise findings array containing only claims supported by inspected evidence.
48
-
49
- Do NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.`;
50
- const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v6-2026-07-14";
51
- //#endregion
52
- //#region src/trace-analyst/analyst.ts
53
- /**
54
- * Run the trace analyst.
55
- *
56
- * Throws:
57
- * - `TraceFileMissingError` if `source` is a path and doesn't exist.
58
- * - `AxAgentClarificationError` if the analyst asks for clarification.
59
- * - Provider errors (auth, rate limits) propagate from the AI service.
60
- */
61
- async function analyzeTraces(input, options) {
62
- if (!input.question || typeof input.question !== "string") throw new TypeError("analyzeTraces: input.question must be a non-empty string");
63
- rejectRemovedOptions(options);
64
- const store = typeof options.source === "string" ? new OtlpFileTraceStore({ path: options.source }) : options.source;
65
- if (store instanceof OtlpFileTraceStore) await store.ensureIndexed();
66
- const tools = buildTraceAnalystTools({ store });
67
- const turns = [];
68
- let progressFs;
69
- if (options.progressLogPath) {
70
- const { createWriteStream } = await import("node:fs");
71
- const { mkdir } = await import("node:fs/promises");
72
- const { dirname } = await import("node:path");
73
- await mkdir(dirname(options.progressLogPath), { recursive: true });
74
- progressFs = createWriteStream(options.progressLogPath, { flags: "a" });
75
- }
76
- const actorTurnCallback = async (turn) => {
77
- const snap = {
78
- stage: turn.stage,
79
- turn: turn.turn,
80
- isError: turn.isError,
81
- code: turn.code,
82
- output: turn.output,
83
- thought: turn.thought
84
- };
85
- turns.push(snap);
86
- if (progressFs) try {
87
- progressFs.write(`${JSON.stringify({
88
- ...snap,
89
- ts: Date.now()
90
- })}\n`);
91
- } catch {}
92
- if (options.onTurn) await options.onTurn(snap);
93
- };
94
- const maxSubqueries = options.maxSubqueries ?? 4;
95
- const maxTurns = options.maxTurns ?? 12;
96
- const maxParallelSubqueries = options.maxParallelSubqueries ?? 2;
97
- const maxRuntimeChars = options.maxRuntimeChars ?? 6e3;
98
- let completed;
99
- try {
100
- completed = await runTraceAnalysisLoop({
101
- id: "TraceAnalyst",
102
- description: "Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes.",
103
- prompt: `${options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION}
104
-
105
- The report must answer the user's question, and findings must be an array of concise evidence-backed strings.`,
106
- question: input.question,
107
- ai: options.ai,
108
- ...options.model ? { model: options.model } : {},
109
- tools,
110
- findingType: "string",
111
- maxSubqueries,
112
- maxParallelSubqueries,
113
- maxTurns,
114
- maxRuntimeChars,
115
- ...options.signal ? { signal: options.signal } : {},
116
- onTurn: actorTurnCallback
117
- });
118
- } finally {
119
- if (progressFs) await new Promise((resolve) => progressFs.end(() => resolve()));
120
- }
121
- return {
122
- answer: completed.report,
123
- findings: completed.findings,
124
- turns,
125
- turnCount: turns.length,
126
- usage: {
127
- actor: normalizeRecordArray(completed.usage),
128
- responder: []
129
- },
130
- chatLog: {
131
- actor: normalizeRecordArray(completed.chatLog),
132
- responder: []
133
- },
134
- actorPromptVersion: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION
135
- };
136
- }
137
- function rejectRemovedOptions(options) {
138
- const supplied = options;
139
- for (const [removed, replacement] of [
140
- ["maxDepth", "maxSubqueries"],
141
- ["maxParallelSubagents", "maxParallelSubqueries"],
142
- ["subagentDescription", "actorDescription"]
143
- ]) if (removed in supplied) throw new TypeError(`analyzeTraces: '${removed}' is unsupported; use '${replacement}'`);
144
- }
145
- function normalizeRecordArray(value) {
146
- if (!Array.isArray(value)) return [];
147
- return value.map((item) => item && typeof item === "object" ? { ...item } : { value: item });
148
- }
149
- //#endregion
150
- export { TRACE_ANALYST_ACTOR_DESCRIPTION as n, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION as r, analyzeTraces as t };
151
-
152
- //# sourceMappingURL=analyst-j5je5J7c.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"analyst-j5je5J7c.js","names":[],"sources":["../src/trace-analyst/prompts.ts","../src/trace-analyst/analyst.ts"],"sourcesContent":["/** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */\n\nexport const TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \\`traces\\` namespace.\n\nDISCOVERY → NARROW → DEEP-READ protocol — follow exactly:\n\n1. ALWAYS call \\`traces.getDatasetOverview({})\\` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids — never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. \\`filters.regex_pattern\\` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces — narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call \\`traces.queryTraces({ filters?, limit, offset? })\\`. Each summary carries raw_jsonl_bytes — use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n - SMALL trace (raw_jsonl_bytes well under 150_000): call \\`traces.viewTrace({ trace_id })\\`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large \\`input.value\\` / \\`output.value\\` / \\`llm.input_messages\\` will show a \\`[trace-analyst truncated: N bytes]\\` marker.\n - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an \\`oversized\\` response): use \\`traces.searchTrace({ trace_id, regex_pattern })\\` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call \\`traces.viewSpans({ trace_id, span_ids: [...] })\\` for surgical reads (~16KB cap, 4× higher than discovery), or \\`traces.searchSpan({ trace_id, span_id, regex_pattern })\\` for one large span. Stays bounded regardless of trace size.\n - Useful regex patterns: \\`STATUS_CODE_ERROR\\` (failures), tool names like \\`grep\\` or \\`view_trace\\`, error strings like \\`MaxTurnsExceeded\\`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** — searchTrace and searchSpan return \\`{ trace_id, hits, total_matches, has_more }\\`. Iterate \\`result.hits\\` (NOT result.matches). Each hit has \\`{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }\\`. viewTrace returns \\`{ trace_id, spans }\\` (or \\`oversized\\`). viewSpans returns \\`{ trace_id, spans, missing_span_ids, truncated_attribute_count }\\`. Never assume a field name — log the result shape first if unsure.\n\n6. If viewTrace returns an \\`oversized\\` summary instead of \\`spans\\`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider — don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If the question splits into independent reasoning branches, use bounded \\`llmQuery(...)\\` calls over evidence you already loaded. Subqueries cannot inspect the trace store, so pass the exact trace excerpts they need. Example:\n\n const reviews = await llmQuery([\n { query: 'Classify the failure mechanism in this excerpt.', context: traceAbcExcerpt },\n { query: 'Classify the failure mechanism in this excerpt.', context: traceDefExcerpt },\n ]);\n\nOBSERVABILITY rules:\n- Each discovery turn must emit at least one concise \\`console.log(...)\\` showing what evidence was learned.\n- Finish gathering evidence before submitting the analysis.\n- Reuse runtime variables across turns; don't recompute.\n\nOUTPUT contract — your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n- A concise findings array containing only claims supported by inspected evidence.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.`\n\nexport const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = 'trace-analyst-actor-v6-2026-07-14'\n","import type { AxAgentActorTurnCallbackArgs, AxAIService, AxFunction } from '@ax-llm/ax'\nimport { runTraceAnalysisLoop, type TraceAnalysisLoopResult } from './loop'\nimport { TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION } from './prompts'\nimport type { TraceAnalysisStore } from './store'\nimport { OtlpFileTraceStore } from './store-otlp'\nimport { buildTraceAnalystTools } from './tools'\n\nexport interface AnalyzeTracesInput {\n /** The user-facing question. Domain framing belongs here, not in the\n * actor description. */\n question: string\n}\n\nexport interface AnalyzeTracesResult {\n /** The actor's submitted prose answer. */\n answer: string\n /** Bulleted findings from the actor's structured completion. */\n findings: string[]\n /** Per-turn snapshots captured via `actorTurnCallback`. */\n turns: AnalyzeTracesTurnSnapshot[]\n /** Total turns the actor took. */\n turnCount: number\n /** Token usage by role. */\n usage: TraceAnalystUsage\n /** Full system + assistant + tool message log by role. */\n chatLog: TraceAnalystChatLog\n /** Prompt version that produced this run. */\n actorPromptVersion: string\n}\n\nexport interface TraceAnalystUsage {\n actor: TraceAnalystUsageEntry[]\n responder: TraceAnalystUsageEntry[]\n}\n\nexport interface TraceAnalystUsageEntry {\n [key: string]: unknown\n}\n\nexport interface TraceAnalystChatLog {\n actor: TraceAnalystChatMessage[]\n responder: TraceAnalystChatMessage[]\n}\n\nexport interface TraceAnalystChatMessage {\n [key: string]: unknown\n}\n\nexport interface AnalyzeTracesTurnSnapshot {\n stage: AxAgentActorTurnCallbackArgs['stage']\n turn: number\n isError: boolean\n /** The JS code the actor produced for this turn. */\n code: string\n /** The formatted action-log entry the actor sees on the next turn. */\n output: string\n /** Provider thought (when `executorOptions.showThoughts` is true and the\n * provider returns it). */\n thought?: string\n}\n\nexport interface AnalyzeTracesOptions {\n /** Trace data source. Pass either an OTLP-JSONL path or a custom store. */\n source: string | TraceAnalysisStore\n /** Caller-provided AxAIService. */\n ai: AxAIService\n /** Model id forwarded to the actor. */\n model?: string\n /** Maximum model subqueries. 0 disables model fan-out. Default 4. */\n maxSubqueries?: number\n /** Maximum actor turns. Default 12. */\n maxTurns?: number\n /** Maximum parallel model subqueries. Default 2. */\n maxParallelSubqueries?: number\n /** Cancels in-flight model and tool work. */\n signal?: AbortSignal\n /** Override the actor description. */\n actorDescription?: string\n /** Per-turn observability hook. */\n onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>\n /** Override max runtime characters per turn. Default 6000. */\n maxRuntimeChars?: number\n /** When set, every turn's snapshot is appended to this JSONL file\n * immediately. If the analyst crashes mid-loop (provider 503,\n * network error, validator reject) the partial reasoning is still\n * on disk for diagnosis and recovery. */\n progressLogPath?: string\n}\n\n/**\n * Run the trace analyst.\n *\n * Throws:\n * - `TraceFileMissingError` if `source` is a path and doesn't exist.\n * - `AxAgentClarificationError` if the analyst asks for clarification.\n * - Provider errors (auth, rate limits) propagate from the AI service.\n */\nexport async function analyzeTraces(\n input: AnalyzeTracesInput,\n options: AnalyzeTracesOptions,\n): Promise<AnalyzeTracesResult> {\n if (!input.question || typeof input.question !== 'string') {\n throw new TypeError('analyzeTraces: input.question must be a non-empty string')\n }\n rejectRemovedOptions(options)\n\n const store: TraceAnalysisStore =\n typeof options.source === 'string'\n ? new OtlpFileTraceStore({ path: options.source })\n : options.source\n\n // Pre-warm file stores so missing inputs fail before the RLM starts.\n if (store instanceof OtlpFileTraceStore) {\n await store.ensureIndexed()\n }\n\n const tools: AxFunction[] = buildTraceAnalystTools({ store })\n const turns: AnalyzeTracesTurnSnapshot[] = []\n\n // Persist each turn as JSONL so interrupted analyst runs keep useful evidence.\n let progressFs: import('node:fs').WriteStream | undefined\n if (options.progressLogPath) {\n const { createWriteStream } = await import('node:fs')\n const { mkdir } = await import('node:fs/promises')\n const { dirname } = await import('node:path')\n await mkdir(dirname(options.progressLogPath), { recursive: true })\n progressFs = createWriteStream(options.progressLogPath, { flags: 'a' })\n }\n\n const actorTurnCallback = async (turn: AxAgentActorTurnCallbackArgs): Promise<void> => {\n const snap: AnalyzeTracesTurnSnapshot = {\n stage: turn.stage,\n turn: turn.turn,\n isError: turn.isError,\n code: turn.code,\n output: turn.output,\n thought: turn.thought,\n }\n turns.push(snap)\n if (progressFs) {\n try {\n progressFs.write(`${JSON.stringify({ ...snap, ts: Date.now() })}\\n`)\n } catch {\n // Progress logging must never fail the analyst.\n }\n }\n if (options.onTurn) await options.onTurn(snap)\n }\n\n const maxSubqueries = options.maxSubqueries ?? 4\n const maxTurns = options.maxTurns ?? 12\n const maxParallelSubqueries = options.maxParallelSubqueries ?? 2\n const maxRuntimeChars = options.maxRuntimeChars ?? 6000\n let completed: TraceAnalysisLoopResult<string>\n try {\n completed = await runTraceAnalysisLoop({\n id: 'TraceAnalyst',\n description:\n 'Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes.',\n prompt: `${options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION}\n\nThe report must answer the user's question, and findings must be an array of concise evidence-backed strings.`,\n question: input.question,\n ai: options.ai,\n ...(options.model ? { model: options.model } : {}),\n tools,\n findingType: 'string',\n maxSubqueries,\n maxParallelSubqueries,\n maxTurns,\n maxRuntimeChars,\n ...(options.signal ? { signal: options.signal } : {}),\n onTurn: actorTurnCallback,\n })\n } finally {\n if (progressFs) {\n await new Promise<void>((resolve) => progressFs!.end(() => resolve()))\n }\n }\n\n return {\n answer: completed.report,\n findings: completed.findings,\n turns,\n turnCount: turns.length,\n usage: { actor: normalizeRecordArray(completed.usage), responder: [] },\n chatLog: { actor: normalizeRecordArray(completed.chatLog), responder: [] },\n actorPromptVersion: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,\n }\n}\n\nfunction rejectRemovedOptions(options: AnalyzeTracesOptions): void {\n const supplied = options as unknown as Record<string, unknown>\n const migrations = [\n ['maxDepth', 'maxSubqueries'],\n ['maxParallelSubagents', 'maxParallelSubqueries'],\n ['subagentDescription', 'actorDescription'],\n ] as const\n for (const [removed, replacement] of migrations) {\n if (removed in supplied) {\n throw new TypeError(`analyzeTraces: '${removed}' is unsupported; use '${replacement}'`)\n }\n }\n}\n\nfunction normalizeRecordArray(value: unknown): Record<string, unknown>[] {\n if (!Array.isArray(value)) return []\n return value.map((item) =>\n item && typeof item === 'object' ? { ...(item as Record<string, unknown>) } : { value: item },\n )\n}\n"],"mappings":";;;AAEA,MAAa,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+C/C,MAAa,0CAA0C;;;;;;;;;;;ACgDvD,eAAsB,cACpB,OACA,SAC8B;CAC9B,IAAI,CAAC,MAAM,YAAY,OAAO,MAAM,aAAa,UAC/C,MAAM,IAAI,UAAU,0DAA0D;CAEhF,qBAAqB,OAAO;CAE5B,MAAM,QACJ,OAAO,QAAQ,WAAW,WACtB,IAAI,mBAAmB,EAAE,MAAM,QAAQ,OAAO,CAAC,IAC/C,QAAQ;CAGd,IAAI,iBAAiB,oBACnB,MAAM,MAAM,cAAc;CAG5B,MAAM,QAAsB,uBAAuB,EAAE,MAAM,CAAC;CAC5D,MAAM,QAAqC,CAAC;CAG5C,IAAI;CACJ,IAAI,QAAQ,iBAAiB;EAC3B,MAAM,EAAE,sBAAsB,MAAM,OAAO;EAC3C,MAAM,EAAE,UAAU,MAAM,OAAO;EAC/B,MAAM,EAAE,YAAY,MAAM,OAAO;EACjC,MAAM,MAAM,QAAQ,QAAQ,eAAe,GAAG,EAAE,WAAW,KAAK,CAAC;EACjE,aAAa,kBAAkB,QAAQ,iBAAiB,EAAE,OAAO,IAAI,CAAC;CACxE;CAEA,MAAM,oBAAoB,OAAO,SAAsD;EACrF,MAAM,OAAkC;GACtC,OAAO,KAAK;GACZ,MAAM,KAAK;GACX,SAAS,KAAK;GACd,MAAM,KAAK;GACX,QAAQ,KAAK;GACb,SAAS,KAAK;EAChB;EACA,MAAM,KAAK,IAAI;EACf,IAAI,YACF,IAAI;GACF,WAAW,MAAM,GAAG,KAAK,UAAU;IAAE,GAAG;IAAM,IAAI,KAAK,IAAI;GAAE,CAAC,EAAE,GAAG;EACrE,QAAQ,CAER;EAEF,IAAI,QAAQ,QAAQ,MAAM,QAAQ,OAAO,IAAI;CAC/C;CAEA,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,wBAAwB,QAAQ,yBAAyB;CAC/D,MAAM,kBAAkB,QAAQ,mBAAmB;CACnD,IAAI;CACJ,IAAI;EACF,YAAY,MAAM,qBAAqB;GACrC,IAAI;GACJ,aACE;GACF,QAAQ,GAAG,QAAQ,oBAAoB,gCAAgC;;;GAGvE,UAAU,MAAM;GAChB,IAAI,QAAQ;GACZ,GAAI,QAAQ,QAAQ,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;GAChD;GACA,aAAa;GACb;GACA;GACA;GACA;GACA,GAAI,QAAQ,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;GACnD,QAAQ;EACV,CAAC;CACH,UAAU;EACR,IAAI,YACF,MAAM,IAAI,SAAe,YAAY,WAAY,UAAU,QAAQ,CAAC,CAAC;CAEzE;CAEA,OAAO;EACL,QAAQ,UAAU;EAClB,UAAU,UAAU;EACpB;EACA,WAAW,MAAM;EACjB,OAAO;GAAE,OAAO,qBAAqB,UAAU,KAAK;GAAG,WAAW,CAAC;EAAE;EACrE,SAAS;GAAE,OAAO,qBAAqB,UAAU,OAAO;GAAG,WAAW,CAAC;EAAE;EACzE,oBAAoB;CACtB;AACF;AAEA,SAAS,qBAAqB,SAAqC;CACjE,MAAM,WAAW;CAMjB,KAAK,MAAM,CAAC,SAAS,gBAAgB;EAJnC,CAAC,YAAY,eAAe;EAC5B,CAAC,wBAAwB,uBAAuB;EAChD,CAAC,uBAAuB,kBAAkB;CAEE,GAC5C,IAAI,WAAW,UACb,MAAM,IAAI,UAAU,mBAAmB,QAAQ,yBAAyB,YAAY,EAAE;AAG5F;AAEA,SAAS,qBAAqB,OAA2C;CACvE,IAAI,CAAC,MAAM,QAAQ,KAAK,GAAG,OAAO,CAAC;CACnC,OAAO,MAAM,KAAK,SAChB,QAAQ,OAAO,SAAS,WAAW,EAAE,GAAI,KAAiC,IAAI,EAAE,OAAO,KAAK,CAC9F;AACF"}