@opensearch-project/agent-health 0.3.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (518) hide show
  1. package/README.md +77 -6
  2. package/cli/dist/index.js +10072 -4502
  3. package/deployment/cloudformation/agent-health-observability.yaml +762 -0
  4. package/dist/assets/index-CCQRDlO0.js +243 -0
  5. package/dist/assets/index-CNHQVbcj.css +1 -0
  6. package/dist/index.html +2 -2
  7. package/docs/ARCHITECTURE.md +450 -0
  8. package/docs/BACKEND_JOB_QUEUE.md +405 -0
  9. package/docs/CLAUDE_CODE_TELEMETRY.md +283 -0
  10. package/docs/CLI.md +431 -0
  11. package/docs/CODING_AGENT_ANALYTICS.md +298 -0
  12. package/docs/CONFIGURATION.md +388 -0
  13. package/docs/CONNECTORS.md +536 -0
  14. package/docs/INSTRUMENT_WITH_OTEL.md +390 -0
  15. package/docs/ML-COMMONS-SETUP.md +289 -0
  16. package/docs/NPX_PACKAGING.md +195 -0
  17. package/docs/PERFORMANCE-MONITORING.md +200 -0
  18. package/docs/PERFORMANCE.md +390 -0
  19. package/docs/PI_PROFILING.md +169 -0
  20. package/docs/PLAN-non-agui-agent-support.md +525 -0
  21. package/docs/SDK.md +577 -0
  22. package/docs/SKILLS.md +264 -0
  23. package/docs/blogs/2026-02-28-opensearch-agent-health.md +200 -0
  24. package/docs/blogs/getting-started-blog.md +608 -0
  25. package/docs/diagrams/Agent-health.excalidraw +5656 -0
  26. package/docs/diagrams/architecture.png +0 -0
  27. package/docs/plans/field-redesign.md +468 -0
  28. package/docs/rfcs/001-coding-agent-analytics.md +374 -0
  29. package/docs/rfcs/002-enterprise-leaderboard.md +267 -0
  30. package/docs/rfcs/003-remote-aggregation.md +146 -0
  31. package/docs/rfcs/004-test-sdk-v2.md +599 -0
  32. package/docs/skills/AGENT_HEALTH.md +598 -0
  33. package/docs/skills/AGENT_PROFILE.md +191 -0
  34. package/docs/skills/add-connector/SKILL.md +68 -0
  35. package/docs/skills/agent-health-profile/SKILL.md +40 -0
  36. package/docs/skills/config-auth/SKILL.md +194 -0
  37. package/docs/skills/config-auth/evals/evals.json +35 -0
  38. package/docs/skills/create-pr/SKILL.md +73 -0
  39. package/docs/skills/instrument-otel/SKILL.md +84 -0
  40. package/docs/skills/write-test/SKILL.md +124 -0
  41. package/docs/ui prd.md +376 -0
  42. package/examples/README.md +53 -0
  43. package/examples/config/agent-health.config.example.ts +155 -0
  44. package/examples/connectors/echo-connector.ts +131 -0
  45. package/examples/eval-files/demo.eval.js +128 -0
  46. package/examples/eval-files/sdk-hooks-demo.eval.js +99 -0
  47. package/examples/pi-profiling/README.md +77 -0
  48. package/examples/pi-profiling/agent-health-profile.ts +417 -0
  49. package/lib/dist/lib/agentUtils.d.ts +29 -0
  50. package/lib/dist/lib/agentUtils.d.ts.map +1 -0
  51. package/lib/dist/lib/agentUtils.js +43 -0
  52. package/lib/dist/lib/agentUtils.js.map +1 -0
  53. package/lib/dist/lib/benchmarkExport.d.ts +14 -0
  54. package/lib/dist/lib/benchmarkExport.d.ts.map +1 -0
  55. package/lib/dist/lib/benchmarkExport.js +41 -0
  56. package/lib/dist/lib/benchmarkExport.js.map +1 -0
  57. package/lib/dist/lib/benchmarkVersionUtils.d.ts +37 -0
  58. package/lib/dist/lib/benchmarkVersionUtils.d.ts.map +1 -0
  59. package/lib/dist/lib/benchmarkVersionUtils.js +68 -0
  60. package/lib/dist/lib/benchmarkVersionUtils.js.map +1 -0
  61. package/lib/dist/lib/config/defineConfig.d.ts +27 -0
  62. package/lib/dist/lib/config/defineConfig.d.ts.map +1 -0
  63. package/lib/dist/lib/config/defineConfig.js +28 -0
  64. package/lib/dist/lib/config/defineConfig.js.map +1 -0
  65. package/lib/dist/lib/config/index.d.ts +9 -0
  66. package/lib/dist/lib/config/index.d.ts.map +1 -0
  67. package/lib/dist/lib/config/index.js +8 -0
  68. package/lib/dist/lib/config/index.js.map +1 -0
  69. package/lib/dist/lib/config/loader.d.ts +39 -0
  70. package/lib/dist/lib/config/loader.d.ts.map +1 -0
  71. package/lib/dist/lib/config/loader.js +258 -0
  72. package/lib/dist/lib/config/loader.js.map +1 -0
  73. package/lib/dist/lib/config/statePaths.d.ts +61 -0
  74. package/lib/dist/lib/config/statePaths.d.ts.map +1 -0
  75. package/lib/dist/lib/config/statePaths.js +188 -0
  76. package/lib/dist/lib/config/statePaths.js.map +1 -0
  77. package/lib/dist/lib/config/types.d.ts +231 -0
  78. package/lib/dist/lib/config/types.d.ts.map +1 -0
  79. package/lib/dist/lib/config/types.js +6 -0
  80. package/lib/dist/lib/config/types.js.map +1 -0
  81. package/lib/dist/lib/config.d.ts +39 -0
  82. package/lib/dist/lib/config.d.ts.map +1 -0
  83. package/lib/dist/lib/config.js +118 -0
  84. package/lib/dist/lib/config.js.map +1 -0
  85. package/lib/dist/lib/constants.d.ts +70 -0
  86. package/lib/dist/lib/constants.d.ts.map +1 -0
  87. package/lib/dist/lib/constants.js +365 -0
  88. package/lib/dist/lib/constants.js.map +1 -0
  89. package/lib/dist/lib/contextUtilization.d.ts +23 -0
  90. package/lib/dist/lib/contextUtilization.d.ts.map +1 -0
  91. package/lib/dist/lib/contextUtilization.js +72 -0
  92. package/lib/dist/lib/contextUtilization.js.map +1 -0
  93. package/lib/dist/lib/dashboardMetrics.d.ts +87 -0
  94. package/lib/dist/lib/dashboardMetrics.d.ts.map +1 -0
  95. package/lib/dist/lib/dashboardMetrics.js +242 -0
  96. package/lib/dist/lib/dashboardMetrics.js.map +1 -0
  97. package/lib/dist/lib/dataSourceConfig.d.ts +108 -0
  98. package/lib/dist/lib/dataSourceConfig.d.ts.map +1 -0
  99. package/lib/dist/lib/dataSourceConfig.js +166 -0
  100. package/lib/dist/lib/dataSourceConfig.js.map +1 -0
  101. package/lib/dist/lib/debug.d.ts +26 -0
  102. package/lib/dist/lib/debug.d.ts.map +1 -0
  103. package/lib/dist/lib/debug.js +132 -0
  104. package/lib/dist/lib/debug.js.map +1 -0
  105. package/lib/dist/lib/diagnostics.d.ts +28 -0
  106. package/lib/dist/lib/diagnostics.d.ts.map +1 -0
  107. package/lib/dist/lib/diagnostics.js +65 -0
  108. package/lib/dist/lib/diagnostics.js.map +1 -0
  109. package/lib/dist/lib/envCompat.d.ts +27 -0
  110. package/lib/dist/lib/envCompat.d.ts.map +1 -0
  111. package/lib/dist/lib/envCompat.js +73 -0
  112. package/lib/dist/lib/envCompat.js.map +1 -0
  113. package/lib/dist/lib/findPackageRoot.d.ts +7 -0
  114. package/lib/dist/lib/findPackageRoot.d.ts.map +1 -0
  115. package/lib/dist/lib/findPackageRoot.js +57 -0
  116. package/lib/dist/lib/findPackageRoot.js.map +1 -0
  117. package/lib/dist/lib/hooks.d.ts +36 -0
  118. package/lib/dist/lib/hooks.d.ts.map +1 -0
  119. package/lib/dist/lib/hooks.js +112 -0
  120. package/lib/dist/lib/hooks.js.map +1 -0
  121. package/lib/dist/lib/index.d.ts +47 -0
  122. package/lib/dist/lib/index.d.ts.map +1 -0
  123. package/lib/dist/lib/index.js +62 -0
  124. package/lib/dist/lib/index.js.map +1 -0
  125. package/lib/dist/lib/labels.d.ts +90 -0
  126. package/lib/dist/lib/labels.d.ts.map +1 -0
  127. package/lib/dist/lib/labels.js +158 -0
  128. package/lib/dist/lib/labels.js.map +1 -0
  129. package/lib/dist/lib/markdown.d.ts +16 -0
  130. package/lib/dist/lib/markdown.d.ts.map +1 -0
  131. package/lib/dist/lib/markdown.js +42 -0
  132. package/lib/dist/lib/markdown.js.map +1 -0
  133. package/lib/dist/lib/matchers/expect.d.ts +3 -0
  134. package/lib/dist/lib/matchers/expect.d.ts.map +1 -0
  135. package/lib/dist/lib/matchers/expect.js +225 -0
  136. package/lib/dist/lib/matchers/expect.js.map +1 -0
  137. package/lib/dist/lib/matchers/index.d.ts +8 -0
  138. package/lib/dist/lib/matchers/index.d.ts.map +1 -0
  139. package/lib/dist/lib/matchers/index.js +9 -0
  140. package/lib/dist/lib/matchers/index.js.map +1 -0
  141. package/lib/dist/lib/matchers/judgeAccessor.d.ts +113 -0
  142. package/lib/dist/lib/matchers/judgeAccessor.d.ts.map +1 -0
  143. package/lib/dist/lib/matchers/judgeAccessor.js +183 -0
  144. package/lib/dist/lib/matchers/judgeAccessor.js.map +1 -0
  145. package/lib/dist/lib/matchers/session.d.ts +39 -0
  146. package/lib/dist/lib/matchers/session.d.ts.map +1 -0
  147. package/lib/dist/lib/matchers/session.js +116 -0
  148. package/lib/dist/lib/matchers/session.js.map +1 -0
  149. package/lib/dist/lib/matchers/traces.d.ts +55 -0
  150. package/lib/dist/lib/matchers/traces.d.ts.map +1 -0
  151. package/lib/dist/lib/matchers/traces.js +116 -0
  152. package/lib/dist/lib/matchers/traces.js.map +1 -0
  153. package/lib/dist/lib/matchers/types.d.ts +75 -0
  154. package/lib/dist/lib/matchers/types.d.ts.map +1 -0
  155. package/lib/dist/lib/matchers/types.js +6 -0
  156. package/lib/dist/lib/matchers/types.js.map +1 -0
  157. package/lib/dist/lib/packagePaths.d.ts +29 -0
  158. package/lib/dist/lib/packagePaths.d.ts.map +1 -0
  159. package/lib/dist/lib/packagePaths.js +63 -0
  160. package/lib/dist/lib/packagePaths.js.map +1 -0
  161. package/lib/dist/lib/performance.d.ts +51 -0
  162. package/lib/dist/lib/performance.d.ts.map +1 -0
  163. package/lib/dist/lib/performance.js +159 -0
  164. package/lib/dist/lib/performance.js.map +1 -0
  165. package/lib/dist/lib/portConfig.d.ts +29 -0
  166. package/lib/dist/lib/portConfig.d.ts.map +1 -0
  167. package/lib/dist/lib/portConfig.js +64 -0
  168. package/lib/dist/lib/portConfig.js.map +1 -0
  169. package/lib/dist/lib/preferences.d.ts +63 -0
  170. package/lib/dist/lib/preferences.d.ts.map +1 -0
  171. package/lib/dist/lib/preferences.js +117 -0
  172. package/lib/dist/lib/preferences.js.map +1 -0
  173. package/lib/dist/lib/resolveAgentModel.d.ts +22 -0
  174. package/lib/dist/lib/resolveAgentModel.d.ts.map +1 -0
  175. package/lib/dist/lib/resolveAgentModel.js +37 -0
  176. package/lib/dist/lib/resolveAgentModel.js.map +1 -0
  177. package/lib/dist/lib/runStats.d.ts +92 -0
  178. package/lib/dist/lib/runStats.d.ts.map +1 -0
  179. package/lib/dist/lib/runStats.js +160 -0
  180. package/lib/dist/lib/runStats.js.map +1 -0
  181. package/lib/dist/lib/telemetry/constants.d.ts +60 -0
  182. package/lib/dist/lib/telemetry/constants.d.ts.map +1 -0
  183. package/lib/dist/lib/telemetry/constants.js +87 -0
  184. package/lib/dist/lib/telemetry/constants.js.map +1 -0
  185. package/lib/dist/lib/telemetry/evalSpans.d.ts +61 -0
  186. package/lib/dist/lib/telemetry/evalSpans.d.ts.map +1 -0
  187. package/lib/dist/lib/telemetry/evalSpans.js +254 -0
  188. package/lib/dist/lib/telemetry/evalSpans.js.map +1 -0
  189. package/lib/dist/lib/telemetry/index.d.ts +11 -0
  190. package/lib/dist/lib/telemetry/index.d.ts.map +1 -0
  191. package/lib/dist/lib/telemetry/index.js +15 -0
  192. package/lib/dist/lib/telemetry/index.js.map +1 -0
  193. package/lib/dist/lib/telemetry/opensearchExporter.d.ts +43 -0
  194. package/lib/dist/lib/telemetry/opensearchExporter.d.ts.map +1 -0
  195. package/lib/dist/lib/telemetry/opensearchExporter.js +217 -0
  196. package/lib/dist/lib/telemetry/opensearchExporter.js.map +1 -0
  197. package/lib/dist/lib/telemetry/provider.d.ts +55 -0
  198. package/lib/dist/lib/telemetry/provider.d.ts.map +1 -0
  199. package/lib/dist/lib/telemetry/provider.js +140 -0
  200. package/lib/dist/lib/telemetry/provider.js.map +1 -0
  201. package/lib/dist/lib/testCaseLabels.d.ts +34 -0
  202. package/lib/dist/lib/testCaseLabels.d.ts.map +1 -0
  203. package/lib/dist/lib/testCaseLabels.js +88 -0
  204. package/lib/dist/lib/testCaseLabels.js.map +1 -0
  205. package/lib/dist/lib/testCaseValidation.d.ts +140 -0
  206. package/lib/dist/lib/testCaseValidation.d.ts.map +1 -0
  207. package/lib/dist/lib/testCaseValidation.js +162 -0
  208. package/lib/dist/lib/testCaseValidation.js.map +1 -0
  209. package/lib/dist/lib/testCases/agentFixture.d.ts +80 -0
  210. package/lib/dist/lib/testCases/agentFixture.d.ts.map +1 -0
  211. package/lib/dist/lib/testCases/agentFixture.js +43 -0
  212. package/lib/dist/lib/testCases/agentFixture.js.map +1 -0
  213. package/lib/dist/lib/testCases/authoringSurface.d.ts +10 -0
  214. package/lib/dist/lib/testCases/authoringSurface.d.ts.map +1 -0
  215. package/lib/dist/lib/testCases/authoringSurface.js +54 -0
  216. package/lib/dist/lib/testCases/authoringSurface.js.map +1 -0
  217. package/lib/dist/lib/testCases/codemod.d.ts +13 -0
  218. package/lib/dist/lib/testCases/codemod.d.ts.map +1 -0
  219. package/lib/dist/lib/testCases/codemod.js +169 -0
  220. package/lib/dist/lib/testCases/codemod.js.map +1 -0
  221. package/lib/dist/lib/testCases/define.d.ts +114 -0
  222. package/lib/dist/lib/testCases/define.d.ts.map +1 -0
  223. package/lib/dist/lib/testCases/define.js +253 -0
  224. package/lib/dist/lib/testCases/define.js.map +1 -0
  225. package/lib/dist/lib/testCases/evaluators.d.ts +80 -0
  226. package/lib/dist/lib/testCases/evaluators.d.ts.map +1 -0
  227. package/lib/dist/lib/testCases/evaluators.js +105 -0
  228. package/lib/dist/lib/testCases/evaluators.js.map +1 -0
  229. package/lib/dist/lib/testCases/index.d.ts +14 -0
  230. package/lib/dist/lib/testCases/index.d.ts.map +1 -0
  231. package/lib/dist/lib/testCases/index.js +12 -0
  232. package/lib/dist/lib/testCases/index.js.map +1 -0
  233. package/lib/dist/lib/testCases/judge.d.ts +165 -0
  234. package/lib/dist/lib/testCases/judge.d.ts.map +1 -0
  235. package/lib/dist/lib/testCases/judge.js +359 -0
  236. package/lib/dist/lib/testCases/judge.js.map +1 -0
  237. package/lib/dist/lib/testCases/loader.d.ts +26 -0
  238. package/lib/dist/lib/testCases/loader.d.ts.map +1 -0
  239. package/lib/dist/lib/testCases/loader.js +149 -0
  240. package/lib/dist/lib/testCases/loader.js.map +1 -0
  241. package/lib/dist/lib/testCases/types.d.ts +242 -0
  242. package/lib/dist/lib/testCases/types.d.ts.map +1 -0
  243. package/lib/dist/lib/testCases/types.js +6 -0
  244. package/lib/dist/lib/testCases/types.js.map +1 -0
  245. package/lib/dist/lib/theme.d.ts +6 -0
  246. package/lib/dist/lib/theme.d.ts.map +1 -0
  247. package/lib/dist/lib/theme.js +36 -0
  248. package/lib/dist/lib/theme.js.map +1 -0
  249. package/lib/dist/lib/uiTelemetry.d.ts +7 -0
  250. package/lib/dist/lib/uiTelemetry.d.ts.map +1 -0
  251. package/lib/dist/lib/uiTelemetry.js +25 -0
  252. package/lib/dist/lib/uiTelemetry.js.map +1 -0
  253. package/lib/dist/lib/utils.d.ts +96 -0
  254. package/lib/dist/lib/utils.d.ts.map +1 -0
  255. package/lib/dist/lib/utils.js +232 -0
  256. package/lib/dist/lib/utils.js.map +1 -0
  257. package/lib/dist/lib/workflow/consolidate.d.ts +12 -0
  258. package/lib/dist/lib/workflow/consolidate.d.ts.map +1 -0
  259. package/lib/dist/lib/workflow/consolidate.js +33 -0
  260. package/lib/dist/lib/workflow/consolidate.js.map +1 -0
  261. package/lib/dist/lib/workflow/index.d.ts +13 -0
  262. package/lib/dist/lib/workflow/index.d.ts.map +1 -0
  263. package/lib/dist/lib/workflow/index.js +12 -0
  264. package/lib/dist/lib/workflow/index.js.map +1 -0
  265. package/lib/dist/lib/workflow/ledger.d.ts +30 -0
  266. package/lib/dist/lib/workflow/ledger.d.ts.map +1 -0
  267. package/lib/dist/lib/workflow/ledger.js +41 -0
  268. package/lib/dist/lib/workflow/ledger.js.map +1 -0
  269. package/lib/dist/lib/workflow/pool.d.ts +13 -0
  270. package/lib/dist/lib/workflow/pool.d.ts.map +1 -0
  271. package/lib/dist/lib/workflow/pool.js +44 -0
  272. package/lib/dist/lib/workflow/pool.js.map +1 -0
  273. package/lib/dist/lib/workflow/source.d.ts +22 -0
  274. package/lib/dist/lib/workflow/source.d.ts.map +1 -0
  275. package/lib/dist/lib/workflow/source.js +29 -0
  276. package/lib/dist/lib/workflow/source.js.map +1 -0
  277. package/lib/dist/lib/workflow/stepB.d.ts +71 -0
  278. package/lib/dist/lib/workflow/stepB.d.ts.map +1 -0
  279. package/lib/dist/lib/workflow/stepB.js +99 -0
  280. package/lib/dist/lib/workflow/stepB.js.map +1 -0
  281. package/lib/dist/lib/workflow/types.d.ts +86 -0
  282. package/lib/dist/lib/workflow/types.d.ts.map +1 -0
  283. package/lib/dist/lib/workflow/types.js +6 -0
  284. package/lib/dist/lib/workflow/types.js.map +1 -0
  285. package/lib/dist/lib/workflow/workflow.d.ts +119 -0
  286. package/lib/dist/lib/workflow/workflow.d.ts.map +1 -0
  287. package/lib/dist/lib/workflow/workflow.js +195 -0
  288. package/lib/dist/lib/workflow/workflow.js.map +1 -0
  289. package/lib/dist/services/agent/aguiConverter.d.ts +50 -0
  290. package/lib/dist/services/agent/aguiConverter.d.ts.map +1 -0
  291. package/lib/dist/services/agent/aguiConverter.js +449 -0
  292. package/lib/dist/services/agent/aguiConverter.js.map +1 -0
  293. package/lib/dist/services/agent/index.d.ts +10 -0
  294. package/lib/dist/services/agent/index.d.ts.map +1 -0
  295. package/lib/dist/services/agent/index.js +12 -0
  296. package/lib/dist/services/agent/index.js.map +1 -0
  297. package/lib/dist/services/agent/payloadBuilder.d.ts +33 -0
  298. package/lib/dist/services/agent/payloadBuilder.d.ts.map +1 -0
  299. package/lib/dist/services/agent/payloadBuilder.js +75 -0
  300. package/lib/dist/services/agent/payloadBuilder.js.map +1 -0
  301. package/lib/dist/services/agent/sseStream.d.ts +43 -0
  302. package/lib/dist/services/agent/sseStream.d.ts.map +1 -0
  303. package/lib/dist/services/agent/sseStream.js +223 -0
  304. package/lib/dist/services/agent/sseStream.js.map +1 -0
  305. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.d.ts +44 -0
  306. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.d.ts.map +1 -0
  307. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.js +95 -0
  308. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.js.map +1 -0
  309. package/lib/dist/services/connectors/base/BaseConnector.d.ts +81 -0
  310. package/lib/dist/services/connectors/base/BaseConnector.d.ts.map +1 -0
  311. package/lib/dist/services/connectors/base/BaseConnector.js +170 -0
  312. package/lib/dist/services/connectors/base/BaseConnector.js.map +1 -0
  313. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +116 -0
  314. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -0
  315. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +403 -0
  316. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -0
  317. package/lib/dist/services/connectors/index.d.ts +13 -0
  318. package/lib/dist/services/connectors/index.d.ts.map +1 -0
  319. package/lib/dist/services/connectors/index.js +32 -0
  320. package/lib/dist/services/connectors/index.js.map +1 -0
  321. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts +48 -0
  322. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts.map +1 -0
  323. package/lib/dist/services/connectors/kiro/KiroConnector.js +158 -0
  324. package/lib/dist/services/connectors/kiro/KiroConnector.js.map +1 -0
  325. package/lib/dist/services/connectors/langgraph/LangGraphConnector.d.ts +36 -0
  326. package/lib/dist/services/connectors/langgraph/LangGraphConnector.d.ts.map +1 -0
  327. package/lib/dist/services/connectors/langgraph/LangGraphConnector.js +175 -0
  328. package/lib/dist/services/connectors/langgraph/LangGraphConnector.js.map +1 -0
  329. package/lib/dist/services/connectors/mock/MockConnector.d.ts +37 -0
  330. package/lib/dist/services/connectors/mock/MockConnector.d.ts.map +1 -0
  331. package/lib/dist/services/connectors/mock/MockConnector.js +120 -0
  332. package/lib/dist/services/connectors/mock/MockConnector.js.map +1 -0
  333. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.d.ts +42 -0
  334. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.d.ts.map +1 -0
  335. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.js +133 -0
  336. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.js.map +1 -0
  337. package/lib/dist/services/connectors/pi/PiConnector.d.ts +87 -0
  338. package/lib/dist/services/connectors/pi/PiConnector.d.ts.map +1 -0
  339. package/lib/dist/services/connectors/pi/PiConnector.js +274 -0
  340. package/lib/dist/services/connectors/pi/PiConnector.js.map +1 -0
  341. package/lib/dist/services/connectors/registry.d.ts +57 -0
  342. package/lib/dist/services/connectors/registry.d.ts.map +1 -0
  343. package/lib/dist/services/connectors/registry.js +106 -0
  344. package/lib/dist/services/connectors/registry.js.map +1 -0
  345. package/lib/dist/services/connectors/rest/RESTConnector.d.ts +38 -0
  346. package/lib/dist/services/connectors/rest/RESTConnector.d.ts.map +1 -0
  347. package/lib/dist/services/connectors/rest/RESTConnector.js +117 -0
  348. package/lib/dist/services/connectors/rest/RESTConnector.js.map +1 -0
  349. package/lib/dist/services/connectors/server.d.ts +13 -0
  350. package/lib/dist/services/connectors/server.d.ts.map +1 -0
  351. package/lib/dist/services/connectors/server.js +34 -0
  352. package/lib/dist/services/connectors/server.js.map +1 -0
  353. package/lib/dist/services/connectors/strands/StrandsConnector.d.ts +48 -0
  354. package/lib/dist/services/connectors/strands/StrandsConnector.d.ts.map +1 -0
  355. package/lib/dist/services/connectors/strands/StrandsConnector.js +221 -0
  356. package/lib/dist/services/connectors/strands/StrandsConnector.js.map +1 -0
  357. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts +88 -0
  358. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -0
  359. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +418 -0
  360. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -0
  361. package/lib/dist/services/connectors/types.d.ts +213 -0
  362. package/lib/dist/services/connectors/types.d.ts.map +1 -0
  363. package/lib/dist/services/connectors/types.js +6 -0
  364. package/lib/dist/services/connectors/types.js.map +1 -0
  365. package/lib/dist/services/evaluation/bedrockJudge.d.ts +64 -0
  366. package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -0
  367. package/lib/dist/services/evaluation/bedrockJudge.js +167 -0
  368. package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -0
  369. package/lib/dist/services/evaluation/evaluatorError.d.ts +56 -0
  370. package/lib/dist/services/evaluation/evaluatorError.d.ts.map +1 -0
  371. package/lib/dist/services/evaluation/evaluatorError.js +56 -0
  372. package/lib/dist/services/evaluation/evaluatorError.js.map +1 -0
  373. package/lib/dist/services/evaluation/index.d.ts +106 -0
  374. package/lib/dist/services/evaluation/index.d.ts.map +1 -0
  375. package/lib/dist/services/evaluation/index.js +684 -0
  376. package/lib/dist/services/evaluation/index.js.map +1 -0
  377. package/lib/dist/services/evaluation/mockTrajectory.d.ts +3 -0
  378. package/lib/dist/services/evaluation/mockTrajectory.d.ts.map +1 -0
  379. package/lib/dist/services/evaluation/mockTrajectory.js +72 -0
  380. package/lib/dist/services/evaluation/mockTrajectory.js.map +1 -0
  381. package/lib/dist/services/opensearch/client.d.ts +26 -0
  382. package/lib/dist/services/opensearch/client.d.ts.map +1 -0
  383. package/lib/dist/services/opensearch/client.js +131 -0
  384. package/lib/dist/services/opensearch/client.js.map +1 -0
  385. package/lib/dist/services/opensearch/index.d.ts +16 -0
  386. package/lib/dist/services/opensearch/index.d.ts.map +1 -0
  387. package/lib/dist/services/opensearch/index.js +25 -0
  388. package/lib/dist/services/opensearch/index.js.map +1 -0
  389. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts +123 -0
  390. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -0
  391. package/lib/dist/services/storage/asyncBenchmarkStorage.js +429 -0
  392. package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -0
  393. package/lib/dist/services/storage/asyncRunStorage.d.ts +127 -0
  394. package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -0
  395. package/lib/dist/services/storage/asyncRunStorage.js +448 -0
  396. package/lib/dist/services/storage/asyncRunStorage.js.map +1 -0
  397. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +156 -0
  398. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -0
  399. package/lib/dist/services/storage/asyncTestCaseStorage.js +285 -0
  400. package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -0
  401. package/lib/dist/services/storage/index.d.ts +17 -0
  402. package/lib/dist/services/storage/index.d.ts.map +1 -0
  403. package/lib/dist/services/storage/index.js +20 -0
  404. package/lib/dist/services/storage/index.js.map +1 -0
  405. package/lib/dist/services/storage/migration.d.ts +54 -0
  406. package/lib/dist/services/storage/migration.d.ts.map +1 -0
  407. package/lib/dist/services/storage/migration.js +296 -0
  408. package/lib/dist/services/storage/migration.js.map +1 -0
  409. package/lib/dist/services/storage/opensearchClient.d.ts +924 -0
  410. package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -0
  411. package/lib/dist/services/storage/opensearchClient.js +435 -0
  412. package/lib/dist/services/storage/opensearchClient.js.map +1 -0
  413. package/lib/dist/services/traces/browserRecovery.d.ts +26 -0
  414. package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -0
  415. package/lib/dist/services/traces/browserRecovery.js +81 -0
  416. package/lib/dist/services/traces/browserRecovery.js.map +1 -0
  417. package/lib/dist/services/traces/categoryStyles.d.ts +21 -0
  418. package/lib/dist/services/traces/categoryStyles.d.ts.map +1 -0
  419. package/lib/dist/services/traces/categoryStyles.js +56 -0
  420. package/lib/dist/services/traces/categoryStyles.js.map +1 -0
  421. package/lib/dist/services/traces/executionOrderTransform.d.ts +35 -0
  422. package/lib/dist/services/traces/executionOrderTransform.d.ts.map +1 -0
  423. package/lib/dist/services/traces/executionOrderTransform.js +313 -0
  424. package/lib/dist/services/traces/executionOrderTransform.js.map +1 -0
  425. package/lib/dist/services/traces/fetchSpansForRun.d.ts +86 -0
  426. package/lib/dist/services/traces/fetchSpansForRun.d.ts.map +1 -0
  427. package/lib/dist/services/traces/fetchSpansForRun.js +69 -0
  428. package/lib/dist/services/traces/fetchSpansForRun.js.map +1 -0
  429. package/lib/dist/services/traces/flowTransform.d.ts +24 -0
  430. package/lib/dist/services/traces/flowTransform.d.ts.map +1 -0
  431. package/lib/dist/services/traces/flowTransform.js +228 -0
  432. package/lib/dist/services/traces/flowTransform.js.map +1 -0
  433. package/lib/dist/services/traces/index.d.ts +121 -0
  434. package/lib/dist/services/traces/index.d.ts.map +1 -0
  435. package/lib/dist/services/traces/index.js +255 -0
  436. package/lib/dist/services/traces/index.js.map +1 -0
  437. package/lib/dist/services/traces/intentTransform.d.ts +20 -0
  438. package/lib/dist/services/traces/intentTransform.d.ts.map +1 -0
  439. package/lib/dist/services/traces/intentTransform.js +131 -0
  440. package/lib/dist/services/traces/intentTransform.js.map +1 -0
  441. package/lib/dist/services/traces/judgeAgentsHints.d.ts +63 -0
  442. package/lib/dist/services/traces/judgeAgentsHints.d.ts.map +1 -0
  443. package/lib/dist/services/traces/judgeAgentsHints.js +89 -0
  444. package/lib/dist/services/traces/judgeAgentsHints.js.map +1 -0
  445. package/lib/dist/services/traces/messageExtraction.d.ts +15 -0
  446. package/lib/dist/services/traces/messageExtraction.d.ts.map +1 -0
  447. package/lib/dist/services/traces/messageExtraction.js +251 -0
  448. package/lib/dist/services/traces/messageExtraction.js.map +1 -0
  449. package/lib/dist/services/traces/spanCategorization.d.ts +63 -0
  450. package/lib/dist/services/traces/spanCategorization.d.ts.map +1 -0
  451. package/lib/dist/services/traces/spanCategorization.js +276 -0
  452. package/lib/dist/services/traces/spanCategorization.js.map +1 -0
  453. package/lib/dist/services/traces/spanPreprocessing.d.ts +37 -0
  454. package/lib/dist/services/traces/spanPreprocessing.d.ts.map +1 -0
  455. package/lib/dist/services/traces/spanPreprocessing.js +102 -0
  456. package/lib/dist/services/traces/spanPreprocessing.js.map +1 -0
  457. package/lib/dist/services/traces/spansToTrajectory.d.ts +36 -0
  458. package/lib/dist/services/traces/spansToTrajectory.d.ts.map +1 -0
  459. package/lib/dist/services/traces/spansToTrajectory.js +387 -0
  460. package/lib/dist/services/traces/spansToTrajectory.js.map +1 -0
  461. package/lib/dist/services/traces/toolSimilarity.d.ts +35 -0
  462. package/lib/dist/services/traces/toolSimilarity.d.ts.map +1 -0
  463. package/lib/dist/services/traces/toolSimilarity.js +203 -0
  464. package/lib/dist/services/traces/toolSimilarity.js.map +1 -0
  465. package/lib/dist/services/traces/traceComparison.d.ts +31 -0
  466. package/lib/dist/services/traces/traceComparison.d.ts.map +1 -0
  467. package/lib/dist/services/traces/traceComparison.js +318 -0
  468. package/lib/dist/services/traces/traceComparison.js.map +1 -0
  469. package/lib/dist/services/traces/traceGrouping.d.ts +19 -0
  470. package/lib/dist/services/traces/traceGrouping.d.ts.map +1 -0
  471. package/lib/dist/services/traces/traceGrouping.js +107 -0
  472. package/lib/dist/services/traces/traceGrouping.js.map +1 -0
  473. package/lib/dist/services/traces/tracePoller.d.ts +84 -0
  474. package/lib/dist/services/traces/tracePoller.d.ts.map +1 -0
  475. package/lib/dist/services/traces/tracePoller.js +309 -0
  476. package/lib/dist/services/traces/tracePoller.js.map +1 -0
  477. package/lib/dist/services/traces/traceStats.d.ts +45 -0
  478. package/lib/dist/services/traces/traceStats.d.ts.map +1 -0
  479. package/lib/dist/services/traces/traceStats.js +114 -0
  480. package/lib/dist/services/traces/traceStats.js.map +1 -0
  481. package/lib/dist/services/traces/traceSummary.d.ts +47 -0
  482. package/lib/dist/services/traces/traceSummary.d.ts.map +1 -0
  483. package/lib/dist/services/traces/traceSummary.js +68 -0
  484. package/lib/dist/services/traces/traceSummary.js.map +1 -0
  485. package/lib/dist/services/traces/utils.d.ts +33 -0
  486. package/lib/dist/services/traces/utils.d.ts.map +1 -0
  487. package/lib/dist/services/traces/utils.js +114 -0
  488. package/lib/dist/services/traces/utils.js.map +1 -0
  489. package/lib/dist/types/agui.d.ts +13 -0
  490. package/lib/dist/types/agui.d.ts.map +1 -0
  491. package/lib/dist/types/agui.js +16 -0
  492. package/lib/dist/types/agui.js.map +1 -0
  493. package/lib/dist/types/index.d.ts +1175 -0
  494. package/lib/dist/types/index.d.ts.map +1 -0
  495. package/lib/dist/types/index.js +12 -0
  496. package/lib/dist/types/index.js.map +1 -0
  497. package/lib/dist/types/skills.d.ts +146 -0
  498. package/lib/dist/types/skills.d.ts.map +1 -0
  499. package/lib/dist/types/skills.js +6 -0
  500. package/lib/dist/types/skills.js.map +1 -0
  501. package/observio-sample-agent/pi-package/README.md +112 -0
  502. package/observio-sample-agent/pi-package/extensions/agent-health.ts +373 -0
  503. package/observio-sample-agent/pi-package/package.json +17 -0
  504. package/observio-sample-agent/pi-package/prompts/agent-health.md +37 -0
  505. package/observio-sample-agent/pi-package/skills/create-pr/SKILL.md +88 -0
  506. package/observio-sample-agent/pi-package/skills/fix-bug/SKILL.md +71 -0
  507. package/observio-sample-agent/pi-package/skills/implement-feature/SKILL.md +156 -0
  508. package/observio-sample-agent/pi-package/skills/instrument-otel/SKILL.md +208 -0
  509. package/observio-sample-agent/pi-package/skills/setup-collector/SKILL.md +146 -0
  510. package/observio-sample-agent/pi-package/skills/write-test/SKILL.md +115 -0
  511. package/package.json +64 -13
  512. package/server/dist/app.js +32651 -17637
  513. package/server/dist/index.js +29875 -14638
  514. package/tsconfig.lib.json +71 -0
  515. package/dist/assets/index-EvPLSTAS.js +0 -267
  516. package/dist/assets/index-RXasQKUs.css +0 -1
  517. package/lib/dist/config/index.js +0 -404
  518. package/lib/dist/index.js +0 -1665
@@ -0,0 +1,924 @@
1
+ export interface StorageTestCase {
2
+ id: string;
3
+ name: string;
4
+ description?: string;
5
+ version: number;
6
+ initialPrompt: string;
7
+ tools?: unknown[];
8
+ messages?: unknown[];
9
+ context?: unknown[];
10
+ forwardedProps?: Record<string, unknown>;
11
+ expectedPPL?: string;
12
+ expectedOutcomes?: string[];
13
+ expectedTrajectory?: unknown[];
14
+ labels?: string[];
15
+ category?: string;
16
+ subcategory?: string;
17
+ difficulty?: 'Easy' | 'Medium' | 'Hard';
18
+ tags?: string[];
19
+ author?: string;
20
+ createdAt: string;
21
+ updatedAt: string;
22
+ lastRunAt?: string;
23
+ /** Source provenance for SDK / code-imported test cases. Used by the
24
+ * CollapsibleTestCaseDefinition UI to distinguish SDK tests (path is
25
+ * the source of truth) from JSON tests (the row itself is the source
26
+ * of truth). Already round-trips through OpenSearch — the storage
27
+ * layer was just dropping it on read. */
28
+ sourceFile?: string;
29
+ sourceHash?: string;
30
+ }
31
+ export interface StorageBenchmarkRunConfig {
32
+ id: string;
33
+ name: string;
34
+ description?: string;
35
+ agentKey: string;
36
+ agentId?: string;
37
+ modelId: string;
38
+ headers?: Record<string, string>;
39
+ iterationCount?: number;
40
+ createdAt: string;
41
+ results?: Record<string, {
42
+ reportId: string;
43
+ status: string;
44
+ error?: string;
45
+ }>;
46
+ status?: string;
47
+ error?: string;
48
+ stats?: {
49
+ passed: number;
50
+ failed: number;
51
+ pending: number;
52
+ total: number;
53
+ errored?: number;
54
+ };
55
+ }
56
+ export interface StorageBenchmark {
57
+ id: string;
58
+ name: string;
59
+ description?: string;
60
+ author?: string;
61
+ createdAt: string;
62
+ llmJudgePrompt?: string;
63
+ testCaseIds: string[];
64
+ runs: StorageBenchmarkRunConfig[];
65
+ }
66
+ /** @deprecated Use StorageBenchmarkRunConfig instead */
67
+ export type StorageExperimentRunConfig = StorageBenchmarkRunConfig;
68
+ /** @deprecated Use StorageBenchmark instead */
69
+ export type StorageExperiment = StorageBenchmark;
70
+ export interface StorageRunAnnotation {
71
+ id: string;
72
+ text: string;
73
+ tags?: string[];
74
+ author?: string;
75
+ createdAt: string;
76
+ updatedAt: string;
77
+ }
78
+ export interface StorageRun {
79
+ id: string;
80
+ name?: string;
81
+ description?: string;
82
+ experimentId: string;
83
+ experimentRunId: string;
84
+ testCaseId: string;
85
+ testCaseVersionId: string;
86
+ agentId: string;
87
+ modelId: string;
88
+ iteration: number;
89
+ author?: string;
90
+ createdAt: string;
91
+ status: 'running' | 'completed' | 'failed';
92
+ passFailStatus?: 'passed' | 'failed';
93
+ traceId?: string;
94
+ tags?: string[];
95
+ actualOutcomes?: unknown[];
96
+ llmJudgeReasoning?: string;
97
+ metrics?: {
98
+ accuracy?: number;
99
+ faithfulness?: number;
100
+ latency_score?: number;
101
+ trajectory_alignment_score?: number;
102
+ };
103
+ annotations?: StorageRunAnnotation[];
104
+ trajectory?: unknown[];
105
+ rawEvents?: unknown[];
106
+ logs?: unknown[];
107
+ improvementStrategies?: {
108
+ category: string;
109
+ issue: string;
110
+ recommendation: string;
111
+ priority: 'high' | 'medium' | 'low';
112
+ }[];
113
+ connectorProtocol?: string;
114
+ }
115
+ export interface StorageAnalyticsRecord {
116
+ analyticsId: string;
117
+ runId: string;
118
+ experimentId: string;
119
+ experimentRunId: string;
120
+ testCaseId: string;
121
+ testCaseVersionId?: string;
122
+ traceId?: string;
123
+ experimentName?: string;
124
+ testCaseName?: string;
125
+ testCaseCategory?: string;
126
+ testCaseDifficulty?: string;
127
+ agentId: string;
128
+ modelId: string;
129
+ iteration: number;
130
+ tags?: string[];
131
+ passFailStatus?: string;
132
+ status?: string;
133
+ createdAt: string;
134
+ author?: string;
135
+ [key: string]: unknown;
136
+ }
137
+ export declare const storageAdmin: {
138
+ /**
139
+ * Check storage health/connectivity
140
+ */
141
+ health(): Promise<{
142
+ status: string;
143
+ cluster?: unknown;
144
+ error?: string;
145
+ /** Active storage backend. 'file' means OpenSearch is unavailable / not configured; 'error' means configured-but-unreachable. */
146
+ backend?: "file" | "opensearch" | "error";
147
+ /**
148
+ * Real OpenSearch connectivity, present when OpenSearch is configured but
149
+ * the active backend fell back to file storage. Top-level `status` reflects
150
+ * the active (possibly file) backend, NOT OpenSearch — use this to report
151
+ * true OpenSearch connectivity. Shape mirrors testStorageConnection().
152
+ */
153
+ opensearch?: {
154
+ status: string;
155
+ message?: string;
156
+ latencyMs?: number;
157
+ clusterName?: string;
158
+ clusterStatus?: string;
159
+ };
160
+ }>;
161
+ /**
162
+ * Initialize all indexes with mappings
163
+ */
164
+ initIndexes(): Promise<{
165
+ success: boolean;
166
+ results: Record<string, {
167
+ status: string;
168
+ error?: string;
169
+ }>;
170
+ }>;
171
+ /**
172
+ * Get storage statistics
173
+ */
174
+ stats(): Promise<{
175
+ stats: Record<string, {
176
+ count: number;
177
+ error?: string;
178
+ }>;
179
+ }>;
180
+ };
181
+ export declare const testCaseStorage: {
182
+ /**
183
+ * Get all test cases (latest versions only)
184
+ * @param options.fields - 'summary' for lightweight list-view payload
185
+ * @param options.size - page size for pagination
186
+ * @param options.after - cursor token for next page
187
+ * @param options.includeSample - whether to include sample/demo data
188
+ */
189
+ getAll(options?: {
190
+ fields?: "summary";
191
+ size?: number;
192
+ after?: string;
193
+ includeSample?: boolean;
194
+ }): Promise<{
195
+ testCases: StorageTestCase[];
196
+ total: number;
197
+ after?: string | null;
198
+ hasMore?: boolean;
199
+ }>;
200
+ /**
201
+ * Get test cases by specific IDs (latest versions only)
202
+ * Used for efficient filtered fetching (e.g., only test cases in a benchmark)
203
+ */
204
+ getByIds(ids: string[]): Promise<StorageTestCase[]>;
205
+ /**
206
+ * Get test case by ID (latest version)
207
+ */
208
+ getById(id: string): Promise<StorageTestCase | null>;
209
+ /**
210
+ * Get all versions of a test case
211
+ */
212
+ getVersions(id: string): Promise<StorageTestCase[]>;
213
+ /**
214
+ * Get specific version of a test case
215
+ */
216
+ getVersion(id: string, version: number): Promise<StorageTestCase | null>;
217
+ /**
218
+ * Create new test case (starts at version 1)
219
+ */
220
+ create(testCase: Omit<StorageTestCase, "id" | "version" | "createdAt" | "updatedAt">): Promise<StorageTestCase>;
221
+ /**
222
+ * Update test case (creates new version)
223
+ */
224
+ update(id: string, testCase: Partial<StorageTestCase>): Promise<StorageTestCase>;
225
+ /**
226
+ * Delete test case (all versions)
227
+ */
228
+ delete(id: string): Promise<{
229
+ deleted: number;
230
+ }>;
231
+ /**
232
+ * Bulk create test cases
233
+ */
234
+ bulkCreate(testCases: Partial<StorageTestCase>[]): Promise<{
235
+ created: number;
236
+ errors: boolean;
237
+ }>;
238
+ };
239
+ export declare const benchmarkStorage: {
240
+ /**
241
+ * Get all benchmarks
242
+ * @param options.includeSample - whether to include sample/demo data
243
+ */
244
+ getAll(options?: {
245
+ includeSample?: boolean;
246
+ }): Promise<StorageBenchmark[]>;
247
+ /**
248
+ * Get benchmark by ID
249
+ * @param options.fields - 'polling' for lightweight payload (excludes versions, testCaseSnapshots, headers)
250
+ * @param options.runsSize - max number of runs to return
251
+ * @param options.runsOffset - offset into runs array for pagination
252
+ */
253
+ getById(id: string, options?: {
254
+ fields?: "polling";
255
+ runsSize?: number;
256
+ runsOffset?: number;
257
+ }): Promise<StorageBenchmark | null>;
258
+ /**
259
+ * Create benchmark
260
+ */
261
+ create(benchmark: Omit<StorageBenchmark, "id" | "createdAt">): Promise<StorageBenchmark>;
262
+ /**
263
+ * Update benchmark (for run management)
264
+ */
265
+ update(id: string, benchmark: Partial<StorageBenchmark>): Promise<StorageBenchmark>;
266
+ /**
267
+ * Delete benchmark
268
+ */
269
+ delete(id: string): Promise<{
270
+ deleted: boolean;
271
+ }>;
272
+ /**
273
+ * Bulk create benchmarks
274
+ */
275
+ bulkCreate(benchmarks: Partial<StorageBenchmark>[]): Promise<{
276
+ created: number;
277
+ errors: boolean;
278
+ }>;
279
+ /**
280
+ * Update benchmark metadata only (name, description)
281
+ */
282
+ updateMetadata(id: string, updates: {
283
+ name?: string;
284
+ description?: string;
285
+ }): Promise<StorageBenchmark>;
286
+ /**
287
+ * Get all versions of a benchmark
288
+ */
289
+ getVersions(id: string): Promise<{
290
+ versions: any[];
291
+ total: number;
292
+ }>;
293
+ /**
294
+ * Get specific version of a benchmark
295
+ */
296
+ getVersion(id: string, version: number): Promise<any>;
297
+ };
298
+ /** @deprecated Use benchmarkStorage instead */
299
+ export declare const experimentStorage: {
300
+ /**
301
+ * Get all benchmarks
302
+ * @param options.includeSample - whether to include sample/demo data
303
+ */
304
+ getAll(options?: {
305
+ includeSample?: boolean;
306
+ }): Promise<StorageBenchmark[]>;
307
+ /**
308
+ * Get benchmark by ID
309
+ * @param options.fields - 'polling' for lightweight payload (excludes versions, testCaseSnapshots, headers)
310
+ * @param options.runsSize - max number of runs to return
311
+ * @param options.runsOffset - offset into runs array for pagination
312
+ */
313
+ getById(id: string, options?: {
314
+ fields?: "polling";
315
+ runsSize?: number;
316
+ runsOffset?: number;
317
+ }): Promise<StorageBenchmark | null>;
318
+ /**
319
+ * Create benchmark
320
+ */
321
+ create(benchmark: Omit<StorageBenchmark, "id" | "createdAt">): Promise<StorageBenchmark>;
322
+ /**
323
+ * Update benchmark (for run management)
324
+ */
325
+ update(id: string, benchmark: Partial<StorageBenchmark>): Promise<StorageBenchmark>;
326
+ /**
327
+ * Delete benchmark
328
+ */
329
+ delete(id: string): Promise<{
330
+ deleted: boolean;
331
+ }>;
332
+ /**
333
+ * Bulk create benchmarks
334
+ */
335
+ bulkCreate(benchmarks: Partial<StorageBenchmark>[]): Promise<{
336
+ created: number;
337
+ errors: boolean;
338
+ }>;
339
+ /**
340
+ * Update benchmark metadata only (name, description)
341
+ */
342
+ updateMetadata(id: string, updates: {
343
+ name?: string;
344
+ description?: string;
345
+ }): Promise<StorageBenchmark>;
346
+ /**
347
+ * Get all versions of a benchmark
348
+ */
349
+ getVersions(id: string): Promise<{
350
+ versions: any[];
351
+ total: number;
352
+ }>;
353
+ /**
354
+ * Get specific version of a benchmark
355
+ */
356
+ getVersion(id: string, version: number): Promise<any>;
357
+ };
358
+ export declare const runStorage: {
359
+ /**
360
+ * Get all runs with pagination
361
+ */
362
+ getAll(options?: {
363
+ size?: number;
364
+ from?: number;
365
+ _source?: string[];
366
+ }): Promise<{
367
+ runs: StorageRun[];
368
+ total: number;
369
+ size: number;
370
+ from: number;
371
+ }>;
372
+ /**
373
+ * Batch-get runs by ID in one request (server fans out in parallel).
374
+ * Used by the comparison page to load every cell's report at once.
375
+ */
376
+ getByIds(ids: string[]): Promise<StorageRun[]>;
377
+ /**
378
+ * Get run by ID
379
+ */
380
+ getById(id: string): Promise<StorageRun | null>;
381
+ /**
382
+ * Create run
383
+ */
384
+ create(run: Omit<StorageRun, "id" | "createdAt" | "annotations">, options?: {
385
+ analytics?: boolean;
386
+ }): Promise<StorageRun>;
387
+ /**
388
+ * Delete run
389
+ */
390
+ delete(id: string): Promise<{
391
+ deleted: boolean;
392
+ }>;
393
+ /**
394
+ * Partial update of a run
395
+ * Used for updating trace-mode runs after traces become available
396
+ */
397
+ partialUpdate(id: string, updates: Partial<StorageRun>): Promise<StorageRun>;
398
+ /**
399
+ * Get run counts grouped by test case ID (single aggregation query)
400
+ */
401
+ getCountsByTestCase(): Promise<Record<string, number>>;
402
+ /**
403
+ * Get runs by test case ID
404
+ */
405
+ getByTestCase(testCaseId: string, size?: number, from?: number): Promise<{
406
+ runs: StorageRun[];
407
+ total: number;
408
+ }>;
409
+ /**
410
+ * Get runs by benchmark ID
411
+ */
412
+ getByBenchmark(benchmarkId: string, size?: number): Promise<StorageRun[]>;
413
+ /**
414
+ * Get runs by benchmark run config ID
415
+ */
416
+ getByBenchmarkRun(benchmarkId: string, runId: string, size?: number): Promise<StorageRun[]>;
417
+ /**
418
+ * Get all iterations for a test case in a benchmark
419
+ */
420
+ getIterations(benchmarkId: string, testCaseId: string, benchmarkRunId?: string): Promise<{
421
+ runs: StorageRun[];
422
+ total: number;
423
+ maxIteration: number;
424
+ }>;
425
+ /** @deprecated Use getByBenchmark instead */
426
+ getByExperiment(experimentId: string, size?: number): Promise<StorageRun[]>;
427
+ /** @deprecated Use getByBenchmarkRun instead */
428
+ getByExperimentRun(experimentId: string, runId: string, size?: number): Promise<StorageRun[]>;
429
+ /**
430
+ * Search runs with filters
431
+ */
432
+ search(filters: {
433
+ experimentId?: string;
434
+ testCaseId?: string;
435
+ experimentRunId?: string;
436
+ agentId?: string;
437
+ modelId?: string;
438
+ status?: string;
439
+ passFailStatus?: string;
440
+ tags?: string[];
441
+ dateRange?: {
442
+ start: string;
443
+ end: string;
444
+ };
445
+ size?: number;
446
+ from?: number;
447
+ }): Promise<{
448
+ runs: StorageRun[];
449
+ total: number;
450
+ }>;
451
+ /**
452
+ * Add annotation to run
453
+ */
454
+ addAnnotation(runId: string, annotation: Omit<StorageRunAnnotation, "id" | "createdAt" | "updatedAt">): Promise<StorageRunAnnotation>;
455
+ /**
456
+ * Update annotation
457
+ */
458
+ updateAnnotation(runId: string, annotationId: string, updates: Partial<StorageRunAnnotation>): Promise<StorageRunAnnotation>;
459
+ /**
460
+ * Delete annotation
461
+ */
462
+ deleteAnnotation(runId: string, annotationId: string): Promise<{
463
+ deleted: boolean;
464
+ }>;
465
+ /**
466
+ * Bulk create runs
467
+ */
468
+ bulkCreate(runs: Partial<StorageRun>[]): Promise<{
469
+ created: number;
470
+ errors: boolean;
471
+ }>;
472
+ };
473
+ export declare const analyticsStorage: {
474
+ /**
475
+ * Query analytics records
476
+ */
477
+ query(filters: {
478
+ experimentId?: string;
479
+ testCaseId?: string;
480
+ agentId?: string;
481
+ modelId?: string;
482
+ passFailStatus?: string;
483
+ size?: number;
484
+ from?: number;
485
+ }): Promise<{
486
+ records: StorageAnalyticsRecord[];
487
+ total: number;
488
+ }>;
489
+ /**
490
+ * Get aggregated metrics
491
+ */
492
+ aggregations(experimentId?: string, groupBy?: string): Promise<{
493
+ aggregations: Array<{
494
+ key: string;
495
+ metrics: {
496
+ avgAccuracy?: number;
497
+ avgFaithfulness?: number;
498
+ avgLatency?: number;
499
+ avgTrajectory?: number;
500
+ };
501
+ passCount: number;
502
+ failCount: number;
503
+ totalRuns: number;
504
+ }>;
505
+ groupBy: string;
506
+ }>;
507
+ /**
508
+ * Complex search with custom filters and aggregations
509
+ */
510
+ search(options: {
511
+ filters?: Record<string, unknown>;
512
+ aggs?: Record<string, unknown>;
513
+ size?: number;
514
+ from?: number;
515
+ }): Promise<{
516
+ records: StorageAnalyticsRecord[];
517
+ total: number;
518
+ aggregations: Record<string, unknown>;
519
+ }>;
520
+ /**
521
+ * Backfill analytics from existing runs
522
+ */
523
+ backfill(): Promise<{
524
+ backfilled: number;
525
+ errors: number;
526
+ total: number;
527
+ }>;
528
+ };
529
+ export declare const opensearchStorage: {
530
+ admin: {
531
+ /**
532
+ * Check storage health/connectivity
533
+ */
534
+ health(): Promise<{
535
+ status: string;
536
+ cluster?: unknown;
537
+ error?: string;
538
+ /** Active storage backend. 'file' means OpenSearch is unavailable / not configured; 'error' means configured-but-unreachable. */
539
+ backend?: "file" | "opensearch" | "error";
540
+ /**
541
+ * Real OpenSearch connectivity, present when OpenSearch is configured but
542
+ * the active backend fell back to file storage. Top-level `status` reflects
543
+ * the active (possibly file) backend, NOT OpenSearch — use this to report
544
+ * true OpenSearch connectivity. Shape mirrors testStorageConnection().
545
+ */
546
+ opensearch?: {
547
+ status: string;
548
+ message?: string;
549
+ latencyMs?: number;
550
+ clusterName?: string;
551
+ clusterStatus?: string;
552
+ };
553
+ }>;
554
+ /**
555
+ * Initialize all indexes with mappings
556
+ */
557
+ initIndexes(): Promise<{
558
+ success: boolean;
559
+ results: Record<string, {
560
+ status: string;
561
+ error?: string;
562
+ }>;
563
+ }>;
564
+ /**
565
+ * Get storage statistics
566
+ */
567
+ stats(): Promise<{
568
+ stats: Record<string, {
569
+ count: number;
570
+ error?: string;
571
+ }>;
572
+ }>;
573
+ };
574
+ testCases: {
575
+ /**
576
+ * Get all test cases (latest versions only)
577
+ * @param options.fields - 'summary' for lightweight list-view payload
578
+ * @param options.size - page size for pagination
579
+ * @param options.after - cursor token for next page
580
+ * @param options.includeSample - whether to include sample/demo data
581
+ */
582
+ getAll(options?: {
583
+ fields?: "summary";
584
+ size?: number;
585
+ after?: string;
586
+ includeSample?: boolean;
587
+ }): Promise<{
588
+ testCases: StorageTestCase[];
589
+ total: number;
590
+ after?: string | null;
591
+ hasMore?: boolean;
592
+ }>;
593
+ /**
594
+ * Get test cases by specific IDs (latest versions only)
595
+ * Used for efficient filtered fetching (e.g., only test cases in a benchmark)
596
+ */
597
+ getByIds(ids: string[]): Promise<StorageTestCase[]>;
598
+ /**
599
+ * Get test case by ID (latest version)
600
+ */
601
+ getById(id: string): Promise<StorageTestCase | null>;
602
+ /**
603
+ * Get all versions of a test case
604
+ */
605
+ getVersions(id: string): Promise<StorageTestCase[]>;
606
+ /**
607
+ * Get specific version of a test case
608
+ */
609
+ getVersion(id: string, version: number): Promise<StorageTestCase | null>;
610
+ /**
611
+ * Create new test case (starts at version 1)
612
+ */
613
+ create(testCase: Omit<StorageTestCase, "id" | "version" | "createdAt" | "updatedAt">): Promise<StorageTestCase>;
614
+ /**
615
+ * Update test case (creates new version)
616
+ */
617
+ update(id: string, testCase: Partial<StorageTestCase>): Promise<StorageTestCase>;
618
+ /**
619
+ * Delete test case (all versions)
620
+ */
621
+ delete(id: string): Promise<{
622
+ deleted: number;
623
+ }>;
624
+ /**
625
+ * Bulk create test cases
626
+ */
627
+ bulkCreate(testCases: Partial<StorageTestCase>[]): Promise<{
628
+ created: number;
629
+ errors: boolean;
630
+ }>;
631
+ };
632
+ benchmarks: {
633
+ /**
634
+ * Get all benchmarks
635
+ * @param options.includeSample - whether to include sample/demo data
636
+ */
637
+ getAll(options?: {
638
+ includeSample?: boolean;
639
+ }): Promise<StorageBenchmark[]>;
640
+ /**
641
+ * Get benchmark by ID
642
+ * @param options.fields - 'polling' for lightweight payload (excludes versions, testCaseSnapshots, headers)
643
+ * @param options.runsSize - max number of runs to return
644
+ * @param options.runsOffset - offset into runs array for pagination
645
+ */
646
+ getById(id: string, options?: {
647
+ fields?: "polling";
648
+ runsSize?: number;
649
+ runsOffset?: number;
650
+ }): Promise<StorageBenchmark | null>;
651
+ /**
652
+ * Create benchmark
653
+ */
654
+ create(benchmark: Omit<StorageBenchmark, "id" | "createdAt">): Promise<StorageBenchmark>;
655
+ /**
656
+ * Update benchmark (for run management)
657
+ */
658
+ update(id: string, benchmark: Partial<StorageBenchmark>): Promise<StorageBenchmark>;
659
+ /**
660
+ * Delete benchmark
661
+ */
662
+ delete(id: string): Promise<{
663
+ deleted: boolean;
664
+ }>;
665
+ /**
666
+ * Bulk create benchmarks
667
+ */
668
+ bulkCreate(benchmarks: Partial<StorageBenchmark>[]): Promise<{
669
+ created: number;
670
+ errors: boolean;
671
+ }>;
672
+ /**
673
+ * Update benchmark metadata only (name, description)
674
+ */
675
+ updateMetadata(id: string, updates: {
676
+ name?: string;
677
+ description?: string;
678
+ }): Promise<StorageBenchmark>;
679
+ /**
680
+ * Get all versions of a benchmark
681
+ */
682
+ getVersions(id: string): Promise<{
683
+ versions: any[];
684
+ total: number;
685
+ }>;
686
+ /**
687
+ * Get specific version of a benchmark
688
+ */
689
+ getVersion(id: string, version: number): Promise<any>;
690
+ };
691
+ runs: {
692
+ /**
693
+ * Get all runs with pagination
694
+ */
695
+ getAll(options?: {
696
+ size?: number;
697
+ from?: number;
698
+ _source?: string[];
699
+ }): Promise<{
700
+ runs: StorageRun[];
701
+ total: number;
702
+ size: number;
703
+ from: number;
704
+ }>;
705
+ /**
706
+ * Batch-get runs by ID in one request (server fans out in parallel).
707
+ * Used by the comparison page to load every cell's report at once.
708
+ */
709
+ getByIds(ids: string[]): Promise<StorageRun[]>;
710
+ /**
711
+ * Get run by ID
712
+ */
713
+ getById(id: string): Promise<StorageRun | null>;
714
+ /**
715
+ * Create run
716
+ */
717
+ create(run: Omit<StorageRun, "id" | "createdAt" | "annotations">, options?: {
718
+ analytics?: boolean;
719
+ }): Promise<StorageRun>;
720
+ /**
721
+ * Delete run
722
+ */
723
+ delete(id: string): Promise<{
724
+ deleted: boolean;
725
+ }>;
726
+ /**
727
+ * Partial update of a run
728
+ * Used for updating trace-mode runs after traces become available
729
+ */
730
+ partialUpdate(id: string, updates: Partial<StorageRun>): Promise<StorageRun>;
731
+ /**
732
+ * Get run counts grouped by test case ID (single aggregation query)
733
+ */
734
+ getCountsByTestCase(): Promise<Record<string, number>>;
735
+ /**
736
+ * Get runs by test case ID
737
+ */
738
+ getByTestCase(testCaseId: string, size?: number, from?: number): Promise<{
739
+ runs: StorageRun[];
740
+ total: number;
741
+ }>;
742
+ /**
743
+ * Get runs by benchmark ID
744
+ */
745
+ getByBenchmark(benchmarkId: string, size?: number): Promise<StorageRun[]>;
746
+ /**
747
+ * Get runs by benchmark run config ID
748
+ */
749
+ getByBenchmarkRun(benchmarkId: string, runId: string, size?: number): Promise<StorageRun[]>;
750
+ /**
751
+ * Get all iterations for a test case in a benchmark
752
+ */
753
+ getIterations(benchmarkId: string, testCaseId: string, benchmarkRunId?: string): Promise<{
754
+ runs: StorageRun[];
755
+ total: number;
756
+ maxIteration: number;
757
+ }>;
758
+ /** @deprecated Use getByBenchmark instead */
759
+ getByExperiment(experimentId: string, size?: number): Promise<StorageRun[]>;
760
+ /** @deprecated Use getByBenchmarkRun instead */
761
+ getByExperimentRun(experimentId: string, runId: string, size?: number): Promise<StorageRun[]>;
762
+ /**
763
+ * Search runs with filters
764
+ */
765
+ search(filters: {
766
+ experimentId?: string;
767
+ testCaseId?: string;
768
+ experimentRunId?: string;
769
+ agentId?: string;
770
+ modelId?: string;
771
+ status?: string;
772
+ passFailStatus?: string;
773
+ tags?: string[];
774
+ dateRange?: {
775
+ start: string;
776
+ end: string;
777
+ };
778
+ size?: number;
779
+ from?: number;
780
+ }): Promise<{
781
+ runs: StorageRun[];
782
+ total: number;
783
+ }>;
784
+ /**
785
+ * Add annotation to run
786
+ */
787
+ addAnnotation(runId: string, annotation: Omit<StorageRunAnnotation, "id" | "createdAt" | "updatedAt">): Promise<StorageRunAnnotation>;
788
+ /**
789
+ * Update annotation
790
+ */
791
+ updateAnnotation(runId: string, annotationId: string, updates: Partial<StorageRunAnnotation>): Promise<StorageRunAnnotation>;
792
+ /**
793
+ * Delete annotation
794
+ */
795
+ deleteAnnotation(runId: string, annotationId: string): Promise<{
796
+ deleted: boolean;
797
+ }>;
798
+ /**
799
+ * Bulk create runs
800
+ */
801
+ bulkCreate(runs: Partial<StorageRun>[]): Promise<{
802
+ created: number;
803
+ errors: boolean;
804
+ }>;
805
+ };
806
+ analytics: {
807
+ /**
808
+ * Query analytics records
809
+ */
810
+ query(filters: {
811
+ experimentId?: string;
812
+ testCaseId?: string;
813
+ agentId?: string;
814
+ modelId?: string;
815
+ passFailStatus?: string;
816
+ size?: number;
817
+ from?: number;
818
+ }): Promise<{
819
+ records: StorageAnalyticsRecord[];
820
+ total: number;
821
+ }>;
822
+ /**
823
+ * Get aggregated metrics
824
+ */
825
+ aggregations(experimentId?: string, groupBy?: string): Promise<{
826
+ aggregations: Array<{
827
+ key: string;
828
+ metrics: {
829
+ avgAccuracy?: number;
830
+ avgFaithfulness?: number;
831
+ avgLatency?: number;
832
+ avgTrajectory?: number;
833
+ };
834
+ passCount: number;
835
+ failCount: number;
836
+ totalRuns: number;
837
+ }>;
838
+ groupBy: string;
839
+ }>;
840
+ /**
841
+ * Complex search with custom filters and aggregations
842
+ */
843
+ search(options: {
844
+ filters?: Record<string, unknown>;
845
+ aggs?: Record<string, unknown>;
846
+ size?: number;
847
+ from?: number;
848
+ }): Promise<{
849
+ records: StorageAnalyticsRecord[];
850
+ total: number;
851
+ aggregations: Record<string, unknown>;
852
+ }>;
853
+ /**
854
+ * Backfill analytics from existing runs
855
+ */
856
+ backfill(): Promise<{
857
+ backfilled: number;
858
+ errors: number;
859
+ total: number;
860
+ }>;
861
+ };
862
+ /** @deprecated Use benchmarks instead */
863
+ experiments: {
864
+ /**
865
+ * Get all benchmarks
866
+ * @param options.includeSample - whether to include sample/demo data
867
+ */
868
+ getAll(options?: {
869
+ includeSample?: boolean;
870
+ }): Promise<StorageBenchmark[]>;
871
+ /**
872
+ * Get benchmark by ID
873
+ * @param options.fields - 'polling' for lightweight payload (excludes versions, testCaseSnapshots, headers)
874
+ * @param options.runsSize - max number of runs to return
875
+ * @param options.runsOffset - offset into runs array for pagination
876
+ */
877
+ getById(id: string, options?: {
878
+ fields?: "polling";
879
+ runsSize?: number;
880
+ runsOffset?: number;
881
+ }): Promise<StorageBenchmark | null>;
882
+ /**
883
+ * Create benchmark
884
+ */
885
+ create(benchmark: Omit<StorageBenchmark, "id" | "createdAt">): Promise<StorageBenchmark>;
886
+ /**
887
+ * Update benchmark (for run management)
888
+ */
889
+ update(id: string, benchmark: Partial<StorageBenchmark>): Promise<StorageBenchmark>;
890
+ /**
891
+ * Delete benchmark
892
+ */
893
+ delete(id: string): Promise<{
894
+ deleted: boolean;
895
+ }>;
896
+ /**
897
+ * Bulk create benchmarks
898
+ */
899
+ bulkCreate(benchmarks: Partial<StorageBenchmark>[]): Promise<{
900
+ created: number;
901
+ errors: boolean;
902
+ }>;
903
+ /**
904
+ * Update benchmark metadata only (name, description)
905
+ */
906
+ updateMetadata(id: string, updates: {
907
+ name?: string;
908
+ description?: string;
909
+ }): Promise<StorageBenchmark>;
910
+ /**
911
+ * Get all versions of a benchmark
912
+ */
913
+ getVersions(id: string): Promise<{
914
+ versions: any[];
915
+ total: number;
916
+ }>;
917
+ /**
918
+ * Get specific version of a benchmark
919
+ */
920
+ getVersion(id: string, version: number): Promise<any>;
921
+ };
922
+ };
923
+ export default opensearchStorage;
924
+ //# sourceMappingURL=opensearchClient.d.ts.map