@opensearch-project/agent-health 0.3.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (518) hide show
  1. package/README.md +77 -6
  2. package/cli/dist/index.js +10072 -4502
  3. package/deployment/cloudformation/agent-health-observability.yaml +762 -0
  4. package/dist/assets/index-CCQRDlO0.js +243 -0
  5. package/dist/assets/index-CNHQVbcj.css +1 -0
  6. package/dist/index.html +2 -2
  7. package/docs/ARCHITECTURE.md +450 -0
  8. package/docs/BACKEND_JOB_QUEUE.md +405 -0
  9. package/docs/CLAUDE_CODE_TELEMETRY.md +283 -0
  10. package/docs/CLI.md +431 -0
  11. package/docs/CODING_AGENT_ANALYTICS.md +298 -0
  12. package/docs/CONFIGURATION.md +388 -0
  13. package/docs/CONNECTORS.md +536 -0
  14. package/docs/INSTRUMENT_WITH_OTEL.md +390 -0
  15. package/docs/ML-COMMONS-SETUP.md +289 -0
  16. package/docs/NPX_PACKAGING.md +195 -0
  17. package/docs/PERFORMANCE-MONITORING.md +200 -0
  18. package/docs/PERFORMANCE.md +390 -0
  19. package/docs/PI_PROFILING.md +169 -0
  20. package/docs/PLAN-non-agui-agent-support.md +525 -0
  21. package/docs/SDK.md +577 -0
  22. package/docs/SKILLS.md +264 -0
  23. package/docs/blogs/2026-02-28-opensearch-agent-health.md +200 -0
  24. package/docs/blogs/getting-started-blog.md +608 -0
  25. package/docs/diagrams/Agent-health.excalidraw +5656 -0
  26. package/docs/diagrams/architecture.png +0 -0
  27. package/docs/plans/field-redesign.md +468 -0
  28. package/docs/rfcs/001-coding-agent-analytics.md +374 -0
  29. package/docs/rfcs/002-enterprise-leaderboard.md +267 -0
  30. package/docs/rfcs/003-remote-aggregation.md +146 -0
  31. package/docs/rfcs/004-test-sdk-v2.md +599 -0
  32. package/docs/skills/AGENT_HEALTH.md +598 -0
  33. package/docs/skills/AGENT_PROFILE.md +191 -0
  34. package/docs/skills/add-connector/SKILL.md +68 -0
  35. package/docs/skills/agent-health-profile/SKILL.md +40 -0
  36. package/docs/skills/config-auth/SKILL.md +194 -0
  37. package/docs/skills/config-auth/evals/evals.json +35 -0
  38. package/docs/skills/create-pr/SKILL.md +73 -0
  39. package/docs/skills/instrument-otel/SKILL.md +84 -0
  40. package/docs/skills/write-test/SKILL.md +124 -0
  41. package/docs/ui prd.md +376 -0
  42. package/examples/README.md +53 -0
  43. package/examples/config/agent-health.config.example.ts +155 -0
  44. package/examples/connectors/echo-connector.ts +131 -0
  45. package/examples/eval-files/demo.eval.js +128 -0
  46. package/examples/eval-files/sdk-hooks-demo.eval.js +99 -0
  47. package/examples/pi-profiling/README.md +77 -0
  48. package/examples/pi-profiling/agent-health-profile.ts +417 -0
  49. package/lib/dist/lib/agentUtils.d.ts +29 -0
  50. package/lib/dist/lib/agentUtils.d.ts.map +1 -0
  51. package/lib/dist/lib/agentUtils.js +43 -0
  52. package/lib/dist/lib/agentUtils.js.map +1 -0
  53. package/lib/dist/lib/benchmarkExport.d.ts +14 -0
  54. package/lib/dist/lib/benchmarkExport.d.ts.map +1 -0
  55. package/lib/dist/lib/benchmarkExport.js +41 -0
  56. package/lib/dist/lib/benchmarkExport.js.map +1 -0
  57. package/lib/dist/lib/benchmarkVersionUtils.d.ts +37 -0
  58. package/lib/dist/lib/benchmarkVersionUtils.d.ts.map +1 -0
  59. package/lib/dist/lib/benchmarkVersionUtils.js +68 -0
  60. package/lib/dist/lib/benchmarkVersionUtils.js.map +1 -0
  61. package/lib/dist/lib/config/defineConfig.d.ts +27 -0
  62. package/lib/dist/lib/config/defineConfig.d.ts.map +1 -0
  63. package/lib/dist/lib/config/defineConfig.js +28 -0
  64. package/lib/dist/lib/config/defineConfig.js.map +1 -0
  65. package/lib/dist/lib/config/index.d.ts +9 -0
  66. package/lib/dist/lib/config/index.d.ts.map +1 -0
  67. package/lib/dist/lib/config/index.js +8 -0
  68. package/lib/dist/lib/config/index.js.map +1 -0
  69. package/lib/dist/lib/config/loader.d.ts +39 -0
  70. package/lib/dist/lib/config/loader.d.ts.map +1 -0
  71. package/lib/dist/lib/config/loader.js +258 -0
  72. package/lib/dist/lib/config/loader.js.map +1 -0
  73. package/lib/dist/lib/config/statePaths.d.ts +61 -0
  74. package/lib/dist/lib/config/statePaths.d.ts.map +1 -0
  75. package/lib/dist/lib/config/statePaths.js +188 -0
  76. package/lib/dist/lib/config/statePaths.js.map +1 -0
  77. package/lib/dist/lib/config/types.d.ts +231 -0
  78. package/lib/dist/lib/config/types.d.ts.map +1 -0
  79. package/lib/dist/lib/config/types.js +6 -0
  80. package/lib/dist/lib/config/types.js.map +1 -0
  81. package/lib/dist/lib/config.d.ts +39 -0
  82. package/lib/dist/lib/config.d.ts.map +1 -0
  83. package/lib/dist/lib/config.js +118 -0
  84. package/lib/dist/lib/config.js.map +1 -0
  85. package/lib/dist/lib/constants.d.ts +70 -0
  86. package/lib/dist/lib/constants.d.ts.map +1 -0
  87. package/lib/dist/lib/constants.js +365 -0
  88. package/lib/dist/lib/constants.js.map +1 -0
  89. package/lib/dist/lib/contextUtilization.d.ts +23 -0
  90. package/lib/dist/lib/contextUtilization.d.ts.map +1 -0
  91. package/lib/dist/lib/contextUtilization.js +72 -0
  92. package/lib/dist/lib/contextUtilization.js.map +1 -0
  93. package/lib/dist/lib/dashboardMetrics.d.ts +87 -0
  94. package/lib/dist/lib/dashboardMetrics.d.ts.map +1 -0
  95. package/lib/dist/lib/dashboardMetrics.js +242 -0
  96. package/lib/dist/lib/dashboardMetrics.js.map +1 -0
  97. package/lib/dist/lib/dataSourceConfig.d.ts +108 -0
  98. package/lib/dist/lib/dataSourceConfig.d.ts.map +1 -0
  99. package/lib/dist/lib/dataSourceConfig.js +166 -0
  100. package/lib/dist/lib/dataSourceConfig.js.map +1 -0
  101. package/lib/dist/lib/debug.d.ts +26 -0
  102. package/lib/dist/lib/debug.d.ts.map +1 -0
  103. package/lib/dist/lib/debug.js +132 -0
  104. package/lib/dist/lib/debug.js.map +1 -0
  105. package/lib/dist/lib/diagnostics.d.ts +28 -0
  106. package/lib/dist/lib/diagnostics.d.ts.map +1 -0
  107. package/lib/dist/lib/diagnostics.js +65 -0
  108. package/lib/dist/lib/diagnostics.js.map +1 -0
  109. package/lib/dist/lib/envCompat.d.ts +27 -0
  110. package/lib/dist/lib/envCompat.d.ts.map +1 -0
  111. package/lib/dist/lib/envCompat.js +73 -0
  112. package/lib/dist/lib/envCompat.js.map +1 -0
  113. package/lib/dist/lib/findPackageRoot.d.ts +7 -0
  114. package/lib/dist/lib/findPackageRoot.d.ts.map +1 -0
  115. package/lib/dist/lib/findPackageRoot.js +57 -0
  116. package/lib/dist/lib/findPackageRoot.js.map +1 -0
  117. package/lib/dist/lib/hooks.d.ts +36 -0
  118. package/lib/dist/lib/hooks.d.ts.map +1 -0
  119. package/lib/dist/lib/hooks.js +112 -0
  120. package/lib/dist/lib/hooks.js.map +1 -0
  121. package/lib/dist/lib/index.d.ts +47 -0
  122. package/lib/dist/lib/index.d.ts.map +1 -0
  123. package/lib/dist/lib/index.js +62 -0
  124. package/lib/dist/lib/index.js.map +1 -0
  125. package/lib/dist/lib/labels.d.ts +90 -0
  126. package/lib/dist/lib/labels.d.ts.map +1 -0
  127. package/lib/dist/lib/labels.js +158 -0
  128. package/lib/dist/lib/labels.js.map +1 -0
  129. package/lib/dist/lib/markdown.d.ts +16 -0
  130. package/lib/dist/lib/markdown.d.ts.map +1 -0
  131. package/lib/dist/lib/markdown.js +42 -0
  132. package/lib/dist/lib/markdown.js.map +1 -0
  133. package/lib/dist/lib/matchers/expect.d.ts +3 -0
  134. package/lib/dist/lib/matchers/expect.d.ts.map +1 -0
  135. package/lib/dist/lib/matchers/expect.js +225 -0
  136. package/lib/dist/lib/matchers/expect.js.map +1 -0
  137. package/lib/dist/lib/matchers/index.d.ts +8 -0
  138. package/lib/dist/lib/matchers/index.d.ts.map +1 -0
  139. package/lib/dist/lib/matchers/index.js +9 -0
  140. package/lib/dist/lib/matchers/index.js.map +1 -0
  141. package/lib/dist/lib/matchers/judgeAccessor.d.ts +113 -0
  142. package/lib/dist/lib/matchers/judgeAccessor.d.ts.map +1 -0
  143. package/lib/dist/lib/matchers/judgeAccessor.js +183 -0
  144. package/lib/dist/lib/matchers/judgeAccessor.js.map +1 -0
  145. package/lib/dist/lib/matchers/session.d.ts +39 -0
  146. package/lib/dist/lib/matchers/session.d.ts.map +1 -0
  147. package/lib/dist/lib/matchers/session.js +116 -0
  148. package/lib/dist/lib/matchers/session.js.map +1 -0
  149. package/lib/dist/lib/matchers/traces.d.ts +55 -0
  150. package/lib/dist/lib/matchers/traces.d.ts.map +1 -0
  151. package/lib/dist/lib/matchers/traces.js +116 -0
  152. package/lib/dist/lib/matchers/traces.js.map +1 -0
  153. package/lib/dist/lib/matchers/types.d.ts +75 -0
  154. package/lib/dist/lib/matchers/types.d.ts.map +1 -0
  155. package/lib/dist/lib/matchers/types.js +6 -0
  156. package/lib/dist/lib/matchers/types.js.map +1 -0
  157. package/lib/dist/lib/packagePaths.d.ts +29 -0
  158. package/lib/dist/lib/packagePaths.d.ts.map +1 -0
  159. package/lib/dist/lib/packagePaths.js +63 -0
  160. package/lib/dist/lib/packagePaths.js.map +1 -0
  161. package/lib/dist/lib/performance.d.ts +51 -0
  162. package/lib/dist/lib/performance.d.ts.map +1 -0
  163. package/lib/dist/lib/performance.js +159 -0
  164. package/lib/dist/lib/performance.js.map +1 -0
  165. package/lib/dist/lib/portConfig.d.ts +29 -0
  166. package/lib/dist/lib/portConfig.d.ts.map +1 -0
  167. package/lib/dist/lib/portConfig.js +64 -0
  168. package/lib/dist/lib/portConfig.js.map +1 -0
  169. package/lib/dist/lib/preferences.d.ts +63 -0
  170. package/lib/dist/lib/preferences.d.ts.map +1 -0
  171. package/lib/dist/lib/preferences.js +117 -0
  172. package/lib/dist/lib/preferences.js.map +1 -0
  173. package/lib/dist/lib/resolveAgentModel.d.ts +22 -0
  174. package/lib/dist/lib/resolveAgentModel.d.ts.map +1 -0
  175. package/lib/dist/lib/resolveAgentModel.js +37 -0
  176. package/lib/dist/lib/resolveAgentModel.js.map +1 -0
  177. package/lib/dist/lib/runStats.d.ts +92 -0
  178. package/lib/dist/lib/runStats.d.ts.map +1 -0
  179. package/lib/dist/lib/runStats.js +160 -0
  180. package/lib/dist/lib/runStats.js.map +1 -0
  181. package/lib/dist/lib/telemetry/constants.d.ts +60 -0
  182. package/lib/dist/lib/telemetry/constants.d.ts.map +1 -0
  183. package/lib/dist/lib/telemetry/constants.js +87 -0
  184. package/lib/dist/lib/telemetry/constants.js.map +1 -0
  185. package/lib/dist/lib/telemetry/evalSpans.d.ts +61 -0
  186. package/lib/dist/lib/telemetry/evalSpans.d.ts.map +1 -0
  187. package/lib/dist/lib/telemetry/evalSpans.js +254 -0
  188. package/lib/dist/lib/telemetry/evalSpans.js.map +1 -0
  189. package/lib/dist/lib/telemetry/index.d.ts +11 -0
  190. package/lib/dist/lib/telemetry/index.d.ts.map +1 -0
  191. package/lib/dist/lib/telemetry/index.js +15 -0
  192. package/lib/dist/lib/telemetry/index.js.map +1 -0
  193. package/lib/dist/lib/telemetry/opensearchExporter.d.ts +43 -0
  194. package/lib/dist/lib/telemetry/opensearchExporter.d.ts.map +1 -0
  195. package/lib/dist/lib/telemetry/opensearchExporter.js +217 -0
  196. package/lib/dist/lib/telemetry/opensearchExporter.js.map +1 -0
  197. package/lib/dist/lib/telemetry/provider.d.ts +55 -0
  198. package/lib/dist/lib/telemetry/provider.d.ts.map +1 -0
  199. package/lib/dist/lib/telemetry/provider.js +140 -0
  200. package/lib/dist/lib/telemetry/provider.js.map +1 -0
  201. package/lib/dist/lib/testCaseLabels.d.ts +34 -0
  202. package/lib/dist/lib/testCaseLabels.d.ts.map +1 -0
  203. package/lib/dist/lib/testCaseLabels.js +88 -0
  204. package/lib/dist/lib/testCaseLabels.js.map +1 -0
  205. package/lib/dist/lib/testCaseValidation.d.ts +140 -0
  206. package/lib/dist/lib/testCaseValidation.d.ts.map +1 -0
  207. package/lib/dist/lib/testCaseValidation.js +162 -0
  208. package/lib/dist/lib/testCaseValidation.js.map +1 -0
  209. package/lib/dist/lib/testCases/agentFixture.d.ts +80 -0
  210. package/lib/dist/lib/testCases/agentFixture.d.ts.map +1 -0
  211. package/lib/dist/lib/testCases/agentFixture.js +43 -0
  212. package/lib/dist/lib/testCases/agentFixture.js.map +1 -0
  213. package/lib/dist/lib/testCases/authoringSurface.d.ts +10 -0
  214. package/lib/dist/lib/testCases/authoringSurface.d.ts.map +1 -0
  215. package/lib/dist/lib/testCases/authoringSurface.js +54 -0
  216. package/lib/dist/lib/testCases/authoringSurface.js.map +1 -0
  217. package/lib/dist/lib/testCases/codemod.d.ts +13 -0
  218. package/lib/dist/lib/testCases/codemod.d.ts.map +1 -0
  219. package/lib/dist/lib/testCases/codemod.js +169 -0
  220. package/lib/dist/lib/testCases/codemod.js.map +1 -0
  221. package/lib/dist/lib/testCases/define.d.ts +114 -0
  222. package/lib/dist/lib/testCases/define.d.ts.map +1 -0
  223. package/lib/dist/lib/testCases/define.js +253 -0
  224. package/lib/dist/lib/testCases/define.js.map +1 -0
  225. package/lib/dist/lib/testCases/evaluators.d.ts +80 -0
  226. package/lib/dist/lib/testCases/evaluators.d.ts.map +1 -0
  227. package/lib/dist/lib/testCases/evaluators.js +105 -0
  228. package/lib/dist/lib/testCases/evaluators.js.map +1 -0
  229. package/lib/dist/lib/testCases/index.d.ts +14 -0
  230. package/lib/dist/lib/testCases/index.d.ts.map +1 -0
  231. package/lib/dist/lib/testCases/index.js +12 -0
  232. package/lib/dist/lib/testCases/index.js.map +1 -0
  233. package/lib/dist/lib/testCases/judge.d.ts +165 -0
  234. package/lib/dist/lib/testCases/judge.d.ts.map +1 -0
  235. package/lib/dist/lib/testCases/judge.js +359 -0
  236. package/lib/dist/lib/testCases/judge.js.map +1 -0
  237. package/lib/dist/lib/testCases/loader.d.ts +26 -0
  238. package/lib/dist/lib/testCases/loader.d.ts.map +1 -0
  239. package/lib/dist/lib/testCases/loader.js +149 -0
  240. package/lib/dist/lib/testCases/loader.js.map +1 -0
  241. package/lib/dist/lib/testCases/types.d.ts +242 -0
  242. package/lib/dist/lib/testCases/types.d.ts.map +1 -0
  243. package/lib/dist/lib/testCases/types.js +6 -0
  244. package/lib/dist/lib/testCases/types.js.map +1 -0
  245. package/lib/dist/lib/theme.d.ts +6 -0
  246. package/lib/dist/lib/theme.d.ts.map +1 -0
  247. package/lib/dist/lib/theme.js +36 -0
  248. package/lib/dist/lib/theme.js.map +1 -0
  249. package/lib/dist/lib/uiTelemetry.d.ts +7 -0
  250. package/lib/dist/lib/uiTelemetry.d.ts.map +1 -0
  251. package/lib/dist/lib/uiTelemetry.js +25 -0
  252. package/lib/dist/lib/uiTelemetry.js.map +1 -0
  253. package/lib/dist/lib/utils.d.ts +96 -0
  254. package/lib/dist/lib/utils.d.ts.map +1 -0
  255. package/lib/dist/lib/utils.js +232 -0
  256. package/lib/dist/lib/utils.js.map +1 -0
  257. package/lib/dist/lib/workflow/consolidate.d.ts +12 -0
  258. package/lib/dist/lib/workflow/consolidate.d.ts.map +1 -0
  259. package/lib/dist/lib/workflow/consolidate.js +33 -0
  260. package/lib/dist/lib/workflow/consolidate.js.map +1 -0
  261. package/lib/dist/lib/workflow/index.d.ts +13 -0
  262. package/lib/dist/lib/workflow/index.d.ts.map +1 -0
  263. package/lib/dist/lib/workflow/index.js +12 -0
  264. package/lib/dist/lib/workflow/index.js.map +1 -0
  265. package/lib/dist/lib/workflow/ledger.d.ts +30 -0
  266. package/lib/dist/lib/workflow/ledger.d.ts.map +1 -0
  267. package/lib/dist/lib/workflow/ledger.js +41 -0
  268. package/lib/dist/lib/workflow/ledger.js.map +1 -0
  269. package/lib/dist/lib/workflow/pool.d.ts +13 -0
  270. package/lib/dist/lib/workflow/pool.d.ts.map +1 -0
  271. package/lib/dist/lib/workflow/pool.js +44 -0
  272. package/lib/dist/lib/workflow/pool.js.map +1 -0
  273. package/lib/dist/lib/workflow/source.d.ts +22 -0
  274. package/lib/dist/lib/workflow/source.d.ts.map +1 -0
  275. package/lib/dist/lib/workflow/source.js +29 -0
  276. package/lib/dist/lib/workflow/source.js.map +1 -0
  277. package/lib/dist/lib/workflow/stepB.d.ts +71 -0
  278. package/lib/dist/lib/workflow/stepB.d.ts.map +1 -0
  279. package/lib/dist/lib/workflow/stepB.js +99 -0
  280. package/lib/dist/lib/workflow/stepB.js.map +1 -0
  281. package/lib/dist/lib/workflow/types.d.ts +86 -0
  282. package/lib/dist/lib/workflow/types.d.ts.map +1 -0
  283. package/lib/dist/lib/workflow/types.js +6 -0
  284. package/lib/dist/lib/workflow/types.js.map +1 -0
  285. package/lib/dist/lib/workflow/workflow.d.ts +119 -0
  286. package/lib/dist/lib/workflow/workflow.d.ts.map +1 -0
  287. package/lib/dist/lib/workflow/workflow.js +195 -0
  288. package/lib/dist/lib/workflow/workflow.js.map +1 -0
  289. package/lib/dist/services/agent/aguiConverter.d.ts +50 -0
  290. package/lib/dist/services/agent/aguiConverter.d.ts.map +1 -0
  291. package/lib/dist/services/agent/aguiConverter.js +449 -0
  292. package/lib/dist/services/agent/aguiConverter.js.map +1 -0
  293. package/lib/dist/services/agent/index.d.ts +10 -0
  294. package/lib/dist/services/agent/index.d.ts.map +1 -0
  295. package/lib/dist/services/agent/index.js +12 -0
  296. package/lib/dist/services/agent/index.js.map +1 -0
  297. package/lib/dist/services/agent/payloadBuilder.d.ts +33 -0
  298. package/lib/dist/services/agent/payloadBuilder.d.ts.map +1 -0
  299. package/lib/dist/services/agent/payloadBuilder.js +75 -0
  300. package/lib/dist/services/agent/payloadBuilder.js.map +1 -0
  301. package/lib/dist/services/agent/sseStream.d.ts +43 -0
  302. package/lib/dist/services/agent/sseStream.d.ts.map +1 -0
  303. package/lib/dist/services/agent/sseStream.js +223 -0
  304. package/lib/dist/services/agent/sseStream.js.map +1 -0
  305. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.d.ts +44 -0
  306. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.d.ts.map +1 -0
  307. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.js +95 -0
  308. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.js.map +1 -0
  309. package/lib/dist/services/connectors/base/BaseConnector.d.ts +81 -0
  310. package/lib/dist/services/connectors/base/BaseConnector.d.ts.map +1 -0
  311. package/lib/dist/services/connectors/base/BaseConnector.js +170 -0
  312. package/lib/dist/services/connectors/base/BaseConnector.js.map +1 -0
  313. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +116 -0
  314. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -0
  315. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +403 -0
  316. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -0
  317. package/lib/dist/services/connectors/index.d.ts +13 -0
  318. package/lib/dist/services/connectors/index.d.ts.map +1 -0
  319. package/lib/dist/services/connectors/index.js +32 -0
  320. package/lib/dist/services/connectors/index.js.map +1 -0
  321. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts +48 -0
  322. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts.map +1 -0
  323. package/lib/dist/services/connectors/kiro/KiroConnector.js +158 -0
  324. package/lib/dist/services/connectors/kiro/KiroConnector.js.map +1 -0
  325. package/lib/dist/services/connectors/langgraph/LangGraphConnector.d.ts +36 -0
  326. package/lib/dist/services/connectors/langgraph/LangGraphConnector.d.ts.map +1 -0
  327. package/lib/dist/services/connectors/langgraph/LangGraphConnector.js +175 -0
  328. package/lib/dist/services/connectors/langgraph/LangGraphConnector.js.map +1 -0
  329. package/lib/dist/services/connectors/mock/MockConnector.d.ts +37 -0
  330. package/lib/dist/services/connectors/mock/MockConnector.d.ts.map +1 -0
  331. package/lib/dist/services/connectors/mock/MockConnector.js +120 -0
  332. package/lib/dist/services/connectors/mock/MockConnector.js.map +1 -0
  333. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.d.ts +42 -0
  334. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.d.ts.map +1 -0
  335. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.js +133 -0
  336. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.js.map +1 -0
  337. package/lib/dist/services/connectors/pi/PiConnector.d.ts +87 -0
  338. package/lib/dist/services/connectors/pi/PiConnector.d.ts.map +1 -0
  339. package/lib/dist/services/connectors/pi/PiConnector.js +274 -0
  340. package/lib/dist/services/connectors/pi/PiConnector.js.map +1 -0
  341. package/lib/dist/services/connectors/registry.d.ts +57 -0
  342. package/lib/dist/services/connectors/registry.d.ts.map +1 -0
  343. package/lib/dist/services/connectors/registry.js +106 -0
  344. package/lib/dist/services/connectors/registry.js.map +1 -0
  345. package/lib/dist/services/connectors/rest/RESTConnector.d.ts +38 -0
  346. package/lib/dist/services/connectors/rest/RESTConnector.d.ts.map +1 -0
  347. package/lib/dist/services/connectors/rest/RESTConnector.js +117 -0
  348. package/lib/dist/services/connectors/rest/RESTConnector.js.map +1 -0
  349. package/lib/dist/services/connectors/server.d.ts +13 -0
  350. package/lib/dist/services/connectors/server.d.ts.map +1 -0
  351. package/lib/dist/services/connectors/server.js +34 -0
  352. package/lib/dist/services/connectors/server.js.map +1 -0
  353. package/lib/dist/services/connectors/strands/StrandsConnector.d.ts +48 -0
  354. package/lib/dist/services/connectors/strands/StrandsConnector.d.ts.map +1 -0
  355. package/lib/dist/services/connectors/strands/StrandsConnector.js +221 -0
  356. package/lib/dist/services/connectors/strands/StrandsConnector.js.map +1 -0
  357. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts +88 -0
  358. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -0
  359. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +418 -0
  360. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -0
  361. package/lib/dist/services/connectors/types.d.ts +213 -0
  362. package/lib/dist/services/connectors/types.d.ts.map +1 -0
  363. package/lib/dist/services/connectors/types.js +6 -0
  364. package/lib/dist/services/connectors/types.js.map +1 -0
  365. package/lib/dist/services/evaluation/bedrockJudge.d.ts +64 -0
  366. package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -0
  367. package/lib/dist/services/evaluation/bedrockJudge.js +167 -0
  368. package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -0
  369. package/lib/dist/services/evaluation/evaluatorError.d.ts +56 -0
  370. package/lib/dist/services/evaluation/evaluatorError.d.ts.map +1 -0
  371. package/lib/dist/services/evaluation/evaluatorError.js +56 -0
  372. package/lib/dist/services/evaluation/evaluatorError.js.map +1 -0
  373. package/lib/dist/services/evaluation/index.d.ts +106 -0
  374. package/lib/dist/services/evaluation/index.d.ts.map +1 -0
  375. package/lib/dist/services/evaluation/index.js +684 -0
  376. package/lib/dist/services/evaluation/index.js.map +1 -0
  377. package/lib/dist/services/evaluation/mockTrajectory.d.ts +3 -0
  378. package/lib/dist/services/evaluation/mockTrajectory.d.ts.map +1 -0
  379. package/lib/dist/services/evaluation/mockTrajectory.js +72 -0
  380. package/lib/dist/services/evaluation/mockTrajectory.js.map +1 -0
  381. package/lib/dist/services/opensearch/client.d.ts +26 -0
  382. package/lib/dist/services/opensearch/client.d.ts.map +1 -0
  383. package/lib/dist/services/opensearch/client.js +131 -0
  384. package/lib/dist/services/opensearch/client.js.map +1 -0
  385. package/lib/dist/services/opensearch/index.d.ts +16 -0
  386. package/lib/dist/services/opensearch/index.d.ts.map +1 -0
  387. package/lib/dist/services/opensearch/index.js +25 -0
  388. package/lib/dist/services/opensearch/index.js.map +1 -0
  389. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts +123 -0
  390. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -0
  391. package/lib/dist/services/storage/asyncBenchmarkStorage.js +429 -0
  392. package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -0
  393. package/lib/dist/services/storage/asyncRunStorage.d.ts +127 -0
  394. package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -0
  395. package/lib/dist/services/storage/asyncRunStorage.js +448 -0
  396. package/lib/dist/services/storage/asyncRunStorage.js.map +1 -0
  397. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +156 -0
  398. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -0
  399. package/lib/dist/services/storage/asyncTestCaseStorage.js +285 -0
  400. package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -0
  401. package/lib/dist/services/storage/index.d.ts +17 -0
  402. package/lib/dist/services/storage/index.d.ts.map +1 -0
  403. package/lib/dist/services/storage/index.js +20 -0
  404. package/lib/dist/services/storage/index.js.map +1 -0
  405. package/lib/dist/services/storage/migration.d.ts +54 -0
  406. package/lib/dist/services/storage/migration.d.ts.map +1 -0
  407. package/lib/dist/services/storage/migration.js +296 -0
  408. package/lib/dist/services/storage/migration.js.map +1 -0
  409. package/lib/dist/services/storage/opensearchClient.d.ts +924 -0
  410. package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -0
  411. package/lib/dist/services/storage/opensearchClient.js +435 -0
  412. package/lib/dist/services/storage/opensearchClient.js.map +1 -0
  413. package/lib/dist/services/traces/browserRecovery.d.ts +26 -0
  414. package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -0
  415. package/lib/dist/services/traces/browserRecovery.js +81 -0
  416. package/lib/dist/services/traces/browserRecovery.js.map +1 -0
  417. package/lib/dist/services/traces/categoryStyles.d.ts +21 -0
  418. package/lib/dist/services/traces/categoryStyles.d.ts.map +1 -0
  419. package/lib/dist/services/traces/categoryStyles.js +56 -0
  420. package/lib/dist/services/traces/categoryStyles.js.map +1 -0
  421. package/lib/dist/services/traces/executionOrderTransform.d.ts +35 -0
  422. package/lib/dist/services/traces/executionOrderTransform.d.ts.map +1 -0
  423. package/lib/dist/services/traces/executionOrderTransform.js +313 -0
  424. package/lib/dist/services/traces/executionOrderTransform.js.map +1 -0
  425. package/lib/dist/services/traces/fetchSpansForRun.d.ts +86 -0
  426. package/lib/dist/services/traces/fetchSpansForRun.d.ts.map +1 -0
  427. package/lib/dist/services/traces/fetchSpansForRun.js +69 -0
  428. package/lib/dist/services/traces/fetchSpansForRun.js.map +1 -0
  429. package/lib/dist/services/traces/flowTransform.d.ts +24 -0
  430. package/lib/dist/services/traces/flowTransform.d.ts.map +1 -0
  431. package/lib/dist/services/traces/flowTransform.js +228 -0
  432. package/lib/dist/services/traces/flowTransform.js.map +1 -0
  433. package/lib/dist/services/traces/index.d.ts +121 -0
  434. package/lib/dist/services/traces/index.d.ts.map +1 -0
  435. package/lib/dist/services/traces/index.js +255 -0
  436. package/lib/dist/services/traces/index.js.map +1 -0
  437. package/lib/dist/services/traces/intentTransform.d.ts +20 -0
  438. package/lib/dist/services/traces/intentTransform.d.ts.map +1 -0
  439. package/lib/dist/services/traces/intentTransform.js +131 -0
  440. package/lib/dist/services/traces/intentTransform.js.map +1 -0
  441. package/lib/dist/services/traces/judgeAgentsHints.d.ts +63 -0
  442. package/lib/dist/services/traces/judgeAgentsHints.d.ts.map +1 -0
  443. package/lib/dist/services/traces/judgeAgentsHints.js +89 -0
  444. package/lib/dist/services/traces/judgeAgentsHints.js.map +1 -0
  445. package/lib/dist/services/traces/messageExtraction.d.ts +15 -0
  446. package/lib/dist/services/traces/messageExtraction.d.ts.map +1 -0
  447. package/lib/dist/services/traces/messageExtraction.js +251 -0
  448. package/lib/dist/services/traces/messageExtraction.js.map +1 -0
  449. package/lib/dist/services/traces/spanCategorization.d.ts +63 -0
  450. package/lib/dist/services/traces/spanCategorization.d.ts.map +1 -0
  451. package/lib/dist/services/traces/spanCategorization.js +276 -0
  452. package/lib/dist/services/traces/spanCategorization.js.map +1 -0
  453. package/lib/dist/services/traces/spanPreprocessing.d.ts +37 -0
  454. package/lib/dist/services/traces/spanPreprocessing.d.ts.map +1 -0
  455. package/lib/dist/services/traces/spanPreprocessing.js +102 -0
  456. package/lib/dist/services/traces/spanPreprocessing.js.map +1 -0
  457. package/lib/dist/services/traces/spansToTrajectory.d.ts +36 -0
  458. package/lib/dist/services/traces/spansToTrajectory.d.ts.map +1 -0
  459. package/lib/dist/services/traces/spansToTrajectory.js +387 -0
  460. package/lib/dist/services/traces/spansToTrajectory.js.map +1 -0
  461. package/lib/dist/services/traces/toolSimilarity.d.ts +35 -0
  462. package/lib/dist/services/traces/toolSimilarity.d.ts.map +1 -0
  463. package/lib/dist/services/traces/toolSimilarity.js +203 -0
  464. package/lib/dist/services/traces/toolSimilarity.js.map +1 -0
  465. package/lib/dist/services/traces/traceComparison.d.ts +31 -0
  466. package/lib/dist/services/traces/traceComparison.d.ts.map +1 -0
  467. package/lib/dist/services/traces/traceComparison.js +318 -0
  468. package/lib/dist/services/traces/traceComparison.js.map +1 -0
  469. package/lib/dist/services/traces/traceGrouping.d.ts +19 -0
  470. package/lib/dist/services/traces/traceGrouping.d.ts.map +1 -0
  471. package/lib/dist/services/traces/traceGrouping.js +107 -0
  472. package/lib/dist/services/traces/traceGrouping.js.map +1 -0
  473. package/lib/dist/services/traces/tracePoller.d.ts +84 -0
  474. package/lib/dist/services/traces/tracePoller.d.ts.map +1 -0
  475. package/lib/dist/services/traces/tracePoller.js +309 -0
  476. package/lib/dist/services/traces/tracePoller.js.map +1 -0
  477. package/lib/dist/services/traces/traceStats.d.ts +45 -0
  478. package/lib/dist/services/traces/traceStats.d.ts.map +1 -0
  479. package/lib/dist/services/traces/traceStats.js +114 -0
  480. package/lib/dist/services/traces/traceStats.js.map +1 -0
  481. package/lib/dist/services/traces/traceSummary.d.ts +47 -0
  482. package/lib/dist/services/traces/traceSummary.d.ts.map +1 -0
  483. package/lib/dist/services/traces/traceSummary.js +68 -0
  484. package/lib/dist/services/traces/traceSummary.js.map +1 -0
  485. package/lib/dist/services/traces/utils.d.ts +33 -0
  486. package/lib/dist/services/traces/utils.d.ts.map +1 -0
  487. package/lib/dist/services/traces/utils.js +114 -0
  488. package/lib/dist/services/traces/utils.js.map +1 -0
  489. package/lib/dist/types/agui.d.ts +13 -0
  490. package/lib/dist/types/agui.d.ts.map +1 -0
  491. package/lib/dist/types/agui.js +16 -0
  492. package/lib/dist/types/agui.js.map +1 -0
  493. package/lib/dist/types/index.d.ts +1175 -0
  494. package/lib/dist/types/index.d.ts.map +1 -0
  495. package/lib/dist/types/index.js +12 -0
  496. package/lib/dist/types/index.js.map +1 -0
  497. package/lib/dist/types/skills.d.ts +146 -0
  498. package/lib/dist/types/skills.d.ts.map +1 -0
  499. package/lib/dist/types/skills.js +6 -0
  500. package/lib/dist/types/skills.js.map +1 -0
  501. package/observio-sample-agent/pi-package/README.md +112 -0
  502. package/observio-sample-agent/pi-package/extensions/agent-health.ts +373 -0
  503. package/observio-sample-agent/pi-package/package.json +17 -0
  504. package/observio-sample-agent/pi-package/prompts/agent-health.md +37 -0
  505. package/observio-sample-agent/pi-package/skills/create-pr/SKILL.md +88 -0
  506. package/observio-sample-agent/pi-package/skills/fix-bug/SKILL.md +71 -0
  507. package/observio-sample-agent/pi-package/skills/implement-feature/SKILL.md +156 -0
  508. package/observio-sample-agent/pi-package/skills/instrument-otel/SKILL.md +208 -0
  509. package/observio-sample-agent/pi-package/skills/setup-collector/SKILL.md +146 -0
  510. package/observio-sample-agent/pi-package/skills/write-test/SKILL.md +115 -0
  511. package/package.json +64 -13
  512. package/server/dist/app.js +32651 -17637
  513. package/server/dist/index.js +29875 -14638
  514. package/tsconfig.lib.json +71 -0
  515. package/dist/assets/index-EvPLSTAS.js +0 -267
  516. package/dist/assets/index-RXasQKUs.css +0 -1
  517. package/lib/dist/config/index.js +0 -404
  518. package/lib/dist/index.js +0 -1665
@@ -0,0 +1,429 @@
1
+ /*
2
+ * Copyright OpenSearch Contributors
3
+ * SPDX-License-Identifier: Apache-2.0
4
+ */
5
+ /**
6
+ * Async Benchmark Storage
7
+ *
8
+ * Async wrapper around OpenSearch storage for benchmarks.
9
+ * Maps between app's Benchmark type and OpenSearch StorageBenchmark.
10
+ */
11
+ import { benchmarkStorage as opensearchBenchmarks } from './opensearchClient.js';
12
+ /**
13
+ * Convert OpenSearch storage format to app Benchmark format.
14
+ * Normalizes legacy data without version fields.
15
+ */
16
+ function toBenchmark(stored) {
17
+ const version = stored.currentVersion ?? stored.version ?? 1;
18
+ return {
19
+ id: stored.id,
20
+ name: stored.name,
21
+ description: stored.description,
22
+ createdAt: stored.createdAt,
23
+ updatedAt: stored.updatedAt ?? stored.createdAt,
24
+ currentVersion: version,
25
+ versions: stored.versions ?? [{
26
+ version: 1,
27
+ createdAt: stored.createdAt,
28
+ testCaseIds: stored.testCaseIds || [],
29
+ }],
30
+ testCaseIds: stored.testCaseIds,
31
+ runs: (stored.runs || []).map(toBenchmarkRun),
32
+ };
33
+ }
34
+ /**
35
+ * Convert OpenSearch run config to app BenchmarkRun format.
36
+ * Normalizes legacy data without version tracking fields.
37
+ */
38
+ function toBenchmarkRun(stored) {
39
+ // Convert results with proper typing for status field
40
+ const results = {};
41
+ if (stored.results) {
42
+ Object.entries(stored.results).forEach(([key, value]) => {
43
+ results[key] = {
44
+ reportId: value.reportId,
45
+ status: value.status,
46
+ // passFailStatus is the per-case verdict bucketRunResults() reads to
47
+ // compute pass rate (a completed result with no verdict = errored,
48
+ // #242). Dropping it here made every benchmark-embedded run read as
49
+ // all-errored in the comparison panel.
50
+ ...(value.passFailStatus && { passFailStatus: value.passFailStatus }),
51
+ ...(value.error && { error: value.error }),
52
+ ...(value.performanceMetrics && { performanceMetrics: value.performanceMetrics }),
53
+ };
54
+ });
55
+ }
56
+ return {
57
+ id: stored.id,
58
+ name: stored.name,
59
+ description: stored.description,
60
+ createdAt: stored.createdAt,
61
+ // Handle both current (agentKey) and legacy (agentId) field names
62
+ agentKey: stored.agentKey || stored.agentId || '',
63
+ modelId: stored.modelId,
64
+ headers: stored.headers,
65
+ benchmarkVersion: stored.benchmarkVersion ?? 1,
66
+ testCaseSnapshots: stored.testCaseSnapshots ?? [],
67
+ status: stored.status,
68
+ error: stored.error,
69
+ stats: stored.stats,
70
+ results,
71
+ performanceMetrics: stored.performanceMetrics,
72
+ };
73
+ }
74
+ /**
75
+ * Convert app Benchmark format to OpenSearch storage format
76
+ */
77
+ function toStorageFormat(benchmark) {
78
+ const result = {
79
+ name: benchmark.name,
80
+ description: benchmark.description,
81
+ testCaseIds: benchmark.testCaseIds,
82
+ };
83
+ // Include versioning fields
84
+ if (benchmark.currentVersion !== undefined) {
85
+ result.currentVersion = benchmark.currentVersion;
86
+ }
87
+ if (benchmark.versions !== undefined) {
88
+ result.versions = benchmark.versions;
89
+ }
90
+ if (benchmark.updatedAt !== undefined) {
91
+ result.updatedAt = benchmark.updatedAt;
92
+ }
93
+ // Convert runs with version tracking fields
94
+ // Use agentKey to match server route behavior
95
+ if (benchmark.runs) {
96
+ result.runs = benchmark.runs.map(run => ({
97
+ id: run.id,
98
+ name: run.name,
99
+ description: run.description,
100
+ agentKey: run.agentKey,
101
+ modelId: run.modelId,
102
+ headers: run.headers,
103
+ createdAt: run.createdAt,
104
+ benchmarkVersion: run.benchmarkVersion,
105
+ testCaseSnapshots: run.testCaseSnapshots,
106
+ results: run.results,
107
+ ...(run.performanceMetrics && { performanceMetrics: run.performanceMetrics }),
108
+ }));
109
+ }
110
+ return result;
111
+ }
112
+ class AsyncBenchmarkStorage {
113
+ // ==================== Benchmark CRUD Operations ====================
114
+ /**
115
+ * Get all benchmarks, sorted by updatedAt descending (most recently active first)
116
+ * @param options.includeSample - whether to include sample/demo data
117
+ */
118
+ async getAll(options) {
119
+ const stored = await opensearchBenchmarks.getAll(options);
120
+ const benchmarks = stored.map(toBenchmark);
121
+ // Sort by updatedAt descending (most recently active first), fallback to createdAt
122
+ return benchmarks.sort((a, b) => new Date(b.updatedAt || b.createdAt || 0).getTime() - new Date(a.updatedAt || a.createdAt || 0).getTime());
123
+ }
124
+ /**
125
+ * Get a single benchmark by ID
126
+ * @param options.fields - 'polling' for lightweight payload (excludes versions, testCaseSnapshots, headers)
127
+ * @param options.runsSize - max number of runs to return
128
+ * @param options.runsOffset - offset into runs array for pagination
129
+ */
130
+ async getById(id, options) {
131
+ const stored = await opensearchBenchmarks.getById(id, options);
132
+ if (!stored)
133
+ return null;
134
+ const benchmark = toBenchmark(stored);
135
+ // Pass through pagination metadata from server response
136
+ const storedAny = stored;
137
+ if (storedAny.totalRuns !== undefined) {
138
+ benchmark.totalRuns = storedAny.totalRuns;
139
+ benchmark.hasMoreRuns = storedAny.hasMoreRuns;
140
+ }
141
+ return benchmark;
142
+ }
143
+ /**
144
+ * Create a new benchmark (immutable)
145
+ */
146
+ async create(benchmark) {
147
+ const storageData = toStorageFormat(benchmark);
148
+ const created = await opensearchBenchmarks.create(storageData);
149
+ return toBenchmark(created);
150
+ }
151
+ /**
152
+ * Save a benchmark (create only - benchmarks are immutable)
153
+ * For compatibility with existing code that calls save()
154
+ */
155
+ async save(benchmark) {
156
+ // Check if exists
157
+ const existing = await this.getById(benchmark.id);
158
+ if (existing) {
159
+ // Benchmarks are immutable - return existing
160
+ console.warn('Benchmark already exists and cannot be updated:', benchmark.id);
161
+ return existing;
162
+ }
163
+ const storageData = toStorageFormat(benchmark);
164
+ // Include id for existing benchmarks being saved
165
+ const dataWithId = {
166
+ ...storageData,
167
+ id: benchmark.id,
168
+ };
169
+ const created = await opensearchBenchmarks.create(dataWithId);
170
+ return toBenchmark(created);
171
+ }
172
+ /**
173
+ * Delete a benchmark
174
+ */
175
+ async delete(id) {
176
+ const result = await opensearchBenchmarks.delete(id);
177
+ return result.deleted;
178
+ }
179
+ /**
180
+ * Get total count of benchmarks
181
+ */
182
+ async getCount() {
183
+ const benchmarks = await this.getAll();
184
+ return benchmarks.length;
185
+ }
186
+ // ==================== Run Operations ====================
187
+ // Note: In OpenSearch model, runs are embedded in benchmark.
188
+ // Actual execution results go to evals_runs index, not here.
189
+ /**
190
+ * Get all run configs for a benchmark
191
+ */
192
+ async getRuns(benchmarkId) {
193
+ const benchmark = await this.getById(benchmarkId);
194
+ if (!benchmark)
195
+ return [];
196
+ return benchmark.runs || [];
197
+ }
198
+ /**
199
+ * Get a specific run config by ID from a benchmark
200
+ */
201
+ async getRunById(benchmarkId, runId) {
202
+ const benchmark = await this.getById(benchmarkId);
203
+ if (!benchmark)
204
+ return null;
205
+ return benchmark.runs?.find(r => r.id === runId) || null;
206
+ }
207
+ /**
208
+ * Add or update a run in a benchmark
209
+ */
210
+ async addRun(benchmarkId, run) {
211
+ console.log('[asyncBenchmarkStorage] addRun called', { benchmarkId, runId: run.id });
212
+ const benchmark = await this.getById(benchmarkId);
213
+ if (!benchmark) {
214
+ console.error('[asyncBenchmarkStorage] Benchmark not found:', benchmarkId);
215
+ return false;
216
+ }
217
+ console.log('[asyncBenchmarkStorage] Found benchmark, current runs:', benchmark.runs?.length || 0);
218
+ const currentRuns = benchmark.runs || [];
219
+ const existingIndex = currentRuns.findIndex(r => r.id === run.id);
220
+ let updatedRuns;
221
+ if (existingIndex >= 0) {
222
+ // Update existing run
223
+ console.log('[asyncBenchmarkStorage] Updating existing run at index:', existingIndex);
224
+ updatedRuns = [...currentRuns];
225
+ updatedRuns[existingIndex] = run;
226
+ }
227
+ else {
228
+ // Add new run
229
+ console.log('[asyncBenchmarkStorage] Adding new run');
230
+ updatedRuns = [...currentRuns, run];
231
+ }
232
+ // Convert to storage format (preserve all version tracking fields)
233
+ // Use agentKey to match server route behavior
234
+ const storageRuns = updatedRuns.map(r => ({
235
+ id: r.id,
236
+ name: r.name,
237
+ description: r.description,
238
+ agentKey: r.agentKey,
239
+ modelId: r.modelId,
240
+ headers: r.headers,
241
+ createdAt: r.createdAt,
242
+ status: r.status,
243
+ benchmarkVersion: r.benchmarkVersion,
244
+ testCaseSnapshots: r.testCaseSnapshots,
245
+ results: r.results,
246
+ }));
247
+ console.log('[asyncBenchmarkStorage] Saving updated runs:', storageRuns.length);
248
+ await opensearchBenchmarks.update(benchmarkId, { runs: storageRuns });
249
+ console.log('[asyncBenchmarkStorage] Runs saved successfully');
250
+ return true;
251
+ }
252
+ /**
253
+ * Delete a run config from a benchmark using atomic server-side operation.
254
+ * This avoids the read-modify-write pattern that could corrupt other runs.
255
+ */
256
+ async deleteRun(benchmarkId, runId) {
257
+ try {
258
+ const response = await fetch(`/api/storage/benchmarks/${benchmarkId}/runs/${runId}`, {
259
+ method: 'DELETE',
260
+ });
261
+ if (response.status === 404) {
262
+ // Either benchmark or run not found
263
+ return false;
264
+ }
265
+ if (!response.ok) {
266
+ const error = await response.json();
267
+ console.error('[asyncBenchmarkStorage] deleteRun failed:', error);
268
+ return false;
269
+ }
270
+ return true;
271
+ }
272
+ catch (error) {
273
+ console.error('[asyncBenchmarkStorage] deleteRun failed:', error);
274
+ return false;
275
+ }
276
+ }
277
+ // ==================== Utility Functions ====================
278
+ /**
279
+ * Generate a unique benchmark ID
280
+ */
281
+ generateBenchmarkId() {
282
+ return `bench-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
283
+ }
284
+ /**
285
+ * Generate a unique run ID
286
+ */
287
+ generateRunId() {
288
+ return `run-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
289
+ }
290
+ /**
291
+ * Bulk create benchmarks (for migration)
292
+ */
293
+ async bulkCreate(benchmarks) {
294
+ const storageData = benchmarks.map(bench => ({
295
+ ...toStorageFormat(bench),
296
+ id: bench.id,
297
+ createdAt: bench.createdAt,
298
+ }));
299
+ return opensearchBenchmarks.bulkCreate(storageData);
300
+ }
301
+ // ==================== Version Operations ====================
302
+ /**
303
+ * Update benchmark metadata only (name, description) - no version change
304
+ * Uses centralized opensearchClient for consistent API handling
305
+ */
306
+ async updateMetadata(id, updates) {
307
+ try {
308
+ const result = await opensearchBenchmarks.updateMetadata(id, updates);
309
+ return toBenchmark(result);
310
+ }
311
+ catch (error) {
312
+ console.error('[asyncBenchmarkStorage] updateMetadata failed:', error);
313
+ return null;
314
+ }
315
+ }
316
+ /**
317
+ * Update test case list - creates a new version
318
+ * Uses centralized opensearchClient for consistent API handling
319
+ */
320
+ async updateTestCases(id, testCaseIds) {
321
+ try {
322
+ const result = await opensearchBenchmarks.update(id, { testCaseIds });
323
+ return toBenchmark(result);
324
+ }
325
+ catch (error) {
326
+ console.error('[asyncBenchmarkStorage] updateTestCases failed:', error);
327
+ return null;
328
+ }
329
+ }
330
+ /**
331
+ * Get all versions of a benchmark
332
+ * Uses centralized opensearchClient for consistent API handling
333
+ */
334
+ async getVersions(benchmarkId) {
335
+ try {
336
+ const result = await opensearchBenchmarks.getVersions(benchmarkId);
337
+ return result.versions;
338
+ }
339
+ catch (error) {
340
+ // Handle 404 gracefully
341
+ if (error.message?.includes('404') || error.message?.includes('not found')) {
342
+ return [];
343
+ }
344
+ console.error('[asyncBenchmarkStorage] getVersions failed:', error);
345
+ return [];
346
+ }
347
+ }
348
+ /**
349
+ * Get a specific version of a benchmark
350
+ * Uses centralized opensearchClient for consistent API handling
351
+ */
352
+ async getVersion(benchmarkId, version) {
353
+ try {
354
+ return await opensearchBenchmarks.getVersion(benchmarkId, version);
355
+ }
356
+ catch (error) {
357
+ // Handle 404 gracefully
358
+ if (error.message?.includes('404') || error.message?.includes('not found')) {
359
+ return null;
360
+ }
361
+ console.error('[asyncBenchmarkStorage] getVersion failed:', error);
362
+ return null;
363
+ }
364
+ }
365
+ /**
366
+ * Update a benchmark (name, description, or testCaseIds).
367
+ * If testCaseIds changed, creates a new version.
368
+ * Uses centralized opensearchClient for consistent API handling
369
+ */
370
+ async update(id, updates) {
371
+ try {
372
+ const result = await opensearchBenchmarks.update(id, updates);
373
+ return toBenchmark(result);
374
+ }
375
+ catch (error) {
376
+ console.error('[asyncBenchmarkStorage] update failed:', error);
377
+ return null;
378
+ }
379
+ }
380
+ // ==================== Stats Refresh Operations ====================
381
+ /**
382
+ * Manually refresh stats for all runs in a benchmark.
383
+ * Useful for fixing stale stats or after manual data corrections.
384
+ */
385
+ async refreshAllStats(benchmarkId) {
386
+ try {
387
+ const response = await fetch(`/api/storage/benchmarks/${benchmarkId}/refresh-all-stats`, {
388
+ method: 'POST',
389
+ });
390
+ if (!response.ok) {
391
+ const error = await response.json();
392
+ console.error('[asyncBenchmarkStorage] refreshAllStats failed:', error);
393
+ return null;
394
+ }
395
+ return await response.json();
396
+ }
397
+ catch (error) {
398
+ console.error('[asyncBenchmarkStorage] refreshAllStats failed:', error);
399
+ return null;
400
+ }
401
+ }
402
+ /**
403
+ * Manually refresh stats for a single run in a benchmark.
404
+ * Useful for fixing stale stats for a specific run.
405
+ */
406
+ async refreshRunStats(benchmarkId, runId) {
407
+ try {
408
+ const response = await fetch(`/api/storage/benchmarks/${benchmarkId}/runs/${runId}/refresh-stats`, {
409
+ method: 'POST',
410
+ });
411
+ if (!response.ok) {
412
+ const error = await response.json();
413
+ console.error('[asyncBenchmarkStorage] refreshRunStats failed:', error);
414
+ return null;
415
+ }
416
+ return response.json();
417
+ }
418
+ catch (error) {
419
+ console.error('[asyncBenchmarkStorage] refreshRunStats failed:', error);
420
+ return null;
421
+ }
422
+ }
423
+ }
424
+ // Export singleton instance
425
+ export const asyncBenchmarkStorage = new AsyncBenchmarkStorage();
426
+ // Backwards compatibility alias
427
+ /** @deprecated Use asyncBenchmarkStorage instead */
428
+ export const asyncExperimentStorage = asyncBenchmarkStorage;
429
+ //# sourceMappingURL=asyncBenchmarkStorage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"asyncBenchmarkStorage.js","sourceRoot":"","sources":["../../../../services/storage/asyncBenchmarkStorage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,IAAI,oBAAoB,EAA+C,MAAM,oBAAoB,CAAC;AAe3H;;;GAGG;AACH,SAAS,WAAW,CAAC,MAAwB;IAC3C,MAAM,OAAO,GAAI,MAAc,CAAC,cAAc,IAAK,MAAc,CAAC,OAAO,IAAI,CAAC,CAAC;IAC/E,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAG,MAAc,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS;QACxD,cAAc,EAAE,OAAO;QACvB,QAAQ,EAAG,MAAc,CAAC,QAAQ,IAAI,CAAC;gBACrC,OAAO,EAAE,CAAC;gBACV,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;aACtC,CAAC;QACF,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,IAAI,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,MAAiC;IACvD,sDAAsD;IACtD,MAAM,OAAO,GAAkJ,EAAE,CAAC;IAClK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACtD,OAAO,CAAC,GAAG,CAAC,GAAG;gBACb,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,KAAK,CAAC,MAAyB;gBACvC,qEAAqE;gBACrE,mEAAmE;gBACnE,oEAAoE;gBACpE,uCAAuC;gBACvC,GAAG,CAAE,KAAa,CAAC,cAAc,IAAI,EAAE,cAAc,EAAG,KAAa,CAAC,cAAc,EAAE,CAAC;gBACvF,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC1C,GAAG,CAAE,KAAa,CAAC,kBAAkB,IAAI,EAAE,kBAAkB,EAAG,KAAa,CAAC,kBAAkB,EAAE,CAAC;aACpG,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,EAAE;QACb,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,kEAAkE;QAClE,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE;QACjD,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,gBAAgB,EAAG,MAAc,CAAC,gBAAgB,IAAI,CAAC;QACvD,iBAAiB,EAAG,MAAc,CAAC,iBAAiB,IAAI,EAAE;QAC1D,MAAM,EAAE,MAAM,CAAC,MAAwC;QACvD,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,MAAM,CAAC,KAA6B;QAC3C,OAAO;QACP,kBAAkB,EAAG,MAAc,CAAC,kBAAkB;KACvD,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,SAA6B;IACpD,MAAM,MAAM,GAAwB;QAClC,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,WAAW,EAAE,SAAS,CAAC,WAAW;QAClC,WAAW,EAAE,SAAS,CAAC,WAAW;KACnC,CAAC;IAEF,4BAA4B;IAC5B,IAAI,SAAS,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QAC3C,MAAM,CAAC,cAAc,GAAG,SAAS,CAAC,cAAc,CAAC;IACnD,CAAC;IACD,IAAI,SAAS,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;IACvC,CAAC;IACD,IAAI,SAAS,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACtC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;IACzC,CAAC;IAED,4CAA4C;IAC5C,8CAA8C;IAC9C,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;YACtC,iBAAiB,EAAE,GAAG,CAAC,iBAAiB;YACxC,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,GAAG,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE,kBAAkB,EAAE,GAAG,CAAC,kBAAkB,EAAE,CAAC;SAC9E,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,qBAAqB;IACzB,sEAAsE;IAEtE;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,OAAqC;QAChD,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC3C,mFAAmF;QACnF,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC9B,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAC1G,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,OAAwE;QAChG,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QACtC,wDAAwD;QACxD,MAAM,SAAS,GAAG,MAAa,CAAC;QAChC,IAAI,SAAS,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACrC,SAAiB,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;YAClD,SAAiB,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;QACzD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,SAA4D;QACvE,MAAM,WAAW,GAAG,eAAe,CAAC,SAAsB,CAA+C,CAAC;QAC1G,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAC/D,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI,CAAC,SAAoB;QAC7B,kBAAkB;QAClB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClD,IAAI,QAAQ,EAAE,CAAC;YACb,6CAA6C;YAC7C,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;YAC9E,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,MAAM,WAAW,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;QAC/C,iDAAiD;QACjD,MAAM,UAAU,GAAG;YACjB,GAAG,WAAW;YACd,EAAE,EAAE,SAAS,CAAC,EAAE;SACwC,CAAC;QAC3D,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC9D,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACvC,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,2DAA2D;IAC3D,6DAA6D;IAC7D,6DAA6D;IAE7D;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,WAAmB;QAC/B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS;YAAE,OAAO,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,WAAmB,EAAE,KAAa;QACjD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,WAAmB,EAAE,GAAiB;QACjD,OAAO,CAAC,GAAG,CAAC,uCAAuC,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAErF,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,8CAA8C,EAAE,WAAW,CAAC,CAAC;YAC3E,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,wDAAwD,EAAE,SAAS,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;QAEnG,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC;QACzC,MAAM,aAAa,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QAElE,IAAI,WAA2B,CAAC;QAChC,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;YACvB,sBAAsB;YACtB,OAAO,CAAC,GAAG,CAAC,yDAAyD,EAAE,aAAa,CAAC,CAAC;YACtF,WAAW,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;YAC/B,WAAW,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,cAAc;YACd,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;YACtD,WAAW,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,CAAC,CAAC;QACtC,CAAC;QAED,mEAAmE;QACnE,8CAA8C;QAC9C,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxC,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,gBAAgB,EAAE,CAAC,CAAC,gBAAgB;YACpC,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;YACtC,OAAO,EAAE,CAAC,CAAC,OAAO;SACnB,CAAC,CAAC,CAAC;QAEJ,OAAO,CAAC,GAAG,CAAC,8CAA8C,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QAChF,MAAM,oBAAoB,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;QACtE,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;QAC/D,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,WAAmB,EAAE,KAAa;QAChD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,2BAA2B,WAAW,SAAS,KAAK,EAAE,EAAE;gBACnF,MAAM,EAAE,QAAQ;aACjB,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,oCAAoC;gBACpC,OAAO,KAAK,CAAC;YACf,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;gBAClE,OAAO,KAAK,CAAC;YACf,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,KAAK,CAAC,CAAC;YAClE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,8DAA8D;IAE9D;;OAEG;IACH,mBAAmB;QACjB,OAAO,SAAS,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAC9E,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAC5E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,UAAuB;QACtC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC3C,GAAG,eAAe,CAAC,KAAK,CAAC;YACzB,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC,CAAC,CAAC;QACJ,OAAO,oBAAoB,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;IACtD,CAAC;IAED,+DAA+D;IAE/D;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,EAAU,EAAE,OAAgD;QAC/E,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,cAAc,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACtE,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gDAAgD,EAAE,KAAK,CAAC,CAAC;YACvE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,EAAU,EAAE,WAAqB;QACrD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;YACtE,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,WAAmB;QACnC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YACnE,OAAO,MAAM,CAAC,QAAQ,CAAC;QACzB,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,wBAAwB;YACxB,IAAI,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3E,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,KAAK,CAAC,CAAC;YACpE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CAAC,WAAmB,EAAE,OAAe;QACnD,IAAI,CAAC;YACH,OAAO,MAAM,oBAAoB,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,wBAAwB;YACxB,IAAI,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC3E,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE,KAAK,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,OAAwE;QAC/F,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAC9D,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;YAC/D,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,qEAAqE;IAErE;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,WAAmB;QACvC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,2BAA2B,WAAW,oBAAoB,EAAE;gBACvF,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;gBACxE,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,WAAmB,EAAE,KAAa;QACtD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,2BAA2B,WAAW,SAAS,KAAK,gBAAgB,EAAE;gBACjG,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;gBACxE,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;YACxE,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,EAAE,CAAC;AAEjE,gCAAgC;AAChC,oDAAoD;AACpD,MAAM,CAAC,MAAM,sBAAsB,GAAG,qBAAqB,CAAC"}
@@ -0,0 +1,127 @@
1
+ import type { EvaluationReport, RunAnnotation } from '../../types/index.js';
2
+ export interface SearchQuery {
3
+ testCaseIds?: string[];
4
+ dateRange?: {
5
+ start: string;
6
+ end: string;
7
+ };
8
+ agentNames?: string[];
9
+ modelNames?: string[];
10
+ minAccuracy?: number;
11
+ status?: ('running' | 'completed' | 'failed')[];
12
+ hasAnnotations?: boolean;
13
+ annotationTags?: string[];
14
+ }
15
+ export interface GetReportsOptions {
16
+ limit?: number;
17
+ offset?: number;
18
+ sortBy?: 'timestamp' | 'accuracy';
19
+ order?: 'asc' | 'desc';
20
+ fields?: string[];
21
+ }
22
+ declare class AsyncRunStorage {
23
+ /**
24
+ * Save a report/run
25
+ */
26
+ saveReport(report: EvaluationReport, options?: {
27
+ experimentId?: string;
28
+ experimentRunId?: string;
29
+ iteration?: number;
30
+ }): Promise<EvaluationReport>;
31
+ /**
32
+ * Get run counts grouped by test case ID (single bulk query)
33
+ */
34
+ getRunCountsByTestCase(): Promise<Record<string, number>>;
35
+ /**
36
+ * Get all reports for a specific test case
37
+ */
38
+ getReportsByTestCase(testCaseId: string, options?: GetReportsOptions): Promise<{
39
+ reports: EvaluationReport[];
40
+ total: number;
41
+ }>;
42
+ /**
43
+ * Get all reports across all test cases
44
+ */
45
+ getAllReports(options?: GetReportsOptions): Promise<EvaluationReport[]>;
46
+ /**
47
+ * Get a single report by ID
48
+ */
49
+ getReportById(reportId: string): Promise<EvaluationReport | null>;
50
+ /**
51
+ * Batch-fetch reports by id in a SINGLE request (vs N getReportById calls).
52
+ * Returned map is keyed by the requested reportId (the storage doc id).
53
+ */
54
+ getReportsByIds(reportIds: string[]): Promise<Record<string, EvaluationReport>>;
55
+ /**
56
+ * Delete a report and its annotations
57
+ */
58
+ deleteReport(reportId: string): Promise<boolean>;
59
+ /**
60
+ * Partial update of a report
61
+ * Used for updating trace-mode runs after traces become available
62
+ */
63
+ updateReport(reportId: string, updates: Partial<EvaluationReport>): Promise<EvaluationReport | null>;
64
+ /**
65
+ * Get total count of reports
66
+ */
67
+ getReportCount(): Promise<number>;
68
+ /**
69
+ * Get report count for a specific test case
70
+ */
71
+ getReportCountByTestCase(testCaseId: string): Promise<number>;
72
+ /**
73
+ * Get runs for a benchmark
74
+ */
75
+ getByBenchmark(benchmarkId: string, size?: number): Promise<EvaluationReport[]>;
76
+ /**
77
+ * Get runs for a specific benchmark run config
78
+ */
79
+ getByBenchmarkRun(benchmarkId: string, runId: string, size?: number): Promise<EvaluationReport[]>;
80
+ /**
81
+ * Get all iterations for a test case in a benchmark
82
+ */
83
+ getIterations(benchmarkId: string, testCaseId: string, benchmarkRunId?: string): Promise<{
84
+ runs: EvaluationReport[];
85
+ total: number;
86
+ maxIteration: number;
87
+ }>;
88
+ /** @deprecated Use getByBenchmark instead */
89
+ getByExperiment(experimentId: string, size?: number): Promise<EvaluationReport[]>;
90
+ /** @deprecated Use getByBenchmarkRun instead */
91
+ getByExperimentRun(experimentId: string, runId: string, size?: number): Promise<EvaluationReport[]>;
92
+ /**
93
+ * Add an annotation to a report
94
+ */
95
+ addAnnotation(reportId: string, annotation: Omit<RunAnnotation, 'id' | 'timestamp' | 'reportId'>): Promise<RunAnnotation>;
96
+ /**
97
+ * Update an existing annotation
98
+ */
99
+ updateAnnotation(reportId: string, annotationId: string, updates: Partial<RunAnnotation>): Promise<boolean>;
100
+ /**
101
+ * Delete an annotation
102
+ */
103
+ deleteAnnotation(reportId: string, annotationId: string): Promise<boolean>;
104
+ /**
105
+ * Get all annotations for a report
106
+ */
107
+ getAnnotationsByReport(reportId: string): Promise<RunAnnotation[]>;
108
+ /**
109
+ * Search reports with complex filtering
110
+ */
111
+ searchReports(query: SearchQuery): Promise<EvaluationReport[]>;
112
+ /**
113
+ * Generate a unique report ID
114
+ */
115
+ generateReportId(): string;
116
+ /**
117
+ * Bulk create runs (for migration)
118
+ */
119
+ bulkCreate(runs: EvaluationReport[]): Promise<{
120
+ created: number;
121
+ errors: boolean;
122
+ }>;
123
+ }
124
+ export declare const asyncRunStorage: AsyncRunStorage;
125
+ export declare const asyncReportStorage: AsyncRunStorage;
126
+ export {};
127
+ //# sourceMappingURL=asyncRunStorage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"asyncRunStorage.d.ts","sourceRoot":"","sources":["../../../../services/storage/asyncRunStorage.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EACV,gBAAgB,EAEhB,aAAa,EAMd,MAAM,SAAS,CAAC;AAGjB,MAAM,WAAW,WAAW;IAC1B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,CAAC,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC,EAAE,CAAC;IAChD,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,GAAG,UAAU,CAAC;IAClC,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAqKD,cAAM,eAAe;IAGnB;;OAEG;IACG,UAAU,CACd,MAAM,EAAE,gBAAgB,EACxB,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAChF,OAAO,CAAC,gBAAgB,CAAC;IAkB5B;;OAEG;IACG,sBAAsB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAI/D;;OAEG;IACG,oBAAoB,CACxB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC;QAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAM1D;;OAEG;IACG,aAAa,CAAC,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAajF;;OAEG;IACG,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAKvE;;;OAGG;IACG,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAUrF;;OAEG;IACG,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKtD;;;OAGG;IACG,YAAY,CAChB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC,GACjC,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IA0CnC;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,MAAM,CAAC;IAKvC;;OAEG;IACG,wBAAwB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAOnE;;OAEG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAKrF;;OAEG;IACG,iBAAiB,CACrB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAK9B;;OAEG;IACG,aAAa,CACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,MAAM,GACtB,OAAO,CAAC;QAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;IAU7E,6CAA6C;IACvC,eAAe,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAIvF,gDAAgD;IAC1C,kBAAkB,CACtB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAM9B;;OAEG;IACG,aAAa,CACjB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,WAAW,GAAG,UAAU,CAAC,GAC/D,OAAO,CAAC,aAAa,CAAC;IAiBzB;;OAEG;IACG,gBAAgB,CACpB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,GAC9B,OAAO,CAAC,OAAO,CAAC;IAYnB;;OAEG;IACG,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKhF;;OAEG;IACG,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAgBxE;;OAEG;IACG,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAqCpE;;OAEG;IACH,gBAAgB,IAAI,MAAM;IAI1B;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;CAQ1F;AAGD,eAAO,MAAM,eAAe,iBAAwB,CAAC;AAGrD,eAAO,MAAM,kBAAkB,iBAAkB,CAAC"}