@opensearch-project/agent-health 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (551) hide show
  1. package/README.md +36 -5
  2. package/cli/dist/index.js +8292 -2938
  3. package/deployment/cloudformation/agent-health-observability.yaml +223 -32
  4. package/dist/assets/index-BfxtxmKc.css +1 -0
  5. package/dist/assets/index-CrjAfDHu.js +243 -0
  6. package/dist/index.html +2 -2
  7. package/docs/ARCHITECTURE.md +450 -0
  8. package/docs/BACKEND_JOB_QUEUE.md +405 -0
  9. package/docs/CLAUDE_CODE_TELEMETRY.md +283 -0
  10. package/docs/CLI.md +474 -0
  11. package/docs/CODING_AGENT_ANALYTICS.md +298 -0
  12. package/docs/CONFIGURATION.md +388 -0
  13. package/docs/CONNECTORS.md +536 -0
  14. package/docs/INSTRUMENT_WITH_OTEL.md +397 -0
  15. package/docs/ML-COMMONS-SETUP.md +289 -0
  16. package/docs/NPX_PACKAGING.md +195 -0
  17. package/docs/PERFORMANCE-MONITORING.md +200 -0
  18. package/docs/PERFORMANCE.md +390 -0
  19. package/docs/PI_PROFILING.md +169 -0
  20. package/docs/PLAN-non-agui-agent-support.md +525 -0
  21. package/docs/SDK.md +626 -0
  22. package/docs/SKILLS.md +264 -0
  23. package/docs/STORAGE_INDEX_FIELD_LIMITS.md +216 -0
  24. package/docs/blogs/2026-02-28-opensearch-agent-health.md +200 -0
  25. package/docs/blogs/getting-started-blog.md +608 -0
  26. package/docs/diagrams/Agent-health.excalidraw +5656 -0
  27. package/docs/diagrams/architecture.png +0 -0
  28. package/docs/plans/field-redesign.md +468 -0
  29. package/docs/rfcs/001-coding-agent-analytics.md +374 -0
  30. package/docs/rfcs/002-enterprise-leaderboard.md +267 -0
  31. package/docs/rfcs/003-remote-aggregation.md +146 -0
  32. package/docs/rfcs/004-test-sdk-v2.md +599 -0
  33. package/docs/skills/AGENT_HEALTH.md +652 -0
  34. package/docs/skills/AGENT_PROFILE.md +191 -0
  35. package/docs/skills/add-connector/SKILL.md +68 -0
  36. package/docs/skills/agent-health-profile/SKILL.md +40 -0
  37. package/docs/skills/config-auth/SKILL.md +194 -0
  38. package/docs/skills/config-auth/evals/evals.json +35 -0
  39. package/docs/skills/create-pr/SKILL.md +73 -0
  40. package/docs/skills/instrument-otel/SKILL.md +84 -0
  41. package/docs/skills/write-test/SKILL.md +124 -0
  42. package/docs/ui prd.md +376 -0
  43. package/examples/README.md +53 -0
  44. package/examples/config/agent-health.config.example.ts +155 -0
  45. package/examples/connectors/echo-connector.ts +131 -0
  46. package/examples/eval-files/demo.eval.js +128 -0
  47. package/examples/eval-files/ops-rca-classification.eval.js +71 -0
  48. package/examples/eval-files/ops-rca-evaluator.json +15 -0
  49. package/examples/eval-files/sdk-demo.eval.js +72 -0
  50. package/examples/eval-files/sdk-describe-demo.eval.js +50 -0
  51. package/examples/eval-files/sdk-hooks-demo.eval.js +99 -0
  52. package/examples/pi-profiling/README.md +77 -0
  53. package/examples/pi-profiling/agent-health-profile.ts +417 -0
  54. package/lib/dist/lib/agentUtils.d.ts +29 -0
  55. package/lib/dist/lib/agentUtils.d.ts.map +1 -0
  56. package/lib/dist/lib/agentUtils.js +43 -0
  57. package/lib/dist/lib/agentUtils.js.map +1 -0
  58. package/lib/dist/lib/bedrockCompat.d.ts +27 -0
  59. package/lib/dist/lib/bedrockCompat.d.ts.map +1 -0
  60. package/lib/dist/lib/bedrockCompat.js +83 -0
  61. package/lib/dist/lib/bedrockCompat.js.map +1 -0
  62. package/lib/dist/lib/benchmarkExport.d.ts +14 -0
  63. package/lib/dist/lib/benchmarkExport.d.ts.map +1 -0
  64. package/lib/dist/lib/benchmarkExport.js +41 -0
  65. package/lib/dist/lib/benchmarkExport.js.map +1 -0
  66. package/lib/dist/lib/benchmarkImage.d.ts +52 -0
  67. package/lib/dist/lib/benchmarkImage.d.ts.map +1 -0
  68. package/lib/dist/lib/benchmarkImage.js +113 -0
  69. package/lib/dist/lib/benchmarkImage.js.map +1 -0
  70. package/lib/dist/lib/benchmarkVersionUtils.d.ts +50 -0
  71. package/lib/dist/lib/benchmarkVersionUtils.d.ts.map +1 -0
  72. package/lib/dist/lib/benchmarkVersionUtils.js +89 -0
  73. package/lib/dist/lib/benchmarkVersionUtils.js.map +1 -0
  74. package/lib/dist/lib/chunkedFetch.d.ts +18 -0
  75. package/lib/dist/lib/chunkedFetch.d.ts.map +1 -0
  76. package/lib/dist/lib/chunkedFetch.js +40 -0
  77. package/lib/dist/lib/chunkedFetch.js.map +1 -0
  78. package/lib/dist/lib/comparisonInsights.d.ts +104 -0
  79. package/lib/dist/lib/comparisonInsights.d.ts.map +1 -0
  80. package/lib/dist/lib/comparisonInsights.js +212 -0
  81. package/lib/dist/lib/comparisonInsights.js.map +1 -0
  82. package/lib/dist/lib/config/defineConfig.d.ts +27 -0
  83. package/lib/dist/lib/config/defineConfig.d.ts.map +1 -0
  84. package/lib/dist/lib/config/defineConfig.js +28 -0
  85. package/lib/dist/lib/config/defineConfig.js.map +1 -0
  86. package/lib/dist/lib/config/index.d.ts +9 -0
  87. package/lib/dist/lib/config/index.d.ts.map +1 -0
  88. package/lib/dist/lib/config/index.js +8 -0
  89. package/lib/dist/lib/config/index.js.map +1 -0
  90. package/lib/dist/lib/config/loader.d.ts +39 -0
  91. package/lib/dist/lib/config/loader.d.ts.map +1 -0
  92. package/lib/dist/lib/config/loader.js +263 -0
  93. package/lib/dist/lib/config/loader.js.map +1 -0
  94. package/lib/dist/lib/config/statePaths.d.ts +61 -0
  95. package/lib/dist/lib/config/statePaths.d.ts.map +1 -0
  96. package/lib/dist/lib/config/statePaths.js +188 -0
  97. package/lib/dist/lib/config/statePaths.js.map +1 -0
  98. package/lib/dist/lib/config/types.d.ts +245 -0
  99. package/lib/dist/lib/config/types.d.ts.map +1 -0
  100. package/lib/dist/lib/config/types.js +6 -0
  101. package/lib/dist/lib/config/types.js.map +1 -0
  102. package/lib/dist/lib/config.d.ts +39 -0
  103. package/lib/dist/lib/config.d.ts.map +1 -0
  104. package/lib/dist/lib/config.js +118 -0
  105. package/lib/dist/lib/config.js.map +1 -0
  106. package/lib/dist/lib/constants.d.ts +81 -0
  107. package/lib/dist/lib/constants.d.ts.map +1 -0
  108. package/lib/dist/lib/constants.js +374 -0
  109. package/lib/dist/lib/constants.js.map +1 -0
  110. package/lib/dist/lib/contextFormat.d.ts +26 -0
  111. package/lib/dist/lib/contextFormat.d.ts.map +1 -0
  112. package/lib/dist/lib/contextFormat.js +28 -0
  113. package/lib/dist/lib/contextFormat.js.map +1 -0
  114. package/lib/dist/lib/contextUtilization.d.ts +23 -0
  115. package/lib/dist/lib/contextUtilization.d.ts.map +1 -0
  116. package/lib/dist/lib/contextUtilization.js +72 -0
  117. package/lib/dist/lib/contextUtilization.js.map +1 -0
  118. package/lib/dist/lib/dashboardMetrics.d.ts +87 -0
  119. package/lib/dist/lib/dashboardMetrics.d.ts.map +1 -0
  120. package/lib/dist/lib/dashboardMetrics.js +242 -0
  121. package/lib/dist/lib/dashboardMetrics.js.map +1 -0
  122. package/lib/dist/lib/dataSourceConfig.d.ts +108 -0
  123. package/lib/dist/lib/dataSourceConfig.d.ts.map +1 -0
  124. package/lib/dist/lib/dataSourceConfig.js +166 -0
  125. package/lib/dist/lib/dataSourceConfig.js.map +1 -0
  126. package/lib/dist/lib/debug.d.ts +26 -0
  127. package/lib/dist/lib/debug.d.ts.map +1 -0
  128. package/lib/dist/lib/debug.js +132 -0
  129. package/lib/dist/lib/debug.js.map +1 -0
  130. package/lib/dist/lib/diagnostics.d.ts +28 -0
  131. package/lib/dist/lib/diagnostics.d.ts.map +1 -0
  132. package/lib/dist/lib/diagnostics.js +65 -0
  133. package/lib/dist/lib/diagnostics.js.map +1 -0
  134. package/lib/dist/lib/envCompat.d.ts +27 -0
  135. package/lib/dist/lib/envCompat.d.ts.map +1 -0
  136. package/lib/dist/lib/envCompat.js +82 -0
  137. package/lib/dist/lib/envCompat.js.map +1 -0
  138. package/lib/dist/lib/evaluationRerun.d.ts +63 -0
  139. package/lib/dist/lib/evaluationRerun.d.ts.map +1 -0
  140. package/lib/dist/lib/evaluationRerun.js +85 -0
  141. package/lib/dist/lib/evaluationRerun.js.map +1 -0
  142. package/lib/dist/lib/findPackageRoot.d.ts +7 -0
  143. package/lib/dist/lib/findPackageRoot.d.ts.map +1 -0
  144. package/lib/dist/lib/findPackageRoot.js +57 -0
  145. package/lib/dist/lib/findPackageRoot.js.map +1 -0
  146. package/lib/dist/lib/hooks.d.ts +36 -0
  147. package/lib/dist/lib/hooks.d.ts.map +1 -0
  148. package/lib/dist/lib/hooks.js +112 -0
  149. package/lib/dist/lib/hooks.js.map +1 -0
  150. package/lib/dist/lib/index.d.ts +47 -0
  151. package/lib/dist/lib/index.d.ts.map +1 -0
  152. package/lib/dist/lib/index.js +62 -0
  153. package/lib/dist/lib/index.js.map +1 -0
  154. package/lib/dist/lib/labels.d.ts +90 -0
  155. package/lib/dist/lib/labels.d.ts.map +1 -0
  156. package/lib/dist/lib/labels.js +158 -0
  157. package/lib/dist/lib/labels.js.map +1 -0
  158. package/lib/dist/lib/markdown.d.ts +16 -0
  159. package/lib/dist/lib/markdown.d.ts.map +1 -0
  160. package/lib/dist/lib/markdown.js +42 -0
  161. package/lib/dist/lib/markdown.js.map +1 -0
  162. package/lib/dist/lib/matchers/expect.d.ts +3 -0
  163. package/lib/dist/lib/matchers/expect.d.ts.map +1 -0
  164. package/lib/dist/lib/matchers/expect.js +225 -0
  165. package/lib/dist/lib/matchers/expect.js.map +1 -0
  166. package/lib/dist/lib/matchers/index.d.ts +8 -0
  167. package/lib/dist/lib/matchers/index.d.ts.map +1 -0
  168. package/lib/dist/lib/matchers/index.js +9 -0
  169. package/lib/dist/lib/matchers/index.js.map +1 -0
  170. package/lib/dist/lib/matchers/judgeAccessor.d.ts +113 -0
  171. package/lib/dist/lib/matchers/judgeAccessor.d.ts.map +1 -0
  172. package/lib/dist/lib/matchers/judgeAccessor.js +183 -0
  173. package/lib/dist/lib/matchers/judgeAccessor.js.map +1 -0
  174. package/lib/dist/lib/matchers/session.d.ts +39 -0
  175. package/lib/dist/lib/matchers/session.d.ts.map +1 -0
  176. package/lib/dist/lib/matchers/session.js +116 -0
  177. package/lib/dist/lib/matchers/session.js.map +1 -0
  178. package/lib/dist/lib/matchers/traces.d.ts +70 -0
  179. package/lib/dist/lib/matchers/traces.d.ts.map +1 -0
  180. package/lib/dist/lib/matchers/traces.js +236 -0
  181. package/lib/dist/lib/matchers/traces.js.map +1 -0
  182. package/lib/dist/lib/matchers/tracesPricing.d.ts +38 -0
  183. package/lib/dist/lib/matchers/tracesPricing.d.ts.map +1 -0
  184. package/lib/dist/lib/matchers/tracesPricing.js +64 -0
  185. package/lib/dist/lib/matchers/tracesPricing.js.map +1 -0
  186. package/lib/dist/lib/matchers/types.d.ts +75 -0
  187. package/lib/dist/lib/matchers/types.d.ts.map +1 -0
  188. package/lib/dist/lib/matchers/types.js +6 -0
  189. package/lib/dist/lib/matchers/types.js.map +1 -0
  190. package/lib/dist/lib/packagePaths.d.ts +29 -0
  191. package/lib/dist/lib/packagePaths.d.ts.map +1 -0
  192. package/lib/dist/lib/packagePaths.js +63 -0
  193. package/lib/dist/lib/packagePaths.js.map +1 -0
  194. package/lib/dist/lib/performance.d.ts +51 -0
  195. package/lib/dist/lib/performance.d.ts.map +1 -0
  196. package/lib/dist/lib/performance.js +159 -0
  197. package/lib/dist/lib/performance.js.map +1 -0
  198. package/lib/dist/lib/portConfig.d.ts +29 -0
  199. package/lib/dist/lib/portConfig.d.ts.map +1 -0
  200. package/lib/dist/lib/portConfig.js +64 -0
  201. package/lib/dist/lib/portConfig.js.map +1 -0
  202. package/lib/dist/lib/preferences.d.ts +63 -0
  203. package/lib/dist/lib/preferences.d.ts.map +1 -0
  204. package/lib/dist/lib/preferences.js +117 -0
  205. package/lib/dist/lib/preferences.js.map +1 -0
  206. package/lib/dist/lib/resolveAgentModel.d.ts +22 -0
  207. package/lib/dist/lib/resolveAgentModel.d.ts.map +1 -0
  208. package/lib/dist/lib/resolveAgentModel.js +37 -0
  209. package/lib/dist/lib/resolveAgentModel.js.map +1 -0
  210. package/lib/dist/lib/runStats.d.ts +116 -0
  211. package/lib/dist/lib/runStats.d.ts.map +1 -0
  212. package/lib/dist/lib/runStats.js +192 -0
  213. package/lib/dist/lib/runStats.js.map +1 -0
  214. package/lib/dist/lib/telemetry/constants.d.ts +60 -0
  215. package/lib/dist/lib/telemetry/constants.d.ts.map +1 -0
  216. package/lib/dist/lib/telemetry/constants.js +87 -0
  217. package/lib/dist/lib/telemetry/constants.js.map +1 -0
  218. package/lib/dist/lib/telemetry/evalSpans.d.ts +61 -0
  219. package/lib/dist/lib/telemetry/evalSpans.d.ts.map +1 -0
  220. package/lib/dist/lib/telemetry/evalSpans.js +254 -0
  221. package/lib/dist/lib/telemetry/evalSpans.js.map +1 -0
  222. package/lib/dist/lib/telemetry/index.d.ts +11 -0
  223. package/lib/dist/lib/telemetry/index.d.ts.map +1 -0
  224. package/lib/dist/lib/telemetry/index.js +15 -0
  225. package/lib/dist/lib/telemetry/index.js.map +1 -0
  226. package/lib/dist/lib/telemetry/opensearchExporter.d.ts +43 -0
  227. package/lib/dist/lib/telemetry/opensearchExporter.d.ts.map +1 -0
  228. package/lib/dist/lib/telemetry/opensearchExporter.js +217 -0
  229. package/lib/dist/lib/telemetry/opensearchExporter.js.map +1 -0
  230. package/lib/dist/lib/telemetry/provider.d.ts +55 -0
  231. package/lib/dist/lib/telemetry/provider.d.ts.map +1 -0
  232. package/lib/dist/lib/telemetry/provider.js +140 -0
  233. package/lib/dist/lib/telemetry/provider.js.map +1 -0
  234. package/lib/dist/lib/testCaseLabels.d.ts +34 -0
  235. package/lib/dist/lib/testCaseLabels.d.ts.map +1 -0
  236. package/lib/dist/lib/testCaseLabels.js +88 -0
  237. package/lib/dist/lib/testCaseLabels.js.map +1 -0
  238. package/lib/dist/lib/testCaseValidation.d.ts +140 -0
  239. package/lib/dist/lib/testCaseValidation.d.ts.map +1 -0
  240. package/lib/dist/lib/testCaseValidation.js +162 -0
  241. package/lib/dist/lib/testCaseValidation.js.map +1 -0
  242. package/lib/dist/lib/testCases/agentFixture.d.ts +80 -0
  243. package/lib/dist/lib/testCases/agentFixture.d.ts.map +1 -0
  244. package/lib/dist/lib/testCases/agentFixture.js +43 -0
  245. package/lib/dist/lib/testCases/agentFixture.js.map +1 -0
  246. package/lib/dist/lib/testCases/authoringSurface.d.ts +10 -0
  247. package/lib/dist/lib/testCases/authoringSurface.d.ts.map +1 -0
  248. package/lib/dist/lib/testCases/authoringSurface.js +54 -0
  249. package/lib/dist/lib/testCases/authoringSurface.js.map +1 -0
  250. package/lib/dist/lib/testCases/codemod.d.ts +13 -0
  251. package/lib/dist/lib/testCases/codemod.d.ts.map +1 -0
  252. package/lib/dist/lib/testCases/codemod.js +169 -0
  253. package/lib/dist/lib/testCases/codemod.js.map +1 -0
  254. package/lib/dist/lib/testCases/define.d.ts +114 -0
  255. package/lib/dist/lib/testCases/define.d.ts.map +1 -0
  256. package/lib/dist/lib/testCases/define.js +253 -0
  257. package/lib/dist/lib/testCases/define.js.map +1 -0
  258. package/lib/dist/lib/testCases/evaluators.d.ts +80 -0
  259. package/lib/dist/lib/testCases/evaluators.d.ts.map +1 -0
  260. package/lib/dist/lib/testCases/evaluators.js +105 -0
  261. package/lib/dist/lib/testCases/evaluators.js.map +1 -0
  262. package/lib/dist/lib/testCases/index.d.ts +14 -0
  263. package/lib/dist/lib/testCases/index.d.ts.map +1 -0
  264. package/lib/dist/lib/testCases/index.js +12 -0
  265. package/lib/dist/lib/testCases/index.js.map +1 -0
  266. package/lib/dist/lib/testCases/judge.d.ts +165 -0
  267. package/lib/dist/lib/testCases/judge.d.ts.map +1 -0
  268. package/lib/dist/lib/testCases/judge.js +359 -0
  269. package/lib/dist/lib/testCases/judge.js.map +1 -0
  270. package/lib/dist/lib/testCases/loader.d.ts +36 -0
  271. package/lib/dist/lib/testCases/loader.d.ts.map +1 -0
  272. package/lib/dist/lib/testCases/loader.js +179 -0
  273. package/lib/dist/lib/testCases/loader.js.map +1 -0
  274. package/lib/dist/lib/testCases/types.d.ts +242 -0
  275. package/lib/dist/lib/testCases/types.d.ts.map +1 -0
  276. package/lib/dist/lib/testCases/types.js +6 -0
  277. package/lib/dist/lib/testCases/types.js.map +1 -0
  278. package/lib/dist/lib/theme.d.ts +6 -0
  279. package/lib/dist/lib/theme.d.ts.map +1 -0
  280. package/lib/dist/lib/theme.js +36 -0
  281. package/lib/dist/lib/theme.js.map +1 -0
  282. package/lib/dist/lib/uiTelemetry.d.ts +7 -0
  283. package/lib/dist/lib/uiTelemetry.d.ts.map +1 -0
  284. package/lib/dist/lib/uiTelemetry.js +25 -0
  285. package/lib/dist/lib/uiTelemetry.js.map +1 -0
  286. package/lib/dist/lib/utils.d.ts +111 -0
  287. package/lib/dist/lib/utils.d.ts.map +1 -0
  288. package/lib/dist/lib/utils.js +254 -0
  289. package/lib/dist/lib/utils.js.map +1 -0
  290. package/lib/dist/lib/workflow/consolidate.d.ts +12 -0
  291. package/lib/dist/lib/workflow/consolidate.d.ts.map +1 -0
  292. package/lib/dist/lib/workflow/consolidate.js +33 -0
  293. package/lib/dist/lib/workflow/consolidate.js.map +1 -0
  294. package/lib/dist/lib/workflow/index.d.ts +13 -0
  295. package/lib/dist/lib/workflow/index.d.ts.map +1 -0
  296. package/lib/dist/lib/workflow/index.js +12 -0
  297. package/lib/dist/lib/workflow/index.js.map +1 -0
  298. package/lib/dist/lib/workflow/ledger.d.ts +30 -0
  299. package/lib/dist/lib/workflow/ledger.d.ts.map +1 -0
  300. package/lib/dist/lib/workflow/ledger.js +41 -0
  301. package/lib/dist/lib/workflow/ledger.js.map +1 -0
  302. package/lib/dist/lib/workflow/pool.d.ts +13 -0
  303. package/lib/dist/lib/workflow/pool.d.ts.map +1 -0
  304. package/lib/dist/lib/workflow/pool.js +44 -0
  305. package/lib/dist/lib/workflow/pool.js.map +1 -0
  306. package/lib/dist/lib/workflow/source.d.ts +22 -0
  307. package/lib/dist/lib/workflow/source.d.ts.map +1 -0
  308. package/lib/dist/lib/workflow/source.js +29 -0
  309. package/lib/dist/lib/workflow/source.js.map +1 -0
  310. package/lib/dist/lib/workflow/stepB.d.ts +71 -0
  311. package/lib/dist/lib/workflow/stepB.d.ts.map +1 -0
  312. package/lib/dist/lib/workflow/stepB.js +99 -0
  313. package/lib/dist/lib/workflow/stepB.js.map +1 -0
  314. package/lib/dist/lib/workflow/types.d.ts +86 -0
  315. package/lib/dist/lib/workflow/types.d.ts.map +1 -0
  316. package/lib/dist/lib/workflow/types.js +6 -0
  317. package/lib/dist/lib/workflow/types.js.map +1 -0
  318. package/lib/dist/lib/workflow/workflow.d.ts +119 -0
  319. package/lib/dist/lib/workflow/workflow.d.ts.map +1 -0
  320. package/lib/dist/lib/workflow/workflow.js +195 -0
  321. package/lib/dist/lib/workflow/workflow.js.map +1 -0
  322. package/lib/dist/services/agent/aguiConverter.d.ts +50 -0
  323. package/lib/dist/services/agent/aguiConverter.d.ts.map +1 -0
  324. package/lib/dist/services/agent/aguiConverter.js +449 -0
  325. package/lib/dist/services/agent/aguiConverter.js.map +1 -0
  326. package/lib/dist/services/agent/index.d.ts +10 -0
  327. package/lib/dist/services/agent/index.d.ts.map +1 -0
  328. package/lib/dist/services/agent/index.js +12 -0
  329. package/lib/dist/services/agent/index.js.map +1 -0
  330. package/lib/dist/services/agent/payloadBuilder.d.ts +33 -0
  331. package/lib/dist/services/agent/payloadBuilder.d.ts.map +1 -0
  332. package/lib/dist/services/agent/payloadBuilder.js +75 -0
  333. package/lib/dist/services/agent/payloadBuilder.js.map +1 -0
  334. package/lib/dist/services/agent/sseStream.d.ts +43 -0
  335. package/lib/dist/services/agent/sseStream.d.ts.map +1 -0
  336. package/lib/dist/services/agent/sseStream.js +223 -0
  337. package/lib/dist/services/agent/sseStream.js.map +1 -0
  338. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.d.ts +44 -0
  339. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.d.ts.map +1 -0
  340. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.js +95 -0
  341. package/lib/dist/services/connectors/agui/AGUIStreamingConnector.js.map +1 -0
  342. package/lib/dist/services/connectors/base/BaseConnector.d.ts +81 -0
  343. package/lib/dist/services/connectors/base/BaseConnector.d.ts.map +1 -0
  344. package/lib/dist/services/connectors/base/BaseConnector.js +170 -0
  345. package/lib/dist/services/connectors/base/BaseConnector.js.map +1 -0
  346. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +126 -0
  347. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -0
  348. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +417 -0
  349. package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -0
  350. package/lib/dist/services/connectors/index.d.ts +13 -0
  351. package/lib/dist/services/connectors/index.d.ts.map +1 -0
  352. package/lib/dist/services/connectors/index.js +32 -0
  353. package/lib/dist/services/connectors/index.js.map +1 -0
  354. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts +48 -0
  355. package/lib/dist/services/connectors/kiro/KiroConnector.d.ts.map +1 -0
  356. package/lib/dist/services/connectors/kiro/KiroConnector.js +158 -0
  357. package/lib/dist/services/connectors/kiro/KiroConnector.js.map +1 -0
  358. package/lib/dist/services/connectors/langgraph/LangGraphConnector.d.ts +36 -0
  359. package/lib/dist/services/connectors/langgraph/LangGraphConnector.d.ts.map +1 -0
  360. package/lib/dist/services/connectors/langgraph/LangGraphConnector.js +175 -0
  361. package/lib/dist/services/connectors/langgraph/LangGraphConnector.js.map +1 -0
  362. package/lib/dist/services/connectors/mock/MockConnector.d.ts +37 -0
  363. package/lib/dist/services/connectors/mock/MockConnector.d.ts.map +1 -0
  364. package/lib/dist/services/connectors/mock/MockConnector.js +120 -0
  365. package/lib/dist/services/connectors/mock/MockConnector.js.map +1 -0
  366. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.d.ts +42 -0
  367. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.d.ts.map +1 -0
  368. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.js +133 -0
  369. package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.js.map +1 -0
  370. package/lib/dist/services/connectors/pi/PiConnector.d.ts +87 -0
  371. package/lib/dist/services/connectors/pi/PiConnector.d.ts.map +1 -0
  372. package/lib/dist/services/connectors/pi/PiConnector.js +274 -0
  373. package/lib/dist/services/connectors/pi/PiConnector.js.map +1 -0
  374. package/lib/dist/services/connectors/registry.d.ts +57 -0
  375. package/lib/dist/services/connectors/registry.d.ts.map +1 -0
  376. package/lib/dist/services/connectors/registry.js +106 -0
  377. package/lib/dist/services/connectors/registry.js.map +1 -0
  378. package/lib/dist/services/connectors/rest/RESTConnector.d.ts +38 -0
  379. package/lib/dist/services/connectors/rest/RESTConnector.d.ts.map +1 -0
  380. package/lib/dist/services/connectors/rest/RESTConnector.js +117 -0
  381. package/lib/dist/services/connectors/rest/RESTConnector.js.map +1 -0
  382. package/lib/dist/services/connectors/server.d.ts +13 -0
  383. package/lib/dist/services/connectors/server.d.ts.map +1 -0
  384. package/lib/dist/services/connectors/server.js +34 -0
  385. package/lib/dist/services/connectors/server.js.map +1 -0
  386. package/lib/dist/services/connectors/strands/StrandsConnector.d.ts +48 -0
  387. package/lib/dist/services/connectors/strands/StrandsConnector.d.ts.map +1 -0
  388. package/lib/dist/services/connectors/strands/StrandsConnector.js +221 -0
  389. package/lib/dist/services/connectors/strands/StrandsConnector.js.map +1 -0
  390. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts +88 -0
  391. package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -0
  392. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +426 -0
  393. package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -0
  394. package/lib/dist/services/connectors/types.d.ts +213 -0
  395. package/lib/dist/services/connectors/types.d.ts.map +1 -0
  396. package/lib/dist/services/connectors/types.js +6 -0
  397. package/lib/dist/services/connectors/types.js.map +1 -0
  398. package/lib/dist/services/evaluation/bedrockJudge.d.ts +71 -0
  399. package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -0
  400. package/lib/dist/services/evaluation/bedrockJudge.js +169 -0
  401. package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -0
  402. package/lib/dist/services/evaluation/evaluatorError.d.ts +56 -0
  403. package/lib/dist/services/evaluation/evaluatorError.d.ts.map +1 -0
  404. package/lib/dist/services/evaluation/evaluatorError.js +56 -0
  405. package/lib/dist/services/evaluation/evaluatorError.js.map +1 -0
  406. package/lib/dist/services/evaluation/index.d.ts +106 -0
  407. package/lib/dist/services/evaluation/index.d.ts.map +1 -0
  408. package/lib/dist/services/evaluation/index.js +684 -0
  409. package/lib/dist/services/evaluation/index.js.map +1 -0
  410. package/lib/dist/services/evaluation/mockTrajectory.d.ts +3 -0
  411. package/lib/dist/services/evaluation/mockTrajectory.d.ts.map +1 -0
  412. package/lib/dist/services/evaluation/mockTrajectory.js +72 -0
  413. package/lib/dist/services/evaluation/mockTrajectory.js.map +1 -0
  414. package/lib/dist/services/opensearch/client.d.ts +26 -0
  415. package/lib/dist/services/opensearch/client.d.ts.map +1 -0
  416. package/lib/dist/services/opensearch/client.js +131 -0
  417. package/lib/dist/services/opensearch/client.js.map +1 -0
  418. package/lib/dist/services/opensearch/index.d.ts +16 -0
  419. package/lib/dist/services/opensearch/index.d.ts.map +1 -0
  420. package/lib/dist/services/opensearch/index.js +25 -0
  421. package/lib/dist/services/opensearch/index.js.map +1 -0
  422. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts +123 -0
  423. package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -0
  424. package/lib/dist/services/storage/asyncBenchmarkStorage.js +440 -0
  425. package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -0
  426. package/lib/dist/services/storage/asyncRunStorage.d.ts +135 -0
  427. package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -0
  428. package/lib/dist/services/storage/asyncRunStorage.js +524 -0
  429. package/lib/dist/services/storage/asyncRunStorage.js.map +1 -0
  430. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +170 -0
  431. package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -0
  432. package/lib/dist/services/storage/asyncTestCaseStorage.js +301 -0
  433. package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -0
  434. package/lib/dist/services/storage/index.d.ts +17 -0
  435. package/lib/dist/services/storage/index.d.ts.map +1 -0
  436. package/lib/dist/services/storage/index.js +20 -0
  437. package/lib/dist/services/storage/index.js.map +1 -0
  438. package/lib/dist/services/storage/migration.d.ts +54 -0
  439. package/lib/dist/services/storage/migration.d.ts.map +1 -0
  440. package/lib/dist/services/storage/migration.js +296 -0
  441. package/lib/dist/services/storage/migration.js.map +1 -0
  442. package/lib/dist/services/storage/opensearchClient.d.ts +947 -0
  443. package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -0
  444. package/lib/dist/services/storage/opensearchClient.js +442 -0
  445. package/lib/dist/services/storage/opensearchClient.js.map +1 -0
  446. package/lib/dist/services/traces/browserRecovery.d.ts +37 -0
  447. package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -0
  448. package/lib/dist/services/traces/browserRecovery.js +108 -0
  449. package/lib/dist/services/traces/browserRecovery.js.map +1 -0
  450. package/lib/dist/services/traces/categoryStyles.d.ts +21 -0
  451. package/lib/dist/services/traces/categoryStyles.d.ts.map +1 -0
  452. package/lib/dist/services/traces/categoryStyles.js +56 -0
  453. package/lib/dist/services/traces/categoryStyles.js.map +1 -0
  454. package/lib/dist/services/traces/executionOrderTransform.d.ts +35 -0
  455. package/lib/dist/services/traces/executionOrderTransform.d.ts.map +1 -0
  456. package/lib/dist/services/traces/executionOrderTransform.js +313 -0
  457. package/lib/dist/services/traces/executionOrderTransform.js.map +1 -0
  458. package/lib/dist/services/traces/fetchSpansForRun.d.ts +86 -0
  459. package/lib/dist/services/traces/fetchSpansForRun.d.ts.map +1 -0
  460. package/lib/dist/services/traces/fetchSpansForRun.js +69 -0
  461. package/lib/dist/services/traces/fetchSpansForRun.js.map +1 -0
  462. package/lib/dist/services/traces/flowTransform.d.ts +24 -0
  463. package/lib/dist/services/traces/flowTransform.d.ts.map +1 -0
  464. package/lib/dist/services/traces/flowTransform.js +228 -0
  465. package/lib/dist/services/traces/flowTransform.js.map +1 -0
  466. package/lib/dist/services/traces/index.d.ts +121 -0
  467. package/lib/dist/services/traces/index.d.ts.map +1 -0
  468. package/lib/dist/services/traces/index.js +255 -0
  469. package/lib/dist/services/traces/index.js.map +1 -0
  470. package/lib/dist/services/traces/intentTransform.d.ts +20 -0
  471. package/lib/dist/services/traces/intentTransform.d.ts.map +1 -0
  472. package/lib/dist/services/traces/intentTransform.js +131 -0
  473. package/lib/dist/services/traces/intentTransform.js.map +1 -0
  474. package/lib/dist/services/traces/judgeAgentsHints.d.ts +63 -0
  475. package/lib/dist/services/traces/judgeAgentsHints.d.ts.map +1 -0
  476. package/lib/dist/services/traces/judgeAgentsHints.js +89 -0
  477. package/lib/dist/services/traces/judgeAgentsHints.js.map +1 -0
  478. package/lib/dist/services/traces/messageExtraction.d.ts +15 -0
  479. package/lib/dist/services/traces/messageExtraction.d.ts.map +1 -0
  480. package/lib/dist/services/traces/messageExtraction.js +313 -0
  481. package/lib/dist/services/traces/messageExtraction.js.map +1 -0
  482. package/lib/dist/services/traces/spanCategorization.d.ts +63 -0
  483. package/lib/dist/services/traces/spanCategorization.d.ts.map +1 -0
  484. package/lib/dist/services/traces/spanCategorization.js +276 -0
  485. package/lib/dist/services/traces/spanCategorization.js.map +1 -0
  486. package/lib/dist/services/traces/spanPreprocessing.d.ts +37 -0
  487. package/lib/dist/services/traces/spanPreprocessing.d.ts.map +1 -0
  488. package/lib/dist/services/traces/spanPreprocessing.js +102 -0
  489. package/lib/dist/services/traces/spanPreprocessing.js.map +1 -0
  490. package/lib/dist/services/traces/spansToTrajectory.d.ts +36 -0
  491. package/lib/dist/services/traces/spansToTrajectory.d.ts.map +1 -0
  492. package/lib/dist/services/traces/spansToTrajectory.js +425 -0
  493. package/lib/dist/services/traces/spansToTrajectory.js.map +1 -0
  494. package/lib/dist/services/traces/toolSimilarity.d.ts +35 -0
  495. package/lib/dist/services/traces/toolSimilarity.d.ts.map +1 -0
  496. package/lib/dist/services/traces/toolSimilarity.js +203 -0
  497. package/lib/dist/services/traces/toolSimilarity.js.map +1 -0
  498. package/lib/dist/services/traces/traceComparison.d.ts +31 -0
  499. package/lib/dist/services/traces/traceComparison.d.ts.map +1 -0
  500. package/lib/dist/services/traces/traceComparison.js +318 -0
  501. package/lib/dist/services/traces/traceComparison.js.map +1 -0
  502. package/lib/dist/services/traces/traceGrouping.d.ts +19 -0
  503. package/lib/dist/services/traces/traceGrouping.d.ts.map +1 -0
  504. package/lib/dist/services/traces/traceGrouping.js +107 -0
  505. package/lib/dist/services/traces/traceGrouping.js.map +1 -0
  506. package/lib/dist/services/traces/tracePoller.d.ts +108 -0
  507. package/lib/dist/services/traces/tracePoller.d.ts.map +1 -0
  508. package/lib/dist/services/traces/tracePoller.js +475 -0
  509. package/lib/dist/services/traces/tracePoller.js.map +1 -0
  510. package/lib/dist/services/traces/traceStats.d.ts +45 -0
  511. package/lib/dist/services/traces/traceStats.d.ts.map +1 -0
  512. package/lib/dist/services/traces/traceStats.js +114 -0
  513. package/lib/dist/services/traces/traceStats.js.map +1 -0
  514. package/lib/dist/services/traces/traceSummary.d.ts +47 -0
  515. package/lib/dist/services/traces/traceSummary.d.ts.map +1 -0
  516. package/lib/dist/services/traces/traceSummary.js +68 -0
  517. package/lib/dist/services/traces/traceSummary.js.map +1 -0
  518. package/lib/dist/services/traces/utils.d.ts +33 -0
  519. package/lib/dist/services/traces/utils.d.ts.map +1 -0
  520. package/lib/dist/services/traces/utils.js +114 -0
  521. package/lib/dist/services/traces/utils.js.map +1 -0
  522. package/lib/dist/types/agui.d.ts +13 -0
  523. package/lib/dist/types/agui.d.ts.map +1 -0
  524. package/lib/dist/types/agui.js +16 -0
  525. package/lib/dist/types/agui.js.map +1 -0
  526. package/lib/dist/types/index.d.ts +1229 -0
  527. package/lib/dist/types/index.d.ts.map +1 -0
  528. package/lib/dist/types/index.js +12 -0
  529. package/lib/dist/types/index.js.map +1 -0
  530. package/lib/dist/types/skills.d.ts +146 -0
  531. package/lib/dist/types/skills.d.ts.map +1 -0
  532. package/lib/dist/types/skills.js +6 -0
  533. package/lib/dist/types/skills.js.map +1 -0
  534. package/observio-sample-agent/pi-package/README.md +112 -0
  535. package/observio-sample-agent/pi-package/extensions/agent-health.ts +373 -0
  536. package/observio-sample-agent/pi-package/package.json +17 -0
  537. package/observio-sample-agent/pi-package/prompts/agent-health.md +37 -0
  538. package/observio-sample-agent/pi-package/skills/create-pr/SKILL.md +88 -0
  539. package/observio-sample-agent/pi-package/skills/fix-bug/SKILL.md +71 -0
  540. package/observio-sample-agent/pi-package/skills/implement-feature/SKILL.md +156 -0
  541. package/observio-sample-agent/pi-package/skills/instrument-otel/SKILL.md +208 -0
  542. package/observio-sample-agent/pi-package/skills/setup-collector/SKILL.md +146 -0
  543. package/observio-sample-agent/pi-package/skills/write-test/SKILL.md +115 -0
  544. package/package.json +45 -9
  545. package/server/dist/app.js +34031 -18731
  546. package/server/dist/index.js +30737 -15338
  547. package/tsconfig.lib.json +71 -0
  548. package/dist/assets/index-BOIP5L7h.js +0 -246
  549. package/dist/assets/index-CU9YKpAL.css +0 -1
  550. package/lib/dist/config/index.js +0 -452
  551. package/lib/dist/index.js +0 -1725
@@ -0,0 +1,140 @@
1
+ /*
2
+ * Copyright OpenSearch Contributors
3
+ * SPDX-License-Identifier: Apache-2.0
4
+ */
5
+ /**
6
+ * OTel TracerProvider management for evaluation telemetry.
7
+ *
8
+ * Manages a singleton TracerProvider that exports evaluation spans
9
+ * via OTLP/HTTP to a collector, or directly to OpenSearch when an
10
+ * observability data source is configured.
11
+ */
12
+ import { trace } from '@opentelemetry/api';
13
+ import { NodeTracerProvider, BatchSpanProcessor, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-node';
14
+ import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
15
+ import { resourceFromAttributes } from '@opentelemetry/resources';
16
+ import { EVAL_TRACER_NAME } from './constants.js';
17
+ import { OpenSearchSpanExporter } from './opensearchExporter.js';
18
+ let provider = null;
19
+ let telemetryEnabled = false;
20
+ /**
21
+ * Resolve telemetry config from environment variables and optional user config
22
+ */
23
+ export function resolveEvalTelemetryConfig(userConfig) {
24
+ const enabled = userConfig?.enabled
25
+ ?? process.env.OTEL_EVAL_ENABLED === 'true';
26
+ const exporterEndpoint = userConfig?.exporterEndpoint
27
+ ?? process.env.OTEL_EVAL_EXPORTER_ENDPOINT
28
+ ?? 'http://localhost:4318/v1/traces';
29
+ let exporterHeaders = userConfig?.exporterHeaders;
30
+ if (!exporterHeaders && process.env.OTEL_EVAL_EXPORTER_HEADERS) {
31
+ try {
32
+ exporterHeaders = JSON.parse(process.env.OTEL_EVAL_EXPORTER_HEADERS);
33
+ }
34
+ catch {
35
+ console.warn('[Telemetry] Failed to parse OTEL_EVAL_EXPORTER_HEADERS as JSON');
36
+ }
37
+ }
38
+ // Use a dedicated env var for eval service name — do NOT fall back to
39
+ // OTEL_SERVICE_NAME which belongs to the agent's own telemetry (e.g. claude-code-agent).
40
+ const serviceName = userConfig?.serviceName
41
+ ?? process.env.OTEL_EVAL_SERVICE_NAME
42
+ ?? 'agent-health';
43
+ return { enabled, exporterEndpoint, exporterHeaders, serviceName, opensearch: userConfig?.opensearch };
44
+ }
45
+ /**
46
+ * Initialize the evaluation TracerProvider.
47
+ * Safe to call multiple times — subsequent calls are no-ops.
48
+ *
49
+ * When an OpenSearch config is provided (from observability data source),
50
+ * spans are written directly to OpenSearch in OSIS-compatible format.
51
+ * Otherwise, falls back to OTLP/HTTP exporter.
52
+ */
53
+ export function initEvalTracerProvider(config) {
54
+ if (provider) {
55
+ return; // Already initialized
56
+ }
57
+ if (!config.enabled) {
58
+ telemetryEnabled = false;
59
+ console.log('[Telemetry] Evaluation telemetry disabled (OTEL_EVAL_ENABLED != true)');
60
+ return;
61
+ }
62
+ const resource = resourceFromAttributes({
63
+ 'service.name': config.serviceName ?? 'agent-health',
64
+ 'telemetry.sdk.name': 'agent-health',
65
+ 'telemetry.sdk.language': 'nodejs',
66
+ });
67
+ // Choose exporter priority:
68
+ // 1. OTLP/HTTP when exporterEndpoint is explicitly configured (e.g., OSIS pipeline)
69
+ // 2. OpenSearch direct when observability data source is configured
70
+ // 3. OTLP/HTTP with default endpoint as last resort
71
+ const spanProcessors = [];
72
+ const hasExplicitOtlpEndpoint = config.exporterEndpoint !== 'http://localhost:4318/v1/traces';
73
+ if (hasExplicitOtlpEndpoint) {
74
+ // User explicitly configured an OTLP endpoint (e.g., OSIS pipeline) — prefer it
75
+ const otlpExporter = new OTLPTraceExporter({
76
+ url: config.exporterEndpoint,
77
+ headers: config.exporterHeaders,
78
+ });
79
+ spanProcessors.push(new BatchSpanProcessor(otlpExporter));
80
+ console.log(`[Telemetry] Evaluation telemetry enabled → OTLP (${config.exporterEndpoint})`);
81
+ }
82
+ else if (config.opensearch) {
83
+ const osExporter = new OpenSearchSpanExporter(config.opensearch);
84
+ // Use SimpleSpanProcessor for immediate export (eval spans are infrequent)
85
+ spanProcessors.push(new SimpleSpanProcessor(osExporter));
86
+ console.log(`[Telemetry] Evaluation telemetry enabled → OpenSearch direct (${config.opensearch.endpoint})`);
87
+ }
88
+ else {
89
+ const otlpExporter = new OTLPTraceExporter({
90
+ url: config.exporterEndpoint,
91
+ headers: config.exporterHeaders,
92
+ });
93
+ spanProcessors.push(new BatchSpanProcessor(otlpExporter));
94
+ console.log(`[Telemetry] Evaluation telemetry enabled → OTLP (${config.exporterEndpoint})`);
95
+ }
96
+ provider = new NodeTracerProvider({
97
+ resource,
98
+ spanProcessors,
99
+ });
100
+ provider.register();
101
+ telemetryEnabled = true;
102
+ }
103
+ /**
104
+ * Get the evaluation tracer. Returns a no-op tracer if telemetry is disabled.
105
+ */
106
+ export function getEvalTracer() {
107
+ return trace.getTracer(EVAL_TRACER_NAME);
108
+ }
109
+ /**
110
+ * Check if evaluation telemetry is enabled
111
+ */
112
+ export function isEvalTelemetryEnabled() {
113
+ return telemetryEnabled;
114
+ }
115
+ /**
116
+ * Force-flush any buffered spans to the exporter.
117
+ * Call after emitting spans to ensure they are exported promptly.
118
+ */
119
+ export async function flushEvalTracer() {
120
+ if (provider) {
121
+ try {
122
+ await provider.forceFlush();
123
+ }
124
+ catch (err) {
125
+ console.warn('[Telemetry] forceFlush failed:', err instanceof Error ? err.message : err);
126
+ }
127
+ }
128
+ }
129
+ /**
130
+ * Flush pending spans and shut down the TracerProvider.
131
+ */
132
+ export async function shutdownEvalTracer() {
133
+ if (provider) {
134
+ await provider.shutdown();
135
+ provider = null;
136
+ telemetryEnabled = false;
137
+ console.log('[Telemetry] Evaluation telemetry shut down');
138
+ }
139
+ }
140
+ //# sourceMappingURL=provider.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"provider.js","sourceRoot":"","sources":["../../../telemetry/provider.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;GAMG;AAEH,OAAO,EAAE,KAAK,EAAe,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAC5G,OAAO,EAAE,iBAAiB,EAAE,MAAM,yCAAyC,CAAC;AAC5E,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAiC,MAAM,yBAAyB,CAAC;AAkBhG,IAAI,QAAQ,GAA8B,IAAI,CAAC;AAC/C,IAAI,gBAAgB,GAAG,KAAK,CAAC;AAE7B;;GAEG;AACH,MAAM,UAAU,0BAA0B,CAAC,UAAyC;IAClF,MAAM,OAAO,GAAG,UAAU,EAAE,OAAO;WAC9B,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,MAAM,CAAC;IAE9C,MAAM,gBAAgB,GAAG,UAAU,EAAE,gBAAgB;WAChD,OAAO,CAAC,GAAG,CAAC,2BAA2B;WACvC,iCAAiC,CAAC;IAEvC,IAAI,eAAe,GAAG,UAAU,EAAE,eAAe,CAAC;IAClD,IAAI,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,CAAC;QAC/D,IAAI,CAAC;YACH,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;QACvE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IAED,sEAAsE;IACtE,yFAAyF;IACzF,MAAM,WAAW,GAAG,UAAU,EAAE,WAAW;WACtC,OAAO,CAAC,GAAG,CAAC,sBAAsB;WAClC,cAAc,CAAC;IAEpB,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;AACzG,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAA2B;IAChE,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,CAAC,sBAAsB;IAChC,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,gBAAgB,GAAG,KAAK,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,uEAAuE,CAAC,CAAC;QACrF,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,sBAAsB,CAAC;QACtC,cAAc,EAAE,MAAM,CAAC,WAAW,IAAI,cAAc;QACpD,oBAAoB,EAAE,cAAc;QACpC,wBAAwB,EAAE,QAAQ;KACnC,CAAC,CAAC;IAEH,4BAA4B;IAC5B,sFAAsF;IACtF,sEAAsE;IACtE,sDAAsD;IACtD,MAAM,cAAc,GAAG,EAAE,CAAC;IAC1B,MAAM,uBAAuB,GAAG,MAAM,CAAC,gBAAgB,KAAK,iCAAiC,CAAC;IAE9F,IAAI,uBAAuB,EAAE,CAAC;QAC5B,gFAAgF;QAChF,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC;YACzC,GAAG,EAAE,MAAM,CAAC,gBAAgB;YAC5B,OAAO,EAAE,MAAM,CAAC,eAAe;SAChC,CAAC,CAAC;QACH,cAAc,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,oDAAoD,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC;IAC9F,CAAC;SAAM,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAI,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjE,2EAA2E;QAC3E,cAAc,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC,UAAU,CAAC,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,iEAAiE,MAAM,CAAC,UAAU,CAAC,QAAQ,GAAG,CAAC,CAAC;IAC9G,CAAC;SAAM,CAAC;QACN,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC;YACzC,GAAG,EAAE,MAAM,CAAC,gBAAgB;YAC5B,OAAO,EAAE,MAAM,CAAC,eAAe;SAChC,CAAC,CAAC;QACH,cAAc,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,oDAAoD,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC;IAC9F,CAAC;IAED,QAAQ,GAAG,IAAI,kBAAkB,CAAC;QAChC,QAAQ;QACR,cAAc;KACf,CAAC,CAAC;IACH,QAAQ,CAAC,QAAQ,EAAE,CAAC;IAEpB,gBAAgB,GAAG,IAAI,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,OAAO,KAAK,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB;IACpC,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,gCAAgC,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC1B,QAAQ,GAAG,IAAI,CAAC;QAChB,gBAAgB,GAAG,KAAK,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC5D,CAAC;AACH,CAAC"}
@@ -0,0 +1,34 @@
1
+ export type DifficultyLabel = 'Easy' | 'Medium' | 'Hard';
2
+ /**
3
+ * Extract the category from a labels array, or undefined when not present.
4
+ * If multiple `category:` labels are present, the first one wins.
5
+ */
6
+ export declare function getCategoryFromLabels(labels?: string[]): string | undefined;
7
+ /**
8
+ * Extract the difficulty from a labels array, or undefined when not present.
9
+ * Values are normalized to the canonical `Easy | Medium | Hard` form when
10
+ * possible; non-canonical values are returned as-is.
11
+ */
12
+ export declare function getDifficultyFromLabels(labels?: string[]): DifficultyLabel | string | undefined;
13
+ /**
14
+ * Extract the subcategory from a labels array, or undefined when not present.
15
+ */
16
+ export declare function getSubcategoryFromLabels(labels?: string[]): string | undefined;
17
+ /**
18
+ * Build a labels array from the legacy top-level `category` / `difficulty`
19
+ * fields, preserving any existing labels and refusing to add a duplicate
20
+ * facet (e.g. won't add `category:RCA` if a `category:` label is already
21
+ * present).
22
+ */
23
+ export declare function migrateLegacyFieldsToLabels(input: {
24
+ category?: string;
25
+ difficulty?: string;
26
+ subcategory?: string;
27
+ labels?: string[];
28
+ }): string[];
29
+ /**
30
+ * Returns true when the labels array contains a structured facet for the
31
+ * given prefix. Useful in migrations to avoid double-applying.
32
+ */
33
+ export declare function hasLabelPrefix(labels: string[] | undefined, prefix: string): boolean;
34
+ //# sourceMappingURL=testCaseLabels.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"testCaseLabels.d.ts","sourceRoot":"","sources":["../../testCaseLabels.ts"],"names":[],"mappings":"AAwBA,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEzD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAI3E;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,eAAe,GAAG,MAAM,GAAG,SAAS,CAU/F;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAI9E;AAED;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB,GAAG,MAAM,EAAE,CAYX;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAIpF"}
@@ -0,0 +1,88 @@
1
+ /*
2
+ * Copyright OpenSearch Contributors
3
+ * SPDX-License-Identifier: Apache-2.0
4
+ */
5
+ /**
6
+ * Helpers for the unified `labels` tagging system that replaces the legacy
7
+ * top-level `category` and `difficulty` fields on TestCase.
8
+ *
9
+ * Convention: structured labels use a `prefix:value` form so we can extract
10
+ * specific facets (e.g. category, difficulty) when the UI needs them.
11
+ *
12
+ * 'category:RCA'
13
+ * 'difficulty:Medium'
14
+ * 'team:platform'
15
+ * 'tier:p0'
16
+ *
17
+ * Free-form labels without a prefix are also allowed for human tagging.
18
+ */
19
+ const CATEGORY_PREFIX = 'category:';
20
+ const DIFFICULTY_PREFIX = 'difficulty:';
21
+ const SUBCATEGORY_PREFIX = 'subcategory:';
22
+ /**
23
+ * Extract the category from a labels array, or undefined when not present.
24
+ * If multiple `category:` labels are present, the first one wins.
25
+ */
26
+ export function getCategoryFromLabels(labels) {
27
+ if (!labels || labels.length === 0)
28
+ return undefined;
29
+ const hit = labels.find(l => l.startsWith(CATEGORY_PREFIX));
30
+ return hit ? hit.slice(CATEGORY_PREFIX.length) || undefined : undefined;
31
+ }
32
+ /**
33
+ * Extract the difficulty from a labels array, or undefined when not present.
34
+ * Values are normalized to the canonical `Easy | Medium | Hard` form when
35
+ * possible; non-canonical values are returned as-is.
36
+ */
37
+ export function getDifficultyFromLabels(labels) {
38
+ if (!labels || labels.length === 0)
39
+ return undefined;
40
+ const hit = labels.find(l => l.startsWith(DIFFICULTY_PREFIX));
41
+ if (!hit)
42
+ return undefined;
43
+ const raw = hit.slice(DIFFICULTY_PREFIX.length).trim();
44
+ const normalized = raw.charAt(0).toUpperCase() + raw.slice(1).toLowerCase();
45
+ if (normalized === 'Easy' || normalized === 'Medium' || normalized === 'Hard') {
46
+ return normalized;
47
+ }
48
+ return raw || undefined;
49
+ }
50
+ /**
51
+ * Extract the subcategory from a labels array, or undefined when not present.
52
+ */
53
+ export function getSubcategoryFromLabels(labels) {
54
+ if (!labels || labels.length === 0)
55
+ return undefined;
56
+ const hit = labels.find(l => l.startsWith(SUBCATEGORY_PREFIX));
57
+ return hit ? hit.slice(SUBCATEGORY_PREFIX.length) || undefined : undefined;
58
+ }
59
+ /**
60
+ * Build a labels array from the legacy top-level `category` / `difficulty`
61
+ * fields, preserving any existing labels and refusing to add a duplicate
62
+ * facet (e.g. won't add `category:RCA` if a `category:` label is already
63
+ * present).
64
+ */
65
+ export function migrateLegacyFieldsToLabels(input) {
66
+ const out = [...(input.labels ?? [])];
67
+ if (input.category && !out.some(l => l.startsWith(CATEGORY_PREFIX))) {
68
+ out.push(`${CATEGORY_PREFIX}${input.category}`);
69
+ }
70
+ if (input.difficulty && !out.some(l => l.startsWith(DIFFICULTY_PREFIX))) {
71
+ out.push(`${DIFFICULTY_PREFIX}${input.difficulty}`);
72
+ }
73
+ if (input.subcategory && !out.some(l => l.startsWith(SUBCATEGORY_PREFIX))) {
74
+ out.push(`${SUBCATEGORY_PREFIX}${input.subcategory}`);
75
+ }
76
+ return out;
77
+ }
78
+ /**
79
+ * Returns true when the labels array contains a structured facet for the
80
+ * given prefix. Useful in migrations to avoid double-applying.
81
+ */
82
+ export function hasLabelPrefix(labels, prefix) {
83
+ if (!labels)
84
+ return false;
85
+ const normalized = prefix.endsWith(':') ? prefix : `${prefix}:`;
86
+ return labels.some(l => l.startsWith(normalized));
87
+ }
88
+ //# sourceMappingURL=testCaseLabels.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"testCaseLabels.js","sourceRoot":"","sources":["../../testCaseLabels.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;GAaG;AAEH,MAAM,eAAe,GAAG,WAAW,CAAC;AACpC,MAAM,iBAAiB,GAAG,aAAa,CAAC;AACxC,MAAM,kBAAkB,GAAG,cAAc,CAAC;AAI1C;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAiB;IACrD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACrD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC;IAC5D,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1E,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAAiB;IACvD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACrD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC9D,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5E,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,QAAQ,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC9E,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,OAAO,GAAG,IAAI,SAAS,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CAAC,MAAiB;IACxD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACrD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC/D,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7E,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,2BAA2B,CAAC,KAK3C;IACC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,IAAI,CAAC,GAAG,eAAe,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,IAAI,CAAC,GAAG,iBAAiB,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,IAAI,CAAC,GAAG,kBAAkB,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAA4B,EAAE,MAAc;IACzE,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC1B,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC;IAChE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;AACpD,CAAC"}
@@ -0,0 +1,140 @@
1
+ import { z } from 'zod';
2
+ import type { AgentContextItem } from '../types/index.js';
3
+ import type { CreateTestCaseInput } from '../services/storage/index.js';
4
+ /**
5
+ * Zod schema for validating test case JSON input.
6
+ * This validates a subset of CreateTestCaseInput fields that are relevant for the JSON editor.
7
+ * The output is compatible with CreateTestCaseInput.
8
+ */
9
+ export declare const testCaseSchema: z.ZodObject<{
10
+ name: z.ZodString;
11
+ description: z.ZodDefault<z.ZodOptional<z.ZodString>>;
12
+ category: z.ZodString;
13
+ subcategory: z.ZodOptional<z.ZodString>;
14
+ difficulty: z.ZodEnum<["Easy", "Medium", "Hard"]>;
15
+ initialPrompt: z.ZodString;
16
+ context: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodObject<{
17
+ description: z.ZodString;
18
+ value: z.ZodString;
19
+ }, "strip", z.ZodTypeAny, {
20
+ description?: string;
21
+ value?: string;
22
+ }, {
23
+ description?: string;
24
+ value?: string;
25
+ }>, "many">>>;
26
+ expectedOutcomes: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
27
+ }, "strip", z.ZodTypeAny, {
28
+ name?: string;
29
+ description?: string;
30
+ initialPrompt?: string;
31
+ context?: {
32
+ description?: string;
33
+ value?: string;
34
+ }[];
35
+ expectedOutcomes?: string[];
36
+ category?: string;
37
+ subcategory?: string;
38
+ difficulty?: "Easy" | "Medium" | "Hard";
39
+ }, {
40
+ name?: string;
41
+ description?: string;
42
+ initialPrompt?: string;
43
+ context?: {
44
+ description?: string;
45
+ value?: string;
46
+ }[];
47
+ expectedOutcomes?: string[];
48
+ category?: string;
49
+ subcategory?: string;
50
+ difficulty?: "Easy" | "Medium" | "Hard";
51
+ }>;
52
+ export declare const testCasesArraySchema: z.ZodArray<z.ZodObject<{
53
+ name: z.ZodString;
54
+ description: z.ZodDefault<z.ZodOptional<z.ZodString>>;
55
+ category: z.ZodString;
56
+ subcategory: z.ZodOptional<z.ZodString>;
57
+ difficulty: z.ZodEnum<["Easy", "Medium", "Hard"]>;
58
+ initialPrompt: z.ZodString;
59
+ context: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodObject<{
60
+ description: z.ZodString;
61
+ value: z.ZodString;
62
+ }, "strip", z.ZodTypeAny, {
63
+ description?: string;
64
+ value?: string;
65
+ }, {
66
+ description?: string;
67
+ value?: string;
68
+ }>, "many">>>;
69
+ expectedOutcomes: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
70
+ }, "strip", z.ZodTypeAny, {
71
+ name?: string;
72
+ description?: string;
73
+ initialPrompt?: string;
74
+ context?: {
75
+ description?: string;
76
+ value?: string;
77
+ }[];
78
+ expectedOutcomes?: string[];
79
+ category?: string;
80
+ subcategory?: string;
81
+ difficulty?: "Easy" | "Medium" | "Hard";
82
+ }, {
83
+ name?: string;
84
+ description?: string;
85
+ initialPrompt?: string;
86
+ context?: {
87
+ description?: string;
88
+ value?: string;
89
+ }[];
90
+ expectedOutcomes?: string[];
91
+ category?: string;
92
+ subcategory?: string;
93
+ difficulty?: "Easy" | "Medium" | "Hard";
94
+ }>, "many">;
95
+ /**
96
+ * The validated test case input from JSON.
97
+ * This is a subset of CreateTestCaseInput with only the fields the JSON editor handles.
98
+ * It's structurally compatible with CreateTestCaseInput for use with asyncTestCaseStorage.
99
+ */
100
+ export type ValidatedTestCaseInput = Pick<CreateTestCaseInput, 'name' | 'description' | 'category' | 'subcategory' | 'difficulty' | 'initialPrompt' | 'context' | 'expectedOutcomes'>;
101
+ export interface TestCaseFormState {
102
+ name: string;
103
+ description: string;
104
+ category: string;
105
+ subcategory: string;
106
+ difficulty: 'Easy' | 'Medium' | 'Hard';
107
+ initialPrompt: string;
108
+ context: AgentContextItem[];
109
+ expectedOutcomes: string[];
110
+ }
111
+ export interface ValidationError {
112
+ path: string;
113
+ message: string;
114
+ type: 'error' | 'warning';
115
+ }
116
+ export interface ValidationResult<T = unknown> {
117
+ valid: boolean;
118
+ errors: ValidationError[];
119
+ data?: T;
120
+ }
121
+ /**
122
+ * Validates a single test case JSON object.
123
+ * Returns data compatible with CreateTestCaseInput.
124
+ */
125
+ export declare function validateTestCaseJson(json: unknown): ValidationResult<ValidatedTestCaseInput>;
126
+ /**
127
+ * Validates an array of test cases for bulk import.
128
+ * Also handles single objects by wrapping them in an array.
129
+ * Returns data compatible with CreateTestCaseInput[].
130
+ */
131
+ export declare function validateTestCasesArrayJson(json: unknown): ValidationResult<ValidatedTestCaseInput[]>;
132
+ /**
133
+ * Serializes form state to JSON string for the editor.
134
+ */
135
+ export declare function serializeFormToJson(formState: TestCaseFormState): string;
136
+ /**
137
+ * Parses JSON string and returns form state if valid.
138
+ */
139
+ export declare function parseJsonToFormState(jsonString: string): ValidationResult<TestCaseFormState>;
140
+ //# sourceMappingURL=testCaseValidation.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"testCaseValidation.d.ts","sourceRoot":"","sources":["../../testCaseValidation.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAa9D;;;;GAIG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyBzB,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;WAA0D,CAAC;AAI5F;;;;GAIG;AACH,MAAM,MAAM,sBAAsB,GAAG,IAAI,CACvC,mBAAmB,EACnB,MAAM,GAAG,aAAa,GAAG,UAAU,GAAG,aAAa,GAAG,YAAY,GAAG,eAAe,GAAG,SAAS,GAAG,kBAAkB,CACtH,CAAC;AAGF,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,CAAC;IACvC,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAID,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,OAAO,GAAG,SAAS,CAAC;CAC3B;AAED,MAAM,WAAW,gBAAgB,CAAC,CAAC,GAAG,OAAO;IAC3C,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,IAAI,CAAC,EAAE,CAAC,CAAC;CACV;AAcD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,OAAO,GAAG,gBAAgB,CAAC,sBAAsB,CAAC,CAoB5F;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,OAAO,GAAG,gBAAgB,CAAC,sBAAsB,EAAE,CAAC,CAyBpG;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,iBAAiB,GAAG,MAAM,CAuBxE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,gBAAgB,CAAC,iBAAiB,CAAC,CAkC5F"}
@@ -0,0 +1,162 @@
1
+ /*
2
+ * Copyright OpenSearch Contributors
3
+ * SPDX-License-Identifier: Apache-2.0
4
+ */
5
+ import { z } from 'zod';
6
+ // ============ Zod Schemas ============
7
+ const contextItemSchema = z
8
+ .object({
9
+ description: z.string(),
10
+ value: z.string(),
11
+ })
12
+ .required();
13
+ const difficultySchema = z.enum(['Easy', 'Medium', 'Hard']);
14
+ /**
15
+ * Zod schema for validating test case JSON input.
16
+ * This validates a subset of CreateTestCaseInput fields that are relevant for the JSON editor.
17
+ * The output is compatible with CreateTestCaseInput.
18
+ */
19
+ export const testCaseSchema = z.object({
20
+ name: z.string().min(1, 'Name is required'),
21
+ description: z.string().optional().default(''),
22
+ category: z.string().min(1, 'Category is required'),
23
+ subcategory: z.string().optional(),
24
+ difficulty: difficultySchema,
25
+ initialPrompt: z.string().min(1, 'Initial prompt is required'),
26
+ context: z.array(contextItemSchema).optional().default([]),
27
+ // expectedOutcomes is optional. SDK code-only tests (`test()` blocks
28
+ // that drive pass/fail via deterministic body or inline `judge()`
29
+ // calls) have no expectedOutcomes — so requiring "at least one
30
+ // non-empty" here would force UI-created cross-surface-equivalent test
31
+ // cases to invent placeholder outcomes that the runner ignores. The
32
+ // server-side judge contract does still require expectedOutcomes when
33
+ // it actually grades a run (see services/evaluation/evaluatorError.ts
34
+ // — issue #242 surfaces "Missing required field" as an evaluator
35
+ // error), but the test case schema itself shouldn't gate authoring.
36
+ //
37
+ // `expectedTrajectory` is intentionally NOT modelled here. It exists on
38
+ // the persisted `TestCase` shape (set by the SDK loader from `test()`
39
+ // options) but the JSON-paste / form authoring path doesn't surface it,
40
+ // and adding it here without plumbing it through `ValidatedTestCaseInput`
41
+ // / `TestCaseFormState` / the editor save path would silently drop it on
42
+ // round-trip (PR #258 review feedback).
43
+ expectedOutcomes: z.array(z.string()).optional().default([]),
44
+ });
45
+ export const testCasesArraySchema = z.array(testCaseSchema).min(1, 'Array cannot be empty');
46
+ // ============ Helper Functions ============
47
+ function zodErrorToValidationErrors(error) {
48
+ return error.errors.map((e) => ({
49
+ path: e.path.join('.'),
50
+ message: e.message,
51
+ type: 'error',
52
+ }));
53
+ }
54
+ // ============ Validation Functions ============
55
+ /**
56
+ * Validates a single test case JSON object.
57
+ * Returns data compatible with CreateTestCaseInput.
58
+ */
59
+ export function validateTestCaseJson(json) {
60
+ // Handle array case - should use bulk import
61
+ if (Array.isArray(json)) {
62
+ return {
63
+ valid: false,
64
+ errors: [{ path: '', message: 'Received an array. Use Bulk Import mode for multiple test cases', type: 'error' }],
65
+ };
66
+ }
67
+ const result = testCaseSchema.safeParse(json);
68
+ if (!result.success) {
69
+ return {
70
+ valid: false,
71
+ errors: zodErrorToValidationErrors(result.error),
72
+ };
73
+ }
74
+ // Cast is safe here - Zod has validated all required fields
75
+ return { valid: true, errors: [], data: result.data };
76
+ }
77
+ /**
78
+ * Validates an array of test cases for bulk import.
79
+ * Also handles single objects by wrapping them in an array.
80
+ * Returns data compatible with CreateTestCaseInput[].
81
+ */
82
+ export function validateTestCasesArrayJson(json) {
83
+ // Handle single object - auto-wrap in array
84
+ if (json && typeof json === 'object' && !Array.isArray(json)) {
85
+ const singleResult = validateTestCaseJson(json);
86
+ if (singleResult.valid && singleResult.data) {
87
+ return {
88
+ valid: true,
89
+ errors: [],
90
+ data: [singleResult.data],
91
+ };
92
+ }
93
+ return { valid: false, errors: singleResult.errors };
94
+ }
95
+ const result = testCasesArraySchema.safeParse(json);
96
+ if (!result.success) {
97
+ return {
98
+ valid: false,
99
+ errors: zodErrorToValidationErrors(result.error),
100
+ };
101
+ }
102
+ // Cast is safe here - Zod has validated all required fields
103
+ return { valid: true, errors: [], data: result.data };
104
+ }
105
+ /**
106
+ * Serializes form state to JSON string for the editor.
107
+ */
108
+ export function serializeFormToJson(formState) {
109
+ const json = {
110
+ name: formState.name,
111
+ category: formState.category,
112
+ difficulty: formState.difficulty,
113
+ initialPrompt: formState.initialPrompt,
114
+ expectedOutcomes: formState.expectedOutcomes.filter((o) => o.trim()),
115
+ };
116
+ // Only include optional fields if they have values
117
+ if (formState.description.trim()) {
118
+ json.description = formState.description;
119
+ }
120
+ if (formState.subcategory.trim()) {
121
+ json.subcategory = formState.subcategory;
122
+ }
123
+ if (formState.context.length > 0) {
124
+ json.context = formState.context;
125
+ }
126
+ return JSON.stringify(json, null, 2);
127
+ }
128
+ /**
129
+ * Parses JSON string and returns form state if valid.
130
+ */
131
+ export function parseJsonToFormState(jsonString) {
132
+ let parsed;
133
+ try {
134
+ parsed = JSON.parse(jsonString);
135
+ }
136
+ catch (e) {
137
+ return {
138
+ valid: false,
139
+ errors: [{ path: '', message: `Invalid JSON: ${e.message}`, type: 'error' }],
140
+ };
141
+ }
142
+ const validation = validateTestCaseJson(parsed);
143
+ if (!validation.valid || !validation.data) {
144
+ return { valid: false, errors: validation.errors };
145
+ }
146
+ const data = validation.data;
147
+ return {
148
+ valid: true,
149
+ errors: [],
150
+ data: {
151
+ name: data.name,
152
+ description: data.description || '',
153
+ category: data.category,
154
+ subcategory: data.subcategory || '',
155
+ difficulty: data.difficulty,
156
+ initialPrompt: data.initialPrompt,
157
+ context: (data.context || []),
158
+ expectedOutcomes: data.expectedOutcomes.length > 0 ? data.expectedOutcomes : [''],
159
+ },
160
+ };
161
+ }
162
+ //# sourceMappingURL=testCaseValidation.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"testCaseValidation.js","sourceRoot":"","sources":["../../testCaseValidation.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,wCAAwC;AAExC,MAAM,iBAAiB,GAAG,CAAC;KACxB,MAAM,CAAC;IACN,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;CAClB,CAAC;KACD,QAAQ,EAAE,CAAC;AAEd,MAAM,gBAAgB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAE5D;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,kBAAkB,CAAC;IAC3C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,sBAAsB,CAAC;IACnD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,UAAU,EAAE,gBAAgB;IAC5B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,4BAA4B,CAAC;IAC9D,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1D,qEAAqE;IACrE,kEAAkE;IAClE,+DAA+D;IAC/D,uEAAuE;IACvE,oEAAoE;IACpE,sEAAsE;IACtE,sEAAsE;IACtE,iEAAiE;IACjE,oEAAoE;IACpE,EAAE;IACF,wEAAwE;IACxE,sEAAsE;IACtE,wEAAwE;IACxE,0EAA0E;IAC1E,yEAAyE;IACzE,wCAAwC;IACxC,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;CAC7D,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;AAwC5F,6CAA6C;AAE7C,SAAS,0BAA0B,CAAC,KAAiB;IACnD,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9B,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QACtB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,IAAI,EAAE,OAAgB;KACvB,CAAC,CAAC,CAAC;AACN,CAAC;AAED,iDAAiD;AAEjD;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAa;IAChD,6CAA6C;IAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,iEAAiE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;SAClH,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAE9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,0BAA0B,CAAC,MAAM,CAAC,KAAK,CAAC;SACjD,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAA8B,EAAE,CAAC;AAClF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CAAC,IAAa;IACtD,4CAA4C;IAC5C,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7D,MAAM,YAAY,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,YAAY,CAAC,KAAK,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;YAC5C,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC;aAC1B,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC;IACvD,CAAC;IAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,0BAA0B,CAAC,MAAM,CAAC,KAAK,CAAC;SACjD,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAgC,EAAE,CAAC;AACpF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAA4B;IAC9D,MAAM,IAAI,GAA4B;QACpC,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,aAAa,EAAE,SAAS,CAAC,aAAa;QACtC,gBAAgB,EAAE,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;KACrE,CAAC;IAEF,mDAAmD;IACnD,IAAI,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;IAC3C,CAAC;IAED,IAAI,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;IAC3C,CAAC;IAED,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;IACnC,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,IAAI,MAAe,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,iBAAkB,CAAW,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;SACxF,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAEhD,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;IACrD,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;IAE7B,OAAO;QACL,KAAK,EAAE,IAAI;QACX,MAAM,EAAE,EAAE;QACV,IAAI,EAAE;YACJ,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;YACnC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;YACnC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAuB;YACnD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAClF;KACF,CAAC;AACJ,CAAC"}