@terreno/admin-frontend 57.21.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (308) hide show
  1. package/CHANGELOG.md +31 -0
  2. package/dist/index.d.ts +1 -1
  3. package/dist/index.d.ts.map +1 -1
  4. package/dist/index.js +1 -1
  5. package/dist/index.js.map +1 -1
  6. package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.d.ts.map +1 -1
  7. package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js +19 -9
  8. package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js.map +1 -1
  9. package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts +7 -2
  10. package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts.map +1 -1
  11. package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js +155 -43
  12. package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js.map +1 -1
  13. package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts +20 -1
  14. package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts.map +1 -1
  15. package/dist/widgets/aiObservability/datasets/datasetTypes.js +17 -12
  16. package/dist/widgets/aiObservability/datasets/datasetTypes.js.map +1 -1
  17. package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.d.ts.map +1 -1
  18. package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js +12 -8
  19. package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js.map +1 -1
  20. package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.d.ts.map +1 -1
  21. package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js +4 -1
  22. package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js.map +1 -1
  23. package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts +8 -0
  24. package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts.map +1 -1
  25. package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js +68 -37
  26. package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js.map +1 -1
  27. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.d.ts.map +1 -1
  28. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js +4 -2
  29. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js.map +1 -1
  30. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts +8 -0
  31. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts.map +1 -1
  32. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js +156 -33
  33. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js.map +1 -1
  34. package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts +32 -0
  35. package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts.map +1 -1
  36. package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js +49 -0
  37. package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js.map +1 -1
  38. package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.d.ts.map +1 -1
  39. package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js +235 -12
  40. package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js.map +1 -1
  41. package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts +10 -1
  42. package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts.map +1 -1
  43. package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js +113 -56
  44. package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js.map +1 -1
  45. package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.d.ts.map +1 -1
  46. package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js +53 -38
  47. package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js.map +1 -1
  48. package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts +2 -3
  49. package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts.map +1 -1
  50. package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js +117 -32
  51. package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js.map +1 -1
  52. package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.d.ts.map +1 -1
  53. package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js +13 -4
  54. package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js.map +1 -1
  55. package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts +5 -0
  56. package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts.map +1 -1
  57. package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js +157 -55
  58. package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js.map +1 -1
  59. package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts +41 -0
  60. package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts.map +1 -0
  61. package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js +113 -0
  62. package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js.map +1 -0
  63. package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts +30 -0
  64. package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts.map +1 -0
  65. package/dist/widgets/aiObservability/experiments/ExperimentWizard.js +29 -0
  66. package/dist/widgets/aiObservability/experiments/ExperimentWizard.js.map +1 -0
  67. package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts +82 -3
  68. package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts.map +1 -1
  69. package/dist/widgets/aiObservability/experiments/experimentTypes.js +172 -6
  70. package/dist/widgets/aiObservability/experiments/experimentTypes.js.map +1 -1
  71. package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts +34 -4
  72. package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts.map +1 -1
  73. package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js +11 -4
  74. package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js.map +1 -1
  75. package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.d.ts.map +1 -1
  76. package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js +24 -5
  77. package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js.map +1 -1
  78. package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts +2 -0
  79. package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts.map +1 -1
  80. package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js +36 -6
  81. package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js.map +1 -1
  82. package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts +3 -0
  83. package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts.map +1 -1
  84. package/dist/widgets/aiObservability/prompts/AiPromptHubView.js +6 -6
  85. package/dist/widgets/aiObservability/prompts/AiPromptHubView.js.map +1 -1
  86. package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.d.ts.map +1 -1
  87. package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js +12 -12
  88. package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js.map +1 -1
  89. package/dist/widgets/aiObservability/prompts/AiPromptsListView.d.ts.map +1 -1
  90. package/dist/widgets/aiObservability/prompts/AiPromptsListView.js +156 -35
  91. package/dist/widgets/aiObservability/prompts/AiPromptsListView.js.map +1 -1
  92. package/dist/widgets/aiObservability/prompts/promptTypes.d.ts +13 -3
  93. package/dist/widgets/aiObservability/prompts/promptTypes.d.ts.map +1 -1
  94. package/dist/widgets/aiObservability/prompts/promptTypes.js +61 -19
  95. package/dist/widgets/aiObservability/prompts/promptTypes.js.map +1 -1
  96. package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts +11 -0
  97. package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts.map +1 -0
  98. package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js +115 -0
  99. package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js.map +1 -0
  100. package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts +22 -0
  101. package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts.map +1 -0
  102. package/dist/widgets/aiObservability/review/AiReviewBatchView.js +136 -0
  103. package/dist/widgets/aiObservability/review/AiReviewBatchView.js.map +1 -0
  104. package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts +4 -0
  105. package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts.map +1 -0
  106. package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js +92 -0
  107. package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js.map +1 -0
  108. package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts +26 -0
  109. package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts.map +1 -0
  110. package/dist/widgets/aiObservability/review/AiReviewBatchesView.js +115 -0
  111. package/dist/widgets/aiObservability/review/AiReviewBatchesView.js.map +1 -0
  112. package/dist/widgets/aiObservability/review/AiReviewItemScreen.d.ts.map +1 -1
  113. package/dist/widgets/aiObservability/review/AiReviewItemScreen.js +73 -16
  114. package/dist/widgets/aiObservability/review/AiReviewItemScreen.js.map +1 -1
  115. package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts +19 -5
  116. package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts.map +1 -1
  117. package/dist/widgets/aiObservability/review/AiReviewItemView.js +14 -9
  118. package/dist/widgets/aiObservability/review/AiReviewItemView.js.map +1 -1
  119. package/dist/widgets/aiObservability/review/AiReviewQueueScreen.d.ts.map +1 -1
  120. package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js +16 -2
  121. package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js.map +1 -1
  122. package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts +11 -1
  123. package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts.map +1 -1
  124. package/dist/widgets/aiObservability/review/AiReviewQueueView.js +96 -37
  125. package/dist/widgets/aiObservability/review/AiReviewQueueView.js.map +1 -1
  126. package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts +13 -0
  127. package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts.map +1 -0
  128. package/dist/widgets/aiObservability/review/ReviewQuestionList.js +28 -0
  129. package/dist/widgets/aiObservability/review/ReviewQuestionList.js.map +1 -0
  130. package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts +3 -0
  131. package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts.map +1 -1
  132. package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js +2 -2
  133. package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js.map +1 -1
  134. package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts +14 -0
  135. package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts.map +1 -1
  136. package/dist/widgets/aiObservability/review/ReviewScoreField.js +41 -12
  137. package/dist/widgets/aiObservability/review/ReviewScoreField.js.map +1 -1
  138. package/dist/widgets/aiObservability/review/batchTypes.d.ts +103 -0
  139. package/dist/widgets/aiObservability/review/batchTypes.d.ts.map +1 -0
  140. package/dist/widgets/aiObservability/review/batchTypes.js +61 -0
  141. package/dist/widgets/aiObservability/review/batchTypes.js.map +1 -0
  142. package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts +21 -0
  143. package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts.map +1 -0
  144. package/dist/widgets/aiObservability/review/reviewShortcuts.js +141 -0
  145. package/dist/widgets/aiObservability/review/reviewShortcuts.js.map +1 -0
  146. package/dist/widgets/aiObservability/review/reviewTypes.d.ts +23 -2
  147. package/dist/widgets/aiObservability/review/reviewTypes.d.ts.map +1 -1
  148. package/dist/widgets/aiObservability/review/reviewTypes.js +25 -0
  149. package/dist/widgets/aiObservability/review/reviewTypes.js.map +1 -1
  150. package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts +48 -0
  151. package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts.map +1 -1
  152. package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js +49 -0
  153. package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js.map +1 -1
  154. package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts +30 -0
  155. package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts.map +1 -0
  156. package/dist/widgets/aiObservability/shell/AgreementMeter.js +32 -0
  157. package/dist/widgets/aiObservability/shell/AgreementMeter.js.map +1 -0
  158. package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts +10 -0
  159. package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts.map +1 -1
  160. package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js +6 -4
  161. package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js.map +1 -1
  162. package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts +3 -1
  163. package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts.map +1 -1
  164. package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js +5 -1
  165. package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js.map +1 -1
  166. package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.d.ts.map +1 -1
  167. package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js +3 -3
  168. package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js.map +1 -1
  169. package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts +14 -0
  170. package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts.map +1 -0
  171. package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js +26 -0
  172. package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js.map +1 -0
  173. package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts +11 -0
  174. package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts.map +1 -0
  175. package/dist/widgets/aiObservability/shell/EvalStepBanner.js +8 -0
  176. package/dist/widgets/aiObservability/shell/EvalStepBanner.js.map +1 -0
  177. package/dist/widgets/aiObservability/shell/EvalStepper.d.ts +22 -0
  178. package/dist/widgets/aiObservability/shell/EvalStepper.d.ts.map +1 -0
  179. package/dist/widgets/aiObservability/shell/EvalStepper.js +28 -0
  180. package/dist/widgets/aiObservability/shell/EvalStepper.js.map +1 -0
  181. package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts +17 -0
  182. package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts.map +1 -0
  183. package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js +10 -0
  184. package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js.map +1 -0
  185. package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts +24 -0
  186. package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts.map +1 -0
  187. package/dist/widgets/aiObservability/shell/ObservabilityStates.js +11 -0
  188. package/dist/widgets/aiObservability/shell/ObservabilityStates.js.map +1 -0
  189. package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts +11 -0
  190. package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts.map +1 -0
  191. package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js +9 -0
  192. package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js.map +1 -0
  193. package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts +1 -0
  194. package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts.map +1 -1
  195. package/dist/widgets/aiObservability/shell/aiObservabilityNav.js +15 -8
  196. package/dist/widgets/aiObservability/shell/aiObservabilityNav.js.map +1 -1
  197. package/dist/widgets/aiObservability/shell/clientTable.d.ts +48 -0
  198. package/dist/widgets/aiObservability/shell/clientTable.d.ts.map +1 -0
  199. package/dist/widgets/aiObservability/shell/clientTable.js +163 -0
  200. package/dist/widgets/aiObservability/shell/clientTable.js.map +1 -0
  201. package/dist/widgets/aiObservability/shell/evalPipeline.d.ts +24 -0
  202. package/dist/widgets/aiObservability/shell/evalPipeline.d.ts.map +1 -0
  203. package/dist/widgets/aiObservability/shell/evalPipeline.js +43 -0
  204. package/dist/widgets/aiObservability/shell/evalPipeline.js.map +1 -0
  205. package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts +23 -0
  206. package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts.map +1 -0
  207. package/dist/widgets/aiObservability/shell/observabilityStatus.js +72 -0
  208. package/dist/widgets/aiObservability/shell/observabilityStatus.js.map +1 -0
  209. package/dist/widgets/aiObservability/traces/AiTracesListScreen.d.ts.map +1 -1
  210. package/dist/widgets/aiObservability/traces/AiTracesListScreen.js +18 -9
  211. package/dist/widgets/aiObservability/traces/AiTracesListScreen.js.map +1 -1
  212. package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts +18 -7
  213. package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts.map +1 -1
  214. package/dist/widgets/aiObservability/traces/AiTracesListView.js +198 -78
  215. package/dist/widgets/aiObservability/traces/AiTracesListView.js.map +1 -1
  216. package/dist/widgets/aiObservability/traces/traceTypes.d.ts +23 -1
  217. package/dist/widgets/aiObservability/traces/traceTypes.d.ts.map +1 -1
  218. package/dist/widgets/aiObservability/traces/traceTypes.js +92 -1
  219. package/dist/widgets/aiObservability/traces/traceTypes.js.map +1 -1
  220. package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts +2 -0
  221. package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts.map +1 -1
  222. package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js +9 -2
  223. package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js.map +1 -1
  224. package/package.json +3 -3
  225. package/src/index.tsx +2 -0
  226. package/src/isolated/AiPromptEditorScreen.isolated.tsx +24 -0
  227. package/src/isolated/aiListTables.isolated.tsx +654 -0
  228. package/src/isolated/aiObservabilityScreens.isolated.tsx +715 -287
  229. package/src/isolated/aiReviewBatchScreens.isolated.tsx +378 -0
  230. package/src/isolated/aiTracesTable.isolated.tsx +375 -0
  231. package/src/widgets/aiObservability/datasets/AiDatasetsListScreen.tsx +44 -13
  232. package/src/widgets/aiObservability/datasets/AiDatasetsListView.tsx +303 -103
  233. package/src/widgets/aiObservability/datasets/datasetTypes.test.ts +0 -13
  234. package/src/widgets/aiObservability/datasets/datasetTypes.ts +37 -13
  235. package/src/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.tsx +34 -11
  236. package/src/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.tsx +12 -1
  237. package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.test.tsx +97 -9
  238. package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.tsx +235 -131
  239. package/src/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.tsx +21 -2
  240. package/src/widgets/aiObservability/evaluators/AiEvaluatorsListView.tsx +261 -76
  241. package/src/widgets/aiObservability/evaluators/evaluatorTypes.test.ts +7 -0
  242. package/src/widgets/aiObservability/evaluators/evaluatorTypes.ts +82 -0
  243. package/src/widgets/aiObservability/experiments/AiExperimentNewScreen.tsx +271 -13
  244. package/src/widgets/aiObservability/experiments/AiExperimentNewView.test.tsx +164 -2
  245. package/src/widgets/aiObservability/experiments/AiExperimentNewView.tsx +400 -186
  246. package/src/widgets/aiObservability/experiments/AiExperimentResultsScreen.tsx +104 -45
  247. package/src/widgets/aiObservability/experiments/AiExperimentResultsView.test.tsx +290 -0
  248. package/src/widgets/aiObservability/experiments/AiExperimentResultsView.tsx +342 -97
  249. package/src/widgets/aiObservability/experiments/AiExperimentsListScreen.tsx +35 -4
  250. package/src/widgets/aiObservability/experiments/AiExperimentsListView.tsx +302 -112
  251. package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.test.tsx +250 -0
  252. package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.tsx +468 -0
  253. package/src/widgets/aiObservability/experiments/ExperimentWizard.tsx +106 -0
  254. package/src/widgets/aiObservability/experiments/experimentTypes.test.ts +103 -16
  255. package/src/widgets/aiObservability/experiments/experimentTypes.ts +250 -10
  256. package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.test.ts +35 -0
  257. package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.ts +55 -9
  258. package/src/widgets/aiObservability/prompts/AiPromptEditorScreen.tsx +26 -2
  259. package/src/widgets/aiObservability/prompts/AiPromptEditorView.tsx +95 -1
  260. package/src/widgets/aiObservability/prompts/AiPromptHubView.test.tsx +61 -1
  261. package/src/widgets/aiObservability/prompts/AiPromptHubView.tsx +71 -27
  262. package/src/widgets/aiObservability/prompts/AiPromptsListScreen.tsx +34 -15
  263. package/src/widgets/aiObservability/prompts/AiPromptsListView.test.tsx +104 -8
  264. package/src/widgets/aiObservability/prompts/AiPromptsListView.tsx +287 -117
  265. package/src/widgets/aiObservability/prompts/promptTypes.test.ts +63 -1
  266. package/src/widgets/aiObservability/prompts/promptTypes.ts +83 -22
  267. package/src/widgets/aiObservability/review/AiReviewBatchScreen.tsx +217 -0
  268. package/src/widgets/aiObservability/review/AiReviewBatchView.tsx +398 -0
  269. package/src/widgets/aiObservability/review/AiReviewBatchViews.test.tsx +328 -0
  270. package/src/widgets/aiObservability/review/AiReviewBatchesScreen.tsx +137 -0
  271. package/src/widgets/aiObservability/review/AiReviewBatchesView.tsx +333 -0
  272. package/src/widgets/aiObservability/review/AiReviewItemScreen.tsx +121 -25
  273. package/src/widgets/aiObservability/review/AiReviewItemView.tsx +112 -70
  274. package/src/widgets/aiObservability/review/AiReviewQueueScreen.tsx +29 -2
  275. package/src/widgets/aiObservability/review/AiReviewQueueView.tsx +183 -75
  276. package/src/widgets/aiObservability/review/ReviewQuestionList.tsx +128 -0
  277. package/src/widgets/aiObservability/review/ReviewReadOnlyPanel.tsx +21 -13
  278. package/src/widgets/aiObservability/review/ReviewScoreField.tsx +111 -27
  279. package/src/widgets/aiObservability/review/batchTypes.test.ts +207 -0
  280. package/src/widgets/aiObservability/review/batchTypes.ts +162 -0
  281. package/src/widgets/aiObservability/review/reviewShortcuts.ts +195 -0
  282. package/src/widgets/aiObservability/review/reviewTypes.ts +62 -2
  283. package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.test.ts +74 -0
  284. package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.ts +98 -0
  285. package/src/widgets/aiObservability/shell/AgreementMeter.tsx +103 -0
  286. package/src/widgets/aiObservability/shell/AiObservabilityChrome.tsx +48 -14
  287. package/src/widgets/aiObservability/shell/AiObservabilityScreenWidgets.tsx +6 -0
  288. package/src/widgets/aiObservability/shell/AiObservabilityStatusChip.tsx +10 -2
  289. package/src/widgets/aiObservability/shell/EvalPipelineOverview.tsx +68 -0
  290. package/src/widgets/aiObservability/shell/EvalStepBanner.tsx +24 -0
  291. package/src/widgets/aiObservability/shell/EvalStepper.test.tsx +80 -0
  292. package/src/widgets/aiObservability/shell/EvalStepper.tsx +126 -0
  293. package/src/widgets/aiObservability/shell/ObservabilityPageHeader.tsx +50 -0
  294. package/src/widgets/aiObservability/shell/ObservabilitySharedUi.test.tsx +140 -0
  295. package/src/widgets/aiObservability/shell/ObservabilityStates.tsx +81 -0
  296. package/src/widgets/aiObservability/shell/ObservabilityStatusBadge.tsx +29 -0
  297. package/src/widgets/aiObservability/shell/aiObservabilityNav.ts +16 -8
  298. package/src/widgets/aiObservability/shell/clientTable.test.ts +176 -0
  299. package/src/widgets/aiObservability/shell/clientTable.ts +251 -0
  300. package/src/widgets/aiObservability/shell/evalPipeline.ts +74 -0
  301. package/src/widgets/aiObservability/shell/observabilityStatus.test.ts +73 -0
  302. package/src/widgets/aiObservability/shell/observabilityStatus.ts +100 -0
  303. package/src/widgets/aiObservability/traces/AiTracesListScreen.tsx +41 -13
  304. package/src/widgets/aiObservability/traces/AiTracesListView.tsx +423 -300
  305. package/src/widgets/aiObservability/traces/traceTypes.test.ts +52 -1
  306. package/src/widgets/aiObservability/traces/traceTypes.ts +117 -2
  307. package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.test.ts +13 -1
  308. package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.ts +10 -2
@@ -1,12 +1,38 @@
1
- import {Box, Button, CheckBox, SelectField, Text, TextField} from "@terreno/ui";
1
+ import {
2
+ Badge,
3
+ Banner,
4
+ Box,
5
+ Card,
6
+ CheckBox,
7
+ type IconName,
8
+ SegmentedControl,
9
+ SelectField,
10
+ Spinner,
11
+ Text,
12
+ TextField,
13
+ } from "@terreno/ui";
2
14
  import React from "react";
3
15
  import type {DatasetRecord} from "../datasets/datasetTypes";
4
- import type {EvaluatorRecord} from "../evaluators/evaluatorTypes";
16
+ import {
17
+ EVALUATOR_TYPE_LABELS,
18
+ type EvaluatorRecord,
19
+ summarizeCalibration,
20
+ } from "../evaluators/evaluatorTypes";
5
21
  import type {PromptDetail, PromptListItem} from "../prompts/promptTypes";
22
+ import {AgreementMeter} from "../shell/AgreementMeter";
23
+ import {EvalPipelineOverview} from "../shell/EvalPipelineOverview";
24
+ import {EvalStepBanner} from "../shell/EvalStepBanner";
25
+ import {EvalStepper} from "../shell/EvalStepper";
26
+ import {type ExperimentPipeline, PIPELINE_ACTIVITY, PIPELINE_STEP} from "../shell/evalPipeline";
27
+ import {AiJudgeExperimentSteps, type AiJudgeExperimentStepsProps} from "./AiJudgeExperimentSteps";
28
+ import {buildWizardSteps, WizardFooter} from "./ExperimentWizard";
6
29
  import {
7
30
  type ExperimentEstimate,
31
+ formatExperimentEstimate,
32
+ formatVersionList,
8
33
  isAutomaticExperimentEvaluator,
9
34
  isValidExperimentVersionCount,
35
+ promptVersionTags,
10
36
  } from "./experimentTypes";
11
37
 
12
38
  export type ExperimentWizardStep = 1 | 2 | 3 | 4;
@@ -27,11 +53,17 @@ export interface AiExperimentNewViewProps {
27
53
  onEvaluatorToggle: (id: string) => void;
28
54
  onIncludeUnproofreadChange: (value: boolean) => void;
29
55
  onModelOverrideChange: (value: string) => void;
56
+ /** Props for the judge calibration path (step 3); required to offer that path. */
57
+ judge?: AiJudgeExperimentStepsProps;
30
58
  onNameChange: (value: string) => void;
59
+ /** Pick a pipeline, or `undefined` to go back to the choice. */
60
+ onPipelineChange?: (pipeline?: ExperimentPipeline) => void;
31
61
  onPromptChange: (name: string) => void;
32
62
  onRun: () => void;
33
63
  onStepChange: (step: ExperimentWizardStep) => void;
34
64
  onVersionToggle: (version: number) => void;
65
+ /** Which pipeline this experiment follows; unset shows the plan and the choice first. */
66
+ pipeline?: ExperimentPipeline;
35
67
  promptDetail?: PromptDetail;
36
68
  promptName: string;
37
69
  prompts: PromptListItem[];
@@ -40,26 +72,127 @@ export interface AiExperimentNewViewProps {
40
72
  versions: number[];
41
73
  }
42
74
 
43
- const STEP_LABELS = ["Dataset", "Prompt versions", "Evaluators", "Review & run"];
75
+ const STEP_LABELS = ["Dataset", "Prompt versions", "Evaluators", "Review and run"];
44
76
 
45
77
  const EXPERIMENT_VERSION_HELP = "Select 2–3 prompt versions to compare.";
46
78
 
47
- const versionTags = (version: number, detail?: PromptDetail): string[] => {
48
- const tags: string[] = [];
49
- if (!detail) {
50
- return tags;
51
- }
52
- const latest = Math.max(...detail.versions.map((entry) => entry.version));
53
- if (version === latest) {
54
- tags.push("latest");
55
- }
56
- if (detail.labels.some((label) => label.label === "production" && label.version === version)) {
57
- tags.push("production");
58
- }
59
- if (version < latest - 1) {
60
- tags.push("superseded");
61
- }
62
- return tags;
79
+ export const DATASET_REQUIRED_MESSAGE = "Select a dataset.";
80
+
81
+ // Shown in the dataset field while nothing is chosen. Without it, a required SelectField with an
82
+ // empty value displays the first dataset even though none is selected.
83
+ const DATASET_PLACEHOLDER = "Choose a dataset";
84
+
85
+ const PROMPT_STEP_DETAILS: Record<ExperimentWizardStep, string> = {
86
+ 1: "pick the dataset the prompt versions will run on.",
87
+ 2: "pick 2–3 versions of the product prompt to compare side by side.",
88
+ 3: "pick how outputs are scored. Prefer a calibrated judge, since its scores track what people would say.",
89
+ 4: "check the plan and run it. A version that passes every gate can be promoted.",
90
+ };
91
+
92
+ const EVALUATOR_TYPE_ICONS: Record<EvaluatorRecord["type"], IconName> = {
93
+ human: "user",
94
+ "json-assert": "code",
95
+ "llm-judge": "gavel",
96
+ };
97
+
98
+ const PIPELINE_OPTIONS: Array<{label: string; pipeline: ExperimentPipeline}> = [
99
+ {label: "Calibrate a judge", pipeline: "judge"},
100
+ {label: "Compare prompt versions", pipeline: "prompt"},
101
+ ];
102
+
103
+ const PIPELINE_DESCRIPTIONS: Record<ExperimentPipeline, string> = {
104
+ judge:
105
+ "Humans already judged a dataset and you want an LLM judge that agrees with them. Each run compares 2–3 versions of the judge prompt with the human answers.",
106
+ prompt:
107
+ "You changed a product prompt and want to know which version is better. Score it with a calibrated judge so the scores track what people would say.",
108
+ };
109
+
110
+ const PipelinePicker: React.FC<{
111
+ canCalibrate: boolean;
112
+ onPipelineChange?: (pipeline?: ExperimentPipeline) => void;
113
+ pipeline?: ExperimentPipeline;
114
+ }> = ({canCalibrate, onPipelineChange, pipeline}) => {
115
+ const options = PIPELINE_OPTIONS.filter((option) => canCalibrate || option.pipeline !== "judge");
116
+ const selectedIndex = options.findIndex((option) => option.pipeline === pipeline);
117
+ return (
118
+ <Box gap={2} testID="ai-experiment-pipeline-choice">
119
+ <Text bold>What do you want to do?</Text>
120
+ <Box alignItems="start">
121
+ <SegmentedControl
122
+ items={options.map((option) => option.label)}
123
+ onChange={(index) => {
124
+ onPipelineChange?.(options[index]?.pipeline);
125
+ }}
126
+ selectedIndex={selectedIndex >= 0 ? selectedIndex : undefined}
127
+ size="sm"
128
+ testID="ai-experiment-pipeline"
129
+ />
130
+ </Box>
131
+ {pipeline ? (
132
+ <Text color="secondaryLight" size="sm" testID="ai-experiment-pipeline-description">
133
+ {PIPELINE_DESCRIPTIONS[pipeline]}
134
+ </Text>
135
+ ) : null}
136
+ </Box>
137
+ );
138
+ };
139
+
140
+ const EvaluatorChoice: React.FC<{
141
+ evaluator: EvaluatorRecord;
142
+ onToggle: (id: string) => void;
143
+ selected: boolean;
144
+ }> = ({evaluator, onToggle, selected}) => {
145
+ const isJudge = evaluator.type === "llm-judge";
146
+ const calibration = isJudge ? summarizeCalibration(evaluator.calibration) : undefined;
147
+ return (
148
+ <Box
149
+ accessibilityHint={`Toggle evaluator ${evaluator.name}`}
150
+ accessibilityLabel={evaluator.name}
151
+ alignItems="center"
152
+ direction="row"
153
+ gap={2}
154
+ onClick={() => {
155
+ onToggle(evaluator.id);
156
+ }}
157
+ wrap
158
+ >
159
+ <CheckBox selected={selected} />
160
+ <Text>{evaluator.name}</Text>
161
+ <Badge
162
+ iconName={EVALUATOR_TYPE_ICONS[evaluator.type]}
163
+ secondary
164
+ status="neutral"
165
+ testID={`ai-experiment-evaluator-kind-${evaluator.id}`}
166
+ value={EVALUATOR_TYPE_LABELS[evaluator.type]}
167
+ />
168
+ {isJudge ? (
169
+ evaluator.calibration?.rate !== undefined && !evaluator.calibration.stale ? (
170
+ <AgreementMeter
171
+ compact
172
+ rate={evaluator.calibration.rate}
173
+ testID={`ai-experiment-calibration-${evaluator.id}`}
174
+ />
175
+ ) : (
176
+ <Text
177
+ color="secondaryLight"
178
+ size="sm"
179
+ testID={`ai-experiment-calibration-${evaluator.id}`}
180
+ >
181
+ {evaluator.calibration?.stale ? "Calibration out of date" : "Not calibrated"}
182
+ </Text>
183
+ )
184
+ ) : null}
185
+ {calibration?.isTrusted ? (
186
+ <Badge
187
+ iconName="star"
188
+ secondary
189
+ status="success"
190
+ testID={`ai-experiment-recommended-${evaluator.id}`}
191
+ value="Recommended"
192
+ />
193
+ ) : null}
194
+ </Box>
195
+ );
63
196
  };
64
197
 
65
198
  export const AiExperimentNewView: React.FC<AiExperimentNewViewProps> = ({
@@ -73,16 +206,19 @@ export const AiExperimentNewView: React.FC<AiExperimentNewViewProps> = ({
73
206
  isCreating,
74
207
  isEstimating,
75
208
  modelOverride,
209
+ judge,
76
210
  name,
77
211
  onDatasetChange,
78
212
  onEvaluatorToggle,
79
213
  onIncludeUnproofreadChange,
80
214
  onModelOverrideChange,
81
215
  onNameChange,
216
+ onPipelineChange,
82
217
  onPromptChange,
83
218
  onRun,
84
219
  onStepChange,
85
220
  onVersionToggle,
221
+ pipeline,
86
222
  promptDetail,
87
223
  promptName,
88
224
  prompts,
@@ -91,194 +227,272 @@ export const AiExperimentNewView: React.FC<AiExperimentNewViewProps> = ({
91
227
  versions,
92
228
  }) => {
93
229
  const selectedDataset = datasets.find((entry) => entry.id === datasetId);
94
- const automaticEvaluators = evaluators.filter(isAutomaticExperimentEvaluator);
230
+ // Calibrated judges first: their scores are the ones that track human answers.
231
+ const automaticEvaluators = evaluators
232
+ .filter(isAutomaticExperimentEvaluator)
233
+ .map((evaluator, index) => ({
234
+ evaluator,
235
+ index,
236
+ trusted:
237
+ evaluator.type === "llm-judge" && summarizeCalibration(evaluator.calibration).isTrusted,
238
+ }))
239
+ .sort((left, right) => Number(right.trusted) - Number(left.trusted) || left.index - right.index)
240
+ .map((entry) => entry.evaluator);
241
+ const hasJudgeOption = automaticEvaluators.some((evaluator) => evaluator.type === "llm-judge");
242
+ const selectedEvaluators = automaticEvaluators.filter((evaluator) =>
243
+ evaluatorIds.includes(evaluator.id)
244
+ );
245
+ const uncalibratedJudges = selectedEvaluators.filter((evaluator) => {
246
+ return evaluator.type === "llm-judge" && !summarizeCalibration(evaluator.calibration).isTrusted;
247
+ });
248
+ const hasDataset = Boolean(selectedDataset);
95
249
  const hasVersionSelection = isValidExperimentVersionCount(versions.length);
96
- const isNextDisabled = step === 2 && !hasVersionSelection;
250
+ const isNextDisabled = (step === 1 && !hasDataset) || (step === 2 && !hasVersionSelection);
251
+ // Later steps need a dataset, and steps after prompt versions need a valid version selection.
252
+ const isStepLocked = (stepNumber: ExperimentWizardStep): boolean => {
253
+ if (stepNumber > 1 && !hasDataset) {
254
+ return true;
255
+ }
256
+ return stepNumber > 2 && !hasVersionSelection;
257
+ };
258
+
259
+ const pipelinePicker = (
260
+ <PipelinePicker
261
+ canCalibrate={Boolean(judge)}
262
+ onPipelineChange={onPipelineChange}
263
+ pipeline={pipeline}
264
+ />
265
+ );
266
+
267
+ if (!pipeline) {
268
+ return (
269
+ <Box gap={4} testID="ai-experiment-new">
270
+ <EvalPipelineOverview currentSteps={[3, 4]} />
271
+ <Card gap={4} padding={4}>
272
+ {pipelinePicker}
273
+ </Card>
274
+ </Box>
275
+ );
276
+ }
277
+
278
+ const nameField = (
279
+ <TextField
280
+ onChange={onNameChange}
281
+ testID="ai-experiment-name"
282
+ title="Experiment name"
283
+ value={name}
284
+ />
285
+ );
286
+ const validationBanner = validationError ? (
287
+ <Box testID="ai-experiment-validation-error">
288
+ <Banner hasIcon status="alert" text={validationError} />
289
+ </Box>
290
+ ) : null;
291
+
292
+ if (pipeline === "judge" && judge) {
293
+ return (
294
+ <Box gap={4} testID="ai-experiment-new">
295
+ <Card gap={4} padding={4}>
296
+ {pipelinePicker}
297
+ {nameField}
298
+ </Card>
299
+ <AiJudgeExperimentSteps {...judge} />
300
+ {validationBanner}
301
+ </Box>
302
+ );
303
+ }
97
304
 
98
305
  return (
99
306
  <Box gap={4} testID="ai-experiment-new">
100
- <Box direction="row" gap={2} wrap>
101
- {STEP_LABELS.map((label, index) => {
102
- const stepNumber = (index + 1) as ExperimentWizardStep;
103
- const completed = step > stepNumber;
104
- return (
105
- <Button
106
- disabled={stepNumber > 2 && !hasVersionSelection}
107
- key={label}
307
+ <Card gap={4} padding={4}>
308
+ {pipelinePicker}
309
+ {nameField}
310
+ </Card>
311
+ <Card gap={4} padding={4}>
312
+ <EvalStepper
313
+ steps={buildWizardSteps({
314
+ current: step,
315
+ isLocked: isStepLocked,
316
+ labels: STEP_LABELS,
317
+ onStepChange,
318
+ testIDPrefix: "ai-experiment-step",
319
+ })}
320
+ testID="ai-experiment-stepper"
321
+ />
322
+ <EvalStepBanner
323
+ activity={PIPELINE_ACTIVITY.prompt}
324
+ detail={PROMPT_STEP_DETAILS[step]}
325
+ step={PIPELINE_STEP.prompt}
326
+ />
327
+ {step === 1 ? (
328
+ <Box gap={2} testID="ai-experiment-step-dataset">
329
+ <SelectField
330
+ onChange={onDatasetChange}
331
+ options={datasets.map((entry) => {
332
+ return {
333
+ label: `${entry.name} (${entry.counts.total} items)`,
334
+ value: entry.id,
335
+ };
336
+ })}
337
+ placeholder={DATASET_PLACEHOLDER}
338
+ testID="ai-experiment-dataset"
339
+ title="Dataset"
340
+ value={hasDataset ? datasetId : ""}
341
+ />
342
+ {selectedDataset ? (
343
+ <Text color="secondaryLight" size="sm" testID="ai-experiment-dataset-summary">
344
+ {`Input schema from ${selectedDataset.inputSchemaPromptName ?? "no prompt"}. ${
345
+ selectedDataset.counts.needsReview
346
+ } ${selectedDataset.counts.needsReview === 1 ? "item needs" : "items need"} review.`}
347
+ </Text>
348
+ ) : (
349
+ <Text color="secondaryLight" size="sm" testID="ai-experiment-dataset-help">
350
+ {DATASET_REQUIRED_MESSAGE}
351
+ </Text>
352
+ )}
353
+ <Box
354
+ accessibilityHint="Toggle include unproofread items"
355
+ accessibilityLabel="Include unproofread items"
356
+ alignItems="center"
357
+ direction="row"
358
+ gap={1}
108
359
  onClick={() => {
109
- if (stepNumber > 2 && !hasVersionSelection) {
110
- return;
111
- }
112
- onStepChange(stepNumber);
360
+ onIncludeUnproofreadChange(!includeUnproofread);
113
361
  }}
114
- testID={`ai-experiment-step-${stepNumber}`}
115
- text={`${completed ? "✓ " : ""}${label}`}
116
- variant={step === stepNumber ? "primary" : "ghost"}
117
- />
118
- );
119
- })}
120
- </Box>
121
- <TextField
122
- onChange={onNameChange}
123
- testID="ai-experiment-name"
124
- title="Experiment name"
125
- value={name}
126
- />
127
- {step === 1 ? (
128
- <Box gap={2} testID="ai-experiment-step-dataset">
129
- <SelectField
130
- onChange={onDatasetChange}
131
- options={datasets.map((entry) => {
132
- return {
133
- label: `${entry.name} (${entry.counts.total} items)`,
134
- value: entry.id,
135
- };
136
- })}
137
- requireValue
138
- title="Dataset"
139
- value={datasetId}
140
- />
141
- {selectedDataset ? (
142
- <Text color="secondaryDark" size="sm">
143
- Schema binding: {selectedDataset.inputSchemaPromptName ?? "none"} ·{" "}
144
- {selectedDataset.counts.needsReview} need review
145
- </Text>
146
- ) : undefined}
147
- <Box
148
- accessibilityHint="Toggle include unproofread items"
149
- accessibilityLabel="Include unproofread items"
150
- alignItems="center"
151
- direction="row"
152
- gap={1}
153
- onClick={() => {
154
- onIncludeUnproofreadChange(!includeUnproofread);
155
- }}
156
- >
157
- <CheckBox selected={includeUnproofread} />
158
- <Text>Include unproofread items</Text>
362
+ >
363
+ <CheckBox selected={includeUnproofread} />
364
+ <Text>Include unproofread items</Text>
365
+ </Box>
159
366
  </Box>
160
- </Box>
161
- ) : undefined}
162
- {step === 2 ? (
163
- <Box gap={2} testID="ai-experiment-step-prompt">
164
- <SelectField
165
- onChange={onPromptChange}
166
- options={prompts.map((entry) => {
167
- return {label: entry.name, value: entry.name};
168
- })}
169
- requireValue
170
- title="Prompt"
171
- value={promptName}
172
- />
173
- {promptDetail?.versions.map((version) => {
174
- const tags = versionTags(version.version, promptDetail);
175
- const selected = versions.includes(version.version);
176
- return (
177
- <Box
178
- accessibilityHint={`Toggle version ${version.version}`}
179
- accessibilityLabel={`Version ${version.version}`}
180
- alignItems="center"
181
- direction="row"
182
- gap={1}
183
- key={version.version}
184
- onClick={() => {
185
- onVersionToggle(version.version);
186
- }}
187
- wrap
188
- >
189
- <CheckBox selected={selected} />
190
- <Text>
191
- {`v${version.version}${tags.length > 0 ? ` (${tags.join(", ")})` : ""}`}
192
- </Text>
193
- </Box>
194
- );
195
- })}
196
- <Text
197
- color={hasVersionSelection ? "secondaryDark" : "error"}
198
- size="sm"
199
- testID="ai-experiment-version-help"
200
- >
201
- {EXPERIMENT_VERSION_HELP}
202
- </Text>
203
- </Box>
204
- ) : undefined}
205
- {step === 3 ? (
206
- <Box gap={2} testID="ai-experiment-step-evaluators">
207
- {automaticEvaluators.length === 0 ? (
208
- <Text color="secondaryDark" size="sm" testID="ai-experiment-evaluators-empty">
209
- Create an LLM judge or JSON assert evaluator before running an experiment.
210
- </Text>
211
- ) : (
212
- automaticEvaluators.map((evaluator) => {
213
- const selected = evaluatorIds.includes(evaluator.id);
367
+ ) : null}
368
+ {step === 2 ? (
369
+ <Box gap={2} testID="ai-experiment-step-prompt">
370
+ <SelectField
371
+ onChange={onPromptChange}
372
+ options={prompts.map((entry) => {
373
+ return {label: entry.name, value: entry.name};
374
+ })}
375
+ requireValue
376
+ title="Prompt"
377
+ value={promptName}
378
+ />
379
+ {promptDetail?.versions.map((version) => {
380
+ const tags = promptVersionTags(version.version, promptDetail);
381
+ const selected = versions.includes(version.version);
214
382
  return (
215
383
  <Box
216
- accessibilityHint={`Toggle evaluator ${evaluator.name}`}
217
- accessibilityLabel={evaluator.name}
384
+ accessibilityHint={`Toggle version ${version.version}`}
385
+ accessibilityLabel={`Version ${version.version}`}
218
386
  alignItems="center"
219
387
  direction="row"
220
- gap={1}
221
- key={evaluator.id}
388
+ gap={2}
389
+ key={version.version}
222
390
  onClick={() => {
223
- onEvaluatorToggle(evaluator.id);
391
+ onVersionToggle(version.version);
224
392
  }}
393
+ wrap
225
394
  >
226
395
  <CheckBox selected={selected} />
227
- <Text>{`${evaluator.name} (${evaluator.type})`}</Text>
396
+ <Text>{`v${version.version}`}</Text>
397
+ {tags.map((tag) => (
398
+ <Badge key={tag} secondary status="neutral" value={tag} />
399
+ ))}
228
400
  </Box>
229
401
  );
230
- })
231
- )}
232
- </Box>
233
- ) : undefined}
234
- {step === 4 ? (
235
- <Box gap={2} testID="ai-experiment-step-review">
236
- <Text>
237
- Dataset: {selectedDataset?.name ?? "—"} · Versions: {versions.join(", ") || "—"} ·
238
- Evaluators: {evaluatorIds.length}
239
- </Text>
240
- <TextField
241
- helperText="Optional override for generation model"
242
- onChange={onModelOverrideChange}
243
- testID="ai-experiment-model-override"
244
- title="Model override"
245
- value={modelOverride}
246
- />
247
- {isEstimating ? <Text>Estimating…</Text> : undefined}
248
- {estimate ? (
249
- <Text testID="ai-experiment-estimate">
250
- ~{estimate.generations} generations · ~{Math.round(estimate.wallClockSeconds / 60)}{" "}
251
- min
252
- {estimate.costUsd !== undefined ? ` · ~$${estimate.costUsd.toFixed(2)}` : ""}
402
+ })}
403
+ <Text color="secondaryLight" size="sm" testID="ai-experiment-version-help">
404
+ {EXPERIMENT_VERSION_HELP}
253
405
  </Text>
254
- ) : undefined}
255
- {estimateError ? <Text color="error">{estimateError}</Text> : undefined}
256
- <Button
257
- loading={isCreating}
258
- onClick={onRun}
259
- testID="ai-experiment-run"
260
- text="Run experiment"
261
- />
262
- </Box>
263
- ) : undefined}
264
- {validationError ? (
265
- <Text color="error" testID="ai-experiment-validation-error">
266
- {validationError}
267
- </Text>
268
- ) : undefined}
269
- {step < 4 ? (
270
- <Button
271
- disabled={isNextDisabled}
272
- onClick={() => {
273
- if (isNextDisabled) {
274
- return;
275
- }
406
+ </Box>
407
+ ) : null}
408
+ {step === 3 ? (
409
+ <Box gap={3} testID="ai-experiment-step-evaluators">
410
+ <Text color="secondaryLight" size="sm" testID="ai-experiment-evaluators-recommendation">
411
+ {hasJudgeOption
412
+ ? "Pick an LLM judge calibrated against human answers. An uncalibrated judge's scores may not match what people would say."
413
+ : "Calibrate an LLM judge against human answers first so scores track what people would say."}
414
+ </Text>
415
+ {automaticEvaluators.length === 0 ? (
416
+ <Text color="secondaryLight" size="sm" testID="ai-experiment-evaluators-empty">
417
+ Create an LLM judge or JSON assert evaluator before running an experiment.
418
+ </Text>
419
+ ) : (
420
+ automaticEvaluators.map((evaluator) => {
421
+ return (
422
+ <EvaluatorChoice
423
+ evaluator={evaluator}
424
+ key={evaluator.id}
425
+ onToggle={onEvaluatorToggle}
426
+ selected={evaluatorIds.includes(evaluator.id)}
427
+ />
428
+ );
429
+ })
430
+ )}
431
+ </Box>
432
+ ) : null}
433
+ {step === 4 ? (
434
+ <Box gap={3} testID="ai-experiment-step-review">
435
+ <Text testID="ai-experiment-review-summary">
436
+ {`You're comparing ${formatVersionList(versions) || "no versions"} of ${promptName || "the prompt"} on ${selectedDataset?.name ?? "the dataset"}, scored by ${
437
+ selectedEvaluators.map((evaluator) => evaluator.name).join(", ") ||
438
+ "no evaluator yet"
439
+ }. Promote a version only when every one of its gates passes.`}
440
+ </Text>
441
+ {uncalibratedJudges.length > 0 ? (
442
+ <Box testID="ai-experiment-uncalibrated-warning">
443
+ <Banner
444
+ hasIcon
445
+ status="warning"
446
+ text={`${uncalibratedJudges.map((evaluator) => evaluator.name).join(", ")} ${
447
+ uncalibratedJudges.length === 1 ? "is" : "are"
448
+ } not calibrated against current human answers, so scores may not match what people would say. Calibrate first to trust the result.`}
449
+ />
450
+ </Box>
451
+ ) : null}
452
+ <TextField
453
+ helperText="Optional. Leave blank to use each prompt version's model."
454
+ onChange={onModelOverrideChange}
455
+ testID="ai-experiment-model-override"
456
+ title="Model override"
457
+ value={modelOverride}
458
+ />
459
+ {isEstimating ? (
460
+ <Box alignItems="start" testID="ai-experiment-estimating">
461
+ <Spinner size="sm" />
462
+ </Box>
463
+ ) : null}
464
+ {estimate ? (
465
+ <Text bold testID="ai-experiment-estimate">
466
+ {formatExperimentEstimate({estimate})}
467
+ </Text>
468
+ ) : null}
469
+ {estimateError ? (
470
+ <Text color="error" size="sm">
471
+ {estimateError}
472
+ </Text>
473
+ ) : null}
474
+ </Box>
475
+ ) : null}
476
+ {validationBanner}
477
+ <WizardFooter
478
+ isLastStep={step === 4}
479
+ isNextDisabled={isNextDisabled}
480
+ isRunning={isCreating}
481
+ onBack={
482
+ step > 1
483
+ ? () => {
484
+ onStepChange((step - 1) as ExperimentWizardStep);
485
+ }
486
+ : undefined
487
+ }
488
+ onNext={() => {
276
489
  onStepChange((step + 1) as ExperimentWizardStep);
277
490
  }}
278
- testID="ai-experiment-next"
279
- text="Next"
491
+ onRun={onRun}
492
+ runText="Run experiment"
493
+ testIDPrefix="ai-experiment"
280
494
  />
281
- ) : undefined}
495
+ </Card>
282
496
  </Box>
283
497
  );
284
498
  };