@terreno/admin-frontend 57.20.0 → 57.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (361) hide show
  1. package/CHANGELOG.md +81 -0
  2. package/dist/AdminFilterDrawer.d.ts.map +1 -1
  3. package/dist/AdminFilterDrawer.js.map +1 -1
  4. package/dist/index.d.ts +2 -2
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js +1 -1
  7. package/dist/index.js.map +1 -1
  8. package/dist/types.d.ts +2 -0
  9. package/dist/types.d.ts.map +1 -1
  10. package/dist/types.js.map +1 -1
  11. package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.d.ts.map +1 -1
  12. package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js +19 -9
  13. package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js.map +1 -1
  14. package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts +7 -2
  15. package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts.map +1 -1
  16. package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js +155 -43
  17. package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js.map +1 -1
  18. package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts +20 -1
  19. package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts.map +1 -1
  20. package/dist/widgets/aiObservability/datasets/datasetTypes.js +17 -12
  21. package/dist/widgets/aiObservability/datasets/datasetTypes.js.map +1 -1
  22. package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.d.ts.map +1 -1
  23. package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js +12 -8
  24. package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js.map +1 -1
  25. package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.d.ts.map +1 -1
  26. package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js +4 -1
  27. package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js.map +1 -1
  28. package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts +8 -0
  29. package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts.map +1 -1
  30. package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js +68 -37
  31. package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js.map +1 -1
  32. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.d.ts.map +1 -1
  33. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js +4 -2
  34. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js.map +1 -1
  35. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts +8 -0
  36. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts.map +1 -1
  37. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js +156 -33
  38. package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js.map +1 -1
  39. package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts +32 -0
  40. package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts.map +1 -1
  41. package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js +49 -0
  42. package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js.map +1 -1
  43. package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.d.ts.map +1 -1
  44. package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js +235 -12
  45. package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js.map +1 -1
  46. package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts +10 -1
  47. package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts.map +1 -1
  48. package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js +113 -56
  49. package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js.map +1 -1
  50. package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.d.ts.map +1 -1
  51. package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js +53 -38
  52. package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js.map +1 -1
  53. package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts +2 -3
  54. package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts.map +1 -1
  55. package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js +117 -32
  56. package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js.map +1 -1
  57. package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.d.ts.map +1 -1
  58. package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js +13 -4
  59. package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js.map +1 -1
  60. package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts +5 -0
  61. package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts.map +1 -1
  62. package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js +157 -55
  63. package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js.map +1 -1
  64. package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts +41 -0
  65. package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts.map +1 -0
  66. package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js +113 -0
  67. package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js.map +1 -0
  68. package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts +30 -0
  69. package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts.map +1 -0
  70. package/dist/widgets/aiObservability/experiments/ExperimentWizard.js +29 -0
  71. package/dist/widgets/aiObservability/experiments/ExperimentWizard.js.map +1 -0
  72. package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts +82 -3
  73. package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts.map +1 -1
  74. package/dist/widgets/aiObservability/experiments/experimentTypes.js +172 -6
  75. package/dist/widgets/aiObservability/experiments/experimentTypes.js.map +1 -1
  76. package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts +34 -4
  77. package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts.map +1 -1
  78. package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js +11 -4
  79. package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js.map +1 -1
  80. package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.d.ts.map +1 -1
  81. package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js +24 -5
  82. package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js.map +1 -1
  83. package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts +2 -0
  84. package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts.map +1 -1
  85. package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js +36 -6
  86. package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js.map +1 -1
  87. package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts +3 -0
  88. package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts.map +1 -1
  89. package/dist/widgets/aiObservability/prompts/AiPromptHubView.js +6 -6
  90. package/dist/widgets/aiObservability/prompts/AiPromptHubView.js.map +1 -1
  91. package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.d.ts.map +1 -1
  92. package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js +12 -12
  93. package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js.map +1 -1
  94. package/dist/widgets/aiObservability/prompts/AiPromptsListView.d.ts.map +1 -1
  95. package/dist/widgets/aiObservability/prompts/AiPromptsListView.js +156 -35
  96. package/dist/widgets/aiObservability/prompts/AiPromptsListView.js.map +1 -1
  97. package/dist/widgets/aiObservability/prompts/promptTypes.d.ts +13 -3
  98. package/dist/widgets/aiObservability/prompts/promptTypes.d.ts.map +1 -1
  99. package/dist/widgets/aiObservability/prompts/promptTypes.js +61 -19
  100. package/dist/widgets/aiObservability/prompts/promptTypes.js.map +1 -1
  101. package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts +11 -0
  102. package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts.map +1 -0
  103. package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js +115 -0
  104. package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js.map +1 -0
  105. package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts +22 -0
  106. package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts.map +1 -0
  107. package/dist/widgets/aiObservability/review/AiReviewBatchView.js +136 -0
  108. package/dist/widgets/aiObservability/review/AiReviewBatchView.js.map +1 -0
  109. package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts +4 -0
  110. package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts.map +1 -0
  111. package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js +92 -0
  112. package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js.map +1 -0
  113. package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts +26 -0
  114. package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts.map +1 -0
  115. package/dist/widgets/aiObservability/review/AiReviewBatchesView.js +115 -0
  116. package/dist/widgets/aiObservability/review/AiReviewBatchesView.js.map +1 -0
  117. package/dist/widgets/aiObservability/review/AiReviewItemScreen.d.ts.map +1 -1
  118. package/dist/widgets/aiObservability/review/AiReviewItemScreen.js +73 -16
  119. package/dist/widgets/aiObservability/review/AiReviewItemScreen.js.map +1 -1
  120. package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts +19 -5
  121. package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts.map +1 -1
  122. package/dist/widgets/aiObservability/review/AiReviewItemView.js +14 -9
  123. package/dist/widgets/aiObservability/review/AiReviewItemView.js.map +1 -1
  124. package/dist/widgets/aiObservability/review/AiReviewQueueScreen.d.ts.map +1 -1
  125. package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js +16 -2
  126. package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js.map +1 -1
  127. package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts +11 -1
  128. package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts.map +1 -1
  129. package/dist/widgets/aiObservability/review/AiReviewQueueView.js +96 -37
  130. package/dist/widgets/aiObservability/review/AiReviewQueueView.js.map +1 -1
  131. package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts +13 -0
  132. package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts.map +1 -0
  133. package/dist/widgets/aiObservability/review/ReviewQuestionList.js +28 -0
  134. package/dist/widgets/aiObservability/review/ReviewQuestionList.js.map +1 -0
  135. package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts +3 -0
  136. package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts.map +1 -1
  137. package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js +2 -2
  138. package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js.map +1 -1
  139. package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts +14 -0
  140. package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts.map +1 -1
  141. package/dist/widgets/aiObservability/review/ReviewScoreField.js +41 -12
  142. package/dist/widgets/aiObservability/review/ReviewScoreField.js.map +1 -1
  143. package/dist/widgets/aiObservability/review/batchTypes.d.ts +103 -0
  144. package/dist/widgets/aiObservability/review/batchTypes.d.ts.map +1 -0
  145. package/dist/widgets/aiObservability/review/batchTypes.js +61 -0
  146. package/dist/widgets/aiObservability/review/batchTypes.js.map +1 -0
  147. package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts +21 -0
  148. package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts.map +1 -0
  149. package/dist/widgets/aiObservability/review/reviewShortcuts.js +141 -0
  150. package/dist/widgets/aiObservability/review/reviewShortcuts.js.map +1 -0
  151. package/dist/widgets/aiObservability/review/reviewTypes.d.ts +23 -2
  152. package/dist/widgets/aiObservability/review/reviewTypes.d.ts.map +1 -1
  153. package/dist/widgets/aiObservability/review/reviewTypes.js +25 -0
  154. package/dist/widgets/aiObservability/review/reviewTypes.js.map +1 -1
  155. package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts +48 -0
  156. package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts.map +1 -1
  157. package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js +49 -0
  158. package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js.map +1 -1
  159. package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts +30 -0
  160. package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts.map +1 -0
  161. package/dist/widgets/aiObservability/shell/AgreementMeter.js +32 -0
  162. package/dist/widgets/aiObservability/shell/AgreementMeter.js.map +1 -0
  163. package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts +10 -0
  164. package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts.map +1 -1
  165. package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js +6 -4
  166. package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js.map +1 -1
  167. package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts +3 -1
  168. package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts.map +1 -1
  169. package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js +5 -1
  170. package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js.map +1 -1
  171. package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.d.ts.map +1 -1
  172. package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js +3 -3
  173. package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js.map +1 -1
  174. package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts +14 -0
  175. package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts.map +1 -0
  176. package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js +26 -0
  177. package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js.map +1 -0
  178. package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts +11 -0
  179. package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts.map +1 -0
  180. package/dist/widgets/aiObservability/shell/EvalStepBanner.js +8 -0
  181. package/dist/widgets/aiObservability/shell/EvalStepBanner.js.map +1 -0
  182. package/dist/widgets/aiObservability/shell/EvalStepper.d.ts +22 -0
  183. package/dist/widgets/aiObservability/shell/EvalStepper.d.ts.map +1 -0
  184. package/dist/widgets/aiObservability/shell/EvalStepper.js +28 -0
  185. package/dist/widgets/aiObservability/shell/EvalStepper.js.map +1 -0
  186. package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts +17 -0
  187. package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts.map +1 -0
  188. package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js +10 -0
  189. package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js.map +1 -0
  190. package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts +24 -0
  191. package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts.map +1 -0
  192. package/dist/widgets/aiObservability/shell/ObservabilityStates.js +11 -0
  193. package/dist/widgets/aiObservability/shell/ObservabilityStates.js.map +1 -0
  194. package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts +11 -0
  195. package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts.map +1 -0
  196. package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js +9 -0
  197. package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js.map +1 -0
  198. package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts +1 -0
  199. package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts.map +1 -1
  200. package/dist/widgets/aiObservability/shell/aiObservabilityNav.js +15 -8
  201. package/dist/widgets/aiObservability/shell/aiObservabilityNav.js.map +1 -1
  202. package/dist/widgets/aiObservability/shell/clientTable.d.ts +48 -0
  203. package/dist/widgets/aiObservability/shell/clientTable.d.ts.map +1 -0
  204. package/dist/widgets/aiObservability/shell/clientTable.js +163 -0
  205. package/dist/widgets/aiObservability/shell/clientTable.js.map +1 -0
  206. package/dist/widgets/aiObservability/shell/evalPipeline.d.ts +24 -0
  207. package/dist/widgets/aiObservability/shell/evalPipeline.d.ts.map +1 -0
  208. package/dist/widgets/aiObservability/shell/evalPipeline.js +43 -0
  209. package/dist/widgets/aiObservability/shell/evalPipeline.js.map +1 -0
  210. package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts +23 -0
  211. package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts.map +1 -0
  212. package/dist/widgets/aiObservability/shell/observabilityStatus.js +72 -0
  213. package/dist/widgets/aiObservability/shell/observabilityStatus.js.map +1 -0
  214. package/dist/widgets/aiObservability/traces/AiTracesListScreen.d.ts.map +1 -1
  215. package/dist/widgets/aiObservability/traces/AiTracesListScreen.js +18 -9
  216. package/dist/widgets/aiObservability/traces/AiTracesListScreen.js.map +1 -1
  217. package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts +18 -7
  218. package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts.map +1 -1
  219. package/dist/widgets/aiObservability/traces/AiTracesListView.js +198 -78
  220. package/dist/widgets/aiObservability/traces/AiTracesListView.js.map +1 -1
  221. package/dist/widgets/aiObservability/traces/traceTypes.d.ts +23 -1
  222. package/dist/widgets/aiObservability/traces/traceTypes.d.ts.map +1 -1
  223. package/dist/widgets/aiObservability/traces/traceTypes.js +92 -1
  224. package/dist/widgets/aiObservability/traces/traceTypes.js.map +1 -1
  225. package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts +2 -0
  226. package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts.map +1 -1
  227. package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js +9 -2
  228. package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js.map +1 -1
  229. package/package.json +7 -3
  230. package/src/AdminBreadcrumbs.test.tsx +1 -1
  231. package/src/AdminFieldRenderer.test.tsx +2 -1
  232. package/src/AdminFilterDrawer.tsx +1 -2
  233. package/src/AdminHome.test.tsx +2 -1
  234. package/src/AdminModelTable.test.tsx +16 -12
  235. package/src/AdminModelTable.windowed.test.tsx +11 -5
  236. package/src/AdminPrimitiveArrayField.test.tsx +3 -3
  237. package/src/AdminProvider.test.tsx +11 -6
  238. package/src/AdminRolesList.test.tsx +22 -13
  239. package/src/AdminScreenPage.test.tsx +1 -1
  240. package/src/AdminScreenRouter.test.tsx +15 -5
  241. package/src/AdminScriptList.test.tsx +9 -2
  242. package/src/AdminShell.test.tsx +1 -1
  243. package/src/AdminVersionConfig.test.tsx +4 -4
  244. package/src/AnnouncementOverviewScreenWidget.test.tsx +6 -1
  245. package/src/CheckboxListEditor.test.tsx +11 -9
  246. package/src/ConfigurationScreen.test.tsx +2 -2
  247. package/src/adminRequest.test.ts +3 -3
  248. package/src/adminWindowedTable.test.ts +4 -2
  249. package/src/comms/CommsDashboardScreen.test.tsx +5 -5
  250. package/src/comms/CommsMessageDetail.test.tsx +1 -1
  251. package/src/index.tsx +4 -0
  252. package/src/isolated/AdminFieldRenderer.isolated.tsx +11 -6
  253. package/src/isolated/AdminFilterDrawer.isolated.tsx +8 -6
  254. package/src/isolated/AdminFilterDrawerMobile.isolated.tsx +3 -1
  255. package/src/isolated/AdminHomeWidgets.isolated.tsx +2 -0
  256. package/src/isolated/AdminNestedArrayField.isolated.tsx +10 -9
  257. package/src/isolated/AdminObjectPicker.isolated.tsx +9 -10
  258. package/src/isolated/AiPromptEditorScreen.isolated.tsx +24 -0
  259. package/src/isolated/AiPromptsListScreen.isolated.tsx +1 -1
  260. package/src/isolated/DocumentStorageBrowser.isolated.tsx +16 -8
  261. package/src/isolated/RecentActivityWidget.isolated.tsx +2 -0
  262. package/src/isolated/adminWidgets.isolated.tsx +46 -6
  263. package/src/isolated/aiListTables.isolated.tsx +654 -0
  264. package/src/isolated/aiObservabilityScreens.isolated.tsx +731 -294
  265. package/src/isolated/aiReviewBatchScreens.isolated.tsx +378 -0
  266. package/src/isolated/aiTracesTable.isolated.tsx +375 -0
  267. package/src/isolated/generateConsentHistoryPdf.isolated.ts +30 -10
  268. package/src/isolated/generateConsentPdfsMobile.isolated.ts +24 -3
  269. package/src/isolated/hooks.isolated.tsx +84 -60
  270. package/src/isolated/rpcDualRun.isolated.tsx +9 -2
  271. package/src/jobs/JobsDashboardScreen.test.tsx +3 -3
  272. package/src/jobs/JobsHomeWidget.test.tsx +1 -1
  273. package/src/jobs/JobsJobDetail.test.tsx +1 -1
  274. package/src/orgs/OrgManagementScreens.test.tsx +1 -1
  275. package/src/orgs/adminModelEndpoints.test.ts +1 -1
  276. package/src/testing/useAdminApiDouble.ts +2 -1
  277. package/src/types.ts +2 -0
  278. package/src/useAdminRpc.test.tsx +3 -3
  279. package/src/widgets/AIRequestsScreenWidget.test.tsx +20 -3
  280. package/src/widgets/ModelsGridWidget.test.tsx +25 -5
  281. package/src/widgets/aiObservability/datasets/AiDatasetDetailView.test.tsx +4 -2
  282. package/src/widgets/aiObservability/datasets/AiDatasetsListScreen.tsx +44 -13
  283. package/src/widgets/aiObservability/datasets/AiDatasetsListView.tsx +303 -103
  284. package/src/widgets/aiObservability/datasets/datasetTypes.test.ts +0 -14
  285. package/src/widgets/aiObservability/datasets/datasetTypes.ts +37 -13
  286. package/src/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.tsx +34 -11
  287. package/src/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.tsx +12 -1
  288. package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.test.tsx +186 -32
  289. package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.tsx +235 -131
  290. package/src/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.tsx +21 -2
  291. package/src/widgets/aiObservability/evaluators/AiEvaluatorsListView.tsx +261 -76
  292. package/src/widgets/aiObservability/evaluators/evaluatorTypes.test.ts +7 -0
  293. package/src/widgets/aiObservability/evaluators/evaluatorTypes.ts +82 -0
  294. package/src/widgets/aiObservability/experiments/AiExperimentNewScreen.tsx +271 -13
  295. package/src/widgets/aiObservability/experiments/AiExperimentNewView.test.tsx +168 -2
  296. package/src/widgets/aiObservability/experiments/AiExperimentNewView.tsx +400 -186
  297. package/src/widgets/aiObservability/experiments/AiExperimentResultsScreen.tsx +104 -45
  298. package/src/widgets/aiObservability/experiments/AiExperimentResultsView.test.tsx +290 -0
  299. package/src/widgets/aiObservability/experiments/AiExperimentResultsView.tsx +342 -97
  300. package/src/widgets/aiObservability/experiments/AiExperimentsListScreen.tsx +35 -4
  301. package/src/widgets/aiObservability/experiments/AiExperimentsListView.tsx +302 -112
  302. package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.test.tsx +250 -0
  303. package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.tsx +468 -0
  304. package/src/widgets/aiObservability/experiments/ExperimentWizard.tsx +106 -0
  305. package/src/widgets/aiObservability/experiments/experimentTypes.test.ts +103 -16
  306. package/src/widgets/aiObservability/experiments/experimentTypes.ts +250 -10
  307. package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.test.ts +35 -0
  308. package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.ts +55 -9
  309. package/src/widgets/aiObservability/prompts/AiPromptEditorScreen.tsx +26 -2
  310. package/src/widgets/aiObservability/prompts/AiPromptEditorView.tsx +95 -1
  311. package/src/widgets/aiObservability/prompts/AiPromptHubView.test.tsx +61 -1
  312. package/src/widgets/aiObservability/prompts/AiPromptHubView.tsx +71 -27
  313. package/src/widgets/aiObservability/prompts/AiPromptsListScreen.tsx +34 -15
  314. package/src/widgets/aiObservability/prompts/AiPromptsListView.test.tsx +104 -8
  315. package/src/widgets/aiObservability/prompts/AiPromptsListView.tsx +287 -117
  316. package/src/widgets/aiObservability/prompts/promptTypes.test.ts +63 -1
  317. package/src/widgets/aiObservability/prompts/promptTypes.ts +83 -22
  318. package/src/widgets/aiObservability/review/AiReviewBatchScreen.tsx +217 -0
  319. package/src/widgets/aiObservability/review/AiReviewBatchView.tsx +398 -0
  320. package/src/widgets/aiObservability/review/AiReviewBatchViews.test.tsx +328 -0
  321. package/src/widgets/aiObservability/review/AiReviewBatchesScreen.tsx +137 -0
  322. package/src/widgets/aiObservability/review/AiReviewBatchesView.tsx +333 -0
  323. package/src/widgets/aiObservability/review/AiReviewItemScreen.tsx +121 -25
  324. package/src/widgets/aiObservability/review/AiReviewItemView.tsx +112 -70
  325. package/src/widgets/aiObservability/review/AiReviewQueueScreen.tsx +29 -2
  326. package/src/widgets/aiObservability/review/AiReviewQueueView.tsx +183 -75
  327. package/src/widgets/aiObservability/review/ReviewQuestionList.tsx +128 -0
  328. package/src/widgets/aiObservability/review/ReviewReadOnlyPanel.tsx +21 -13
  329. package/src/widgets/aiObservability/review/ReviewScoreField.tsx +111 -27
  330. package/src/widgets/aiObservability/review/batchTypes.test.ts +207 -0
  331. package/src/widgets/aiObservability/review/batchTypes.ts +162 -0
  332. package/src/widgets/aiObservability/review/reviewShortcuts.ts +195 -0
  333. package/src/widgets/aiObservability/review/reviewTypes.ts +62 -2
  334. package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.test.ts +74 -0
  335. package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.ts +98 -0
  336. package/src/widgets/aiObservability/shell/AgreementMeter.tsx +103 -0
  337. package/src/widgets/aiObservability/shell/AiObservabilityChrome.tsx +48 -14
  338. package/src/widgets/aiObservability/shell/AiObservabilityScreenWidgets.tsx +6 -0
  339. package/src/widgets/aiObservability/shell/AiObservabilityStatusChip.tsx +10 -2
  340. package/src/widgets/aiObservability/shell/EvalPipelineOverview.tsx +68 -0
  341. package/src/widgets/aiObservability/shell/EvalStepBanner.tsx +24 -0
  342. package/src/widgets/aiObservability/shell/EvalStepper.test.tsx +80 -0
  343. package/src/widgets/aiObservability/shell/EvalStepper.tsx +126 -0
  344. package/src/widgets/aiObservability/shell/ObservabilityPageHeader.tsx +50 -0
  345. package/src/widgets/aiObservability/shell/ObservabilitySharedUi.test.tsx +140 -0
  346. package/src/widgets/aiObservability/shell/ObservabilityStates.tsx +81 -0
  347. package/src/widgets/aiObservability/shell/ObservabilityStatusBadge.tsx +29 -0
  348. package/src/widgets/aiObservability/shell/ObservabilityTable.test.tsx +7 -1
  349. package/src/widgets/aiObservability/shell/aiObservabilityNav.ts +16 -8
  350. package/src/widgets/aiObservability/shell/clientTable.test.ts +176 -0
  351. package/src/widgets/aiObservability/shell/clientTable.ts +251 -0
  352. package/src/widgets/aiObservability/shell/evalPipeline.ts +74 -0
  353. package/src/widgets/aiObservability/shell/observabilityStatus.test.ts +73 -0
  354. package/src/widgets/aiObservability/shell/observabilityStatus.ts +100 -0
  355. package/src/widgets/aiObservability/traces/AiTracesListScreen.tsx +41 -13
  356. package/src/widgets/aiObservability/traces/AiTracesListView.tsx +423 -300
  357. package/src/widgets/aiObservability/traces/traceTypes.test.ts +52 -1
  358. package/src/widgets/aiObservability/traces/traceTypes.ts +117 -2
  359. package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.test.ts +13 -1
  360. package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.ts +10 -2
  361. package/src/widgets/builtInFieldWidgets.test.tsx +16 -3
@@ -1,13 +1,17 @@
1
1
  import {
2
2
  Badge,
3
3
  Box,
4
- Button,
5
4
  DataTable,
6
5
  type DataTableCellData,
7
- type DataTableColumn,
6
+ type IconName,
7
+ SegmentedControl,
8
+ Spinner,
8
9
  Text,
9
10
  } from "@terreno/ui";
10
- import React, {useMemo} from "react";
11
+ import React, {useCallback, useMemo, useState} from "react";
12
+ import {AgreementMeter} from "../shell/AgreementMeter";
13
+ import {type ClientTableColumn, distinctChoiceOptions, useClientTable} from "../shell/clientTable";
14
+ import {ObservabilityEmptyState, ObservabilityErrorState} from "../shell/ObservabilityStates";
11
15
  import {
12
16
  EVALUATOR_TYPE_LABELS,
13
17
  type EvaluatorRecord,
@@ -16,6 +20,7 @@ import {
16
20
  } from "./evaluatorTypes";
17
21
 
18
22
  export interface AiEvaluatorsListViewProps {
23
+ /** Shows the empty state's New evaluator action; the header owns the main button. */
19
24
  canCreate?: boolean;
20
25
  evaluators: EvaluatorRecord[];
21
26
  isLoading: boolean;
@@ -25,33 +30,155 @@ export interface AiEvaluatorsListViewProps {
25
30
  onRetry: () => void;
26
31
  }
27
32
 
28
- const COLUMNS: DataTableColumn[] = [
29
- {columnType: "text", title: "Name", width: 200},
30
- {columnType: "evaluatorType", title: "Type", width: 120},
31
- {columnType: "text", title: "Dimensions", width: 200},
32
- {columnType: "text", title: "Target", width: 140},
33
- {columnType: "text", title: "Run modes", width: 220},
34
- {columnType: "evaluatorOpen", title: "", width: 88},
33
+ /** Icon per evaluator kind. Kind badges stay neutral: colour means state, not category. */
34
+ const EVALUATOR_TYPE_ICONS: Record<EvaluatorRecord["type"], IconName> = {
35
+ human: "user",
36
+ "json-assert": "code",
37
+ "llm-judge": "gavel",
38
+ };
39
+
40
+ /** Readable target labels, e.g. "full trace" -> "Full trace". */
41
+ export const EVALUATOR_TARGET_LABELS: Record<EvaluatorRecord["target"], string> = {
42
+ "dataset item": "Dataset item",
43
+ "full trace": "Full trace",
44
+ "generation span": "One AI step",
45
+ };
46
+
47
+ const KIND_FILTERS: Array<{label: string; type?: EvaluatorRecord["type"]}> = [
48
+ {label: "All"},
49
+ {label: EVALUATOR_TYPE_LABELS.human, type: "human"},
50
+ {label: EVALUATOR_TYPE_LABELS["llm-judge"], type: "llm-judge"},
51
+ {label: EVALUATOR_TYPE_LABELS["json-assert"], type: "json-assert"},
35
52
  ];
36
53
 
37
- const typeBadgeStatus = (type: EvaluatorRecord["type"]): "info" | "neutral" | "warning" => {
38
- if (type === "human") {
39
- return "neutral";
54
+ const runModesText = (evaluator: EvaluatorRecord): string | undefined => {
55
+ if (evaluator.type === "human") {
56
+ return "People in review";
40
57
  }
41
- if (type === "llm-judge") {
42
- return "info";
58
+ const chips = formatRunModeChips(evaluator.runModes);
59
+ return chips.length > 0 ? chips.join(" · ") : undefined;
60
+ };
61
+
62
+ const evaluatorSearchText = (evaluator: EvaluatorRecord): string => evaluator.name;
63
+
64
+ const buildColumns = (evaluators: EvaluatorRecord[]): ClientTableColumn<EvaluatorRecord>[] => {
65
+ const hasCalibration = evaluators.some((evaluator) => evaluator.calibration);
66
+ const columns: ClientTableColumn<EvaluatorRecord>[] = [
67
+ {
68
+ cell: (evaluator) => ({value: evaluator}),
69
+ columnType: "evaluatorName",
70
+ filter: {kind: "text"},
71
+ key: "name",
72
+ title: "Evaluator",
73
+ value: (evaluator) => evaluator.name,
74
+ width: 280,
75
+ },
76
+ {
77
+ columnType: "evaluatorType",
78
+ filter: {
79
+ kind: "choice",
80
+ options: distinctChoiceOptions(evaluators, (evaluator) => evaluator.type).map((option) => ({
81
+ label: EVALUATOR_TYPE_LABELS[option.value as EvaluatorRecord["type"]] ?? option.value,
82
+ value: option.value,
83
+ })),
84
+ },
85
+ key: "type",
86
+ title: "Kind",
87
+ value: (evaluator) => evaluator.type,
88
+ width: 160,
89
+ },
90
+ {
91
+ cell: (evaluator) => ({value: formatDimensionSummary(evaluator.dimensions)}),
92
+ filter: {kind: "text"},
93
+ key: "dimensions",
94
+ title: "Scores",
95
+ value: (evaluator) =>
96
+ evaluator.dimensions.length > 0 ? formatDimensionSummary(evaluator.dimensions) : undefined,
97
+ width: 220,
98
+ },
99
+ {
100
+ cell: (evaluator) => ({value: EVALUATOR_TARGET_LABELS[evaluator.target] ?? evaluator.target}),
101
+ filter: {
102
+ kind: "choice",
103
+ options: distinctChoiceOptions(evaluators, (evaluator) => evaluator.target).map(
104
+ (option) => ({
105
+ label:
106
+ EVALUATOR_TARGET_LABELS[option.value as EvaluatorRecord["target"]] ?? option.value,
107
+ value: option.value,
108
+ })
109
+ ),
110
+ },
111
+ key: "target",
112
+ title: "Looks at",
113
+ value: (evaluator) => evaluator.target,
114
+ width: 150,
115
+ },
116
+ {
117
+ cell: (evaluator) => ({value: runModesText(evaluator) ?? "—"}),
118
+ filter: {kind: "text"},
119
+ key: "runModes",
120
+ title: "Runs in",
121
+ value: runModesText,
122
+ width: 220,
123
+ },
124
+ ];
125
+ if (hasCalibration) {
126
+ columns.push({
127
+ cell: (evaluator) => ({value: evaluator}),
128
+ columnType: "evaluatorCalibration",
129
+ filter: {kind: "numberRange", label: "Agreement %"},
130
+ key: "calibration",
131
+ title: "Agrees with people",
132
+ value: (evaluator) =>
133
+ evaluator.calibration?.rate !== undefined
134
+ ? Math.round(evaluator.calibration.rate * 100)
135
+ : undefined,
136
+ width: 210,
137
+ });
43
138
  }
44
- return "warning";
139
+ return columns;
140
+ };
141
+
142
+ /** Neutral kind badge with an icon, e.g. a gavel for LLM judges. */
143
+ export const EvaluatorKindBadge: React.FC<{testID?: string; type: EvaluatorRecord["type"]}> = ({
144
+ testID,
145
+ type,
146
+ }) => {
147
+ return (
148
+ <Badge
149
+ iconName={EVALUATOR_TYPE_ICONS[type]}
150
+ secondary
151
+ status="neutral"
152
+ testID={testID}
153
+ value={EVALUATOR_TYPE_LABELS[type] ?? type}
154
+ />
155
+ );
45
156
  };
46
157
 
47
158
  const TypeBadgeCell: React.FC<{cellData: DataTableCellData}> = ({cellData}) => {
48
159
  const type = String(cellData.value) as EvaluatorRecord["type"];
160
+ return (
161
+ <Box alignItems="start" justifyContent="center">
162
+ <EvaluatorKindBadge testID={`ai-evaluator-type-${type}`} type={type} />
163
+ </Box>
164
+ );
165
+ };
166
+
167
+ const CalibrationCell: React.FC<{cellData: DataTableCellData}> = ({cellData}) => {
168
+ const evaluator = cellData.value as EvaluatorRecord;
169
+ if (evaluator.type !== "llm-judge") {
170
+ return (
171
+ <Box justifyContent="center">
172
+ <Text color="secondaryLight">—</Text>
173
+ </Box>
174
+ );
175
+ }
49
176
  return (
50
177
  <Box justifyContent="center">
51
- <Badge
52
- status={typeBadgeStatus(type)}
53
- testID={`ai-evaluator-type-${type}`}
54
- value={EVALUATOR_TYPE_LABELS[type]}
178
+ <AgreementMeter
179
+ compact
180
+ rate={evaluator.calibration?.rate}
181
+ testID={`ai-evaluator-calibration-${evaluator.id}`}
55
182
  />
56
183
  </Box>
57
184
  );
@@ -66,74 +193,132 @@ export const AiEvaluatorsListView: React.FC<AiEvaluatorsListViewProps> = ({
66
193
  onOpen,
67
194
  onRetry,
68
195
  }) => {
196
+ const [kindIndex, setKindIndex] = useState(0);
197
+ const kindType = KIND_FILTERS[kindIndex]?.type;
198
+
199
+ const kindRows = useMemo(() => {
200
+ if (!kindType) {
201
+ return evaluators;
202
+ }
203
+ return evaluators.filter((evaluator) => evaluator.type === kindType);
204
+ }, [evaluators, kindType]);
205
+
206
+ const kindBadges = useMemo(() => {
207
+ return KIND_FILTERS.map((entry) => ({
208
+ count: entry.type
209
+ ? evaluators.filter((evaluator) => evaluator.type === entry.type).length
210
+ : evaluators.length,
211
+ status: "neutral" as const,
212
+ }));
213
+ }, [evaluators]);
214
+
215
+ const handleKindChange = useCallback((index: number): void => {
216
+ setKindIndex(index);
217
+ }, []);
218
+
69
219
  const customColumnComponentMap = useMemo(
70
220
  () => ({
71
- evaluatorOpen: ({cellData}: {cellData: DataTableCellData}) => (
72
- <Box justifyContent="center">
73
- <Button
221
+ evaluatorCalibration: CalibrationCell,
222
+ evaluatorName: ({cellData}: {cellData: DataTableCellData}) => {
223
+ const evaluator = cellData.value as EvaluatorRecord;
224
+ return (
225
+ <Box
226
+ accessibilityHint="Opens the evaluator"
227
+ accessibilityLabel={`Open evaluator ${evaluator.name}`}
228
+ flex="grow"
229
+ gap={1}
230
+ justifyContent="center"
74
231
  onClick={() => {
75
- onOpen(String(cellData.value ?? ""));
232
+ onOpen(evaluator.id);
76
233
  }}
77
- size="sm"
78
- testID={`ai-evaluator-open-${String(cellData.value ?? "")}`}
79
- text="Open"
80
- variant="outline"
81
- />
82
- </Box>
83
- ),
234
+ >
235
+ <Box testID={`ai-evaluator-open-${evaluator.id}`}>
236
+ <Text bold numberOfLines={1}>
237
+ {evaluator.name}
238
+ </Text>
239
+ </Box>
240
+ {evaluator.description ? (
241
+ <Text color="secondaryLight" numberOfLines={1} size="sm">
242
+ {evaluator.description}
243
+ </Text>
244
+ ) : null}
245
+ </Box>
246
+ );
247
+ },
84
248
  evaluatorType: TypeBadgeCell,
85
249
  }),
86
250
  [onOpen]
87
251
  );
88
252
 
89
- const rows: DataTableCellData[][] = useMemo(() => {
90
- return evaluators.map((evaluator) => {
91
- const chips = formatRunModeChips(evaluator.runModes);
92
- return [
93
- {value: evaluator.name},
94
- {value: evaluator.type},
95
- {value: formatDimensionSummary(evaluator.dimensions)},
96
- {value: evaluator.target},
97
- {value: chips.length > 0 ? chips.join(" · ") : "—"},
98
- {value: evaluator.id},
99
- ];
100
- });
101
- }, [evaluators]);
253
+ const columns = useMemo(() => buildColumns(evaluators), [evaluators]);
254
+ const {rows: _sortedEvaluators, ...table} = useClientTable({
255
+ columns,
256
+ rows: kindRows,
257
+ searchText: evaluatorSearchText,
258
+ });
102
259
 
103
- return (
104
- <Box gap={3} testID="ai-evaluators-list">
105
- <Box direction="row" gap={2} justifyContent="between" wrap>
106
- <Text color="secondaryDark" size="sm">
107
- Schema is checked on save. Live sampling bills judge calls at the configured rate.
108
- </Text>
109
- {canCreate ? (
110
- <Button onClick={onCreate} testID="ai-evaluators-create" text="Create evaluator" />
111
- ) : undefined}
112
- </Box>
113
- {loadError ? (
114
- <Box gap={2} testID="ai-evaluators-load-error">
115
- <Text color="error">{loadError}</Text>
116
- <Button onClick={onRetry} text="Retry" variant="secondary" />
260
+ const renderBody = (): React.ReactNode => {
261
+ if (isLoading) {
262
+ return (
263
+ <Box alignItems="center" padding={5} testID="ai-evaluators-loading">
264
+ <Spinner />
117
265
  </Box>
118
- ) : undefined}
119
- {isLoading ? (
120
- <Box padding={4} testID="ai-evaluators-loading">
121
- <Text>Loading evaluators…</Text>
122
- </Box>
123
- ) : evaluators.length === 0 ? (
124
- <Box padding={4} testID="ai-evaluators-empty">
125
- <Text color="secondaryDark">
126
- No evaluators yet. Create one to score traces and experiments.
127
- </Text>
128
- </Box>
129
- ) : (
130
- <DataTable
131
- columns={COLUMNS}
132
- customColumnComponentMap={customColumnComponentMap}
133
- data={rows}
134
- testID="ai-evaluators-table"
266
+ );
267
+ }
268
+ if (loadError) {
269
+ return (
270
+ <ObservabilityErrorState
271
+ onRetry={onRetry}
272
+ testID="ai-evaluators-load-error"
273
+ text={loadError}
274
+ />
275
+ );
276
+ }
277
+ if (evaluators.length === 0) {
278
+ return (
279
+ <ObservabilityEmptyState
280
+ actionIconName="plus"
281
+ actionText={canCreate ? "New evaluator" : undefined}
282
+ description="Create one to score traces, datasets and experiments."
283
+ iconName="gavel"
284
+ onAction={canCreate ? onCreate : undefined}
285
+ testID="ai-evaluators-empty"
286
+ title="No evaluators yet"
135
287
  />
136
- )}
288
+ );
289
+ }
290
+ return (
291
+ <DataTable
292
+ {...table}
293
+ customColumnComponentMap={customColumnComponentMap}
294
+ emptyContent={
295
+ <ObservabilityEmptyState
296
+ description="Pick another kind or clear a filter."
297
+ iconName="filter"
298
+ testID="ai-evaluators-no-matches"
299
+ title="No evaluators match"
300
+ />
301
+ }
302
+ rowHeight={64}
303
+ testID="ai-evaluators-table"
304
+ />
305
+ );
306
+ };
307
+
308
+ return (
309
+ <Box gap={4} testID="ai-evaluators-list">
310
+ {!isLoading && !loadError && evaluators.length > 0 ? (
311
+ <Box alignItems="start" testID="ai-evaluators-kind-filter">
312
+ <SegmentedControl
313
+ badges={kindBadges}
314
+ items={KIND_FILTERS.map((entry) => entry.label)}
315
+ onChange={handleKindChange}
316
+ selectedIndex={kindIndex}
317
+ size="sm"
318
+ />
319
+ </Box>
320
+ ) : null}
321
+ {renderBody()}
137
322
  </Box>
138
323
  );
139
324
  };
@@ -84,6 +84,13 @@ describe("evaluatorTypes helpers", () => {
84
84
  ),
85
85
  {dataType: "boolean", key: "score", required: true}
86
86
  );
87
+ assert.deepEqual(
88
+ dimensionForDataType(
89
+ {dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true},
90
+ "boolean"
91
+ ),
92
+ {dataType: "boolean", key: "faithful", question: "Is it faithful?", required: true}
93
+ );
87
94
  assert.equal(
88
95
  dimensionForDataType({dataType: "boolean", key: "score", required: true}, "categorical")
89
96
  .range,
@@ -1,6 +1,8 @@
1
1
  export interface EvaluatorDimension {
2
2
  dataType: "boolean" | "categorical" | "numeric";
3
3
  key: string;
4
+ /** Question shown to human reviewers in place of the score key. */
5
+ question?: string;
4
6
  range?: string;
5
7
  required: boolean;
6
8
  }
@@ -11,8 +13,26 @@ export interface EvaluatorRunModes {
11
13
  liveSampleRate: number;
12
14
  }
13
15
 
16
+ /** How well an llm-judge agreed with human answers in its latest judge calibration experiment. */
17
+ export interface EvaluatorCalibration {
18
+ datasetName?: string;
19
+ experimentId: string;
20
+ experimentName: string;
21
+ passed: boolean;
22
+ questionSetName?: string;
23
+ /** Agreement rate (0–1) of `version`; unset when nothing could be compared. */
24
+ rate?: number;
25
+ /** True when the question set's questions changed after the experiment ran. */
26
+ stale: boolean;
27
+ targetAgreement?: number;
28
+ /** The judge prompt's production version when it was compared, else the best version. */
29
+ version: number;
30
+ }
31
+
14
32
  export interface EvaluatorRecord {
15
33
  assertion?: {constraint: string; path: string};
34
+ /** llm-judge only: the latest judge calibration against human answers. */
35
+ calibration?: EvaluatorCalibration;
16
36
  confidenceAlertBelow: number;
17
37
  description?: string;
18
38
  dimensions: EvaluatorDimension[];
@@ -100,6 +120,7 @@ export const dimensionForDataType = (
100
120
  return {
101
121
  dataType,
102
122
  key: dimension.key,
123
+ ...(dimension.question ? {question: dimension.question} : {}),
103
124
  required: dimension.required,
104
125
  };
105
126
  }
@@ -196,3 +217,64 @@ export const parseApiErrorTitle = (error: unknown): string | undefined => {
196
217
  }
197
218
  return undefined;
198
219
  };
220
+
221
+ export interface CalibrationSummary {
222
+ /** True when the judge passed its latest calibration and the questions have not changed. */
223
+ isTrusted: boolean;
224
+ label: string;
225
+ status: "neutral" | "success" | "warning";
226
+ }
227
+
228
+ const percent = (rate?: number): string => {
229
+ if (rate === undefined || Number.isNaN(rate)) {
230
+ return "—";
231
+ }
232
+ return `${Math.round(rate * 1000) / 10}%`;
233
+ };
234
+
235
+ /**
236
+ * One-line calibration status for an llm-judge, such as
237
+ * "Calibrated: 95% agreement on summary-review-set (v2)".
238
+ */
239
+ export const summarizeCalibration = (calibration?: EvaluatorCalibration): CalibrationSummary => {
240
+ if (!calibration) {
241
+ return {
242
+ isTrusted: false,
243
+ label: "Not calibrated — run a judge calibration (step 3) first",
244
+ status: "neutral",
245
+ };
246
+ }
247
+ const where = calibration.datasetName ? ` on ${calibration.datasetName}` : "";
248
+ const rate = percent(calibration.rate);
249
+ if (calibration.stale) {
250
+ return {
251
+ isTrusted: false,
252
+ label: `Calibration stale: the questions changed after ${calibration.experimentName} — re-judge and recalibrate`,
253
+ status: "warning",
254
+ };
255
+ }
256
+ if (!calibration.passed) {
257
+ const target = percent(calibration.targetAgreement ?? 0.9);
258
+ return {
259
+ isTrusted: false,
260
+ label: `Below target: ${rate} agreement${where} (v${calibration.version}, needs ${target})`,
261
+ status: "warning",
262
+ };
263
+ }
264
+ return {
265
+ isTrusted: true,
266
+ label: `Calibrated: ${rate} agreement${where} (v${calibration.version})`,
267
+ status: "success",
268
+ };
269
+ };
270
+
271
+ /** Question keys the judge and the question set both answer. */
272
+ export const sharedQuestionKeys = (
273
+ judge: Pick<EvaluatorRecord, "dimensions">,
274
+ questionSet: Pick<EvaluatorRecord, "dimensions">
275
+ ): string[] => {
276
+ const judgeKeys = new Set(judge.dimensions.map((dimension) => dimension.key));
277
+ return questionSet.dimensions
278
+ .map((dimension) => dimension.key)
279
+ .filter((key) => judgeKeys.has(key));
280
+ };