@terreno/admin-frontend 57.20.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/dist/AdminFilterDrawer.d.ts.map +1 -1
- package/dist/AdminFilterDrawer.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js +19 -9
- package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts +7 -2
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js +155 -43
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts +20 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.js +17 -12
- package/dist/widgets/aiObservability/datasets/datasetTypes.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js +12 -8
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js +4 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts +8 -0
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js +68 -37
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js +4 -2
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts +8 -0
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js +156 -33
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts +32 -0
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js +49 -0
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js +235 -12
- package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts +10 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js +113 -56
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js +53 -38
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts +2 -3
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js +117 -32
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js +13 -4
- package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts +5 -0
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js +157 -55
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts +41 -0
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts.map +1 -0
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js +113 -0
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js.map +1 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts +30 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts.map +1 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.js +29 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.js.map +1 -0
- package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts +82 -3
- package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/experimentTypes.js +172 -6
- package/dist/widgets/aiObservability/experiments/experimentTypes.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts +34 -4
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js +11 -4
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js +24 -5
- package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts +2 -0
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js +36 -6
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts +3 -0
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.js +6 -6
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js +12 -12
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.js +156 -35
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/promptTypes.d.ts +13 -3
- package/dist/widgets/aiObservability/prompts/promptTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/promptTypes.js +61 -19
- package/dist/widgets/aiObservability/prompts/promptTypes.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts +11 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js +115 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts +22 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.js +136 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts +4 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js +92 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts +26 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.js +115 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewItemScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewItemScreen.js +73 -16
- package/dist/widgets/aiObservability/review/AiReviewItemScreen.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts +19 -5
- package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewItemView.js +14 -9
- package/dist/widgets/aiObservability/review/AiReviewItemView.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js +16 -2
- package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts +11 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueView.js +96 -37
- package/dist/widgets/aiObservability/review/AiReviewQueueView.js.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts +13 -0
- package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/ReviewQuestionList.js +28 -0
- package/dist/widgets/aiObservability/review/ReviewQuestionList.js.map +1 -0
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts +3 -0
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js +2 -2
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts +14 -0
- package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewScoreField.js +41 -12
- package/dist/widgets/aiObservability/review/ReviewScoreField.js.map +1 -1
- package/dist/widgets/aiObservability/review/batchTypes.d.ts +103 -0
- package/dist/widgets/aiObservability/review/batchTypes.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/batchTypes.js +61 -0
- package/dist/widgets/aiObservability/review/batchTypes.js.map +1 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts +21 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.js +141 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.js.map +1 -0
- package/dist/widgets/aiObservability/review/reviewTypes.d.ts +23 -2
- package/dist/widgets/aiObservability/review/reviewTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/reviewTypes.js +25 -0
- package/dist/widgets/aiObservability/review/reviewTypes.js.map +1 -1
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts +48 -0
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js +49 -0
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts +30 -0
- package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/AgreementMeter.js +32 -0
- package/dist/widgets/aiObservability/shell/AgreementMeter.js.map +1 -0
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts +10 -0
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js +6 -4
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts +3 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js +5 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js +3 -3
- package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js.map +1 -1
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts +14 -0
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js +26 -0
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts +11 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.js +8 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.js.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.d.ts +22 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.js +28 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.js.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts +17 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js +10 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts +24 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.js +11 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.js.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts +11 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js +9 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js.map +1 -0
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts +1 -0
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.js +15 -8
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.js.map +1 -1
- package/dist/widgets/aiObservability/shell/clientTable.d.ts +48 -0
- package/dist/widgets/aiObservability/shell/clientTable.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/clientTable.js +163 -0
- package/dist/widgets/aiObservability/shell/clientTable.js.map +1 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.d.ts +24 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.js +43 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.js.map +1 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts +23 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.js +72 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.js.map +1 -0
- package/dist/widgets/aiObservability/traces/AiTracesListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListScreen.js +18 -9
- package/dist/widgets/aiObservability/traces/AiTracesListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts +18 -7
- package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListView.js +198 -78
- package/dist/widgets/aiObservability/traces/AiTracesListView.js.map +1 -1
- package/dist/widgets/aiObservability/traces/traceTypes.d.ts +23 -1
- package/dist/widgets/aiObservability/traces/traceTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/traceTypes.js +92 -1
- package/dist/widgets/aiObservability/traces/traceTypes.js.map +1 -1
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts +2 -0
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js +9 -2
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js.map +1 -1
- package/package.json +7 -3
- package/src/AdminBreadcrumbs.test.tsx +1 -1
- package/src/AdminFieldRenderer.test.tsx +2 -1
- package/src/AdminFilterDrawer.tsx +1 -2
- package/src/AdminHome.test.tsx +2 -1
- package/src/AdminModelTable.test.tsx +16 -12
- package/src/AdminModelTable.windowed.test.tsx +11 -5
- package/src/AdminPrimitiveArrayField.test.tsx +3 -3
- package/src/AdminProvider.test.tsx +11 -6
- package/src/AdminRolesList.test.tsx +22 -13
- package/src/AdminScreenPage.test.tsx +1 -1
- package/src/AdminScreenRouter.test.tsx +15 -5
- package/src/AdminScriptList.test.tsx +9 -2
- package/src/AdminShell.test.tsx +1 -1
- package/src/AdminVersionConfig.test.tsx +4 -4
- package/src/AnnouncementOverviewScreenWidget.test.tsx +6 -1
- package/src/CheckboxListEditor.test.tsx +11 -9
- package/src/ConfigurationScreen.test.tsx +2 -2
- package/src/adminRequest.test.ts +3 -3
- package/src/adminWindowedTable.test.ts +4 -2
- package/src/comms/CommsDashboardScreen.test.tsx +5 -5
- package/src/comms/CommsMessageDetail.test.tsx +1 -1
- package/src/index.tsx +4 -0
- package/src/isolated/AdminFieldRenderer.isolated.tsx +11 -6
- package/src/isolated/AdminFilterDrawer.isolated.tsx +8 -6
- package/src/isolated/AdminFilterDrawerMobile.isolated.tsx +3 -1
- package/src/isolated/AdminHomeWidgets.isolated.tsx +2 -0
- package/src/isolated/AdminNestedArrayField.isolated.tsx +10 -9
- package/src/isolated/AdminObjectPicker.isolated.tsx +9 -10
- package/src/isolated/AiPromptEditorScreen.isolated.tsx +24 -0
- package/src/isolated/AiPromptsListScreen.isolated.tsx +1 -1
- package/src/isolated/DocumentStorageBrowser.isolated.tsx +16 -8
- package/src/isolated/RecentActivityWidget.isolated.tsx +2 -0
- package/src/isolated/adminWidgets.isolated.tsx +46 -6
- package/src/isolated/aiListTables.isolated.tsx +654 -0
- package/src/isolated/aiObservabilityScreens.isolated.tsx +731 -294
- package/src/isolated/aiReviewBatchScreens.isolated.tsx +378 -0
- package/src/isolated/aiTracesTable.isolated.tsx +375 -0
- package/src/isolated/generateConsentHistoryPdf.isolated.ts +30 -10
- package/src/isolated/generateConsentPdfsMobile.isolated.ts +24 -3
- package/src/isolated/hooks.isolated.tsx +84 -60
- package/src/isolated/rpcDualRun.isolated.tsx +9 -2
- package/src/jobs/JobsDashboardScreen.test.tsx +3 -3
- package/src/jobs/JobsHomeWidget.test.tsx +1 -1
- package/src/jobs/JobsJobDetail.test.tsx +1 -1
- package/src/orgs/OrgManagementScreens.test.tsx +1 -1
- package/src/orgs/adminModelEndpoints.test.ts +1 -1
- package/src/testing/useAdminApiDouble.ts +2 -1
- package/src/types.ts +2 -0
- package/src/useAdminRpc.test.tsx +3 -3
- package/src/widgets/AIRequestsScreenWidget.test.tsx +20 -3
- package/src/widgets/ModelsGridWidget.test.tsx +25 -5
- package/src/widgets/aiObservability/datasets/AiDatasetDetailView.test.tsx +4 -2
- package/src/widgets/aiObservability/datasets/AiDatasetsListScreen.tsx +44 -13
- package/src/widgets/aiObservability/datasets/AiDatasetsListView.tsx +303 -103
- package/src/widgets/aiObservability/datasets/datasetTypes.test.ts +0 -14
- package/src/widgets/aiObservability/datasets/datasetTypes.ts +37 -13
- package/src/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.tsx +34 -11
- package/src/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.tsx +12 -1
- package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.test.tsx +186 -32
- package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.tsx +235 -131
- package/src/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.tsx +21 -2
- package/src/widgets/aiObservability/evaluators/AiEvaluatorsListView.tsx +261 -76
- package/src/widgets/aiObservability/evaluators/evaluatorTypes.test.ts +7 -0
- package/src/widgets/aiObservability/evaluators/evaluatorTypes.ts +82 -0
- package/src/widgets/aiObservability/experiments/AiExperimentNewScreen.tsx +271 -13
- package/src/widgets/aiObservability/experiments/AiExperimentNewView.test.tsx +168 -2
- package/src/widgets/aiObservability/experiments/AiExperimentNewView.tsx +400 -186
- package/src/widgets/aiObservability/experiments/AiExperimentResultsScreen.tsx +104 -45
- package/src/widgets/aiObservability/experiments/AiExperimentResultsView.test.tsx +290 -0
- package/src/widgets/aiObservability/experiments/AiExperimentResultsView.tsx +342 -97
- package/src/widgets/aiObservability/experiments/AiExperimentsListScreen.tsx +35 -4
- package/src/widgets/aiObservability/experiments/AiExperimentsListView.tsx +302 -112
- package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.test.tsx +250 -0
- package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.tsx +468 -0
- package/src/widgets/aiObservability/experiments/ExperimentWizard.tsx +106 -0
- package/src/widgets/aiObservability/experiments/experimentTypes.test.ts +103 -16
- package/src/widgets/aiObservability/experiments/experimentTypes.ts +250 -10
- package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.test.ts +35 -0
- package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.ts +55 -9
- package/src/widgets/aiObservability/prompts/AiPromptEditorScreen.tsx +26 -2
- package/src/widgets/aiObservability/prompts/AiPromptEditorView.tsx +95 -1
- package/src/widgets/aiObservability/prompts/AiPromptHubView.test.tsx +61 -1
- package/src/widgets/aiObservability/prompts/AiPromptHubView.tsx +71 -27
- package/src/widgets/aiObservability/prompts/AiPromptsListScreen.tsx +34 -15
- package/src/widgets/aiObservability/prompts/AiPromptsListView.test.tsx +104 -8
- package/src/widgets/aiObservability/prompts/AiPromptsListView.tsx +287 -117
- package/src/widgets/aiObservability/prompts/promptTypes.test.ts +63 -1
- package/src/widgets/aiObservability/prompts/promptTypes.ts +83 -22
- package/src/widgets/aiObservability/review/AiReviewBatchScreen.tsx +217 -0
- package/src/widgets/aiObservability/review/AiReviewBatchView.tsx +398 -0
- package/src/widgets/aiObservability/review/AiReviewBatchViews.test.tsx +328 -0
- package/src/widgets/aiObservability/review/AiReviewBatchesScreen.tsx +137 -0
- package/src/widgets/aiObservability/review/AiReviewBatchesView.tsx +333 -0
- package/src/widgets/aiObservability/review/AiReviewItemScreen.tsx +121 -25
- package/src/widgets/aiObservability/review/AiReviewItemView.tsx +112 -70
- package/src/widgets/aiObservability/review/AiReviewQueueScreen.tsx +29 -2
- package/src/widgets/aiObservability/review/AiReviewQueueView.tsx +183 -75
- package/src/widgets/aiObservability/review/ReviewQuestionList.tsx +128 -0
- package/src/widgets/aiObservability/review/ReviewReadOnlyPanel.tsx +21 -13
- package/src/widgets/aiObservability/review/ReviewScoreField.tsx +111 -27
- package/src/widgets/aiObservability/review/batchTypes.test.ts +207 -0
- package/src/widgets/aiObservability/review/batchTypes.ts +162 -0
- package/src/widgets/aiObservability/review/reviewShortcuts.ts +195 -0
- package/src/widgets/aiObservability/review/reviewTypes.ts +62 -2
- package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.test.ts +74 -0
- package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.ts +98 -0
- package/src/widgets/aiObservability/shell/AgreementMeter.tsx +103 -0
- package/src/widgets/aiObservability/shell/AiObservabilityChrome.tsx +48 -14
- package/src/widgets/aiObservability/shell/AiObservabilityScreenWidgets.tsx +6 -0
- package/src/widgets/aiObservability/shell/AiObservabilityStatusChip.tsx +10 -2
- package/src/widgets/aiObservability/shell/EvalPipelineOverview.tsx +68 -0
- package/src/widgets/aiObservability/shell/EvalStepBanner.tsx +24 -0
- package/src/widgets/aiObservability/shell/EvalStepper.test.tsx +80 -0
- package/src/widgets/aiObservability/shell/EvalStepper.tsx +126 -0
- package/src/widgets/aiObservability/shell/ObservabilityPageHeader.tsx +50 -0
- package/src/widgets/aiObservability/shell/ObservabilitySharedUi.test.tsx +140 -0
- package/src/widgets/aiObservability/shell/ObservabilityStates.tsx +81 -0
- package/src/widgets/aiObservability/shell/ObservabilityStatusBadge.tsx +29 -0
- package/src/widgets/aiObservability/shell/ObservabilityTable.test.tsx +7 -1
- package/src/widgets/aiObservability/shell/aiObservabilityNav.ts +16 -8
- package/src/widgets/aiObservability/shell/clientTable.test.ts +176 -0
- package/src/widgets/aiObservability/shell/clientTable.ts +251 -0
- package/src/widgets/aiObservability/shell/evalPipeline.ts +74 -0
- package/src/widgets/aiObservability/shell/observabilityStatus.test.ts +73 -0
- package/src/widgets/aiObservability/shell/observabilityStatus.ts +100 -0
- package/src/widgets/aiObservability/traces/AiTracesListScreen.tsx +41 -13
- package/src/widgets/aiObservability/traces/AiTracesListView.tsx +423 -300
- package/src/widgets/aiObservability/traces/traceTypes.test.ts +52 -1
- package/src/widgets/aiObservability/traces/traceTypes.ts +117 -2
- package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.test.ts +13 -1
- package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.ts +10 -2
- package/src/widgets/builtInFieldWidgets.test.tsx +16 -3
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import {describe, it} from "bun:test";
|
|
2
2
|
import {assert} from "chai";
|
|
3
|
-
import type {ExperimentRecord} from "./experimentTypes";
|
|
3
|
+
import type {ExperimentRecord, VersionGateSummary} from "./experimentTypes";
|
|
4
4
|
import {
|
|
5
|
-
|
|
5
|
+
experimentItemColumnWidths,
|
|
6
6
|
failingGateCount,
|
|
7
|
-
|
|
7
|
+
formatGateActual,
|
|
8
|
+
formatGateName,
|
|
9
|
+
formatGateThreshold,
|
|
10
|
+
formatPromoteBlockedReason,
|
|
11
|
+
formatVersionGateVerdict,
|
|
8
12
|
isValidExperimentVersionCount,
|
|
9
13
|
parsePromoteBlockedTitle,
|
|
14
|
+
summarizeGatesByVersion,
|
|
10
15
|
unwrapExperimentList,
|
|
11
16
|
unwrapExperimentRecord,
|
|
12
17
|
unwrapObservabilityPayload,
|
|
@@ -55,6 +60,14 @@ const experiment: ExperimentRecord = {
|
|
|
55
60
|
versions: [1, 2],
|
|
56
61
|
};
|
|
57
62
|
|
|
63
|
+
const summaryFor = (version: number): VersionGateSummary => {
|
|
64
|
+
const summary = summarizeGatesByVersion(experiment).find((entry) => entry.version === version);
|
|
65
|
+
if (!summary) {
|
|
66
|
+
throw new Error(`No gate summary for v${version}`);
|
|
67
|
+
}
|
|
68
|
+
return summary;
|
|
69
|
+
};
|
|
70
|
+
|
|
58
71
|
describe("experimentTypes helpers", () => {
|
|
59
72
|
it("unwraps experiment payloads", () => {
|
|
60
73
|
assert.deepEqual(unwrapObservabilityPayload({data: experiment}), experiment);
|
|
@@ -64,21 +77,13 @@ describe("experimentTypes helpers", () => {
|
|
|
64
77
|
assert.isUndefined(unwrapExperimentRecord({bad: true}));
|
|
65
78
|
});
|
|
66
79
|
|
|
67
|
-
it("counts failing gates and
|
|
80
|
+
it("counts failing gates and groups gates by version", () => {
|
|
68
81
|
assert.equal(failingGateCount(experiment), 1);
|
|
69
82
|
assert.equal(failingGateCount({...experiment, results: undefined}), 0);
|
|
70
|
-
assert.equal(
|
|
71
|
-
assert.
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
it("computes progress percent with zero-safe denominator", () => {
|
|
75
|
-
assert.equal(experimentProgressPercent(experiment), 50);
|
|
76
|
-
assert.equal(
|
|
77
|
-
experimentProgressPercent({
|
|
78
|
-
...experiment,
|
|
79
|
-
results: {...experiment.results!, progress: {completed: 0, total: 0}},
|
|
80
|
-
}),
|
|
81
|
-
0
|
|
83
|
+
assert.equal(summaryFor(2).gates.length, 1);
|
|
84
|
+
assert.deepEqual(
|
|
85
|
+
summarizeGatesByVersion({...experiment, versions: [3]}).map((summary) => summary.gates),
|
|
86
|
+
[[]]
|
|
82
87
|
);
|
|
83
88
|
});
|
|
84
89
|
|
|
@@ -97,4 +102,86 @@ describe("experimentTypes helpers", () => {
|
|
|
97
102
|
assert.isTrue(isValidExperimentVersionCount(3));
|
|
98
103
|
assert.isFalse(isValidExperimentVersionCount(4));
|
|
99
104
|
});
|
|
105
|
+
|
|
106
|
+
it("stretches output columns to fill the measured table width", () => {
|
|
107
|
+
assert.deepEqual(experimentItemColumnWidths({containerWidth: 1200, versionCount: 2}), {
|
|
108
|
+
item: 120,
|
|
109
|
+
output: 460,
|
|
110
|
+
pass: 80,
|
|
111
|
+
});
|
|
112
|
+
assert.equal(experimentItemColumnWidths({containerWidth: 1200, versionCount: 3}).output, 280);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it("keeps output columns readable when the table is narrow or unmeasured", () => {
|
|
116
|
+
assert.equal(experimentItemColumnWidths({containerWidth: 0, versionCount: 2}).output, 180);
|
|
117
|
+
assert.equal(experimentItemColumnWidths({containerWidth: 500, versionCount: 3}).output, 180);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("shows gate operators as symbols, never API words", () => {
|
|
121
|
+
assert.equal(formatGateThreshold({op: "gte", value: 0.8}), "≥ 0.8");
|
|
122
|
+
assert.equal(formatGateThreshold({op: "lte", value: 0.2}), "≤ 0.2");
|
|
123
|
+
assert.equal(formatGateThreshold({op: "gt" as never, value: 1}), "> 1");
|
|
124
|
+
assert.equal(formatGateThreshold({op: "lt" as never, value: 1}), "< 1");
|
|
125
|
+
assert.equal(formatGateThreshold({op: "eq", value: 1}), "= 1");
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it("names gates in plain words without repeating a shared name", () => {
|
|
129
|
+
assert.equal(
|
|
130
|
+
formatGateName({dimension: "helpfulness", evaluatorName: "helpfulness"}),
|
|
131
|
+
"helpfulness"
|
|
132
|
+
);
|
|
133
|
+
assert.equal(
|
|
134
|
+
formatGateName({dimension: "correct", evaluatorName: "correctness"}),
|
|
135
|
+
"correctness: correct"
|
|
136
|
+
);
|
|
137
|
+
assert.equal(
|
|
138
|
+
formatGateName({dimension: "schema_valid", evaluatorName: "schema-assert"}),
|
|
139
|
+
"schema-assert: schema valid"
|
|
140
|
+
);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("formats measured values", () => {
|
|
144
|
+
assert.equal(formatGateActual(0.625), "0.63");
|
|
145
|
+
assert.equal(formatGateActual(undefined), "—");
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it("summarizes gates per version in version order", () => {
|
|
149
|
+
const summaries = summarizeGatesByVersion(experiment);
|
|
150
|
+
assert.deepEqual(
|
|
151
|
+
summaries.map((summary) => [summary.version, summary.passed, summary.failing.length]),
|
|
152
|
+
[
|
|
153
|
+
[1, true, 0],
|
|
154
|
+
[2, false, 1],
|
|
155
|
+
]
|
|
156
|
+
);
|
|
157
|
+
assert.equal(formatVersionGateVerdict(summaryFor(1)), "Passes its gate");
|
|
158
|
+
assert.equal(formatVersionGateVerdict(summaryFor(2)), "Fails 1 of 1 gate");
|
|
159
|
+
assert.equal(
|
|
160
|
+
formatVersionGateVerdict({failing: [], gates: [], passed: false, version: 3}),
|
|
161
|
+
"No gates"
|
|
162
|
+
);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it("explains which gates block promoting a version, with correct plurals", () => {
|
|
166
|
+
const failing = summaryFor(2);
|
|
167
|
+
assert.isUndefined(formatPromoteBlockedReason(summaryFor(1)));
|
|
168
|
+
assert.equal(
|
|
169
|
+
formatPromoteBlockedReason(failing),
|
|
170
|
+
"v2 fails 1 gate: quality: correct ≥ 0.8 (got 0.50) — can't promote v2."
|
|
171
|
+
);
|
|
172
|
+
const [gate] = failing.gates;
|
|
173
|
+
if (!gate) {
|
|
174
|
+
throw new Error("Expected a v2 gate");
|
|
175
|
+
}
|
|
176
|
+
const twoFailing: VersionGateSummary = {
|
|
177
|
+
failing: [gate, {...gate, dimension: "helpfulness", evaluatorName: "helpfulness"}],
|
|
178
|
+
gates: [gate],
|
|
179
|
+
passed: false,
|
|
180
|
+
version: 2,
|
|
181
|
+
};
|
|
182
|
+
assert.equal(
|
|
183
|
+
formatPromoteBlockedReason(twoFailing),
|
|
184
|
+
"v2 fails 2 gates: quality: correct ≥ 0.8 (got 0.50); helpfulness ≥ 0.8 (got 0.50) — can't promote v2."
|
|
185
|
+
);
|
|
186
|
+
});
|
|
100
187
|
});
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import type {PromptDetail} from "../prompts/promptTypes";
|
|
2
|
+
import type {JudgeAgreement, ScoreValue} from "../review/batchTypes";
|
|
3
|
+
|
|
4
|
+
export type ExperimentKind = "judge" | "prompt";
|
|
5
|
+
|
|
1
6
|
export interface ExperimentEstimate {
|
|
2
7
|
costUsd?: number;
|
|
3
8
|
generations: number;
|
|
@@ -5,7 +10,8 @@ export interface ExperimentEstimate {
|
|
|
5
10
|
}
|
|
6
11
|
|
|
7
12
|
export interface ScoreThreshold {
|
|
8
|
-
|
|
13
|
+
/** `agreement` gates compare a judge prompt version's answers with human answers. */
|
|
14
|
+
aggregate: "agreement" | "mean" | "trueRate";
|
|
9
15
|
dimension: string;
|
|
10
16
|
evaluatorName: string;
|
|
11
17
|
op: "eq" | "gte" | "lte";
|
|
@@ -19,6 +25,8 @@ export interface ExperimentGateResult extends ScoreThreshold {
|
|
|
19
25
|
}
|
|
20
26
|
|
|
21
27
|
export interface ExperimentAggregates {
|
|
28
|
+
/** Judge experiments: agreement with the human answers, keyed by judge prompt version. */
|
|
29
|
+
agreement?: Record<string, JudgeAgreement>;
|
|
22
30
|
gates: ExperimentGateResult[];
|
|
23
31
|
lowConfidenceItemIds: string[];
|
|
24
32
|
outlierItemIds: string[];
|
|
@@ -41,6 +49,8 @@ export interface ExperimentVersionResult {
|
|
|
41
49
|
export interface ExperimentItemRecord {
|
|
42
50
|
datasetItemId: string;
|
|
43
51
|
failed: boolean;
|
|
52
|
+
/** Judge experiments: the human answers each judge version is compared with. */
|
|
53
|
+
humanScores?: Record<string, ScoreValue>;
|
|
44
54
|
id: string;
|
|
45
55
|
versionResults: Record<string, ExperimentVersionResult>;
|
|
46
56
|
}
|
|
@@ -49,16 +59,30 @@ export interface ExperimentRecord {
|
|
|
49
59
|
backgroundTaskId?: string;
|
|
50
60
|
created: string;
|
|
51
61
|
datasetId: string;
|
|
62
|
+
datasetName?: string;
|
|
52
63
|
estimate?: ExperimentEstimate;
|
|
53
64
|
evaluatorIds: string[];
|
|
54
65
|
id: string;
|
|
55
66
|
includeUnproofread: boolean;
|
|
56
67
|
items: ExperimentItemRecord[];
|
|
68
|
+
/** Judge experiments: the llm-judge evaluator whose prompt versions are compared. */
|
|
69
|
+
judgeEvaluatorId?: string;
|
|
70
|
+
judgeName?: string;
|
|
71
|
+
/** Older payloads leave it out; treat a missing kind as a prompt comparison. */
|
|
72
|
+
kind?: ExperimentKind;
|
|
57
73
|
modelOverride?: string;
|
|
58
74
|
name: string;
|
|
75
|
+
/** The compared prompt; for judge experiments, the judge's prompt. */
|
|
59
76
|
promptName: string;
|
|
77
|
+
/** Judge experiments: the human question set whose answers are the reference. */
|
|
78
|
+
questionSetId?: string;
|
|
79
|
+
questionSetName?: string;
|
|
80
|
+
/** Judge experiments: true when the questions changed after the run. */
|
|
81
|
+
questionsStale?: boolean;
|
|
60
82
|
results?: ExperimentAggregates;
|
|
61
83
|
status: "completed" | "failed" | "pending" | "running";
|
|
84
|
+
/** Judge experiments: agreement rate (0–1) a version must reach to pass. */
|
|
85
|
+
targetAgreement?: number;
|
|
62
86
|
thresholds: ScoreThreshold[];
|
|
63
87
|
updated: string;
|
|
64
88
|
versions: number[];
|
|
@@ -84,6 +108,25 @@ export const unwrapExperimentList = (raw: unknown): ExperimentRecord[] => {
|
|
|
84
108
|
});
|
|
85
109
|
};
|
|
86
110
|
|
|
111
|
+
/** Newest version that passes every gate, else the newest compared version (0 when none). */
|
|
112
|
+
export const defaultPromoteVersion = (experiment: ExperimentRecord): number => {
|
|
113
|
+
const passing = summarizeGatesByVersion(experiment).filter((summary) => summary.passed);
|
|
114
|
+
return (
|
|
115
|
+
passing[passing.length - 1]?.version ?? experiment.versions[experiment.versions.length - 1] ?? 0
|
|
116
|
+
);
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
/** Largest page the experiments list route returns. */
|
|
120
|
+
export const EXPERIMENT_LIST_MAX_LIMIT = 100;
|
|
121
|
+
|
|
122
|
+
/** True when a paginated experiments list response says more rows exist past this page. */
|
|
123
|
+
export const experimentListHasMore = (raw: unknown): boolean => {
|
|
124
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
return (raw as {more?: unknown}).more === true;
|
|
128
|
+
};
|
|
129
|
+
|
|
87
130
|
export const unwrapExperimentRecord = (raw: unknown): ExperimentRecord | undefined => {
|
|
88
131
|
const payload = unwrapObservabilityPayload<ExperimentRecord>(raw);
|
|
89
132
|
if (!payload || typeof payload.id !== "string") {
|
|
@@ -99,22 +142,107 @@ export const failingGateCount = (experiment: ExperimentRecord): number => {
|
|
|
99
142
|
return experiment.results.gates.filter((gate) => !gate.passed).length;
|
|
100
143
|
};
|
|
101
144
|
|
|
102
|
-
|
|
103
|
-
experiment: ExperimentRecord,
|
|
104
|
-
version: number
|
|
105
|
-
): ExperimentGateResult[] => {
|
|
145
|
+
const gatesForVersion = (experiment: ExperimentRecord, version: number): ExperimentGateResult[] => {
|
|
106
146
|
if (!experiment.results) {
|
|
107
147
|
return [];
|
|
108
148
|
}
|
|
109
149
|
return experiment.results.gates.filter((gate) => gate.version === version);
|
|
110
150
|
};
|
|
111
151
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
152
|
+
const GATE_OP_SYMBOLS: Record<string, string> = {
|
|
153
|
+
eq: "=",
|
|
154
|
+
gt: ">",
|
|
155
|
+
gte: "≥",
|
|
156
|
+
lt: "<",
|
|
157
|
+
lte: "≤",
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
/** Show a gate comparison operator as the math symbol people read, never the API word. */
|
|
161
|
+
const formatGateOp = (op: string): string => {
|
|
162
|
+
return GATE_OP_SYMBOLS[op] ?? op;
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
/** Show a measured gate value with two decimals, or a dash before any scores arrive. */
|
|
166
|
+
export const formatGateActual = (actual?: number): string => {
|
|
167
|
+
if (actual === undefined || Number.isNaN(actual)) {
|
|
168
|
+
return "—";
|
|
169
|
+
}
|
|
170
|
+
return actual.toFixed(2);
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
/** The threshold a gate needs, such as "≥ 0.8". */
|
|
174
|
+
export const formatGateThreshold = (gate: Pick<ScoreThreshold, "op" | "value">): string => {
|
|
175
|
+
return `${formatGateOp(gate.op)} ${gate.value}`;
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const humanizeIdentifier = (value: string): string => {
|
|
179
|
+
return value.replace(/[_-]+/g, " ").trim();
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Name a gate in plain words. When the evaluator and dimension share a name ("helpfulness"),
|
|
184
|
+
* say it once; otherwise "correctness: correct" or "schema-assert: schema valid".
|
|
185
|
+
*/
|
|
186
|
+
export const formatGateName = (
|
|
187
|
+
gate: Pick<ScoreThreshold, "dimension" | "evaluatorName">
|
|
188
|
+
): string => {
|
|
189
|
+
const dimension = humanizeIdentifier(gate.dimension);
|
|
190
|
+
if (humanizeIdentifier(gate.evaluatorName).toLowerCase() === dimension.toLowerCase()) {
|
|
191
|
+
return dimension;
|
|
192
|
+
}
|
|
193
|
+
return `${gate.evaluatorName}: ${dimension}`;
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
/** One failing gate in a sentence, such as "helpfulness ≥ 0.8 (got 0.62)". */
|
|
197
|
+
const describeGateResult = (gate: ExperimentGateResult): string => {
|
|
198
|
+
return `${formatGateName(gate)} ${formatGateThreshold(gate)} (got ${formatGateActual(gate.actual)})`;
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
const pluralize = (count: number, singular: string, plural: string): string => {
|
|
202
|
+
return `${count} ${count === 1 ? singular : plural}`;
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
export interface VersionGateSummary {
|
|
206
|
+
failing: ExperimentGateResult[];
|
|
207
|
+
gates: ExperimentGateResult[];
|
|
208
|
+
passed: boolean;
|
|
209
|
+
version: number;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Gate results for every compared version, in version order, with pass/fail rolled up. */
|
|
213
|
+
export const summarizeGatesByVersion = (experiment: ExperimentRecord): VersionGateSummary[] => {
|
|
214
|
+
return [...experiment.versions]
|
|
215
|
+
.sort((left, right) => left - right)
|
|
216
|
+
.map((version) => {
|
|
217
|
+
const gates = gatesForVersion(experiment, version);
|
|
218
|
+
const failing = gates.filter((gate) => !gate.passed);
|
|
219
|
+
return {failing, gates, passed: gates.length > 0 && failing.length === 0, version};
|
|
220
|
+
});
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
/** Short verdict for one version's gates, such as "Fails 1 of 3 gates". */
|
|
224
|
+
export const formatVersionGateVerdict = (summary: VersionGateSummary): string => {
|
|
225
|
+
if (summary.gates.length === 0) {
|
|
226
|
+
return "No gates";
|
|
116
227
|
}
|
|
117
|
-
|
|
228
|
+
if (summary.failing.length === 0) {
|
|
229
|
+
return summary.gates.length === 1
|
|
230
|
+
? "Passes its gate"
|
|
231
|
+
: `Passes all ${summary.gates.length} gates`;
|
|
232
|
+
}
|
|
233
|
+
return `Fails ${summary.failing.length} of ${pluralize(summary.gates.length, "gate", "gates")}`;
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Explain why a version cannot be promoted, such as
|
|
238
|
+
* "v1 fails 1 gate: helpfulness ≥ 0.8 (got 0.62) — can't promote v1."
|
|
239
|
+
*/
|
|
240
|
+
export const formatPromoteBlockedReason = (summary: VersionGateSummary): string | undefined => {
|
|
241
|
+
if (summary.failing.length === 0) {
|
|
242
|
+
return undefined;
|
|
243
|
+
}
|
|
244
|
+
const details = summary.failing.map(describeGateResult).join("; ");
|
|
245
|
+
return `v${summary.version} fails ${pluralize(summary.failing.length, "gate", "gates")}: ${details} — can't promote v${summary.version}.`;
|
|
118
246
|
};
|
|
119
247
|
|
|
120
248
|
export const parsePromoteBlockedTitle = (error: unknown): string | undefined => {
|
|
@@ -140,3 +268,115 @@ export const isAutomaticExperimentEvaluator = (evaluator: {type: string}): boole
|
|
|
140
268
|
export const isValidExperimentVersionCount = (count: number): boolean => {
|
|
141
269
|
return count >= MIN_EXPERIMENT_VERSIONS && count <= MAX_EXPERIMENT_VERSIONS;
|
|
142
270
|
};
|
|
271
|
+
|
|
272
|
+
const ITEM_COLUMN_WIDTH = 120;
|
|
273
|
+
const PASS_COLUMN_WIDTH = 80;
|
|
274
|
+
const MIN_OUTPUT_COLUMN_WIDTH = 180;
|
|
275
|
+
|
|
276
|
+
export interface ExperimentItemColumnWidths {
|
|
277
|
+
item: number;
|
|
278
|
+
output: number;
|
|
279
|
+
pass: number;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Split the measured table width so the per-version output columns take every pixel the item and
|
|
284
|
+
* pass columns leave over, never narrower than the readable minimum.
|
|
285
|
+
*/
|
|
286
|
+
export const experimentItemColumnWidths = ({
|
|
287
|
+
containerWidth,
|
|
288
|
+
versionCount,
|
|
289
|
+
}: {
|
|
290
|
+
containerWidth: number;
|
|
291
|
+
versionCount: number;
|
|
292
|
+
}): ExperimentItemColumnWidths => {
|
|
293
|
+
const fixedWidth = ITEM_COLUMN_WIDTH + PASS_COLUMN_WIDTH * versionCount;
|
|
294
|
+
const available = containerWidth - fixedWidth;
|
|
295
|
+
const output =
|
|
296
|
+
versionCount > 0
|
|
297
|
+
? Math.max(MIN_OUTPUT_COLUMN_WIDTH, Math.floor(available / versionCount))
|
|
298
|
+
: MIN_OUTPUT_COLUMN_WIDTH;
|
|
299
|
+
return {item: ITEM_COLUMN_WIDTH, output, pass: PASS_COLUMN_WIDTH};
|
|
300
|
+
};
|
|
301
|
+
|
|
302
|
+
export const DEFAULT_TARGET_AGREEMENT = 0.9;
|
|
303
|
+
|
|
304
|
+
export const EXPERIMENT_KIND_LABELS: Record<ExperimentKind, string> = {
|
|
305
|
+
judge: "Judge calibration",
|
|
306
|
+
prompt: "Prompt comparison",
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
export const experimentKind = (experiment: Pick<ExperimentRecord, "kind">): ExperimentKind => {
|
|
310
|
+
return experiment.kind === "judge" ? "judge" : "prompt";
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
/** Tags shown next to a prompt version in pickers: latest, production, superseded. */
|
|
314
|
+
export const promptVersionTags = (version: number, detail?: PromptDetail): string[] => {
|
|
315
|
+
const tags: string[] = [];
|
|
316
|
+
if (!detail || detail.versions.length === 0) {
|
|
317
|
+
return tags;
|
|
318
|
+
}
|
|
319
|
+
const latest = Math.max(...detail.versions.map((entry) => entry.version));
|
|
320
|
+
if (version === latest) {
|
|
321
|
+
tags.push("latest");
|
|
322
|
+
}
|
|
323
|
+
if (detail.labels.some((label) => label.label === "production" && label.version === version)) {
|
|
324
|
+
tags.push("production");
|
|
325
|
+
}
|
|
326
|
+
if (version < latest - 1) {
|
|
327
|
+
tags.push("superseded");
|
|
328
|
+
}
|
|
329
|
+
return tags;
|
|
330
|
+
};
|
|
331
|
+
|
|
332
|
+
/** "v1 and v2" or "v1, v2, and v3". */
|
|
333
|
+
export const formatVersionList = (versions: number[]): string => {
|
|
334
|
+
const labels = [...versions].sort((left, right) => left - right).map((version) => `v${version}`);
|
|
335
|
+
if (labels.length <= 2) {
|
|
336
|
+
return labels.join(" and ");
|
|
337
|
+
}
|
|
338
|
+
return `${labels.slice(0, -1).join(", ")}, and ${labels[labels.length - 1]}`;
|
|
339
|
+
};
|
|
340
|
+
|
|
341
|
+
/** Parse a target agreement typed as a percent ("90") into a rate (0.9), or undefined if invalid. */
|
|
342
|
+
export const parseTargetAgreementPercent = (value: string): number | undefined => {
|
|
343
|
+
const trimmed = value.trim();
|
|
344
|
+
if (!trimmed) {
|
|
345
|
+
return undefined;
|
|
346
|
+
}
|
|
347
|
+
const parsed = Number(trimmed);
|
|
348
|
+
if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 100) {
|
|
349
|
+
return undefined;
|
|
350
|
+
}
|
|
351
|
+
return parsed / 100;
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
/** Cost with two decimals; anything above zero but under a cent reads "<$0.01". */
|
|
355
|
+
export const formatExperimentCost = (cost?: number): string => {
|
|
356
|
+
if (cost === undefined || Number.isNaN(cost)) {
|
|
357
|
+
return "—";
|
|
358
|
+
}
|
|
359
|
+
if (cost > 0 && cost < 0.01) {
|
|
360
|
+
return "<$0.01";
|
|
361
|
+
}
|
|
362
|
+
return `$${cost.toFixed(2)}`;
|
|
363
|
+
};
|
|
364
|
+
|
|
365
|
+
/** Plain estimate such as "About 6 generations, 2 minutes, $0.42". */
|
|
366
|
+
export const formatExperimentEstimate = ({
|
|
367
|
+
estimate,
|
|
368
|
+
unit = "generations",
|
|
369
|
+
}: {
|
|
370
|
+
estimate: ExperimentEstimate;
|
|
371
|
+
unit?: "generations" | "judge calls";
|
|
372
|
+
}): string => {
|
|
373
|
+
const minutes = Math.max(1, Math.round(estimate.wallClockSeconds / 60));
|
|
374
|
+
const parts = [
|
|
375
|
+
`About ${estimate.generations} ${estimate.generations === 1 ? unit.replace(/s$/, "") : unit}`,
|
|
376
|
+
`${minutes} ${minutes === 1 ? "minute" : "minutes"}`,
|
|
377
|
+
];
|
|
378
|
+
if (estimate.costUsd !== undefined) {
|
|
379
|
+
parts.push(formatExperimentCost(estimate.costUsd));
|
|
380
|
+
}
|
|
381
|
+
return parts.join(", ");
|
|
382
|
+
};
|
|
@@ -50,6 +50,27 @@ describe("useAiObservabilityExperimentsApi", () => {
|
|
|
50
50
|
method: "GET",
|
|
51
51
|
url: "/ai/observability/experiments",
|
|
52
52
|
});
|
|
53
|
+
expect(endpoints.aiObservabilityExperiments.query(undefined as never)).toEqual({
|
|
54
|
+
method: "GET",
|
|
55
|
+
url: "/ai/observability/experiments",
|
|
56
|
+
});
|
|
57
|
+
expect(endpoints.aiObservabilityExperiments.query({kind: "judge"} as never)).toEqual({
|
|
58
|
+
method: "GET",
|
|
59
|
+
params: {kind: "judge"},
|
|
60
|
+
url: "/ai/observability/experiments",
|
|
61
|
+
} as never);
|
|
62
|
+
expect(
|
|
63
|
+
endpoints.aiObservabilityExperiments.query({kind: "judge", limit: 100} as never)
|
|
64
|
+
).toEqual({
|
|
65
|
+
method: "GET",
|
|
66
|
+
params: {kind: "judge", limit: 100},
|
|
67
|
+
url: "/ai/observability/experiments",
|
|
68
|
+
} as never);
|
|
69
|
+
expect(endpoints.aiObservabilityExperiments.query({limit: 100} as never)).toEqual({
|
|
70
|
+
method: "GET",
|
|
71
|
+
params: {limit: 100},
|
|
72
|
+
url: "/ai/observability/experiments",
|
|
73
|
+
} as never);
|
|
53
74
|
expect(endpoints.aiObservabilityExperiment.query("exp-1" as never)).toEqual({
|
|
54
75
|
method: "GET",
|
|
55
76
|
url: "/ai/observability/experiments/exp-1",
|
|
@@ -66,6 +87,20 @@ describe("useAiObservabilityExperimentsApi", () => {
|
|
|
66
87
|
method: "POST",
|
|
67
88
|
url: "/ai/observability/experiments",
|
|
68
89
|
});
|
|
90
|
+
const judgeBody = {
|
|
91
|
+
datasetId: "ds-1",
|
|
92
|
+
judgeEvaluatorId: "judge-1",
|
|
93
|
+
kind: "judge",
|
|
94
|
+
name: "calibrate",
|
|
95
|
+
questionSetId: "qs-1",
|
|
96
|
+
targetAgreement: 0.9,
|
|
97
|
+
versions: [1, 2],
|
|
98
|
+
};
|
|
99
|
+
expect(endpoints.createAiObservabilityExperiment.query(judgeBody as never)).toEqual({
|
|
100
|
+
body: judgeBody,
|
|
101
|
+
method: "POST",
|
|
102
|
+
url: "/ai/observability/experiments",
|
|
103
|
+
});
|
|
69
104
|
expect(
|
|
70
105
|
endpoints.promoteAiObservabilityExperiment.query({id: "exp-1", version: 2} as never)
|
|
71
106
|
).toEqual({
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import {useMemo} from "react";
|
|
2
2
|
import {asDynamicHookApi} from "../../../dynamicHookApi";
|
|
3
3
|
import type {AdminApi, EndpointBuilder} from "../../../types";
|
|
4
|
-
import type {
|
|
4
|
+
import type {
|
|
5
|
+
ExperimentEstimate,
|
|
6
|
+
ExperimentKind,
|
|
7
|
+
ExperimentRecord,
|
|
8
|
+
ScoreThreshold,
|
|
9
|
+
} from "./experimentTypes";
|
|
5
10
|
|
|
6
11
|
const LIST_KEY = "aiObservabilityExperiments";
|
|
7
12
|
const DETAIL_KEY = "aiObservabilityExperiment";
|
|
@@ -9,10 +14,12 @@ const CREATE_KEY = "createAiObservabilityExperiment";
|
|
|
9
14
|
const ESTIMATE_KEY = "estimateAiObservabilityExperiment";
|
|
10
15
|
const PROMOTE_KEY = "promoteAiObservabilityExperiment";
|
|
11
16
|
|
|
12
|
-
|
|
17
|
+
/** Step 4: compare product prompt versions, scored by evaluators. */
|
|
18
|
+
export interface PromptExperimentCreateBody {
|
|
13
19
|
datasetId: string;
|
|
14
20
|
evaluatorIds: string[];
|
|
15
21
|
includeUnproofread?: boolean;
|
|
22
|
+
kind?: "prompt";
|
|
16
23
|
modelOverride?: string;
|
|
17
24
|
name: string;
|
|
18
25
|
promptName: string;
|
|
@@ -20,14 +27,46 @@ export interface ExperimentCreateBody {
|
|
|
20
27
|
versions: number[];
|
|
21
28
|
}
|
|
22
29
|
|
|
23
|
-
|
|
30
|
+
/** Step 3: compare judge prompt versions by agreement with human answers on a judged dataset. */
|
|
31
|
+
export interface JudgeExperimentCreateBody {
|
|
32
|
+
datasetId: string;
|
|
33
|
+
judgeEvaluatorId: string;
|
|
34
|
+
kind: "judge";
|
|
35
|
+
modelOverride?: string;
|
|
36
|
+
name: string;
|
|
37
|
+
questionSetId: string;
|
|
38
|
+
/** Agreement rate (0–1) a version must reach to pass. The server defaults to 0.9. */
|
|
39
|
+
targetAgreement?: number;
|
|
40
|
+
versions: number[];
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export type ExperimentCreateBody = JudgeExperimentCreateBody | PromptExperimentCreateBody;
|
|
44
|
+
|
|
45
|
+
export interface PromptExperimentEstimateBody {
|
|
24
46
|
datasetId: string;
|
|
25
47
|
evaluatorIds: string[];
|
|
26
48
|
includeUnproofread?: boolean;
|
|
49
|
+
kind?: "prompt";
|
|
50
|
+
modelOverride?: string;
|
|
51
|
+
versions: number[];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface JudgeExperimentEstimateBody {
|
|
55
|
+
datasetId: string;
|
|
56
|
+
kind: "judge";
|
|
27
57
|
modelOverride?: string;
|
|
58
|
+
questionSetId: string;
|
|
28
59
|
versions: number[];
|
|
29
60
|
}
|
|
30
61
|
|
|
62
|
+
export type ExperimentEstimateBody = JudgeExperimentEstimateBody | PromptExperimentEstimateBody;
|
|
63
|
+
|
|
64
|
+
export interface ExperimentListArgs {
|
|
65
|
+
kind?: ExperimentKind;
|
|
66
|
+
/** Page size; the server caps it at EXPERIMENT_LIST_MAX_LIMIT. */
|
|
67
|
+
limit?: number;
|
|
68
|
+
}
|
|
69
|
+
|
|
31
70
|
interface QueryOptions {
|
|
32
71
|
skip?: boolean;
|
|
33
72
|
}
|
|
@@ -41,10 +80,17 @@ const createExperimentsApi = (api: AdminApi) => {
|
|
|
41
80
|
endpoints: (build: EndpointBuilder) => ({
|
|
42
81
|
[LIST_KEY]: build.query({
|
|
43
82
|
providesTags: ["aiObservabilityExperiments"],
|
|
44
|
-
query: () =>
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
83
|
+
query: (args?: ExperimentListArgs) => {
|
|
84
|
+
const params = {
|
|
85
|
+
...(args?.kind ? {kind: args.kind} : {}),
|
|
86
|
+
...(args?.limit ? {limit: args.limit} : {}),
|
|
87
|
+
};
|
|
88
|
+
return {
|
|
89
|
+
method: "GET",
|
|
90
|
+
...(Object.keys(params).length > 0 ? {params} : {}),
|
|
91
|
+
url: "/ai/observability/experiments",
|
|
92
|
+
};
|
|
93
|
+
},
|
|
48
94
|
}),
|
|
49
95
|
[DETAIL_KEY]: build.query({
|
|
50
96
|
providesTags: (_result: unknown, _error: unknown, id: string) => [
|
|
@@ -109,8 +155,8 @@ export const useAiObservabilityExperimentsApi = (api: AdminApi) => {
|
|
|
109
155
|
(body: ExperimentEstimateBody) => {unwrap: () => Promise<ExperimentEstimate>},
|
|
110
156
|
{error?: unknown; isError: boolean; isLoading: boolean},
|
|
111
157
|
],
|
|
112
|
-
useListQuery: hooks.useAiObservabilityExperimentsQuery as () => {
|
|
113
|
-
data?: ExperimentRecord[] | {data: ExperimentRecord[]};
|
|
158
|
+
useListQuery: hooks.useAiObservabilityExperimentsQuery as (args?: ExperimentListArgs) => {
|
|
159
|
+
data?: ExperimentRecord[] | {data: ExperimentRecord[]; more?: boolean};
|
|
114
160
|
error?: unknown;
|
|
115
161
|
isError: boolean;
|
|
116
162
|
isLoading: boolean;
|