@terreno/admin-frontend 58.0.0-beta.3 → 58.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +166 -0
- package/dist/AdminModelForm.js +12 -12
- package/dist/AdminModelForm.js.map +1 -1
- package/dist/AdminModelTable.d.ts.map +1 -1
- package/dist/AdminModelTable.js +11 -0
- package/dist/AdminModelTable.js.map +1 -1
- package/dist/ConfigurationScreen.d.ts.map +1 -1
- package/dist/ConfigurationScreen.js +1 -2
- package/dist/ConfigurationScreen.js.map +1 -1
- package/dist/adminModelUrls.d.ts +7 -0
- package/dist/adminModelUrls.d.ts.map +1 -0
- package/dist/adminModelUrls.js +9 -0
- package/dist/adminModelUrls.js.map +1 -0
- package/dist/adminRpc.d.ts.map +1 -1
- package/dist/adminRpc.js +9 -0
- package/dist/adminRpc.js.map +1 -1
- package/dist/adminRpcSeed.d.ts +25 -0
- package/dist/adminRpcSeed.d.ts.map +1 -0
- package/dist/adminRpcSeed.js +49 -0
- package/dist/adminRpcSeed.js.map +1 -0
- package/dist/comms/commsDashboardParams.d.ts.map +1 -1
- package/dist/comms/commsDashboardParams.js +7 -1
- package/dist/comms/commsDashboardParams.js.map +1 -1
- package/dist/index.d.ts +9 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -2
- package/dist/index.js.map +1 -1
- package/dist/useAdminModelApi.d.ts.map +1 -1
- package/dist/useAdminModelApi.js +2 -1
- package/dist/useAdminModelApi.js.map +1 -1
- package/dist/useAdminRpc.d.ts.map +1 -1
- package/dist/useAdminRpc.js +50 -19
- package/dist/useAdminRpc.js.map +1 -1
- package/dist/useConfigurationApi.d.ts.map +1 -1
- package/dist/useConfigurationApi.js +8 -2
- package/dist/useConfigurationApi.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js +19 -9
- package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts +7 -2
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js +155 -43
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts +20 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.js +17 -12
- package/dist/widgets/aiObservability/datasets/datasetTypes.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js +12 -8
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js +4 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts +8 -0
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js +68 -37
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js +4 -2
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts +8 -0
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js +156 -33
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts +32 -0
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js +49 -0
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js +235 -12
- package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts +10 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js +113 -56
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js +53 -38
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts +2 -3
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js +117 -32
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js +13 -4
- package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts +5 -0
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js +157 -55
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts +41 -0
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts.map +1 -0
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js +113 -0
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js.map +1 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts +30 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts.map +1 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.js +29 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.js.map +1 -0
- package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts +82 -3
- package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/experimentTypes.js +172 -6
- package/dist/widgets/aiObservability/experiments/experimentTypes.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts +34 -4
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js +8 -2
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js +24 -5
- package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts +2 -0
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js +36 -6
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts +3 -0
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.js +6 -6
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js +12 -12
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.js +156 -35
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/promptTypes.d.ts +13 -3
- package/dist/widgets/aiObservability/prompts/promptTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/promptTypes.js +61 -19
- package/dist/widgets/aiObservability/prompts/promptTypes.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts +11 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js +115 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts +22 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.js +136 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts +4 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js +92 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts +26 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.js +115 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewItemScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewItemScreen.js +73 -18
- package/dist/widgets/aiObservability/review/AiReviewItemScreen.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts +19 -5
- package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewItemView.js +14 -9
- package/dist/widgets/aiObservability/review/AiReviewItemView.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js +16 -2
- package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts +11 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueView.js +96 -37
- package/dist/widgets/aiObservability/review/AiReviewQueueView.js.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts +13 -0
- package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/ReviewQuestionList.js +28 -0
- package/dist/widgets/aiObservability/review/ReviewQuestionList.js.map +1 -0
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts +3 -0
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js +2 -2
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts +14 -0
- package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewScoreField.js +40 -12
- package/dist/widgets/aiObservability/review/ReviewScoreField.js.map +1 -1
- package/dist/widgets/aiObservability/review/batchTypes.d.ts +103 -0
- package/dist/widgets/aiObservability/review/batchTypes.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/batchTypes.js +61 -0
- package/dist/widgets/aiObservability/review/batchTypes.js.map +1 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts +21 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.js +141 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.js.map +1 -0
- package/dist/widgets/aiObservability/review/reviewTypes.d.ts +23 -2
- package/dist/widgets/aiObservability/review/reviewTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/reviewTypes.js +25 -0
- package/dist/widgets/aiObservability/review/reviewTypes.js.map +1 -1
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts +27 -1
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js +31 -1
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts +30 -0
- package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/AgreementMeter.js +32 -0
- package/dist/widgets/aiObservability/shell/AgreementMeter.js.map +1 -0
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts +10 -0
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js +6 -4
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts +3 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js +5 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js +3 -3
- package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js.map +1 -1
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts +14 -0
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js +26 -0
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts +11 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.js +8 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.js.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.d.ts +22 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.js +28 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.js.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts +17 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js +10 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts +24 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.js +11 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.js.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts +11 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js +9 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js.map +1 -0
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts +1 -0
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.js +15 -8
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.js.map +1 -1
- package/dist/widgets/aiObservability/shell/clientTable.d.ts +48 -0
- package/dist/widgets/aiObservability/shell/clientTable.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/clientTable.js +163 -0
- package/dist/widgets/aiObservability/shell/clientTable.js.map +1 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.d.ts +24 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.js +43 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.js.map +1 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts +23 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.js +72 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.js.map +1 -0
- package/dist/widgets/aiObservability/traces/AiTracesListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListScreen.js +18 -9
- package/dist/widgets/aiObservability/traces/AiTracesListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts +18 -7
- package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListView.js +198 -78
- package/dist/widgets/aiObservability/traces/AiTracesListView.js.map +1 -1
- package/dist/widgets/aiObservability/traces/traceTypes.d.ts +23 -1
- package/dist/widgets/aiObservability/traces/traceTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/traceTypes.js +92 -1
- package/dist/widgets/aiObservability/traces/traceTypes.js.map +1 -1
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts +2 -0
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js +9 -2
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js.map +1 -1
- package/dist/widgets/homeWidgetUrls.d.ts +13 -0
- package/dist/widgets/homeWidgetUrls.d.ts.map +1 -0
- package/dist/widgets/homeWidgetUrls.js +71 -0
- package/dist/widgets/homeWidgetUrls.js.map +1 -0
- package/package.json +4 -4
- package/src/AdminModelForm.tsx +16 -16
- package/src/AdminModelTable.tsx +12 -0
- package/src/ConfigurationScreen.tsx +1 -2
- package/src/adminModelUrls.test.ts +31 -0
- package/src/adminModelUrls.ts +8 -0
- package/src/adminRpc.test.ts +10 -0
- package/src/adminRpc.ts +9 -0
- package/src/adminRpcSeed.tsx +65 -0
- package/src/comms/commsDashboardParams.ts +7 -1
- package/src/index.tsx +10 -0
- package/src/isolated/AiPromptEditorScreen.isolated.tsx +19 -0
- package/src/isolated/adminModelForm.ssr.isolated.tsx +170 -0
- package/src/isolated/adminModelTable.ssr.isolated.tsx +112 -0
- package/src/isolated/adminModelTable.windowedSeed.isolated.tsx +244 -0
- package/src/isolated/adminRouteSeeds.ssr.isolated.tsx +263 -0
- package/src/isolated/aiListTables.isolated.tsx +648 -0
- package/src/isolated/aiObservabilityDatasets.isolated.tsx +26 -16
- package/src/isolated/aiObservabilityEvaluators.isolated.tsx +90 -15
- package/src/isolated/aiObservabilityExperiments.isolated.tsx +322 -36
- package/src/isolated/aiObservabilityPrompts.isolated.tsx +82 -35
- package/src/isolated/aiObservabilityReview.isolated.tsx +235 -243
- package/src/isolated/aiObservabilityTraces.isolated.tsx +155 -120
- package/src/isolated/aiReviewBatchScreens.isolated.tsx +426 -0
- package/src/isolated/aiTracesTable.isolated.tsx +368 -0
- package/src/isolated/hooks.isolated.tsx +89 -0
- package/src/isolated/jobsCommsRouteSeeds.ssr.isolated.tsx +142 -0
- package/src/isolated/orgsRouteSeeds.ssr.isolated.tsx +204 -0
- package/src/useAdminModelApi.ts +2 -1
- package/src/useAdminRpc.test.tsx +538 -2
- package/src/useAdminRpc.ts +50 -19
- package/src/useConfigurationApi.ts +9 -1
- package/src/widgets/aiObservability/datasets/AiDatasetsListScreen.tsx +44 -13
- package/src/widgets/aiObservability/datasets/AiDatasetsListView.tsx +303 -103
- package/src/widgets/aiObservability/datasets/datasetTypes.test.ts +0 -13
- package/src/widgets/aiObservability/datasets/datasetTypes.ts +37 -13
- package/src/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.tsx +34 -11
- package/src/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.tsx +12 -1
- package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.test.tsx +97 -9
- package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.tsx +235 -131
- package/src/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.tsx +21 -2
- package/src/widgets/aiObservability/evaluators/AiEvaluatorsListView.tsx +261 -76
- package/src/widgets/aiObservability/evaluators/evaluatorTypes.test.ts +7 -0
- package/src/widgets/aiObservability/evaluators/evaluatorTypes.ts +82 -0
- package/src/widgets/aiObservability/experiments/AiExperimentNewScreen.tsx +271 -13
- package/src/widgets/aiObservability/experiments/AiExperimentNewView.test.tsx +164 -2
- package/src/widgets/aiObservability/experiments/AiExperimentNewView.tsx +400 -186
- package/src/widgets/aiObservability/experiments/AiExperimentResultsScreen.tsx +104 -45
- package/src/widgets/aiObservability/experiments/AiExperimentResultsView.test.tsx +290 -0
- package/src/widgets/aiObservability/experiments/AiExperimentResultsView.tsx +342 -97
- package/src/widgets/aiObservability/experiments/AiExperimentsListScreen.tsx +35 -4
- package/src/widgets/aiObservability/experiments/AiExperimentsListView.tsx +302 -112
- package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.test.tsx +250 -0
- package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.tsx +468 -0
- package/src/widgets/aiObservability/experiments/ExperimentWizard.tsx +106 -0
- package/src/widgets/aiObservability/experiments/experimentTypes.test.ts +103 -16
- package/src/widgets/aiObservability/experiments/experimentTypes.ts +250 -10
- package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.test.ts +41 -0
- package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.ts +54 -7
- package/src/widgets/aiObservability/prompts/AiPromptEditorScreen.tsx +26 -2
- package/src/widgets/aiObservability/prompts/AiPromptEditorView.tsx +95 -1
- package/src/widgets/aiObservability/prompts/AiPromptHubView.test.tsx +61 -45
- package/src/widgets/aiObservability/prompts/AiPromptHubView.tsx +71 -27
- package/src/widgets/aiObservability/prompts/AiPromptsListScreen.tsx +34 -15
- package/src/widgets/aiObservability/prompts/AiPromptsListView.test.tsx +104 -8
- package/src/widgets/aiObservability/prompts/AiPromptsListView.tsx +287 -117
- package/src/widgets/aiObservability/prompts/promptTypes.test.ts +63 -1
- package/src/widgets/aiObservability/prompts/promptTypes.ts +83 -22
- package/src/widgets/aiObservability/review/AiReviewBatchScreen.tsx +217 -0
- package/src/widgets/aiObservability/review/AiReviewBatchView.tsx +398 -0
- package/src/widgets/aiObservability/review/AiReviewBatchViews.test.tsx +328 -0
- package/src/widgets/aiObservability/review/AiReviewBatchesScreen.tsx +136 -0
- package/src/widgets/aiObservability/review/AiReviewBatchesView.tsx +333 -0
- package/src/widgets/aiObservability/review/AiReviewItemScreen.tsx +121 -30
- package/src/widgets/aiObservability/review/AiReviewItemView.tsx +112 -70
- package/src/widgets/aiObservability/review/AiReviewQueueScreen.tsx +29 -2
- package/src/widgets/aiObservability/review/AiReviewQueueView.tsx +183 -75
- package/src/widgets/aiObservability/review/ReviewQuestionList.tsx +128 -0
- package/src/widgets/aiObservability/review/ReviewReadOnlyPanel.tsx +21 -13
- package/src/widgets/aiObservability/review/ReviewScoreField.tsx +113 -27
- package/src/widgets/aiObservability/review/batchTypes.test.ts +207 -0
- package/src/widgets/aiObservability/review/batchTypes.ts +162 -0
- package/src/widgets/aiObservability/review/reviewShortcuts.ts +195 -0
- package/src/widgets/aiObservability/review/reviewTypes.ts +62 -2
- package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.test.ts +59 -0
- package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.ts +63 -1
- package/src/widgets/aiObservability/shell/AgreementMeter.tsx +103 -0
- package/src/widgets/aiObservability/shell/AiObservabilityChrome.tsx +43 -14
- package/src/widgets/aiObservability/shell/AiObservabilityScreenWidgets.tsx +6 -0
- package/src/widgets/aiObservability/shell/AiObservabilityStatusChip.test.tsx +2 -6
- package/src/widgets/aiObservability/shell/AiObservabilityStatusChip.tsx +10 -2
- package/src/widgets/aiObservability/shell/EvalPipelineOverview.tsx +68 -0
- package/src/widgets/aiObservability/shell/EvalStepBanner.tsx +24 -0
- package/src/widgets/aiObservability/shell/EvalStepper.test.tsx +80 -0
- package/src/widgets/aiObservability/shell/EvalStepper.tsx +126 -0
- package/src/widgets/aiObservability/shell/ObservabilityPageHeader.tsx +50 -0
- package/src/widgets/aiObservability/shell/ObservabilitySharedUi.test.tsx +140 -0
- package/src/widgets/aiObservability/shell/ObservabilityStates.tsx +81 -0
- package/src/widgets/aiObservability/shell/ObservabilityStatusBadge.tsx +29 -0
- package/src/widgets/aiObservability/shell/aiObservabilityNav.ts +16 -8
- package/src/widgets/aiObservability/shell/clientTable.test.ts +176 -0
- package/src/widgets/aiObservability/shell/clientTable.ts +251 -0
- package/src/widgets/aiObservability/shell/evalPipeline.ts +74 -0
- package/src/widgets/aiObservability/shell/observabilityStatus.test.ts +73 -0
- package/src/widgets/aiObservability/shell/observabilityStatus.ts +100 -0
- package/src/widgets/aiObservability/traces/AiTracesListScreen.tsx +41 -13
- package/src/widgets/aiObservability/traces/AiTracesListView.tsx +423 -300
- package/src/widgets/aiObservability/traces/traceTypes.test.ts +52 -1
- package/src/widgets/aiObservability/traces/traceTypes.ts +117 -2
- package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.test.ts +19 -1
- package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.ts +10 -2
- package/src/widgets/homeWidgetUrls.test.ts +201 -0
- package/src/widgets/homeWidgetUrls.ts +87 -0
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import {describe, it} from "bun:test";
|
|
2
2
|
import {assert} from "chai";
|
|
3
|
-
import type {ExperimentRecord} from "./experimentTypes";
|
|
3
|
+
import type {ExperimentRecord, VersionGateSummary} from "./experimentTypes";
|
|
4
4
|
import {
|
|
5
|
-
|
|
5
|
+
experimentItemColumnWidths,
|
|
6
6
|
failingGateCount,
|
|
7
|
-
|
|
7
|
+
formatGateActual,
|
|
8
|
+
formatGateName,
|
|
9
|
+
formatGateThreshold,
|
|
10
|
+
formatPromoteBlockedReason,
|
|
11
|
+
formatVersionGateVerdict,
|
|
8
12
|
isValidExperimentVersionCount,
|
|
9
13
|
parsePromoteBlockedTitle,
|
|
14
|
+
summarizeGatesByVersion,
|
|
10
15
|
unwrapExperimentList,
|
|
11
16
|
unwrapExperimentRecord,
|
|
12
17
|
unwrapObservabilityPayload,
|
|
@@ -55,6 +60,14 @@ const experiment: ExperimentRecord = {
|
|
|
55
60
|
versions: [1, 2],
|
|
56
61
|
};
|
|
57
62
|
|
|
63
|
+
const summaryFor = (version: number): VersionGateSummary => {
|
|
64
|
+
const summary = summarizeGatesByVersion(experiment).find((entry) => entry.version === version);
|
|
65
|
+
if (!summary) {
|
|
66
|
+
throw new Error(`No gate summary for v${version}`);
|
|
67
|
+
}
|
|
68
|
+
return summary;
|
|
69
|
+
};
|
|
70
|
+
|
|
58
71
|
describe("experimentTypes helpers", () => {
|
|
59
72
|
it("unwraps experiment payloads", () => {
|
|
60
73
|
assert.deepEqual(unwrapObservabilityPayload({data: experiment}), experiment);
|
|
@@ -64,21 +77,13 @@ describe("experimentTypes helpers", () => {
|
|
|
64
77
|
assert.isUndefined(unwrapExperimentRecord({bad: true}));
|
|
65
78
|
});
|
|
66
79
|
|
|
67
|
-
it("counts failing gates and
|
|
80
|
+
it("counts failing gates and groups gates by version", () => {
|
|
68
81
|
assert.equal(failingGateCount(experiment), 1);
|
|
69
82
|
assert.equal(failingGateCount({...experiment, results: undefined}), 0);
|
|
70
|
-
assert.equal(
|
|
71
|
-
assert.
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
it("computes progress percent with zero-safe denominator", () => {
|
|
75
|
-
assert.equal(experimentProgressPercent(experiment), 50);
|
|
76
|
-
assert.equal(
|
|
77
|
-
experimentProgressPercent({
|
|
78
|
-
...experiment,
|
|
79
|
-
results: {...experiment.results!, progress: {completed: 0, total: 0}},
|
|
80
|
-
}),
|
|
81
|
-
0
|
|
83
|
+
assert.equal(summaryFor(2).gates.length, 1);
|
|
84
|
+
assert.deepEqual(
|
|
85
|
+
summarizeGatesByVersion({...experiment, versions: [3]}).map((summary) => summary.gates),
|
|
86
|
+
[[]]
|
|
82
87
|
);
|
|
83
88
|
});
|
|
84
89
|
|
|
@@ -97,4 +102,86 @@ describe("experimentTypes helpers", () => {
|
|
|
97
102
|
assert.isTrue(isValidExperimentVersionCount(3));
|
|
98
103
|
assert.isFalse(isValidExperimentVersionCount(4));
|
|
99
104
|
});
|
|
105
|
+
|
|
106
|
+
it("stretches output columns to fill the measured table width", () => {
|
|
107
|
+
assert.deepEqual(experimentItemColumnWidths({containerWidth: 1200, versionCount: 2}), {
|
|
108
|
+
item: 120,
|
|
109
|
+
output: 460,
|
|
110
|
+
pass: 80,
|
|
111
|
+
});
|
|
112
|
+
assert.equal(experimentItemColumnWidths({containerWidth: 1200, versionCount: 3}).output, 280);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it("keeps output columns readable when the table is narrow or unmeasured", () => {
|
|
116
|
+
assert.equal(experimentItemColumnWidths({containerWidth: 0, versionCount: 2}).output, 180);
|
|
117
|
+
assert.equal(experimentItemColumnWidths({containerWidth: 500, versionCount: 3}).output, 180);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("shows gate operators as symbols, never API words", () => {
|
|
121
|
+
assert.equal(formatGateThreshold({op: "gte", value: 0.8}), "≥ 0.8");
|
|
122
|
+
assert.equal(formatGateThreshold({op: "lte", value: 0.2}), "≤ 0.2");
|
|
123
|
+
assert.equal(formatGateThreshold({op: "gt" as never, value: 1}), "> 1");
|
|
124
|
+
assert.equal(formatGateThreshold({op: "lt" as never, value: 1}), "< 1");
|
|
125
|
+
assert.equal(formatGateThreshold({op: "eq", value: 1}), "= 1");
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it("names gates in plain words without repeating a shared name", () => {
|
|
129
|
+
assert.equal(
|
|
130
|
+
formatGateName({dimension: "helpfulness", evaluatorName: "helpfulness"}),
|
|
131
|
+
"helpfulness"
|
|
132
|
+
);
|
|
133
|
+
assert.equal(
|
|
134
|
+
formatGateName({dimension: "correct", evaluatorName: "correctness"}),
|
|
135
|
+
"correctness: correct"
|
|
136
|
+
);
|
|
137
|
+
assert.equal(
|
|
138
|
+
formatGateName({dimension: "schema_valid", evaluatorName: "schema-assert"}),
|
|
139
|
+
"schema-assert: schema valid"
|
|
140
|
+
);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("formats measured values", () => {
|
|
144
|
+
assert.equal(formatGateActual(0.625), "0.63");
|
|
145
|
+
assert.equal(formatGateActual(undefined), "—");
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it("summarizes gates per version in version order", () => {
|
|
149
|
+
const summaries = summarizeGatesByVersion(experiment);
|
|
150
|
+
assert.deepEqual(
|
|
151
|
+
summaries.map((summary) => [summary.version, summary.passed, summary.failing.length]),
|
|
152
|
+
[
|
|
153
|
+
[1, true, 0],
|
|
154
|
+
[2, false, 1],
|
|
155
|
+
]
|
|
156
|
+
);
|
|
157
|
+
assert.equal(formatVersionGateVerdict(summaryFor(1)), "Passes its gate");
|
|
158
|
+
assert.equal(formatVersionGateVerdict(summaryFor(2)), "Fails 1 of 1 gate");
|
|
159
|
+
assert.equal(
|
|
160
|
+
formatVersionGateVerdict({failing: [], gates: [], passed: false, version: 3}),
|
|
161
|
+
"No gates"
|
|
162
|
+
);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it("explains which gates block promoting a version, with correct plurals", () => {
|
|
166
|
+
const failing = summaryFor(2);
|
|
167
|
+
assert.isUndefined(formatPromoteBlockedReason(summaryFor(1)));
|
|
168
|
+
assert.equal(
|
|
169
|
+
formatPromoteBlockedReason(failing),
|
|
170
|
+
"v2 fails 1 gate: quality: correct ≥ 0.8 (got 0.50) — can't promote v2."
|
|
171
|
+
);
|
|
172
|
+
const [gate] = failing.gates;
|
|
173
|
+
if (!gate) {
|
|
174
|
+
throw new Error("Expected a v2 gate");
|
|
175
|
+
}
|
|
176
|
+
const twoFailing: VersionGateSummary = {
|
|
177
|
+
failing: [gate, {...gate, dimension: "helpfulness", evaluatorName: "helpfulness"}],
|
|
178
|
+
gates: [gate],
|
|
179
|
+
passed: false,
|
|
180
|
+
version: 2,
|
|
181
|
+
};
|
|
182
|
+
assert.equal(
|
|
183
|
+
formatPromoteBlockedReason(twoFailing),
|
|
184
|
+
"v2 fails 2 gates: quality: correct ≥ 0.8 (got 0.50); helpfulness ≥ 0.8 (got 0.50) — can't promote v2."
|
|
185
|
+
);
|
|
186
|
+
});
|
|
100
187
|
});
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import type {PromptDetail} from "../prompts/promptTypes";
|
|
2
|
+
import type {JudgeAgreement, ScoreValue} from "../review/batchTypes";
|
|
3
|
+
|
|
4
|
+
export type ExperimentKind = "judge" | "prompt";
|
|
5
|
+
|
|
1
6
|
export interface ExperimentEstimate {
|
|
2
7
|
costUsd?: number;
|
|
3
8
|
generations: number;
|
|
@@ -5,7 +10,8 @@ export interface ExperimentEstimate {
|
|
|
5
10
|
}
|
|
6
11
|
|
|
7
12
|
export interface ScoreThreshold {
|
|
8
|
-
|
|
13
|
+
/** `agreement` gates compare a judge prompt version's answers with human answers. */
|
|
14
|
+
aggregate: "agreement" | "mean" | "trueRate";
|
|
9
15
|
dimension: string;
|
|
10
16
|
evaluatorName: string;
|
|
11
17
|
op: "eq" | "gte" | "lte";
|
|
@@ -19,6 +25,8 @@ export interface ExperimentGateResult extends ScoreThreshold {
|
|
|
19
25
|
}
|
|
20
26
|
|
|
21
27
|
export interface ExperimentAggregates {
|
|
28
|
+
/** Judge experiments: agreement with the human answers, keyed by judge prompt version. */
|
|
29
|
+
agreement?: Record<string, JudgeAgreement>;
|
|
22
30
|
gates: ExperimentGateResult[];
|
|
23
31
|
lowConfidenceItemIds: string[];
|
|
24
32
|
outlierItemIds: string[];
|
|
@@ -41,6 +49,8 @@ export interface ExperimentVersionResult {
|
|
|
41
49
|
export interface ExperimentItemRecord {
|
|
42
50
|
datasetItemId: string;
|
|
43
51
|
failed: boolean;
|
|
52
|
+
/** Judge experiments: the human answers each judge version is compared with. */
|
|
53
|
+
humanScores?: Record<string, ScoreValue>;
|
|
44
54
|
id: string;
|
|
45
55
|
versionResults: Record<string, ExperimentVersionResult>;
|
|
46
56
|
}
|
|
@@ -49,16 +59,30 @@ export interface ExperimentRecord {
|
|
|
49
59
|
backgroundTaskId?: string;
|
|
50
60
|
created: string;
|
|
51
61
|
datasetId: string;
|
|
62
|
+
datasetName?: string;
|
|
52
63
|
estimate?: ExperimentEstimate;
|
|
53
64
|
evaluatorIds: string[];
|
|
54
65
|
id: string;
|
|
55
66
|
includeUnproofread: boolean;
|
|
56
67
|
items: ExperimentItemRecord[];
|
|
68
|
+
/** Judge experiments: the llm-judge evaluator whose prompt versions are compared. */
|
|
69
|
+
judgeEvaluatorId?: string;
|
|
70
|
+
judgeName?: string;
|
|
71
|
+
/** Older payloads leave it out; treat a missing kind as a prompt comparison. */
|
|
72
|
+
kind?: ExperimentKind;
|
|
57
73
|
modelOverride?: string;
|
|
58
74
|
name: string;
|
|
75
|
+
/** The compared prompt; for judge experiments, the judge's prompt. */
|
|
59
76
|
promptName: string;
|
|
77
|
+
/** Judge experiments: the human question set whose answers are the reference. */
|
|
78
|
+
questionSetId?: string;
|
|
79
|
+
questionSetName?: string;
|
|
80
|
+
/** Judge experiments: true when the questions changed after the run. */
|
|
81
|
+
questionsStale?: boolean;
|
|
60
82
|
results?: ExperimentAggregates;
|
|
61
83
|
status: "completed" | "failed" | "pending" | "running";
|
|
84
|
+
/** Judge experiments: agreement rate (0–1) a version must reach to pass. */
|
|
85
|
+
targetAgreement?: number;
|
|
62
86
|
thresholds: ScoreThreshold[];
|
|
63
87
|
updated: string;
|
|
64
88
|
versions: number[];
|
|
@@ -84,6 +108,25 @@ export const unwrapExperimentList = (raw: unknown): ExperimentRecord[] => {
|
|
|
84
108
|
});
|
|
85
109
|
};
|
|
86
110
|
|
|
111
|
+
/** Newest version that passes every gate, else the newest compared version (0 when none). */
|
|
112
|
+
export const defaultPromoteVersion = (experiment: ExperimentRecord): number => {
|
|
113
|
+
const passing = summarizeGatesByVersion(experiment).filter((summary) => summary.passed);
|
|
114
|
+
return (
|
|
115
|
+
passing[passing.length - 1]?.version ?? experiment.versions[experiment.versions.length - 1] ?? 0
|
|
116
|
+
);
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
/** Largest page the experiments list route returns. */
|
|
120
|
+
export const EXPERIMENT_LIST_MAX_LIMIT = 100;
|
|
121
|
+
|
|
122
|
+
/** True when a paginated experiments list response says more rows exist past this page. */
|
|
123
|
+
export const experimentListHasMore = (raw: unknown): boolean => {
|
|
124
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
return (raw as {more?: unknown}).more === true;
|
|
128
|
+
};
|
|
129
|
+
|
|
87
130
|
export const unwrapExperimentRecord = (raw: unknown): ExperimentRecord | undefined => {
|
|
88
131
|
const payload = unwrapObservabilityPayload<ExperimentRecord>(raw);
|
|
89
132
|
if (!payload || typeof payload.id !== "string") {
|
|
@@ -99,22 +142,107 @@ export const failingGateCount = (experiment: ExperimentRecord): number => {
|
|
|
99
142
|
return experiment.results.gates.filter((gate) => !gate.passed).length;
|
|
100
143
|
};
|
|
101
144
|
|
|
102
|
-
|
|
103
|
-
experiment: ExperimentRecord,
|
|
104
|
-
version: number
|
|
105
|
-
): ExperimentGateResult[] => {
|
|
145
|
+
const gatesForVersion = (experiment: ExperimentRecord, version: number): ExperimentGateResult[] => {
|
|
106
146
|
if (!experiment.results) {
|
|
107
147
|
return [];
|
|
108
148
|
}
|
|
109
149
|
return experiment.results.gates.filter((gate) => gate.version === version);
|
|
110
150
|
};
|
|
111
151
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
152
|
+
const GATE_OP_SYMBOLS: Record<string, string> = {
|
|
153
|
+
eq: "=",
|
|
154
|
+
gt: ">",
|
|
155
|
+
gte: "≥",
|
|
156
|
+
lt: "<",
|
|
157
|
+
lte: "≤",
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
/** Show a gate comparison operator as the math symbol people read, never the API word. */
|
|
161
|
+
const formatGateOp = (op: string): string => {
|
|
162
|
+
return GATE_OP_SYMBOLS[op] ?? op;
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
/** Show a measured gate value with two decimals, or a dash before any scores arrive. */
|
|
166
|
+
export const formatGateActual = (actual?: number): string => {
|
|
167
|
+
if (actual === undefined || Number.isNaN(actual)) {
|
|
168
|
+
return "—";
|
|
169
|
+
}
|
|
170
|
+
return actual.toFixed(2);
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
/** The threshold a gate needs, such as "≥ 0.8". */
|
|
174
|
+
export const formatGateThreshold = (gate: Pick<ScoreThreshold, "op" | "value">): string => {
|
|
175
|
+
return `${formatGateOp(gate.op)} ${gate.value}`;
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const humanizeIdentifier = (value: string): string => {
|
|
179
|
+
return value.replace(/[_-]+/g, " ").trim();
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Name a gate in plain words. When the evaluator and dimension share a name ("helpfulness"),
|
|
184
|
+
* say it once; otherwise "correctness: correct" or "schema-assert: schema valid".
|
|
185
|
+
*/
|
|
186
|
+
export const formatGateName = (
|
|
187
|
+
gate: Pick<ScoreThreshold, "dimension" | "evaluatorName">
|
|
188
|
+
): string => {
|
|
189
|
+
const dimension = humanizeIdentifier(gate.dimension);
|
|
190
|
+
if (humanizeIdentifier(gate.evaluatorName).toLowerCase() === dimension.toLowerCase()) {
|
|
191
|
+
return dimension;
|
|
192
|
+
}
|
|
193
|
+
return `${gate.evaluatorName}: ${dimension}`;
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
/** One failing gate in a sentence, such as "helpfulness ≥ 0.8 (got 0.62)". */
|
|
197
|
+
const describeGateResult = (gate: ExperimentGateResult): string => {
|
|
198
|
+
return `${formatGateName(gate)} ${formatGateThreshold(gate)} (got ${formatGateActual(gate.actual)})`;
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
const pluralize = (count: number, singular: string, plural: string): string => {
|
|
202
|
+
return `${count} ${count === 1 ? singular : plural}`;
|
|
203
|
+
};
|
|
204
|
+
|
|
205
|
+
export interface VersionGateSummary {
|
|
206
|
+
failing: ExperimentGateResult[];
|
|
207
|
+
gates: ExperimentGateResult[];
|
|
208
|
+
passed: boolean;
|
|
209
|
+
version: number;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Gate results for every compared version, in version order, with pass/fail rolled up. */
|
|
213
|
+
export const summarizeGatesByVersion = (experiment: ExperimentRecord): VersionGateSummary[] => {
|
|
214
|
+
return [...experiment.versions]
|
|
215
|
+
.sort((left, right) => left - right)
|
|
216
|
+
.map((version) => {
|
|
217
|
+
const gates = gatesForVersion(experiment, version);
|
|
218
|
+
const failing = gates.filter((gate) => !gate.passed);
|
|
219
|
+
return {failing, gates, passed: gates.length > 0 && failing.length === 0, version};
|
|
220
|
+
});
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
/** Short verdict for one version's gates, such as "Fails 1 of 3 gates". */
|
|
224
|
+
export const formatVersionGateVerdict = (summary: VersionGateSummary): string => {
|
|
225
|
+
if (summary.gates.length === 0) {
|
|
226
|
+
return "No gates";
|
|
116
227
|
}
|
|
117
|
-
|
|
228
|
+
if (summary.failing.length === 0) {
|
|
229
|
+
return summary.gates.length === 1
|
|
230
|
+
? "Passes its gate"
|
|
231
|
+
: `Passes all ${summary.gates.length} gates`;
|
|
232
|
+
}
|
|
233
|
+
return `Fails ${summary.failing.length} of ${pluralize(summary.gates.length, "gate", "gates")}`;
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Explain why a version cannot be promoted, such as
|
|
238
|
+
* "v1 fails 1 gate: helpfulness ≥ 0.8 (got 0.62) — can't promote v1."
|
|
239
|
+
*/
|
|
240
|
+
export const formatPromoteBlockedReason = (summary: VersionGateSummary): string | undefined => {
|
|
241
|
+
if (summary.failing.length === 0) {
|
|
242
|
+
return undefined;
|
|
243
|
+
}
|
|
244
|
+
const details = summary.failing.map(describeGateResult).join("; ");
|
|
245
|
+
return `v${summary.version} fails ${pluralize(summary.failing.length, "gate", "gates")}: ${details} — can't promote v${summary.version}.`;
|
|
118
246
|
};
|
|
119
247
|
|
|
120
248
|
export const parsePromoteBlockedTitle = (error: unknown): string | undefined => {
|
|
@@ -140,3 +268,115 @@ export const isAutomaticExperimentEvaluator = (evaluator: {type: string}): boole
|
|
|
140
268
|
export const isValidExperimentVersionCount = (count: number): boolean => {
|
|
141
269
|
return count >= MIN_EXPERIMENT_VERSIONS && count <= MAX_EXPERIMENT_VERSIONS;
|
|
142
270
|
};
|
|
271
|
+
|
|
272
|
+
const ITEM_COLUMN_WIDTH = 120;
|
|
273
|
+
const PASS_COLUMN_WIDTH = 80;
|
|
274
|
+
const MIN_OUTPUT_COLUMN_WIDTH = 180;
|
|
275
|
+
|
|
276
|
+
export interface ExperimentItemColumnWidths {
|
|
277
|
+
item: number;
|
|
278
|
+
output: number;
|
|
279
|
+
pass: number;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Split the measured table width so the per-version output columns take every pixel the item and
|
|
284
|
+
* pass columns leave over, never narrower than the readable minimum.
|
|
285
|
+
*/
|
|
286
|
+
export const experimentItemColumnWidths = ({
|
|
287
|
+
containerWidth,
|
|
288
|
+
versionCount,
|
|
289
|
+
}: {
|
|
290
|
+
containerWidth: number;
|
|
291
|
+
versionCount: number;
|
|
292
|
+
}): ExperimentItemColumnWidths => {
|
|
293
|
+
const fixedWidth = ITEM_COLUMN_WIDTH + PASS_COLUMN_WIDTH * versionCount;
|
|
294
|
+
const available = containerWidth - fixedWidth;
|
|
295
|
+
const output =
|
|
296
|
+
versionCount > 0
|
|
297
|
+
? Math.max(MIN_OUTPUT_COLUMN_WIDTH, Math.floor(available / versionCount))
|
|
298
|
+
: MIN_OUTPUT_COLUMN_WIDTH;
|
|
299
|
+
return {item: ITEM_COLUMN_WIDTH, output, pass: PASS_COLUMN_WIDTH};
|
|
300
|
+
};
|
|
301
|
+
|
|
302
|
+
export const DEFAULT_TARGET_AGREEMENT = 0.9;
|
|
303
|
+
|
|
304
|
+
export const EXPERIMENT_KIND_LABELS: Record<ExperimentKind, string> = {
|
|
305
|
+
judge: "Judge calibration",
|
|
306
|
+
prompt: "Prompt comparison",
|
|
307
|
+
};
|
|
308
|
+
|
|
309
|
+
export const experimentKind = (experiment: Pick<ExperimentRecord, "kind">): ExperimentKind => {
|
|
310
|
+
return experiment.kind === "judge" ? "judge" : "prompt";
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
/** Tags shown next to a prompt version in pickers: latest, production, superseded. */
|
|
314
|
+
export const promptVersionTags = (version: number, detail?: PromptDetail): string[] => {
|
|
315
|
+
const tags: string[] = [];
|
|
316
|
+
if (!detail || detail.versions.length === 0) {
|
|
317
|
+
return tags;
|
|
318
|
+
}
|
|
319
|
+
const latest = Math.max(...detail.versions.map((entry) => entry.version));
|
|
320
|
+
if (version === latest) {
|
|
321
|
+
tags.push("latest");
|
|
322
|
+
}
|
|
323
|
+
if (detail.labels.some((label) => label.label === "production" && label.version === version)) {
|
|
324
|
+
tags.push("production");
|
|
325
|
+
}
|
|
326
|
+
if (version < latest - 1) {
|
|
327
|
+
tags.push("superseded");
|
|
328
|
+
}
|
|
329
|
+
return tags;
|
|
330
|
+
};
|
|
331
|
+
|
|
332
|
+
/** "v1 and v2" or "v1, v2, and v3". */
|
|
333
|
+
export const formatVersionList = (versions: number[]): string => {
|
|
334
|
+
const labels = [...versions].sort((left, right) => left - right).map((version) => `v${version}`);
|
|
335
|
+
if (labels.length <= 2) {
|
|
336
|
+
return labels.join(" and ");
|
|
337
|
+
}
|
|
338
|
+
return `${labels.slice(0, -1).join(", ")}, and ${labels[labels.length - 1]}`;
|
|
339
|
+
};
|
|
340
|
+
|
|
341
|
+
/** Parse a target agreement typed as a percent ("90") into a rate (0.9), or undefined if invalid. */
|
|
342
|
+
export const parseTargetAgreementPercent = (value: string): number | undefined => {
|
|
343
|
+
const trimmed = value.trim();
|
|
344
|
+
if (!trimmed) {
|
|
345
|
+
return undefined;
|
|
346
|
+
}
|
|
347
|
+
const parsed = Number(trimmed);
|
|
348
|
+
if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 100) {
|
|
349
|
+
return undefined;
|
|
350
|
+
}
|
|
351
|
+
return parsed / 100;
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
/** Cost with two decimals; anything above zero but under a cent reads "<$0.01". */
|
|
355
|
+
export const formatExperimentCost = (cost?: number): string => {
|
|
356
|
+
if (cost === undefined || Number.isNaN(cost)) {
|
|
357
|
+
return "—";
|
|
358
|
+
}
|
|
359
|
+
if (cost > 0 && cost < 0.01) {
|
|
360
|
+
return "<$0.01";
|
|
361
|
+
}
|
|
362
|
+
return `$${cost.toFixed(2)}`;
|
|
363
|
+
};
|
|
364
|
+
|
|
365
|
+
/** Plain estimate such as "About 6 generations, 2 minutes, $0.42". */
|
|
366
|
+
export const formatExperimentEstimate = ({
|
|
367
|
+
estimate,
|
|
368
|
+
unit = "generations",
|
|
369
|
+
}: {
|
|
370
|
+
estimate: ExperimentEstimate;
|
|
371
|
+
unit?: "generations" | "judge calls";
|
|
372
|
+
}): string => {
|
|
373
|
+
const minutes = Math.max(1, Math.round(estimate.wallClockSeconds / 60));
|
|
374
|
+
const parts = [
|
|
375
|
+
`About ${estimate.generations} ${estimate.generations === 1 ? unit.replace(/s$/, "") : unit}`,
|
|
376
|
+
`${minutes} ${minutes === 1 ? "minute" : "minutes"}`,
|
|
377
|
+
];
|
|
378
|
+
if (estimate.costUsd !== undefined) {
|
|
379
|
+
parts.push(formatExperimentCost(estimate.costUsd));
|
|
380
|
+
}
|
|
381
|
+
return parts.join(", ");
|
|
382
|
+
};
|
|
@@ -74,4 +74,45 @@ describe("useAiObservabilityExperimentsApi", () => {
|
|
|
74
74
|
{body: {version: 2}, method: "POST", url: "/ai/observability/experiments/exp-1/promote"},
|
|
75
75
|
]);
|
|
76
76
|
});
|
|
77
|
+
|
|
78
|
+
it("filters the list by kind and page size", async () => {
|
|
79
|
+
const calls = stub();
|
|
80
|
+
renderHook(
|
|
81
|
+
() => {
|
|
82
|
+
const api = useAiObservabilityExperimentsApi();
|
|
83
|
+
return {
|
|
84
|
+
judge: api.useListQuery({kind: "judge", limit: 5}),
|
|
85
|
+
prompt: api.useListQuery({kind: "prompt"}),
|
|
86
|
+
};
|
|
87
|
+
},
|
|
88
|
+
{wrapper: ObservabilityTestProvider}
|
|
89
|
+
);
|
|
90
|
+
await flushObservabilityRequests();
|
|
91
|
+
|
|
92
|
+
expect(calls.map((call) => call.url).sort()).toEqual([
|
|
93
|
+
"/ai/observability/experiments?kind=judge&limit=5",
|
|
94
|
+
"/ai/observability/experiments?kind=prompt",
|
|
95
|
+
]);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it("creates a judge experiment", async () => {
|
|
99
|
+
const calls = stub();
|
|
100
|
+
const {result} = renderHook(() => useAiObservabilityExperimentsApi().useCreateMutation(), {
|
|
101
|
+
wrapper: ObservabilityTestProvider,
|
|
102
|
+
});
|
|
103
|
+
const body = {
|
|
104
|
+
datasetId: "ds-1",
|
|
105
|
+
judgeEvaluatorId: "judge-1",
|
|
106
|
+
kind: "judge" as const,
|
|
107
|
+
name: "Judge v2",
|
|
108
|
+
questionSetId: "qs-1",
|
|
109
|
+
versions: [1, 2],
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
await act(async () => {
|
|
113
|
+
await result.current[0](body).unwrap();
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
expect(calls[0]).toMatchObject({body, method: "POST", url: "/ai/observability/experiments"});
|
|
117
|
+
});
|
|
77
118
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {asJsonBody} from "../../../adminRpc";
|
|
1
|
+
import {asJsonBody, withQueryString} from "../../../adminRpc";
|
|
2
2
|
import type {AdminRpcQueryState} from "../../../useAdminRpc";
|
|
3
3
|
import {
|
|
4
4
|
type ObservabilityMutation,
|
|
@@ -6,15 +6,22 @@ import {
|
|
|
6
6
|
useObservabilityMutation,
|
|
7
7
|
useObservabilityQuery,
|
|
8
8
|
} from "../shell/observabilityRequest";
|
|
9
|
-
import type {
|
|
9
|
+
import type {
|
|
10
|
+
ExperimentEstimate,
|
|
11
|
+
ExperimentKind,
|
|
12
|
+
ExperimentRecord,
|
|
13
|
+
ScoreThreshold,
|
|
14
|
+
} from "./experimentTypes";
|
|
10
15
|
|
|
11
16
|
const EXPERIMENTS_TAGS = ["aiObservabilityExperiments"] as const;
|
|
12
17
|
const EXPERIMENTS_URL = "/ai/observability/experiments";
|
|
13
18
|
|
|
14
|
-
|
|
19
|
+
/** Step 4: compare product prompt versions, scored by evaluators. */
|
|
20
|
+
export interface PromptExperimentCreateBody {
|
|
15
21
|
datasetId: string;
|
|
16
22
|
evaluatorIds: string[];
|
|
17
23
|
includeUnproofread?: boolean;
|
|
24
|
+
kind?: "prompt";
|
|
18
25
|
modelOverride?: string;
|
|
19
26
|
name: string;
|
|
20
27
|
promptName: string;
|
|
@@ -22,14 +29,46 @@ export interface ExperimentCreateBody {
|
|
|
22
29
|
versions: number[];
|
|
23
30
|
}
|
|
24
31
|
|
|
25
|
-
|
|
32
|
+
/** Step 3: compare judge prompt versions by agreement with human answers on a judged dataset. */
|
|
33
|
+
export interface JudgeExperimentCreateBody {
|
|
34
|
+
datasetId: string;
|
|
35
|
+
judgeEvaluatorId: string;
|
|
36
|
+
kind: "judge";
|
|
37
|
+
modelOverride?: string;
|
|
38
|
+
name: string;
|
|
39
|
+
questionSetId: string;
|
|
40
|
+
/** Agreement rate (0–1) a version must reach to pass. The server defaults to 0.9. */
|
|
41
|
+
targetAgreement?: number;
|
|
42
|
+
versions: number[];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export type ExperimentCreateBody = JudgeExperimentCreateBody | PromptExperimentCreateBody;
|
|
46
|
+
|
|
47
|
+
export interface PromptExperimentEstimateBody {
|
|
26
48
|
datasetId: string;
|
|
27
49
|
evaluatorIds: string[];
|
|
28
50
|
includeUnproofread?: boolean;
|
|
51
|
+
kind?: "prompt";
|
|
52
|
+
modelOverride?: string;
|
|
53
|
+
versions: number[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface JudgeExperimentEstimateBody {
|
|
57
|
+
datasetId: string;
|
|
58
|
+
kind: "judge";
|
|
29
59
|
modelOverride?: string;
|
|
60
|
+
questionSetId: string;
|
|
30
61
|
versions: number[];
|
|
31
62
|
}
|
|
32
63
|
|
|
64
|
+
export type ExperimentEstimateBody = JudgeExperimentEstimateBody | PromptExperimentEstimateBody;
|
|
65
|
+
|
|
66
|
+
export interface ExperimentListArgs {
|
|
67
|
+
kind?: ExperimentKind;
|
|
68
|
+
/** Page size; the server caps it at EXPERIMENT_LIST_MAX_LIMIT. */
|
|
69
|
+
limit?: number;
|
|
70
|
+
}
|
|
71
|
+
|
|
33
72
|
export interface AiObservabilityExperimentsApi {
|
|
34
73
|
useCreateMutation: () => ObservabilityMutation<ExperimentCreateBody, ExperimentRecord>;
|
|
35
74
|
useDetailQuery: (
|
|
@@ -37,7 +76,9 @@ export interface AiObservabilityExperimentsApi {
|
|
|
37
76
|
options?: ObservabilityQueryOptions
|
|
38
77
|
) => AdminRpcQueryState<ExperimentRecord | {data: ExperimentRecord}>;
|
|
39
78
|
useEstimateMutation: () => ObservabilityMutation<ExperimentEstimateBody, ExperimentEstimate>;
|
|
40
|
-
useListQuery: (
|
|
79
|
+
useListQuery: (
|
|
80
|
+
args?: ExperimentListArgs
|
|
81
|
+
) => AdminRpcQueryState<ExperimentRecord[] | {data: ExperimentRecord[]; more?: boolean}>;
|
|
41
82
|
usePromoteMutation: () => ObservabilityMutation<
|
|
42
83
|
{id: string; version: number},
|
|
43
84
|
{label: string; outgoingVersion?: number}
|
|
@@ -72,8 +113,14 @@ export const useAiObservabilityExperimentsApi = (): AiObservabilityExperimentsAp
|
|
|
72
113
|
url: `${EXPERIMENTS_URL}/estimate`,
|
|
73
114
|
}),
|
|
74
115
|
}),
|
|
75
|
-
useListQuery: () =>
|
|
76
|
-
useObservabilityQuery({
|
|
116
|
+
useListQuery: (args) =>
|
|
117
|
+
useObservabilityQuery({
|
|
118
|
+
providesTags: EXPERIMENTS_TAGS,
|
|
119
|
+
url: withQueryString({
|
|
120
|
+
params: {kind: args?.kind, limit: args?.limit || undefined},
|
|
121
|
+
url: EXPERIMENTS_URL,
|
|
122
|
+
}),
|
|
123
|
+
}),
|
|
77
124
|
usePromoteMutation: () =>
|
|
78
125
|
useObservabilityMutation({
|
|
79
126
|
invalidatesTags: EXPERIMENTS_TAGS,
|
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
type PlaygroundRunResult,
|
|
13
13
|
type PromptDetail,
|
|
14
14
|
unwrapPromptDetail,
|
|
15
|
+
unwrapPromptList,
|
|
15
16
|
unwrapPromptPayload,
|
|
16
17
|
} from "./promptTypes";
|
|
17
18
|
import {useAiObservabilityPromptsApi} from "./useAiObservabilityPromptsApi";
|
|
@@ -33,6 +34,7 @@ export const AiPromptEditorScreenWidget: React.FC<AiPromptEditorScreenWidgetProp
|
|
|
33
34
|
const {
|
|
34
35
|
useCreateVersionMutation,
|
|
35
36
|
useDetailQuery,
|
|
37
|
+
useListQuery,
|
|
36
38
|
usePlaygroundMutation,
|
|
37
39
|
useSetLabelMutation,
|
|
38
40
|
useStatusQuery,
|
|
@@ -42,16 +44,34 @@ export const AiPromptEditorScreenWidget: React.FC<AiPromptEditorScreenWidgetProp
|
|
|
42
44
|
const [cachedDetail, setCachedDetail] = useState<
|
|
43
45
|
{detail: PromptDetail; name: string} | undefined
|
|
44
46
|
>(undefined);
|
|
47
|
+
const [scopeName, setScopeName] = useState(name);
|
|
48
|
+
const nameChanged = name !== scopeName;
|
|
49
|
+
// Used by navigates to another prompt on this same screen. Drop the previous
|
|
50
|
+
// version pin during render so the new name is not queried at that version.
|
|
51
|
+
if (nameChanged) {
|
|
52
|
+
setScopeName(name);
|
|
53
|
+
setSelectedVersion(undefined);
|
|
54
|
+
setPinnedQueryVersion(undefined);
|
|
55
|
+
setCachedDetail(undefined);
|
|
56
|
+
}
|
|
45
57
|
|
|
46
58
|
const detailQueryArg = useMemo(() => {
|
|
47
59
|
if (!name) {
|
|
48
60
|
return {name: ""};
|
|
49
61
|
}
|
|
50
|
-
if (pinnedQueryVersion === undefined) {
|
|
62
|
+
if (nameChanged || pinnedQueryVersion === undefined) {
|
|
51
63
|
return {name};
|
|
52
64
|
}
|
|
53
65
|
return {name, promptVersion: pinnedQueryVersion};
|
|
54
|
-
}, [name, pinnedQueryVersion]);
|
|
66
|
+
}, [name, nameChanged, pinnedQueryVersion]);
|
|
67
|
+
|
|
68
|
+
const promptListQuery = useListQuery({});
|
|
69
|
+
const promptNames = useMemo(() => {
|
|
70
|
+
return unwrapPromptList(promptListQuery.data)
|
|
71
|
+
.map((prompt) => prompt.name)
|
|
72
|
+
.filter((promptName) => promptName !== name)
|
|
73
|
+
.sort((left, right) => left.localeCompare(right));
|
|
74
|
+
}, [name, promptListQuery.data]);
|
|
55
75
|
|
|
56
76
|
const {data, isError, isFetching, isLoading, refetch} = useDetailQuery(detailQueryArg, {
|
|
57
77
|
skip: !name,
|
|
@@ -220,6 +240,9 @@ export const AiPromptEditorScreenWidget: React.FC<AiPromptEditorScreenWidgetProp
|
|
|
220
240
|
onOpenExperiment={(experimentId) => {
|
|
221
241
|
router.push(`${prefix}/ai-experiment-results?id=${encodeURIComponent(experimentId)}`);
|
|
222
242
|
}}
|
|
243
|
+
onOpenPrompt={(promptName) => {
|
|
244
|
+
router.push(`${prefix}/ai-prompt-editor?name=${encodeURIComponent(promptName)}`);
|
|
245
|
+
}}
|
|
223
246
|
onOpenTrace={(traceId) => {
|
|
224
247
|
router.push(`${prefix}/ai-trace-detail?id=${encodeURIComponent(traceId)}`);
|
|
225
248
|
}}
|
|
@@ -232,6 +255,7 @@ export const AiPromptEditorScreenWidget: React.FC<AiPromptEditorScreenWidgetProp
|
|
|
232
255
|
playgroundError={playgroundError}
|
|
233
256
|
playgroundResult={playgroundResult}
|
|
234
257
|
productionError={labelState.isError ? "Could not set production." : undefined}
|
|
258
|
+
promptNames={promptNames}
|
|
235
259
|
relationshipsError={relationshipsError}
|
|
236
260
|
saveError={createState.isError ? "Could not save a new version." : undefined}
|
|
237
261
|
selectedVersion={version}
|