@terreno/admin-frontend 57.20.0 → 57.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/dist/AdminFilterDrawer.d.ts.map +1 -1
- package/dist/AdminFilterDrawer.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js +19 -9
- package/dist/widgets/aiObservability/datasets/AiDatasetsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts +7 -2
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js +155 -43
- package/dist/widgets/aiObservability/datasets/AiDatasetsListView.js.map +1 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts +20 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/datasets/datasetTypes.js +17 -12
- package/dist/widgets/aiObservability/datasets/datasetTypes.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js +12 -8
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js +4 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts +8 -0
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js +68 -37
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorPanels.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js +4 -2
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts +8 -0
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js +156 -33
- package/dist/widgets/aiObservability/evaluators/AiEvaluatorsListView.js.map +1 -1
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts +32 -0
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js +49 -0
- package/dist/widgets/aiObservability/evaluators/evaluatorTypes.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js +235 -12
- package/dist/widgets/aiObservability/experiments/AiExperimentNewScreen.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts +10 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js +113 -56
- package/dist/widgets/aiObservability/experiments/AiExperimentNewView.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js +53 -38
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsScreen.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts +2 -3
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js +117 -32
- package/dist/widgets/aiObservability/experiments/AiExperimentResultsView.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js +13 -4
- package/dist/widgets/aiObservability/experiments/AiExperimentsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts +5 -0
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js +157 -55
- package/dist/widgets/aiObservability/experiments/AiExperimentsListView.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts +41 -0
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.d.ts.map +1 -0
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js +113 -0
- package/dist/widgets/aiObservability/experiments/AiJudgeExperimentSteps.js.map +1 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts +30 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.d.ts.map +1 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.js +29 -0
- package/dist/widgets/aiObservability/experiments/ExperimentWizard.js.map +1 -0
- package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts +82 -3
- package/dist/widgets/aiObservability/experiments/experimentTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/experimentTypes.js +172 -6
- package/dist/widgets/aiObservability/experiments/experimentTypes.js.map +1 -1
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts +34 -4
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js +11 -4
- package/dist/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js +24 -5
- package/dist/widgets/aiObservability/prompts/AiPromptEditorScreen.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts +2 -0
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js +36 -6
- package/dist/widgets/aiObservability/prompts/AiPromptEditorView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts +3 -0
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.js +6 -6
- package/dist/widgets/aiObservability/prompts/AiPromptHubView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js +12 -12
- package/dist/widgets/aiObservability/prompts/AiPromptsListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.js +156 -35
- package/dist/widgets/aiObservability/prompts/AiPromptsListView.js.map +1 -1
- package/dist/widgets/aiObservability/prompts/promptTypes.d.ts +13 -3
- package/dist/widgets/aiObservability/prompts/promptTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/prompts/promptTypes.js +61 -19
- package/dist/widgets/aiObservability/prompts/promptTypes.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts +11 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js +115 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchScreen.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts +22 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.js +136 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchView.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts +4 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js +92 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesScreen.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts +26 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.js +115 -0
- package/dist/widgets/aiObservability/review/AiReviewBatchesView.js.map +1 -0
- package/dist/widgets/aiObservability/review/AiReviewItemScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewItemScreen.js +73 -16
- package/dist/widgets/aiObservability/review/AiReviewItemScreen.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts +19 -5
- package/dist/widgets/aiObservability/review/AiReviewItemView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewItemView.js +14 -9
- package/dist/widgets/aiObservability/review/AiReviewItemView.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js +16 -2
- package/dist/widgets/aiObservability/review/AiReviewQueueScreen.js.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts +11 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/AiReviewQueueView.js +96 -37
- package/dist/widgets/aiObservability/review/AiReviewQueueView.js.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts +13 -0
- package/dist/widgets/aiObservability/review/ReviewQuestionList.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/ReviewQuestionList.js +28 -0
- package/dist/widgets/aiObservability/review/ReviewQuestionList.js.map +1 -0
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts +3 -0
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js +2 -2
- package/dist/widgets/aiObservability/review/ReviewReadOnlyPanel.js.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts +14 -0
- package/dist/widgets/aiObservability/review/ReviewScoreField.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/ReviewScoreField.js +41 -12
- package/dist/widgets/aiObservability/review/ReviewScoreField.js.map +1 -1
- package/dist/widgets/aiObservability/review/batchTypes.d.ts +103 -0
- package/dist/widgets/aiObservability/review/batchTypes.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/batchTypes.js +61 -0
- package/dist/widgets/aiObservability/review/batchTypes.js.map +1 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts +21 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.d.ts.map +1 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.js +141 -0
- package/dist/widgets/aiObservability/review/reviewShortcuts.js.map +1 -0
- package/dist/widgets/aiObservability/review/reviewTypes.d.ts +23 -2
- package/dist/widgets/aiObservability/review/reviewTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/reviewTypes.js +25 -0
- package/dist/widgets/aiObservability/review/reviewTypes.js.map +1 -1
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts +48 -0
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js +49 -0
- package/dist/widgets/aiObservability/review/useAiObservabilityReviewApi.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts +30 -0
- package/dist/widgets/aiObservability/shell/AgreementMeter.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/AgreementMeter.js +32 -0
- package/dist/widgets/aiObservability/shell/AgreementMeter.js.map +1 -0
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts +10 -0
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js +6 -4
- package/dist/widgets/aiObservability/shell/AiObservabilityChrome.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts +3 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js +5 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityScreenWidgets.js.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js +3 -3
- package/dist/widgets/aiObservability/shell/AiObservabilityStatusChip.js.map +1 -1
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts +14 -0
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js +26 -0
- package/dist/widgets/aiObservability/shell/EvalPipelineOverview.js.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts +11 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.js +8 -0
- package/dist/widgets/aiObservability/shell/EvalStepBanner.js.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.d.ts +22 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.js +28 -0
- package/dist/widgets/aiObservability/shell/EvalStepper.js.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts +17 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js +10 -0
- package/dist/widgets/aiObservability/shell/ObservabilityPageHeader.js.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts +24 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.js +11 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStates.js.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts +11 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js +9 -0
- package/dist/widgets/aiObservability/shell/ObservabilityStatusBadge.js.map +1 -0
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts +1 -0
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.d.ts.map +1 -1
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.js +15 -8
- package/dist/widgets/aiObservability/shell/aiObservabilityNav.js.map +1 -1
- package/dist/widgets/aiObservability/shell/clientTable.d.ts +48 -0
- package/dist/widgets/aiObservability/shell/clientTable.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/clientTable.js +163 -0
- package/dist/widgets/aiObservability/shell/clientTable.js.map +1 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.d.ts +24 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.js +43 -0
- package/dist/widgets/aiObservability/shell/evalPipeline.js.map +1 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts +23 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.d.ts.map +1 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.js +72 -0
- package/dist/widgets/aiObservability/shell/observabilityStatus.js.map +1 -0
- package/dist/widgets/aiObservability/traces/AiTracesListScreen.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListScreen.js +18 -9
- package/dist/widgets/aiObservability/traces/AiTracesListScreen.js.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts +18 -7
- package/dist/widgets/aiObservability/traces/AiTracesListView.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/AiTracesListView.js +198 -78
- package/dist/widgets/aiObservability/traces/AiTracesListView.js.map +1 -1
- package/dist/widgets/aiObservability/traces/traceTypes.d.ts +23 -1
- package/dist/widgets/aiObservability/traces/traceTypes.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/traceTypes.js +92 -1
- package/dist/widgets/aiObservability/traces/traceTypes.js.map +1 -1
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts +2 -0
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.d.ts.map +1 -1
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js +9 -2
- package/dist/widgets/aiObservability/traces/useAiObservabilityTracesApi.js.map +1 -1
- package/package.json +7 -3
- package/src/AdminBreadcrumbs.test.tsx +1 -1
- package/src/AdminFieldRenderer.test.tsx +2 -1
- package/src/AdminFilterDrawer.tsx +1 -2
- package/src/AdminHome.test.tsx +2 -1
- package/src/AdminModelTable.test.tsx +16 -12
- package/src/AdminModelTable.windowed.test.tsx +11 -5
- package/src/AdminPrimitiveArrayField.test.tsx +3 -3
- package/src/AdminProvider.test.tsx +11 -6
- package/src/AdminRolesList.test.tsx +22 -13
- package/src/AdminScreenPage.test.tsx +1 -1
- package/src/AdminScreenRouter.test.tsx +15 -5
- package/src/AdminScriptList.test.tsx +9 -2
- package/src/AdminShell.test.tsx +1 -1
- package/src/AdminVersionConfig.test.tsx +4 -4
- package/src/AnnouncementOverviewScreenWidget.test.tsx +6 -1
- package/src/CheckboxListEditor.test.tsx +11 -9
- package/src/ConfigurationScreen.test.tsx +2 -2
- package/src/adminRequest.test.ts +3 -3
- package/src/adminWindowedTable.test.ts +4 -2
- package/src/comms/CommsDashboardScreen.test.tsx +5 -5
- package/src/comms/CommsMessageDetail.test.tsx +1 -1
- package/src/index.tsx +4 -0
- package/src/isolated/AdminFieldRenderer.isolated.tsx +11 -6
- package/src/isolated/AdminFilterDrawer.isolated.tsx +8 -6
- package/src/isolated/AdminFilterDrawerMobile.isolated.tsx +3 -1
- package/src/isolated/AdminHomeWidgets.isolated.tsx +2 -0
- package/src/isolated/AdminNestedArrayField.isolated.tsx +10 -9
- package/src/isolated/AdminObjectPicker.isolated.tsx +9 -10
- package/src/isolated/AiPromptEditorScreen.isolated.tsx +24 -0
- package/src/isolated/AiPromptsListScreen.isolated.tsx +1 -1
- package/src/isolated/DocumentStorageBrowser.isolated.tsx +16 -8
- package/src/isolated/RecentActivityWidget.isolated.tsx +2 -0
- package/src/isolated/adminWidgets.isolated.tsx +46 -6
- package/src/isolated/aiListTables.isolated.tsx +654 -0
- package/src/isolated/aiObservabilityScreens.isolated.tsx +731 -294
- package/src/isolated/aiReviewBatchScreens.isolated.tsx +378 -0
- package/src/isolated/aiTracesTable.isolated.tsx +375 -0
- package/src/isolated/generateConsentHistoryPdf.isolated.ts +30 -10
- package/src/isolated/generateConsentPdfsMobile.isolated.ts +24 -3
- package/src/isolated/hooks.isolated.tsx +84 -60
- package/src/isolated/rpcDualRun.isolated.tsx +9 -2
- package/src/jobs/JobsDashboardScreen.test.tsx +3 -3
- package/src/jobs/JobsHomeWidget.test.tsx +1 -1
- package/src/jobs/JobsJobDetail.test.tsx +1 -1
- package/src/orgs/OrgManagementScreens.test.tsx +1 -1
- package/src/orgs/adminModelEndpoints.test.ts +1 -1
- package/src/testing/useAdminApiDouble.ts +2 -1
- package/src/types.ts +2 -0
- package/src/useAdminRpc.test.tsx +3 -3
- package/src/widgets/AIRequestsScreenWidget.test.tsx +20 -3
- package/src/widgets/ModelsGridWidget.test.tsx +25 -5
- package/src/widgets/aiObservability/datasets/AiDatasetDetailView.test.tsx +4 -2
- package/src/widgets/aiObservability/datasets/AiDatasetsListScreen.tsx +44 -13
- package/src/widgets/aiObservability/datasets/AiDatasetsListView.tsx +303 -103
- package/src/widgets/aiObservability/datasets/datasetTypes.test.ts +0 -14
- package/src/widgets/aiObservability/datasets/datasetTypes.ts +37 -13
- package/src/widgets/aiObservability/evaluators/AiEvaluatorDetailScreen.tsx +34 -11
- package/src/widgets/aiObservability/evaluators/AiEvaluatorNewScreen.tsx +12 -1
- package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.test.tsx +186 -32
- package/src/widgets/aiObservability/evaluators/AiEvaluatorPanels.tsx +235 -131
- package/src/widgets/aiObservability/evaluators/AiEvaluatorsListScreen.tsx +21 -2
- package/src/widgets/aiObservability/evaluators/AiEvaluatorsListView.tsx +261 -76
- package/src/widgets/aiObservability/evaluators/evaluatorTypes.test.ts +7 -0
- package/src/widgets/aiObservability/evaluators/evaluatorTypes.ts +82 -0
- package/src/widgets/aiObservability/experiments/AiExperimentNewScreen.tsx +271 -13
- package/src/widgets/aiObservability/experiments/AiExperimentNewView.test.tsx +168 -2
- package/src/widgets/aiObservability/experiments/AiExperimentNewView.tsx +400 -186
- package/src/widgets/aiObservability/experiments/AiExperimentResultsScreen.tsx +104 -45
- package/src/widgets/aiObservability/experiments/AiExperimentResultsView.test.tsx +290 -0
- package/src/widgets/aiObservability/experiments/AiExperimentResultsView.tsx +342 -97
- package/src/widgets/aiObservability/experiments/AiExperimentsListScreen.tsx +35 -4
- package/src/widgets/aiObservability/experiments/AiExperimentsListView.tsx +302 -112
- package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.test.tsx +250 -0
- package/src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.tsx +468 -0
- package/src/widgets/aiObservability/experiments/ExperimentWizard.tsx +106 -0
- package/src/widgets/aiObservability/experiments/experimentTypes.test.ts +103 -16
- package/src/widgets/aiObservability/experiments/experimentTypes.ts +250 -10
- package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.test.ts +35 -0
- package/src/widgets/aiObservability/experiments/useAiObservabilityExperimentsApi.ts +55 -9
- package/src/widgets/aiObservability/prompts/AiPromptEditorScreen.tsx +26 -2
- package/src/widgets/aiObservability/prompts/AiPromptEditorView.tsx +95 -1
- package/src/widgets/aiObservability/prompts/AiPromptHubView.test.tsx +61 -1
- package/src/widgets/aiObservability/prompts/AiPromptHubView.tsx +71 -27
- package/src/widgets/aiObservability/prompts/AiPromptsListScreen.tsx +34 -15
- package/src/widgets/aiObservability/prompts/AiPromptsListView.test.tsx +104 -8
- package/src/widgets/aiObservability/prompts/AiPromptsListView.tsx +287 -117
- package/src/widgets/aiObservability/prompts/promptTypes.test.ts +63 -1
- package/src/widgets/aiObservability/prompts/promptTypes.ts +83 -22
- package/src/widgets/aiObservability/review/AiReviewBatchScreen.tsx +217 -0
- package/src/widgets/aiObservability/review/AiReviewBatchView.tsx +398 -0
- package/src/widgets/aiObservability/review/AiReviewBatchViews.test.tsx +328 -0
- package/src/widgets/aiObservability/review/AiReviewBatchesScreen.tsx +137 -0
- package/src/widgets/aiObservability/review/AiReviewBatchesView.tsx +333 -0
- package/src/widgets/aiObservability/review/AiReviewItemScreen.tsx +121 -25
- package/src/widgets/aiObservability/review/AiReviewItemView.tsx +112 -70
- package/src/widgets/aiObservability/review/AiReviewQueueScreen.tsx +29 -2
- package/src/widgets/aiObservability/review/AiReviewQueueView.tsx +183 -75
- package/src/widgets/aiObservability/review/ReviewQuestionList.tsx +128 -0
- package/src/widgets/aiObservability/review/ReviewReadOnlyPanel.tsx +21 -13
- package/src/widgets/aiObservability/review/ReviewScoreField.tsx +111 -27
- package/src/widgets/aiObservability/review/batchTypes.test.ts +207 -0
- package/src/widgets/aiObservability/review/batchTypes.ts +162 -0
- package/src/widgets/aiObservability/review/reviewShortcuts.ts +195 -0
- package/src/widgets/aiObservability/review/reviewTypes.ts +62 -2
- package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.test.ts +74 -0
- package/src/widgets/aiObservability/review/useAiObservabilityReviewApi.ts +98 -0
- package/src/widgets/aiObservability/shell/AgreementMeter.tsx +103 -0
- package/src/widgets/aiObservability/shell/AiObservabilityChrome.tsx +48 -14
- package/src/widgets/aiObservability/shell/AiObservabilityScreenWidgets.tsx +6 -0
- package/src/widgets/aiObservability/shell/AiObservabilityStatusChip.tsx +10 -2
- package/src/widgets/aiObservability/shell/EvalPipelineOverview.tsx +68 -0
- package/src/widgets/aiObservability/shell/EvalStepBanner.tsx +24 -0
- package/src/widgets/aiObservability/shell/EvalStepper.test.tsx +80 -0
- package/src/widgets/aiObservability/shell/EvalStepper.tsx +126 -0
- package/src/widgets/aiObservability/shell/ObservabilityPageHeader.tsx +50 -0
- package/src/widgets/aiObservability/shell/ObservabilitySharedUi.test.tsx +140 -0
- package/src/widgets/aiObservability/shell/ObservabilityStates.tsx +81 -0
- package/src/widgets/aiObservability/shell/ObservabilityStatusBadge.tsx +29 -0
- package/src/widgets/aiObservability/shell/ObservabilityTable.test.tsx +7 -1
- package/src/widgets/aiObservability/shell/aiObservabilityNav.ts +16 -8
- package/src/widgets/aiObservability/shell/clientTable.test.ts +176 -0
- package/src/widgets/aiObservability/shell/clientTable.ts +251 -0
- package/src/widgets/aiObservability/shell/evalPipeline.ts +74 -0
- package/src/widgets/aiObservability/shell/observabilityStatus.test.ts +73 -0
- package/src/widgets/aiObservability/shell/observabilityStatus.ts +100 -0
- package/src/widgets/aiObservability/traces/AiTracesListScreen.tsx +41 -13
- package/src/widgets/aiObservability/traces/AiTracesListView.tsx +423 -300
- package/src/widgets/aiObservability/traces/traceTypes.test.ts +52 -1
- package/src/widgets/aiObservability/traces/traceTypes.ts +117 -2
- package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.test.ts +13 -1
- package/src/widgets/aiObservability/traces/useAiObservabilityTracesApi.ts +10 -2
- package/src/widgets/builtInFieldWidgets.test.tsx +16 -3
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import React from "react";
|
|
2
|
+
import { type DatasetRecord } from "../datasets/datasetTypes";
|
|
3
|
+
import { type EvaluatorRecord } from "../evaluators/evaluatorTypes";
|
|
4
|
+
import type { PromptDetail } from "../prompts/promptTypes";
|
|
5
|
+
import { type ExperimentEstimate } from "./experimentTypes";
|
|
6
|
+
export type JudgeWizardStep = 1 | 2 | 3 | 4;
|
|
7
|
+
export interface AiJudgeExperimentStepsProps {
|
|
8
|
+
datasetId: string;
|
|
9
|
+
datasets: DatasetRecord[];
|
|
10
|
+
estimate?: ExperimentEstimate;
|
|
11
|
+
estimateError?: string;
|
|
12
|
+
/** Every evaluator: human question sets and llm-judges. */
|
|
13
|
+
evaluators: EvaluatorRecord[];
|
|
14
|
+
isCreating: boolean;
|
|
15
|
+
isEstimating: boolean;
|
|
16
|
+
judgeEvaluatorId: string;
|
|
17
|
+
judgePromptDetail?: PromptDetail;
|
|
18
|
+
modelOverride: string;
|
|
19
|
+
onDatasetChange: (id: string) => void;
|
|
20
|
+
onJudgeChange: (id: string) => void;
|
|
21
|
+
onModelOverrideChange: (value: string) => void;
|
|
22
|
+
/** Opens the dataset detail, where the dataset can be sent to human review. */
|
|
23
|
+
onOpenDataset: (id: string) => void;
|
|
24
|
+
onQuestionSetChange: (id: string) => void;
|
|
25
|
+
onRun: () => void;
|
|
26
|
+
onStepChange: (step: JudgeWizardStep) => void;
|
|
27
|
+
onTargetAgreementChange: (value: string) => void;
|
|
28
|
+
onVersionToggle: (version: number) => void;
|
|
29
|
+
questionSetId: string;
|
|
30
|
+
step: JudgeWizardStep;
|
|
31
|
+
/** Target agreement as typed, in percent ("90"). */
|
|
32
|
+
targetAgreement: string;
|
|
33
|
+
versions: number[];
|
|
34
|
+
}
|
|
35
|
+
/** Llm-judges that answer at least one of the question set's questions, most overlap first. */
|
|
36
|
+
export declare const judgesForQuestionSet: (evaluators: EvaluatorRecord[], questionSet?: EvaluatorRecord) => Array<{
|
|
37
|
+
judge: EvaluatorRecord;
|
|
38
|
+
shared: string[];
|
|
39
|
+
}>;
|
|
40
|
+
export declare const AiJudgeExperimentSteps: React.FC<AiJudgeExperimentStepsProps>;
|
|
41
|
+
//# sourceMappingURL=AiJudgeExperimentSteps.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AiJudgeExperimentSteps.d.ts","sourceRoot":"","sources":["../../../../src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.tsx"],"names":[],"mappings":"AAaA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAEL,KAAK,aAAa,EAInB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,KAAK,eAAe,EAGrB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,wBAAwB,CAAC;AAOzD,OAAO,EACL,KAAK,kBAAkB,EAMxB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE5C,MAAM,WAAW,2BAA2B;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,2DAA2D;IAC3D,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,YAAY,CAAC;IACjC,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IACtC,aAAa,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,qBAAqB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC/C,+EAA+E;IAC/E,aAAa,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,mBAAmB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IAC1C,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,YAAY,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;IAC9C,uBAAuB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACjD,eAAe,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAC3C,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,eAAe,CAAC;IACtB,oDAAoD;IACpD,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAmBD,+FAA+F;AAC/F,eAAO,MAAM,oBAAoB,GAC/B,YAAY,eAAe,EAAE,EAC7B,cAAc,eAAe,KAC5B,KAAK,CAAC;IAAC,KAAK,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAC,CASlD,CAAC;AA2DF,eAAO,MAAM,sBAAsB,EAAE,KAAK,CAAC,EAAE,CAAC,2BAA2B,CAiTxE,CAAC"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
|
+
import { Badge, Banner, Box, Button, Card, CheckBox, Heading, SelectField, Spinner, Text, TextField, } from "@terreno/ui";
|
|
3
|
+
import { formatJudgmentCoverage, formatStaleAnswers, judgedQuestionSets, } from "../datasets/datasetTypes";
|
|
4
|
+
import { sharedQuestionKeys, summarizeCalibration, } from "../evaluators/evaluatorTypes";
|
|
5
|
+
import { dimensionLabel } from "../review/reviewTypes";
|
|
6
|
+
import { AgreementMeter } from "../shell/AgreementMeter";
|
|
7
|
+
import { EvalStepBanner } from "../shell/EvalStepBanner";
|
|
8
|
+
import { EvalStepper } from "../shell/EvalStepper";
|
|
9
|
+
import { PIPELINE_ACTIVITY, PIPELINE_STEP } from "../shell/evalPipeline";
|
|
10
|
+
import { buildWizardSteps, WizardFooter } from "./ExperimentWizard";
|
|
11
|
+
import { formatExperimentEstimate, formatVersionList, isValidExperimentVersionCount, parseTargetAgreementPercent, promptVersionTags, } from "./experimentTypes";
|
|
12
|
+
const JUDGE_STEP_LABELS = [
|
|
13
|
+
"Judged dataset",
|
|
14
|
+
"Question set",
|
|
15
|
+
"Judge and versions",
|
|
16
|
+
"Review and run",
|
|
17
|
+
];
|
|
18
|
+
const JUDGE_STEP_DETAILS = {
|
|
19
|
+
1: "pick a dataset humans already judged. Their answers are the answer key the LLM judge is graded against.",
|
|
20
|
+
2: "choose the questions the humans answered. The judge is graded only on these questions.",
|
|
21
|
+
3: "pick the LLM judge and 2–3 versions of its prompt to compare against the human answers.",
|
|
22
|
+
4: "check the plan and run it. Each version's agreement with the humans decides which one to promote.",
|
|
23
|
+
};
|
|
24
|
+
const JUDGE_REUSE_MESSAGE = "Humans already answered these questions for this dataset. This experiment reuses their answers, so no new human work is needed.";
|
|
25
|
+
/** Llm-judges that answer at least one of the question set's questions, most overlap first. */
|
|
26
|
+
export const judgesForQuestionSet = (evaluators, questionSet) => {
|
|
27
|
+
if (!questionSet) {
|
|
28
|
+
return [];
|
|
29
|
+
}
|
|
30
|
+
return evaluators
|
|
31
|
+
.filter((evaluator) => evaluator.type === "llm-judge" && Boolean(evaluator.judgePromptName))
|
|
32
|
+
.map((judge) => ({ judge, shared: sharedQuestionKeys(judge, questionSet) }))
|
|
33
|
+
.filter((entry) => entry.shared.length > 0)
|
|
34
|
+
.sort((left, right) => right.shared.length - left.shared.length);
|
|
35
|
+
};
|
|
36
|
+
const DatasetChoiceBody = ({ dataset, isSelected, onOpenDataset }) => {
|
|
37
|
+
const judgments = dataset.judgments ?? [];
|
|
38
|
+
const isJudged = judgedQuestionSets(dataset).length > 0;
|
|
39
|
+
return (_jsxs(Box, { gap: 1, children: [_jsxs(Box, { alignItems: "center", direction: "row", gap: 2, wrap: true, children: [isJudged ? _jsx(CheckBox, { selected: isSelected }) : undefined, _jsx(Text, { bold: true, children: dataset.name }), _jsx(Text, { color: "secondaryDark", size: "sm", children: `${dataset.counts.total} items` })] }), judgments.map((judgment) => {
|
|
40
|
+
const stale = formatStaleAnswers(judgment);
|
|
41
|
+
return (_jsxs(Box, { gap: 1, children: [_jsx(Text, { color: judgment.current > 0 ? "primary" : "secondaryDark", size: "sm", testID: `ai-judge-dataset-coverage-${dataset.id}-${judgment.evaluatorId}`, children: formatJudgmentCoverage(judgment) }), stale ? (_jsx(Text, { color: "warning", size: "sm", testID: `ai-judge-dataset-stale-${dataset.id}-${judgment.evaluatorId}`, children: stale })) : undefined] }, judgment.evaluatorId));
|
|
42
|
+
}), isJudged ? undefined : (_jsxs(Box, { alignItems: "center", direction: "row", gap: 2, wrap: true, children: [_jsx(Text, { color: "secondaryDark", size: "sm", children: "No current human answers yet, so there is nothing to grade a judge against." }), _jsx(Button, { onClick: () => {
|
|
43
|
+
onOpenDataset(dataset.id);
|
|
44
|
+
}, size: "sm", testID: `ai-judge-dataset-send-review-${dataset.id}`, text: "Send to human review first", variant: "outline" })] }))] }));
|
|
45
|
+
};
|
|
46
|
+
export const AiJudgeExperimentSteps = ({ datasetId, datasets, estimate, estimateError, evaluators, isCreating, isEstimating, judgeEvaluatorId, judgePromptDetail, modelOverride, onDatasetChange, onJudgeChange, onModelOverrideChange, onOpenDataset, onQuestionSetChange, onRun, onStepChange, onTargetAgreementChange, onVersionToggle, questionSetId, step, targetAgreement, versions, }) => {
|
|
47
|
+
const dataset = datasets.find((entry) => entry.id === datasetId);
|
|
48
|
+
const judgedSets = judgedQuestionSets(dataset);
|
|
49
|
+
const judgment = judgedSets.find((entry) => entry.evaluatorId === questionSetId);
|
|
50
|
+
const questionSet = evaluators.find((entry) => entry.id === questionSetId);
|
|
51
|
+
const judgeOptions = judgesForQuestionSet(evaluators, questionSet);
|
|
52
|
+
const judgeEntry = judgeOptions.find((entry) => entry.judge.id === judgeEvaluatorId);
|
|
53
|
+
const judge = judgeEntry?.judge;
|
|
54
|
+
const calibration = judge ? summarizeCalibration(judge.calibration) : undefined;
|
|
55
|
+
const uncoveredQuestions = questionSet
|
|
56
|
+
? questionSet.dimensions.filter((dimension) => !judgeEntry?.shared.includes(dimension.key))
|
|
57
|
+
: [];
|
|
58
|
+
const targetRate = parseTargetAgreementPercent(targetAgreement);
|
|
59
|
+
const hasDataset = judgedSets.length > 0;
|
|
60
|
+
const hasQuestionSet = Boolean(judgment);
|
|
61
|
+
const hasJudgeSetup = Boolean(judge) && isValidExperimentVersionCount(versions.length);
|
|
62
|
+
const isStepLocked = (stepNumber) => {
|
|
63
|
+
if (stepNumber > 1 && !hasDataset) {
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
if (stepNumber > 2 && !hasQuestionSet) {
|
|
67
|
+
return true;
|
|
68
|
+
}
|
|
69
|
+
return stepNumber > 3 && !hasJudgeSetup;
|
|
70
|
+
};
|
|
71
|
+
const isNextDisabled = step < 4 && isStepLocked((step + 1));
|
|
72
|
+
return (_jsxs(Card, { gap: 4, padding: 4, testID: "ai-judge-wizard", children: [_jsx(EvalStepper, { steps: buildWizardSteps({
|
|
73
|
+
current: step,
|
|
74
|
+
isLocked: isStepLocked,
|
|
75
|
+
labels: JUDGE_STEP_LABELS,
|
|
76
|
+
onStepChange,
|
|
77
|
+
testIDPrefix: "ai-judge-step",
|
|
78
|
+
}), testID: "ai-judge-stepper" }), _jsx(EvalStepBanner, { activity: PIPELINE_ACTIVITY.judge, detail: JUDGE_STEP_DETAILS[step], step: PIPELINE_STEP.judge }), step === 1 ? (_jsxs(Box, { gap: 2, testID: "ai-judge-step-dataset", children: [datasets.length === 0 ? (_jsx(Text, { color: "secondaryDark", testID: "ai-judge-datasets-empty", children: "No datasets yet. On Traces, choose Add to dataset, then send the dataset to human review." })) : undefined, datasets.map((entry) => {
|
|
79
|
+
const isJudged = judgedQuestionSets(entry).length > 0;
|
|
80
|
+
const isSelected = entry.id === datasetId && isJudged;
|
|
81
|
+
const body = (_jsx(DatasetChoiceBody, { dataset: entry, isSelected: isSelected, onOpenDataset: onOpenDataset }));
|
|
82
|
+
if (!isJudged) {
|
|
83
|
+
return (_jsx(Box, { border: "default", color: "baseAlternate", padding: 3, rounding: "md", testID: `ai-judge-dataset-${entry.id}-disabled`, children: body }, entry.id));
|
|
84
|
+
}
|
|
85
|
+
return (_jsx(Box, { accessibilityHint: "Use this judged dataset for the calibration", accessibilityLabel: `Dataset ${entry.name}`, accessibilityState: { checked: isSelected }, border: isSelected ? "dark" : "default", onClick: () => {
|
|
86
|
+
onDatasetChange(entry.id);
|
|
87
|
+
}, padding: 3, rounding: "md", testID: `ai-judge-dataset-${entry.id}`, children: body }, entry.id));
|
|
88
|
+
}), hasDataset ? (_jsx(Box, { testID: "ai-judge-dataset-reuse", children: _jsx(Banner, { hasIcon: true, status: "info", text: JUDGE_REUSE_MESSAGE }) })) : (_jsx(Text, { color: "secondaryLight", size: "sm", testID: "ai-judge-dataset-help", children: "Pick a dataset with human answers. Datasets without them are greyed out." }))] })) : undefined, step === 2 ? (_jsxs(Box, { gap: 2, testID: "ai-judge-step-question-set", children: [_jsx(SelectField, { onChange: onQuestionSetChange, options: judgedSets.map((entry) => {
|
|
89
|
+
return {
|
|
90
|
+
label: `${entry.evaluatorName} (${entry.current} of ${entry.total} answered)`,
|
|
91
|
+
value: entry.evaluatorId,
|
|
92
|
+
};
|
|
93
|
+
}), placeholder: "Choose a question set", testID: "ai-judge-question-set", title: "Question set the humans answered", value: judgment ? questionSetId : "" }), judgment ? (_jsx(Text, { color: "secondaryDark", size: "sm", testID: "ai-judge-question-set-summary", children: `The judge is graded on the ${judgment.current} items of ${dataset?.name ?? "this dataset"} with current answers.${judgment.stale > 0
|
|
94
|
+
? ` ${judgment.stale} stale answers (given to older questions) are left out.`
|
|
95
|
+
: ""}` })) : undefined, questionSet ? (_jsx(Box, { gap: 1, testID: "ai-judge-questions", children: questionSet.dimensions.map((dimension, index) => (_jsx(Text, { size: "sm", children: `Question ${index + 1}: ${dimensionLabel(dimension)}` }, dimension.key))) })) : undefined] })) : undefined, step === 3 ? (_jsxs(Box, { gap: 3, testID: "ai-judge-step-judge", children: [judgeOptions.length === 0 ? (_jsx(Text, { color: "secondaryDark", testID: "ai-judge-judges-empty", children: "No LLM judge answers these questions yet. On Evaluators, create an LLM judge whose questions use the same keys." })) : (_jsx(SelectField, { onChange: onJudgeChange, options: judgeOptions.map((entry) => {
|
|
96
|
+
return {
|
|
97
|
+
label: `${entry.judge.name} (answers ${entry.shared.length} of ${questionSet?.dimensions.length ?? 0} questions)`,
|
|
98
|
+
value: entry.judge.id,
|
|
99
|
+
};
|
|
100
|
+
}), placeholder: "Choose an LLM judge", testID: "ai-judge-evaluator", title: "LLM judge to calibrate", value: judge ? judgeEvaluatorId : "" })), judge && calibration ? (_jsxs(Box, { alignItems: "center", direction: "row", gap: 2, testID: "ai-judge-current-calibration", wrap: true, children: [_jsx(Text, { size: "sm", children: "Last calibration" }), judge.calibration?.rate !== undefined && !judge.calibration.stale ? (_jsx(AgreementMeter, { compact: true, rate: judge.calibration.rate })) : (_jsx(Text, { color: "secondaryLight", size: "sm", children: judge.calibration?.stale ? "Out of date" : "Not calibrated yet" }))] })) : undefined, judge && uncoveredQuestions.length > 0 ? (_jsx(Text, { color: "secondaryDark", size: "sm", testID: "ai-judge-uncovered", children: `Not graded on ${uncoveredQuestions.map((dimension) => dimensionLabel(dimension)).join(", ")}. This judge does not answer ${uncoveredQuestions.length === 1 ? "it" : "them"}.` })) : undefined, judge ? (_jsxs(Box, { gap: 2, children: [_jsx(Heading, { size: "sm", children: `Versions of ${judge.judgePromptName ?? "the judge prompt"}` }), judgePromptDetail?.versions.map((version) => {
|
|
101
|
+
const tags = promptVersionTags(version.version, judgePromptDetail);
|
|
102
|
+
return (_jsxs(Box, { accessibilityHint: `Toggle judge version ${version.version}`, accessibilityLabel: `Judge version ${version.version}`, alignItems: "center", direction: "row", gap: 1, onClick: () => {
|
|
103
|
+
onVersionToggle(version.version);
|
|
104
|
+
}, testID: `ai-judge-version-${version.version}`, wrap: true, children: [_jsx(CheckBox, { selected: versions.includes(version.version) }), _jsx(Text, { children: `v${version.version}` }), tags.map((tag) => (_jsx(Badge, { secondary: true, status: "neutral", value: tag }, tag)))] }, version.version));
|
|
105
|
+
}), _jsx(Text, { color: "secondaryLight", size: "sm", testID: "ai-judge-version-help", children: "Select 2\u20133 judge prompt versions to compare. Iterate on the prompt and run again until one agrees with the humans." })] })) : undefined] })) : undefined, step === 4 ? (_jsxs(Box, { gap: 3, testID: "ai-judge-step-review", children: [_jsx(Box, { testID: "ai-judge-review-summary", children: _jsx(Text, { children: `You're calibrating ${judge?.name ?? "the judge"}: ${formatVersionList(versions)} of ${judge?.judgePromptName ?? "its prompt"} will answer ${questionSet?.name ?? "the question set"}'s questions for the ${judgment?.current ?? 0} items of ${dataset?.name ?? "the dataset"} that humans already answered. Each version's answers are compared with the humans'; a version passes at ${targetRate !== undefined ? `${Math.round(targetRate * 1000) / 10}%` : "the target"} agreement or higher, and you promote the best one from the results page. No new human work is needed.` }) }), _jsx(TextField, { errorText: targetRate === undefined ? "Enter a percent between 1 and 100." : undefined, helperText: "A version passes when its answers match the humans' at least this often. Default 90%.", onChange: onTargetAgreementChange, testID: "ai-judge-target-agreement", title: "Target agreement (%)", value: targetAgreement }), _jsx(TextField, { helperText: "Optional. Leave blank to use the judge prompt's model.", onChange: onModelOverrideChange, testID: "ai-judge-model-override", title: "Model override", value: modelOverride }), isEstimating ? (_jsx(Box, { alignItems: "start", testID: "ai-judge-estimating", children: _jsx(Spinner, { size: "sm" }) })) : undefined, estimate ? (_jsx(Text, { bold: true, testID: "ai-judge-estimate", children: formatExperimentEstimate({ estimate, unit: "judge calls" }) })) : undefined, estimateError ? (_jsx(Text, { color: "error", size: "sm", children: estimateError })) : undefined] })) : undefined, _jsx(WizardFooter, { isLastStep: step === 4, isNextDisabled: isNextDisabled, isRunDisabled: targetRate === undefined, isRunning: isCreating, onBack: step > 1
|
|
106
|
+
? () => {
|
|
107
|
+
onStepChange((step - 1));
|
|
108
|
+
}
|
|
109
|
+
: undefined, onNext: () => {
|
|
110
|
+
onStepChange((step + 1));
|
|
111
|
+
}, onRun: onRun, runText: "Run calibration", testIDPrefix: "ai-judge" })] }));
|
|
112
|
+
};
|
|
113
|
+
//# sourceMappingURL=AiJudgeExperimentSteps.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AiJudgeExperimentSteps.js","sourceRoot":"","sources":["../../../../src/widgets/aiObservability/experiments/AiJudgeExperimentSteps.tsx"],"names":[],"mappings":";AAAA,OAAO,EACL,KAAK,EACL,MAAM,EACN,GAAG,EACH,MAAM,EACN,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,WAAW,EACX,OAAO,EACP,IAAI,EACJ,SAAS,GACV,MAAM,aAAa,CAAC;AAErB,OAAO,EAGL,sBAAsB,EACtB,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAAC,cAAc,EAAC,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAC,cAAc,EAAC,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAC,cAAc,EAAC,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAC,WAAW,EAAC,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAC,iBAAiB,EAAE,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACvE,OAAO,EAAC,gBAAgB,EAAE,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAEL,wBAAwB,EACxB,iBAAiB,EACjB,6BAA6B,EAC7B,2BAA2B,EAC3B,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAiC3B,MAAM,iBAAiB,GAAG;IACxB,gBAAgB;IAChB,cAAc;IACd,oBAAoB;IACpB,gBAAgB;CACjB,CAAC;AAEF,MAAM,kBAAkB,GAAoC;IAC1D,CAAC,EAAE,yGAAyG;IAC5G,CAAC,EAAE,wFAAwF;IAC3F,CAAC,EAAE,yFAAyF;IAC5F,CAAC,EAAE,mGAAmG;CACvG,CAAC;AAEF,MAAM,mBAAmB,GACvB,iIAAiI,CAAC;AAEpI,+FAA+F;AAC/F,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,UAA6B,EAC7B,WAA6B,EACsB,EAAE;IACrD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,UAAU;SACd,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;SAC3F,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAC,KAAK,EAAE,MAAM,EAAE,kBAAkB,CAAC,KAAK,EAAE,WAAW,CAAC,EAAC,CAAC,CAAC;SACzE,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;SAC1C,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACrE,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAIlB,CAAC,EAAC,OAAO,EAAE,UAAU,EAAE,aAAa,EAAC,EAAE,EAAE;IAC5C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACxD,OAAO,CACL,MAAC,GAAG,IAAC,GAAG,EAAE,CAAC,aACT,MAAC,GAAG,IAAC,UAAU,EAAC,QAAQ,EAAC,SAAS,EAAC,KAAK,EAAC,GAAG,EAAE,CAAC,EAAE,IAAI,mBAClD,QAAQ,CAAC,CAAC,CAAC,KAAC,QAAQ,IAAC,QAAQ,EAAE,UAAU,GAAI,CAAC,CAAC,CAAC,SAAS,EAC1D,KAAC,IAAI,IAAC,IAAI,kBAAE,OAAO,CAAC,IAAI,GAAQ,EAChC,KAAC,IAAI,IAAC,KAAK,EAAC,eAAe,EAAC,IAAI,EAAC,IAAI,YAAE,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,QAAQ,GAAQ,IAC1E,EACL,SAAS,CAAC,GAAG,CAAC,CAAC,QAAyB,EAAE,EAAE;gBAC3C,MAAM,KAAK,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;gBAC3C,OAAO,CACL,MAAC,GAAG,IAAC,GAAG,EAAE,CAAC,aACT,KAAC,IAAI,IACH,KAAK,EAAE,QAAQ,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,EACzD,IAAI,EAAC,IAAI,EACT,MAAM,EAAE,6BAA6B,OAAO,CAAC,EAAE,IAAI,QAAQ,CAAC,WAAW,EAAE,YAExE,sBAAsB,CAAC,QAAQ,CAAC,GAC5B,EACN,KAAK,CAAC,CAAC,CAAC,CACP,KAAC,IAAI,IACH,KAAK,EAAC,SAAS,EACf,IAAI,EAAC,IAAI,EACT,MAAM,EAAE,0BAA0B,OAAO,CAAC,EAAE,IAAI,QAAQ,CAAC,WAAW,EAAE,YAErE,KAAK,GACD,CACR,CAAC,CAAC,CAAC,SAAS,KAhBG,QAAQ,CAAC,WAAW,CAiBhC,CACP,CAAC;YACJ,CAAC,CAAC,EACD,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CACtB,MAAC,GAAG,IAAC,UAAU,EAAC,QAAQ,EAAC,SAAS,EAAC,KAAK,EAAC,GAAG,EAAE,CAAC,EAAE,IAAI,mBACnD,KAAC,IAAI,IAAC,KAAK,EAAC,eAAe,EAAC,IAAI,EAAC,IAAI,4FAE9B,EACP,KAAC,MAAM,IACL,OAAO,EAAE,GAAG,EAAE;4BACZ,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;wBAC5B,CAAC,EACD,IAAI,EAAC,IAAI,EACT,MAAM,EAAE,gCAAgC,OAAO,CAAC,EAAE,EAAE,EACpD,IAAI,EAAC,4BAA4B,EACjC,OAAO,EAAC,SAAS,GACjB,IACE,CACP,IACG,CACP,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAA0C,CAAC,EAC5E,SAAS,EACT,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,UAAU,EACV,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,aAAa,EACb,eAAe,EACf,aAAa,EACb,qBAAqB,EACrB,aAAa,EACb,mBAAmB,EACnB,KAAK,EACL,YAAY,EACZ,uBAAuB,EACvB,eAAe,EACf,aAAa,EACb,IAAI,EACJ,eAAe,EACf,QAAQ,GACT,EAAE,EAAE;IACH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,aAAa,CAAC,CAAC;IACjF,MAAM,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,aAAa,CAAC,CAAC;IAC3E,MAAM,YAAY,GAAG,oBAAoB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IACnE,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,gBAAgB,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,UAAU,EAAE,KAAK,CAAC;IAChC,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAChF,MAAM,kBAAkB,GAAG,WAAW;QACpC,CAAC,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3F,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,UAAU,GAAG,2BAA2B,CAAC,eAAe,CAAC,CAAC;IAEhE,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;IACzC,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,6BAA6B,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACvF,MAAM,YAAY,GAAG,CAAC,UAA2B,EAAW,EAAE;QAC5D,IAAI,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,UAAU,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,UAAU,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC;IAC1C,CAAC,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,YAAY,CAAC,CAAC,IAAI,GAAG,CAAC,CAAoB,CAAC,CAAC;IAE/E,OAAO,CACL,MAAC,IAAI,IAAC,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAC,iBAAiB,aAChD,KAAC,WAAW,IACV,KAAK,EAAE,gBAAgB,CAAC;oBACtB,OAAO,EAAE,IAAI;oBACb,QAAQ,EAAE,YAAY;oBACtB,MAAM,EAAE,iBAAiB;oBACzB,YAAY;oBACZ,YAAY,EAAE,eAAe;iBAC9B,CAAC,EACF,MAAM,EAAC,kBAAkB,GACzB,EACF,KAAC,cAAc,IACb,QAAQ,EAAE,iBAAiB,CAAC,KAAK,EACjC,MAAM,EAAE,kBAAkB,CAAC,IAAI,CAAC,EAChC,IAAI,EAAE,aAAa,CAAC,KAAK,GACzB,EACD,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CACZ,MAAC,GAAG,IAAC,GAAG,EAAE,CAAC,EAAE,MAAM,EAAC,uBAAuB,aACxC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CACvB,KAAC,IAAI,IAAC,KAAK,EAAC,eAAe,EAAC,MAAM,EAAC,yBAAyB,0GAGrD,CACR,CAAC,CAAC,CAAC,SAAS,EACZ,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;wBACtB,MAAM,QAAQ,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;wBACtD,MAAM,UAAU,GAAG,KAAK,CAAC,EAAE,KAAK,SAAS,IAAI,QAAQ,CAAC;wBACtD,MAAM,IAAI,GAAG,CACX,KAAC,iBAAiB,IAChB,OAAO,EAAE,KAAK,EACd,UAAU,EAAE,UAAU,EACtB,aAAa,EAAE,aAAa,GAC5B,CACH,CAAC;wBACF,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACd,OAAO,CACL,KAAC,GAAG,IACF,MAAM,EAAC,SAAS,EAChB,KAAK,EAAC,eAAe,EAErB,OAAO,EAAE,CAAC,EACV,QAAQ,EAAC,IAAI,EACb,MAAM,EAAE,oBAAoB,KAAK,CAAC,EAAE,WAAW,YAE9C,IAAI,IALA,KAAK,CAAC,EAAE,CAMT,CACP,CAAC;wBACJ,CAAC;wBACD,OAAO,CACL,KAAC,GAAG,IACF,iBAAiB,EAAC,6CAA6C,EAC/D,kBAAkB,EAAE,WAAW,KAAK,CAAC,IAAI,EAAE,EAC3C,kBAAkB,EAAE,EAAC,OAAO,EAAE,UAAU,EAAC,EACzC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAEvC,OAAO,EAAE,GAAG,EAAE;gCACZ,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;4BAC5B,CAAC,EACD,OAAO,EAAE,CAAC,EACV,QAAQ,EAAC,IAAI,EACb,MAAM,EAAE,oBAAoB,KAAK,CAAC,EAAE,EAAE,YAErC,IAAI,IARA,KAAK,CAAC,EAAE,CAST,CACP,CAAC;oBACJ,CAAC,CAAC,EACD,UAAU,CAAC,CAAC,CAAC,CACZ,KAAC,GAAG,IAAC,MAAM,EAAC,wBAAwB,YAClC,KAAC,MAAM,IAAC,OAAO,QAAC,MAAM,EAAC,MAAM,EAAC,IAAI,EAAE,mBAAmB,GAAI,GACvD,CACP,CAAC,CAAC,CAAC,CACF,KAAC,IAAI,IAAC,KAAK,EAAC,gBAAgB,EAAC,IAAI,EAAC,IAAI,EAAC,MAAM,EAAC,uBAAuB,yFAE9D,CACR,IACG,CACP,CAAC,CAAC,CAAC,SAAS,EACZ,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CACZ,MAAC,GAAG,IAAC,GAAG,EAAE,CAAC,EAAE,MAAM,EAAC,4BAA4B,aAC9C,KAAC,WAAW,IACV,QAAQ,EAAE,mBAAmB,EAC7B,OAAO,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;4BAChC,OAAO;gCACL,KAAK,EAAE,GAAG,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,OAAO,OAAO,KAAK,CAAC,KAAK,YAAY;gCAC7E,KAAK,EAAE,KAAK,CAAC,WAAW;6BACzB,CAAC;wBACJ,CAAC,CAAC,EACF,WAAW,EAAC,uBAAuB,EACnC,MAAM,EAAC,uBAAuB,EAC9B,KAAK,EAAC,kCAAkC,EACxC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,GACpC,EACD,QAAQ,CAAC,CAAC,CAAC,CACV,KAAC,IAAI,IAAC,KAAK,EAAC,eAAe,EAAC,IAAI,EAAC,IAAI,EAAC,MAAM,EAAC,+BAA+B,YACzE,8BAA8B,QAAQ,CAAC,OAAO,aAAa,OAAO,EAAE,IAAI,IAAI,cAAc,yBACzF,QAAQ,CAAC,KAAK,GAAG,CAAC;4BAChB,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,yDAAyD;4BAC7E,CAAC,CAAC,EACN,EAAE,GACG,CACR,CAAC,CAAC,CAAC,SAAS,EACZ,WAAW,CAAC,CAAC,CAAC,CACb,KAAC,GAAG,IAAC,GAAG,EAAE,CAAC,EAAE,MAAM,EAAC,oBAAoB,YACrC,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAChD,KAAC,IAAI,IAAqB,IAAI,EAAC,IAAI,YAChC,YAAY,KAAK,GAAG,CAAC,KAAK,cAAc,CAAC,SAAS,CAAC,EAAE,IAD7C,SAAS,CAAC,GAAG,CAEjB,CACR,CAAC,GACE,CACP,CAAC,CAAC,CAAC,SAAS,IACT,CACP,CAAC,CAAC,CAAC,SAAS,EACZ,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CACZ,MAAC,GAAG,IAAC,GAAG,EAAE,CAAC,EAAE,MAAM,EAAC,qBAAqB,aACtC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAC3B,KAAC,IAAI,IAAC,KAAK,EAAC,eAAe,EAAC,MAAM,EAAC,uBAAuB,gIAGnD,CACR,CAAC,CAAC,CAAC,CACF,KAAC,WAAW,IACV,QAAQ,EAAE,aAAa,EACvB,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;4BAClC,OAAO;gCACL,KAAK,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,aAAa,KAAK,CAAC,MAAM,CAAC,MAAM,OAAO,WAAW,EAAE,UAAU,CAAC,MAAM,IAAI,CAAC,aAAa;gCACjH,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE;6BACtB,CAAC;wBACJ,CAAC,CAAC,EACF,WAAW,EAAC,qBAAqB,EACjC,MAAM,EAAC,oBAAoB,EAC3B,KAAK,EAAC,wBAAwB,EAC9B,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,GACpC,CACH,EACA,KAAK,IAAI,WAAW,CAAC,CAAC,CAAC,CACtB,MAAC,GAAG,IACF,UAAU,EAAC,QAAQ,EACnB,SAAS,EAAC,KAAK,EACf,GAAG,EAAE,CAAC,EACN,MAAM,EAAC,8BAA8B,EACrC,IAAI,mBAEJ,KAAC,IAAI,IAAC,IAAI,EAAC,IAAI,iCAAwB,EACtC,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CACnE,KAAC,cAAc,IAAC,OAAO,QAAC,IAAI,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI,GAAI,CACzD,CAAC,CAAC,CAAC,CACF,KAAC,IAAI,IAAC,KAAK,EAAC,gBAAgB,EAAC,IAAI,EAAC,IAAI,YACnC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,oBAAoB,GAC3D,CACR,IACG,CACP,CAAC,CAAC,CAAC,SAAS,EACZ,KAAK,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CACxC,KAAC,IAAI,IAAC,KAAK,EAAC,eAAe,EAAC,IAAI,EAAC,IAAI,EAAC,MAAM,EAAC,oBAAoB,YAC9D,iBAAiB,kBAAkB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,gCAAgC,kBAAkB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,GAC1K,CACR,CAAC,CAAC,CAAC,SAAS,EACZ,KAAK,CAAC,CAAC,CAAC,CACP,MAAC,GAAG,IAAC,GAAG,EAAE,CAAC,aACT,KAAC,OAAO,IAAC,IAAI,EAAC,IAAI,YAAE,eAAe,KAAK,CAAC,eAAe,IAAI,kBAAkB,EAAE,GAAW,EAC1F,iBAAiB,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;gCAC3C,MAAM,IAAI,GAAG,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;gCACnE,OAAO,CACL,MAAC,GAAG,IACF,iBAAiB,EAAE,wBAAwB,OAAO,CAAC,OAAO,EAAE,EAC5D,kBAAkB,EAAE,iBAAiB,OAAO,CAAC,OAAO,EAAE,EACtD,UAAU,EAAC,QAAQ,EACnB,SAAS,EAAC,KAAK,EACf,GAAG,EAAE,CAAC,EAEN,OAAO,EAAE,GAAG,EAAE;wCACZ,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;oCACnC,CAAC,EACD,MAAM,EAAE,oBAAoB,OAAO,CAAC,OAAO,EAAE,EAC7C,IAAI,mBAEJ,KAAC,QAAQ,IAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,GAAI,EAC1D,KAAC,IAAI,cAAE,IAAI,OAAO,CAAC,OAAO,EAAE,GAAQ,EACnC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CACjB,KAAC,KAAK,IAAW,SAAS,QAAC,MAAM,EAAC,SAAS,EAAC,KAAK,EAAE,GAAG,IAA1C,GAAG,CAA2C,CAC3D,CAAC,KAXG,OAAO,CAAC,OAAO,CAYhB,CACP,CAAC;4BACJ,CAAC,CAAC,EACF,KAAC,IAAI,IAAC,KAAK,EAAC,gBAAgB,EAAC,IAAI,EAAC,IAAI,EAAC,MAAM,EAAC,uBAAuB,wIAG9D,IACH,CACP,CAAC,CAAC,CAAC,SAAS,IACT,CACP,CAAC,CAAC,CAAC,SAAS,EACZ,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CACZ,MAAC,GAAG,IAAC,GAAG,EAAE,CAAC,EAAE,MAAM,EAAC,sBAAsB,aACxC,KAAC,GAAG,IAAC,MAAM,EAAC,yBAAyB,YACnC,KAAC,IAAI,cACF,sBAAsB,KAAK,EAAE,IAAI,IAAI,WAAW,KAAK,iBAAiB,CAAC,QAAQ,CAAC,OAAO,KAAK,EAAE,eAAe,IAAI,YAAY,gBAAgB,WAAW,EAAE,IAAI,IAAI,kBAAkB,wBAAwB,QAAQ,EAAE,OAAO,IAAI,CAAC,aAAa,OAAO,EAAE,IAAI,IAAI,aAAa,4GAA4G,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY,wGAAwG,GAC/iB,GACH,EACN,KAAC,SAAS,IACR,SAAS,EAAE,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC,SAAS,EACtF,UAAU,EAAC,uFAAuF,EAClG,QAAQ,EAAE,uBAAuB,EACjC,MAAM,EAAC,2BAA2B,EAClC,KAAK,EAAC,sBAAsB,EAC5B,KAAK,EAAE,eAAe,GACtB,EACF,KAAC,SAAS,IACR,UAAU,EAAC,wDAAwD,EACnE,QAAQ,EAAE,qBAAqB,EAC/B,MAAM,EAAC,yBAAyB,EAChC,KAAK,EAAC,gBAAgB,EACtB,KAAK,EAAE,aAAa,GACpB,EACD,YAAY,CAAC,CAAC,CAAC,CACd,KAAC,GAAG,IAAC,UAAU,EAAC,OAAO,EAAC,MAAM,EAAC,qBAAqB,YAClD,KAAC,OAAO,IAAC,IAAI,EAAC,IAAI,GAAG,GACjB,CACP,CAAC,CAAC,CAAC,SAAS,EACZ,QAAQ,CAAC,CAAC,CAAC,CACV,KAAC,IAAI,IAAC,IAAI,QAAC,MAAM,EAAC,mBAAmB,YAClC,wBAAwB,CAAC,EAAC,QAAQ,EAAE,IAAI,EAAE,aAAa,EAAC,CAAC,GACrD,CACR,CAAC,CAAC,CAAC,SAAS,EACZ,aAAa,CAAC,CAAC,CAAC,CACf,KAAC,IAAI,IAAC,KAAK,EAAC,OAAO,EAAC,IAAI,EAAC,IAAI,YAC1B,aAAa,GACT,CACR,CAAC,CAAC,CAAC,SAAS,IACT,CACP,CAAC,CAAC,CAAC,SAAS,EACb,KAAC,YAAY,IACX,UAAU,EAAE,IAAI,KAAK,CAAC,EACtB,cAAc,EAAE,cAAc,EAC9B,aAAa,EAAE,UAAU,KAAK,SAAS,EACvC,SAAS,EAAE,UAAU,EACrB,MAAM,EACJ,IAAI,GAAG,CAAC;oBACN,CAAC,CAAC,GAAG,EAAE;wBACH,YAAY,CAAC,CAAC,IAAI,GAAG,CAAC,CAAoB,CAAC,CAAC;oBAC9C,CAAC;oBACH,CAAC,CAAC,SAAS,EAEf,MAAM,EAAE,GAAG,EAAE;oBACX,YAAY,CAAC,CAAC,IAAI,GAAG,CAAC,CAAoB,CAAC,CAAC;gBAC9C,CAAC,EACD,KAAK,EAAE,KAAK,EACZ,OAAO,EAAC,iBAAiB,EACzB,YAAY,EAAC,UAAU,GACvB,IACG,CACR,CAAC;AACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import React from "react";
|
|
2
|
+
import type { EvalStepState } from "../shell/EvalStepper";
|
|
3
|
+
export interface WizardFooterProps {
|
|
4
|
+
isNextDisabled?: boolean;
|
|
5
|
+
isRunning?: boolean;
|
|
6
|
+
isRunDisabled?: boolean;
|
|
7
|
+
/** Last step shows the run button instead of Continue. */
|
|
8
|
+
isLastStep: boolean;
|
|
9
|
+
onBack?: () => void;
|
|
10
|
+
onNext: () => void;
|
|
11
|
+
onRun: () => void;
|
|
12
|
+
runText: string;
|
|
13
|
+
testIDPrefix: string;
|
|
14
|
+
}
|
|
15
|
+
/** Ghost Back on the left of a primary Continue, or the run button on the last step. */
|
|
16
|
+
export declare const WizardFooter: React.FC<WizardFooterProps>;
|
|
17
|
+
/** Stepper entries for a wizard: done before the current step, pressable unless locked. */
|
|
18
|
+
export declare const buildWizardSteps: <Step extends number>({ current, isLocked, labels, onStepChange, testIDPrefix, }: {
|
|
19
|
+
current: Step;
|
|
20
|
+
isLocked: (step: Step) => boolean;
|
|
21
|
+
labels: string[];
|
|
22
|
+
onStepChange: (step: Step) => void;
|
|
23
|
+
testIDPrefix: string;
|
|
24
|
+
}) => Array<{
|
|
25
|
+
onPress?: () => void;
|
|
26
|
+
state: EvalStepState;
|
|
27
|
+
testID: string;
|
|
28
|
+
title: string;
|
|
29
|
+
}>;
|
|
30
|
+
//# sourceMappingURL=ExperimentWizard.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ExperimentWizard.d.ts","sourceRoot":"","sources":["../../../../src/widgets/aiObservability/experiments/ExperimentWizard.tsx"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,sBAAsB,CAAC;AAExD,MAAM,WAAW,iBAAiB;IAChC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,0DAA0D;IAC1D,UAAU,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,IAAI,CAAC;IACpB,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,wFAAwF;AACxF,eAAO,MAAM,YAAY,EAAE,KAAK,CAAC,EAAE,CAAC,iBAAiB,CAkDpD,CAAC;AAEF,2FAA2F;AAC3F,eAAO,MAAM,gBAAgB,GAAI,IAAI,SAAS,MAAM,EAAE,4DAMnD;IACD,OAAO,EAAE,IAAI,CAAC;IACd,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,OAAO,CAAC;IAClC,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,YAAY,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC;IACnC,YAAY,EAAE,MAAM,CAAC;CACtB,KAAG,KAAK,CAAC;IACR,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf,CAiBA,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
|
+
import { Box, Button } from "@terreno/ui";
|
|
3
|
+
/** Ghost Back on the left of a primary Continue, or the run button on the last step. */
|
|
4
|
+
export const WizardFooter = ({ isLastStep, isNextDisabled, isRunDisabled, isRunning, onBack, onNext, onRun, runText, testIDPrefix, }) => {
|
|
5
|
+
return (_jsxs(Box, { alignItems: "center", direction: "row", gap: 2, justifyContent: "between", wrap: true, children: [_jsx(Box, { children: onBack ? (_jsx(Button, { iconName: "arrow-left", onClick: onBack, testID: `${testIDPrefix}-back`, text: "Back", variant: "ghost" })) : null }), isLastStep ? (_jsx(Button, { disabled: isRunDisabled, iconName: "play", loading: isRunning, onClick: onRun, testID: `${testIDPrefix}-run`, text: runText })) : (_jsx(Button, { disabled: isNextDisabled, iconName: "arrow-right", iconPosition: "right", onClick: () => {
|
|
6
|
+
if (isNextDisabled) {
|
|
7
|
+
return;
|
|
8
|
+
}
|
|
9
|
+
onNext();
|
|
10
|
+
}, testID: `${testIDPrefix}-next`, text: "Continue" }))] }));
|
|
11
|
+
};
|
|
12
|
+
/** Stepper entries for a wizard: done before the current step, pressable unless locked. */
|
|
13
|
+
export const buildWizardSteps = ({ current, isLocked, labels, onStepChange, testIDPrefix, }) => {
|
|
14
|
+
return labels.map((label, index) => {
|
|
15
|
+
const stepNumber = (index + 1);
|
|
16
|
+
const state = stepNumber === current ? "now" : stepNumber < current ? "done" : "later";
|
|
17
|
+
return {
|
|
18
|
+
onPress: isLocked(stepNumber) || stepNumber === current
|
|
19
|
+
? undefined
|
|
20
|
+
: () => {
|
|
21
|
+
onStepChange(stepNumber);
|
|
22
|
+
},
|
|
23
|
+
state,
|
|
24
|
+
testID: `${testIDPrefix}-${stepNumber}`,
|
|
25
|
+
title: label,
|
|
26
|
+
};
|
|
27
|
+
});
|
|
28
|
+
};
|
|
29
|
+
//# sourceMappingURL=ExperimentWizard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ExperimentWizard.js","sourceRoot":"","sources":["../../../../src/widgets/aiObservability/experiments/ExperimentWizard.tsx"],"names":[],"mappings":";AAAA,OAAO,EAAC,GAAG,EAAE,MAAM,EAAC,MAAM,aAAa,CAAC;AAiBxC,wFAAwF;AACxF,MAAM,CAAC,MAAM,YAAY,GAAgC,CAAC,EACxD,UAAU,EACV,cAAc,EACd,aAAa,EACb,SAAS,EACT,MAAM,EACN,MAAM,EACN,KAAK,EACL,OAAO,EACP,YAAY,GACb,EAAE,EAAE;IACH,OAAO,CACL,MAAC,GAAG,IAAC,UAAU,EAAC,QAAQ,EAAC,SAAS,EAAC,KAAK,EAAC,GAAG,EAAE,CAAC,EAAE,cAAc,EAAC,SAAS,EAAC,IAAI,mBAC5E,KAAC,GAAG,cACD,MAAM,CAAC,CAAC,CAAC,CACR,KAAC,MAAM,IACL,QAAQ,EAAC,YAAY,EACrB,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,GAAG,YAAY,OAAO,EAC9B,IAAI,EAAC,MAAM,EACX,OAAO,EAAC,OAAO,GACf,CACH,CAAC,CAAC,CAAC,IAAI,GACJ,EACL,UAAU,CAAC,CAAC,CAAC,CACZ,KAAC,MAAM,IACL,QAAQ,EAAE,aAAa,EACvB,QAAQ,EAAC,MAAM,EACf,OAAO,EAAE,SAAS,EAClB,OAAO,EAAE,KAAK,EACd,MAAM,EAAE,GAAG,YAAY,MAAM,EAC7B,IAAI,EAAE,OAAO,GACb,CACH,CAAC,CAAC,CAAC,CACF,KAAC,MAAM,IACL,QAAQ,EAAE,cAAc,EACxB,QAAQ,EAAC,aAAa,EACtB,YAAY,EAAC,OAAO,EACpB,OAAO,EAAE,GAAG,EAAE;oBACZ,IAAI,cAAc,EAAE,CAAC;wBACnB,OAAO;oBACT,CAAC;oBACD,MAAM,EAAE,CAAC;gBACX,CAAC,EACD,MAAM,EAAE,GAAG,YAAY,OAAO,EAC9B,IAAI,EAAC,UAAU,GACf,CACH,IACG,CACP,CAAC;AACJ,CAAC,CAAC;AAEF,2FAA2F;AAC3F,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAsB,EACpD,OAAO,EACP,QAAQ,EACR,MAAM,EACN,YAAY,EACZ,YAAY,GAOb,EAKE,EAAE;IACH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACjC,MAAM,UAAU,GAAG,CAAC,KAAK,GAAG,CAAC,CAAS,CAAC;QACvC,MAAM,KAAK,GACT,UAAU,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QAC3E,OAAO;YACL,OAAO,EACL,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,KAAK,OAAO;gBAC5C,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,GAAG,EAAE;oBACH,YAAY,CAAC,UAAU,CAAC,CAAC;gBAC3B,CAAC;YACP,KAAK;YACL,MAAM,EAAE,GAAG,YAAY,IAAI,UAAU,EAAE;YACvC,KAAK,EAAE,KAAK;SACb,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC"}
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
+
import type { PromptDetail } from "../prompts/promptTypes";
|
|
2
|
+
import type { JudgeAgreement, ScoreValue } from "../review/batchTypes";
|
|
3
|
+
export type ExperimentKind = "judge" | "prompt";
|
|
1
4
|
export interface ExperimentEstimate {
|
|
2
5
|
costUsd?: number;
|
|
3
6
|
generations: number;
|
|
4
7
|
wallClockSeconds: number;
|
|
5
8
|
}
|
|
6
9
|
export interface ScoreThreshold {
|
|
7
|
-
|
|
10
|
+
/** `agreement` gates compare a judge prompt version's answers with human answers. */
|
|
11
|
+
aggregate: "agreement" | "mean" | "trueRate";
|
|
8
12
|
dimension: string;
|
|
9
13
|
evaluatorName: string;
|
|
10
14
|
op: "eq" | "gte" | "lte";
|
|
@@ -16,6 +20,8 @@ export interface ExperimentGateResult extends ScoreThreshold {
|
|
|
16
20
|
version: number;
|
|
17
21
|
}
|
|
18
22
|
export interface ExperimentAggregates {
|
|
23
|
+
/** Judge experiments: agreement with the human answers, keyed by judge prompt version. */
|
|
24
|
+
agreement?: Record<string, JudgeAgreement>;
|
|
19
25
|
gates: ExperimentGateResult[];
|
|
20
26
|
lowConfidenceItemIds: string[];
|
|
21
27
|
outlierItemIds: string[];
|
|
@@ -38,6 +44,8 @@ export interface ExperimentVersionResult {
|
|
|
38
44
|
export interface ExperimentItemRecord {
|
|
39
45
|
datasetItemId: string;
|
|
40
46
|
failed: boolean;
|
|
47
|
+
/** Judge experiments: the human answers each judge version is compared with. */
|
|
48
|
+
humanScores?: Record<string, ScoreValue>;
|
|
41
49
|
id: string;
|
|
42
50
|
versionResults: Record<string, ExperimentVersionResult>;
|
|
43
51
|
}
|
|
@@ -45,29 +53,100 @@ export interface ExperimentRecord {
|
|
|
45
53
|
backgroundTaskId?: string;
|
|
46
54
|
created: string;
|
|
47
55
|
datasetId: string;
|
|
56
|
+
datasetName?: string;
|
|
48
57
|
estimate?: ExperimentEstimate;
|
|
49
58
|
evaluatorIds: string[];
|
|
50
59
|
id: string;
|
|
51
60
|
includeUnproofread: boolean;
|
|
52
61
|
items: ExperimentItemRecord[];
|
|
62
|
+
/** Judge experiments: the llm-judge evaluator whose prompt versions are compared. */
|
|
63
|
+
judgeEvaluatorId?: string;
|
|
64
|
+
judgeName?: string;
|
|
65
|
+
/** Older payloads leave it out; treat a missing kind as a prompt comparison. */
|
|
66
|
+
kind?: ExperimentKind;
|
|
53
67
|
modelOverride?: string;
|
|
54
68
|
name: string;
|
|
69
|
+
/** The compared prompt; for judge experiments, the judge's prompt. */
|
|
55
70
|
promptName: string;
|
|
71
|
+
/** Judge experiments: the human question set whose answers are the reference. */
|
|
72
|
+
questionSetId?: string;
|
|
73
|
+
questionSetName?: string;
|
|
74
|
+
/** Judge experiments: true when the questions changed after the run. */
|
|
75
|
+
questionsStale?: boolean;
|
|
56
76
|
results?: ExperimentAggregates;
|
|
57
77
|
status: "completed" | "failed" | "pending" | "running";
|
|
78
|
+
/** Judge experiments: agreement rate (0–1) a version must reach to pass. */
|
|
79
|
+
targetAgreement?: number;
|
|
58
80
|
thresholds: ScoreThreshold[];
|
|
59
81
|
updated: string;
|
|
60
82
|
versions: number[];
|
|
61
83
|
}
|
|
62
84
|
export declare const unwrapObservabilityPayload: <T>(raw: unknown) => T | undefined;
|
|
63
85
|
export declare const unwrapExperimentList: (raw: unknown) => ExperimentRecord[];
|
|
86
|
+
/** Newest version that passes every gate, else the newest compared version (0 when none). */
|
|
87
|
+
export declare const defaultPromoteVersion: (experiment: ExperimentRecord) => number;
|
|
88
|
+
/** Largest page the experiments list route returns. */
|
|
89
|
+
export declare const EXPERIMENT_LIST_MAX_LIMIT = 100;
|
|
90
|
+
/** True when a paginated experiments list response says more rows exist past this page. */
|
|
91
|
+
export declare const experimentListHasMore: (raw: unknown) => boolean;
|
|
64
92
|
export declare const unwrapExperimentRecord: (raw: unknown) => ExperimentRecord | undefined;
|
|
65
93
|
export declare const failingGateCount: (experiment: ExperimentRecord) => number;
|
|
66
|
-
|
|
67
|
-
export declare const
|
|
94
|
+
/** Show a measured gate value with two decimals, or a dash before any scores arrive. */
|
|
95
|
+
export declare const formatGateActual: (actual?: number) => string;
|
|
96
|
+
/** The threshold a gate needs, such as "≥ 0.8". */
|
|
97
|
+
export declare const formatGateThreshold: (gate: Pick<ScoreThreshold, "op" | "value">) => string;
|
|
98
|
+
/**
|
|
99
|
+
* Name a gate in plain words. When the evaluator and dimension share a name ("helpfulness"),
|
|
100
|
+
* say it once; otherwise "correctness: correct" or "schema-assert: schema valid".
|
|
101
|
+
*/
|
|
102
|
+
export declare const formatGateName: (gate: Pick<ScoreThreshold, "dimension" | "evaluatorName">) => string;
|
|
103
|
+
export interface VersionGateSummary {
|
|
104
|
+
failing: ExperimentGateResult[];
|
|
105
|
+
gates: ExperimentGateResult[];
|
|
106
|
+
passed: boolean;
|
|
107
|
+
version: number;
|
|
108
|
+
}
|
|
109
|
+
/** Gate results for every compared version, in version order, with pass/fail rolled up. */
|
|
110
|
+
export declare const summarizeGatesByVersion: (experiment: ExperimentRecord) => VersionGateSummary[];
|
|
111
|
+
/** Short verdict for one version's gates, such as "Fails 1 of 3 gates". */
|
|
112
|
+
export declare const formatVersionGateVerdict: (summary: VersionGateSummary) => string;
|
|
113
|
+
/**
|
|
114
|
+
* Explain why a version cannot be promoted, such as
|
|
115
|
+
* "v1 fails 1 gate: helpfulness ≥ 0.8 (got 0.62) — can't promote v1."
|
|
116
|
+
*/
|
|
117
|
+
export declare const formatPromoteBlockedReason: (summary: VersionGateSummary) => string | undefined;
|
|
68
118
|
export declare const parsePromoteBlockedTitle: (error: unknown) => string | undefined;
|
|
69
119
|
export declare const isAutomaticExperimentEvaluator: (evaluator: {
|
|
70
120
|
type: string;
|
|
71
121
|
}) => boolean;
|
|
72
122
|
export declare const isValidExperimentVersionCount: (count: number) => boolean;
|
|
123
|
+
export interface ExperimentItemColumnWidths {
|
|
124
|
+
item: number;
|
|
125
|
+
output: number;
|
|
126
|
+
pass: number;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Split the measured table width so the per-version output columns take every pixel the item and
|
|
130
|
+
* pass columns leave over, never narrower than the readable minimum.
|
|
131
|
+
*/
|
|
132
|
+
export declare const experimentItemColumnWidths: ({ containerWidth, versionCount, }: {
|
|
133
|
+
containerWidth: number;
|
|
134
|
+
versionCount: number;
|
|
135
|
+
}) => ExperimentItemColumnWidths;
|
|
136
|
+
export declare const DEFAULT_TARGET_AGREEMENT = 0.9;
|
|
137
|
+
export declare const EXPERIMENT_KIND_LABELS: Record<ExperimentKind, string>;
|
|
138
|
+
export declare const experimentKind: (experiment: Pick<ExperimentRecord, "kind">) => ExperimentKind;
|
|
139
|
+
/** Tags shown next to a prompt version in pickers: latest, production, superseded. */
|
|
140
|
+
export declare const promptVersionTags: (version: number, detail?: PromptDetail) => string[];
|
|
141
|
+
/** "v1 and v2" or "v1, v2, and v3". */
|
|
142
|
+
export declare const formatVersionList: (versions: number[]) => string;
|
|
143
|
+
/** Parse a target agreement typed as a percent ("90") into a rate (0.9), or undefined if invalid. */
|
|
144
|
+
export declare const parseTargetAgreementPercent: (value: string) => number | undefined;
|
|
145
|
+
/** Cost with two decimals; anything above zero but under a cent reads "<$0.01". */
|
|
146
|
+
export declare const formatExperimentCost: (cost?: number) => string;
|
|
147
|
+
/** Plain estimate such as "About 6 generations, 2 minutes, $0.42". */
|
|
148
|
+
export declare const formatExperimentEstimate: ({ estimate, unit, }: {
|
|
149
|
+
estimate: ExperimentEstimate;
|
|
150
|
+
unit?: "generations" | "judge calls";
|
|
151
|
+
}) => string;
|
|
73
152
|
//# sourceMappingURL=experimentTypes.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experimentTypes.d.ts","sourceRoot":"","sources":["../../../../src/widgets/aiObservability/experiments/experimentTypes.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,GAAG,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"experimentTypes.d.ts","sourceRoot":"","sources":["../../../../src/widgets/aiObservability/experiments/experimentTypes.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EAAC,cAAc,EAAE,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAErE,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEhD,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,qFAAqF;IACrF,SAAS,EAAE,WAAW,GAAG,MAAM,GAAG,UAAU,CAAC;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,EAAE,EAAE,IAAI,GAAG,KAAK,GAAG,KAAK,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,oBAAqB,SAAQ,cAAc;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,0FAA0F;IAC1F,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC3C,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAC9B,oBAAoB,EAAE,MAAM,EAAE,CAAC;IAC/B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,QAAQ,EAAE;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAC,CAAC;IAC7C,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,+BAA+B;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC;CACpD;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IACjE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,OAAO,CAAC;IAChB,gFAAgF;IAChF,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,gBAAgB;IAC/B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,kBAAkB,CAAC;IAC9B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,EAAE,OAAO,CAAC;IAC5B,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAC9B,qFAAqF;IACrF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gFAAgF;IAChF,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,sEAAsE;IACtE,UAAU,EAAE,MAAM,CAAC;IACnB,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wEAAwE;IACxE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;IACvD,4EAA4E;IAC5E,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,eAAO,MAAM,0BAA0B,GAAI,CAAC,EAAE,KAAK,OAAO,KAAG,CAAC,GAAG,SAQhE,CAAC;AAEF,eAAO,MAAM,oBAAoB,GAAI,KAAK,OAAO,KAAG,gBAAgB,EAQnE,CAAC;AAEF,6FAA6F;AAC7F,eAAO,MAAM,qBAAqB,GAAI,YAAY,gBAAgB,KAAG,MAKpE,CAAC;AAEF,uDAAuD;AACvD,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C,2FAA2F;AAC3F,eAAO,MAAM,qBAAqB,GAAI,KAAK,OAAO,KAAG,OAKpD,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,KAAK,OAAO,KAAG,gBAAgB,GAAG,SAMxE,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,YAAY,gBAAgB,KAAG,MAK/D,CAAC;AAsBF,wFAAwF;AACxF,eAAO,MAAM,gBAAgB,GAAI,SAAS,MAAM,KAAG,MAKlD,CAAC;AAEF,mDAAmD;AACnD,eAAO,MAAM,mBAAmB,GAAI,MAAM,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,OAAO,CAAC,KAAG,MAEhF,CAAC;AAMF;;;GAGG;AACH,eAAO,MAAM,cAAc,GACzB,MAAM,IAAI,CAAC,cAAc,EAAE,WAAW,GAAG,eAAe,CAAC,KACxD,MAMF,CAAC;AAWF,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,oBAAoB,EAAE,CAAC;IAChC,KAAK,EAAE,oBAAoB,EAAE,CAAC;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,2FAA2F;AAC3F,eAAO,MAAM,uBAAuB,GAAI,YAAY,gBAAgB,KAAG,kBAAkB,EAQxF,CAAC;AAEF,2EAA2E;AAC3E,eAAO,MAAM,wBAAwB,GAAI,SAAS,kBAAkB,KAAG,MAUtE,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,0BAA0B,GAAI,SAAS,kBAAkB,KAAG,MAAM,GAAG,SAMjF,CAAC;AAEF,eAAO,MAAM,wBAAwB,GAAI,OAAO,OAAO,KAAG,MAAM,GAAG,SAWlE,CAAC;AAKF,eAAO,MAAM,8BAA8B,GAAI,WAAW;IAAC,IAAI,EAAE,MAAM,CAAA;CAAC,KAAG,OAE1E,CAAC;AAEF,eAAO,MAAM,6BAA6B,GAAI,OAAO,MAAM,KAAG,OAE7D,CAAC;AAMF,MAAM,WAAW,0BAA0B;IACzC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,eAAO,MAAM,0BAA0B,GAAI,mCAGxC;IACD,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,KAAG,0BAQH,CAAC;AAEF,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAE5C,eAAO,MAAM,sBAAsB,EAAE,MAAM,CAAC,cAAc,EAAE,MAAM,CAGjE,CAAC;AAEF,eAAO,MAAM,cAAc,GAAI,YAAY,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,KAAG,cAE3E,CAAC;AAEF,sFAAsF;AACtF,eAAO,MAAM,iBAAiB,GAAI,SAAS,MAAM,EAAE,SAAS,YAAY,KAAG,MAAM,EAgBhF,CAAC;AAEF,uCAAuC;AACvC,eAAO,MAAM,iBAAiB,GAAI,UAAU,MAAM,EAAE,KAAG,MAMtD,CAAC;AAEF,qGAAqG;AACrG,eAAO,MAAM,2BAA2B,GAAI,OAAO,MAAM,KAAG,MAAM,GAAG,SAUpE,CAAC;AAEF,mFAAmF;AACnF,eAAO,MAAM,oBAAoB,GAAI,OAAO,MAAM,KAAG,MAQpD,CAAC;AAEF,sEAAsE;AACtE,eAAO,MAAM,wBAAwB,GAAI,qBAGtC;IACD,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,IAAI,CAAC,EAAE,aAAa,GAAG,aAAa,CAAC;CACtC,KAAG,MAUH,CAAC"}
|
|
@@ -16,6 +16,20 @@ export const unwrapExperimentList = (raw) => {
|
|
|
16
16
|
return Boolean(entry && typeof entry.id === "string");
|
|
17
17
|
});
|
|
18
18
|
};
|
|
19
|
+
/** Newest version that passes every gate, else the newest compared version (0 when none). */
|
|
20
|
+
export const defaultPromoteVersion = (experiment) => {
|
|
21
|
+
const passing = summarizeGatesByVersion(experiment).filter((summary) => summary.passed);
|
|
22
|
+
return (passing[passing.length - 1]?.version ?? experiment.versions[experiment.versions.length - 1] ?? 0);
|
|
23
|
+
};
|
|
24
|
+
/** Largest page the experiments list route returns. */
|
|
25
|
+
export const EXPERIMENT_LIST_MAX_LIMIT = 100;
|
|
26
|
+
/** True when a paginated experiments list response says more rows exist past this page. */
|
|
27
|
+
export const experimentListHasMore = (raw) => {
|
|
28
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
return raw.more === true;
|
|
32
|
+
};
|
|
19
33
|
export const unwrapExperimentRecord = (raw) => {
|
|
20
34
|
const payload = unwrapObservabilityPayload(raw);
|
|
21
35
|
if (!payload || typeof payload.id !== "string") {
|
|
@@ -29,18 +43,87 @@ export const failingGateCount = (experiment) => {
|
|
|
29
43
|
}
|
|
30
44
|
return experiment.results.gates.filter((gate) => !gate.passed).length;
|
|
31
45
|
};
|
|
32
|
-
|
|
46
|
+
const gatesForVersion = (experiment, version) => {
|
|
33
47
|
if (!experiment.results) {
|
|
34
48
|
return [];
|
|
35
49
|
}
|
|
36
50
|
return experiment.results.gates.filter((gate) => gate.version === version);
|
|
37
51
|
};
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
52
|
+
const GATE_OP_SYMBOLS = {
|
|
53
|
+
eq: "=",
|
|
54
|
+
gt: ">",
|
|
55
|
+
gte: "≥",
|
|
56
|
+
lt: "<",
|
|
57
|
+
lte: "≤",
|
|
58
|
+
};
|
|
59
|
+
/** Show a gate comparison operator as the math symbol people read, never the API word. */
|
|
60
|
+
const formatGateOp = (op) => {
|
|
61
|
+
return GATE_OP_SYMBOLS[op] ?? op;
|
|
62
|
+
};
|
|
63
|
+
/** Show a measured gate value with two decimals, or a dash before any scores arrive. */
|
|
64
|
+
export const formatGateActual = (actual) => {
|
|
65
|
+
if (actual === undefined || Number.isNaN(actual)) {
|
|
66
|
+
return "—";
|
|
67
|
+
}
|
|
68
|
+
return actual.toFixed(2);
|
|
69
|
+
};
|
|
70
|
+
/** The threshold a gate needs, such as "≥ 0.8". */
|
|
71
|
+
export const formatGateThreshold = (gate) => {
|
|
72
|
+
return `${formatGateOp(gate.op)} ${gate.value}`;
|
|
73
|
+
};
|
|
74
|
+
const humanizeIdentifier = (value) => {
|
|
75
|
+
return value.replace(/[_-]+/g, " ").trim();
|
|
76
|
+
};
|
|
77
|
+
/**
|
|
78
|
+
* Name a gate in plain words. When the evaluator and dimension share a name ("helpfulness"),
|
|
79
|
+
* say it once; otherwise "correctness: correct" or "schema-assert: schema valid".
|
|
80
|
+
*/
|
|
81
|
+
export const formatGateName = (gate) => {
|
|
82
|
+
const dimension = humanizeIdentifier(gate.dimension);
|
|
83
|
+
if (humanizeIdentifier(gate.evaluatorName).toLowerCase() === dimension.toLowerCase()) {
|
|
84
|
+
return dimension;
|
|
85
|
+
}
|
|
86
|
+
return `${gate.evaluatorName}: ${dimension}`;
|
|
87
|
+
};
|
|
88
|
+
/** One failing gate in a sentence, such as "helpfulness ≥ 0.8 (got 0.62)". */
|
|
89
|
+
const describeGateResult = (gate) => {
|
|
90
|
+
return `${formatGateName(gate)} ${formatGateThreshold(gate)} (got ${formatGateActual(gate.actual)})`;
|
|
91
|
+
};
|
|
92
|
+
const pluralize = (count, singular, plural) => {
|
|
93
|
+
return `${count} ${count === 1 ? singular : plural}`;
|
|
94
|
+
};
|
|
95
|
+
/** Gate results for every compared version, in version order, with pass/fail rolled up. */
|
|
96
|
+
export const summarizeGatesByVersion = (experiment) => {
|
|
97
|
+
return [...experiment.versions]
|
|
98
|
+
.sort((left, right) => left - right)
|
|
99
|
+
.map((version) => {
|
|
100
|
+
const gates = gatesForVersion(experiment, version);
|
|
101
|
+
const failing = gates.filter((gate) => !gate.passed);
|
|
102
|
+
return { failing, gates, passed: gates.length > 0 && failing.length === 0, version };
|
|
103
|
+
});
|
|
104
|
+
};
|
|
105
|
+
/** Short verdict for one version's gates, such as "Fails 1 of 3 gates". */
|
|
106
|
+
export const formatVersionGateVerdict = (summary) => {
|
|
107
|
+
if (summary.gates.length === 0) {
|
|
108
|
+
return "No gates";
|
|
109
|
+
}
|
|
110
|
+
if (summary.failing.length === 0) {
|
|
111
|
+
return summary.gates.length === 1
|
|
112
|
+
? "Passes its gate"
|
|
113
|
+
: `Passes all ${summary.gates.length} gates`;
|
|
114
|
+
}
|
|
115
|
+
return `Fails ${summary.failing.length} of ${pluralize(summary.gates.length, "gate", "gates")}`;
|
|
116
|
+
};
|
|
117
|
+
/**
|
|
118
|
+
* Explain why a version cannot be promoted, such as
|
|
119
|
+
* "v1 fails 1 gate: helpfulness ≥ 0.8 (got 0.62) — can't promote v1."
|
|
120
|
+
*/
|
|
121
|
+
export const formatPromoteBlockedReason = (summary) => {
|
|
122
|
+
if (summary.failing.length === 0) {
|
|
123
|
+
return undefined;
|
|
42
124
|
}
|
|
43
|
-
|
|
125
|
+
const details = summary.failing.map(describeGateResult).join("; ");
|
|
126
|
+
return `v${summary.version} fails ${pluralize(summary.failing.length, "gate", "gates")}: ${details} — can't promote v${summary.version}.`;
|
|
44
127
|
};
|
|
45
128
|
export const parsePromoteBlockedTitle = (error) => {
|
|
46
129
|
if (!error || typeof error !== "object") {
|
|
@@ -62,4 +145,87 @@ export const isAutomaticExperimentEvaluator = (evaluator) => {
|
|
|
62
145
|
export const isValidExperimentVersionCount = (count) => {
|
|
63
146
|
return count >= MIN_EXPERIMENT_VERSIONS && count <= MAX_EXPERIMENT_VERSIONS;
|
|
64
147
|
};
|
|
148
|
+
const ITEM_COLUMN_WIDTH = 120;
|
|
149
|
+
const PASS_COLUMN_WIDTH = 80;
|
|
150
|
+
const MIN_OUTPUT_COLUMN_WIDTH = 180;
|
|
151
|
+
/**
|
|
152
|
+
* Split the measured table width so the per-version output columns take every pixel the item and
|
|
153
|
+
* pass columns leave over, never narrower than the readable minimum.
|
|
154
|
+
*/
|
|
155
|
+
export const experimentItemColumnWidths = ({ containerWidth, versionCount, }) => {
|
|
156
|
+
const fixedWidth = ITEM_COLUMN_WIDTH + PASS_COLUMN_WIDTH * versionCount;
|
|
157
|
+
const available = containerWidth - fixedWidth;
|
|
158
|
+
const output = versionCount > 0
|
|
159
|
+
? Math.max(MIN_OUTPUT_COLUMN_WIDTH, Math.floor(available / versionCount))
|
|
160
|
+
: MIN_OUTPUT_COLUMN_WIDTH;
|
|
161
|
+
return { item: ITEM_COLUMN_WIDTH, output, pass: PASS_COLUMN_WIDTH };
|
|
162
|
+
};
|
|
163
|
+
export const DEFAULT_TARGET_AGREEMENT = 0.9;
|
|
164
|
+
export const EXPERIMENT_KIND_LABELS = {
|
|
165
|
+
judge: "Judge calibration",
|
|
166
|
+
prompt: "Prompt comparison",
|
|
167
|
+
};
|
|
168
|
+
export const experimentKind = (experiment) => {
|
|
169
|
+
return experiment.kind === "judge" ? "judge" : "prompt";
|
|
170
|
+
};
|
|
171
|
+
/** Tags shown next to a prompt version in pickers: latest, production, superseded. */
|
|
172
|
+
export const promptVersionTags = (version, detail) => {
|
|
173
|
+
const tags = [];
|
|
174
|
+
if (!detail || detail.versions.length === 0) {
|
|
175
|
+
return tags;
|
|
176
|
+
}
|
|
177
|
+
const latest = Math.max(...detail.versions.map((entry) => entry.version));
|
|
178
|
+
if (version === latest) {
|
|
179
|
+
tags.push("latest");
|
|
180
|
+
}
|
|
181
|
+
if (detail.labels.some((label) => label.label === "production" && label.version === version)) {
|
|
182
|
+
tags.push("production");
|
|
183
|
+
}
|
|
184
|
+
if (version < latest - 1) {
|
|
185
|
+
tags.push("superseded");
|
|
186
|
+
}
|
|
187
|
+
return tags;
|
|
188
|
+
};
|
|
189
|
+
/** "v1 and v2" or "v1, v2, and v3". */
|
|
190
|
+
export const formatVersionList = (versions) => {
|
|
191
|
+
const labels = [...versions].sort((left, right) => left - right).map((version) => `v${version}`);
|
|
192
|
+
if (labels.length <= 2) {
|
|
193
|
+
return labels.join(" and ");
|
|
194
|
+
}
|
|
195
|
+
return `${labels.slice(0, -1).join(", ")}, and ${labels[labels.length - 1]}`;
|
|
196
|
+
};
|
|
197
|
+
/** Parse a target agreement typed as a percent ("90") into a rate (0.9), or undefined if invalid. */
|
|
198
|
+
export const parseTargetAgreementPercent = (value) => {
|
|
199
|
+
const trimmed = value.trim();
|
|
200
|
+
if (!trimmed) {
|
|
201
|
+
return undefined;
|
|
202
|
+
}
|
|
203
|
+
const parsed = Number(trimmed);
|
|
204
|
+
if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 100) {
|
|
205
|
+
return undefined;
|
|
206
|
+
}
|
|
207
|
+
return parsed / 100;
|
|
208
|
+
};
|
|
209
|
+
/** Cost with two decimals; anything above zero but under a cent reads "<$0.01". */
|
|
210
|
+
export const formatExperimentCost = (cost) => {
|
|
211
|
+
if (cost === undefined || Number.isNaN(cost)) {
|
|
212
|
+
return "—";
|
|
213
|
+
}
|
|
214
|
+
if (cost > 0 && cost < 0.01) {
|
|
215
|
+
return "<$0.01";
|
|
216
|
+
}
|
|
217
|
+
return `$${cost.toFixed(2)}`;
|
|
218
|
+
};
|
|
219
|
+
/** Plain estimate such as "About 6 generations, 2 minutes, $0.42". */
|
|
220
|
+
export const formatExperimentEstimate = ({ estimate, unit = "generations", }) => {
|
|
221
|
+
const minutes = Math.max(1, Math.round(estimate.wallClockSeconds / 60));
|
|
222
|
+
const parts = [
|
|
223
|
+
`About ${estimate.generations} ${estimate.generations === 1 ? unit.replace(/s$/, "") : unit}`,
|
|
224
|
+
`${minutes} ${minutes === 1 ? "minute" : "minutes"}`,
|
|
225
|
+
];
|
|
226
|
+
if (estimate.costUsd !== undefined) {
|
|
227
|
+
parts.push(formatExperimentCost(estimate.costUsd));
|
|
228
|
+
}
|
|
229
|
+
return parts.join(", ");
|
|
230
|
+
};
|
|
65
231
|
//# sourceMappingURL=experimentTypes.js.map
|