@terreno/ai 57.9.0 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +133 -0
- package/dist/harness/errors.js +107 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +522 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +191 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +64 -0
- package/dist/harness/runners/inProcessRunner.js +227 -0
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +52 -0
- package/dist/harness/runtime.js +604 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +952 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +28 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +125 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +769 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +778 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +206 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +907 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +530 -0
- package/src/harness/runners/inProcessRunner.ts +260 -0
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +802 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1119 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsDatasetItemDocument, ObsDatasetItemModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsDatasetItemSchema = new mongoose.Schema<ObsDatasetItemDocument, ObsDatasetItemModel>(
|
|
7
|
+
{
|
|
8
|
+
annotatedBy: {
|
|
9
|
+
description: "Who approved the expected output for provenance display",
|
|
10
|
+
type: {
|
|
11
|
+
label: {description: "Human-readable attribution label", required: true, type: String},
|
|
12
|
+
reviewItemId: {description: "Optional review queue item id", type: String},
|
|
13
|
+
userId: {description: "Optional user id for the annotator", type: String},
|
|
14
|
+
},
|
|
15
|
+
},
|
|
16
|
+
datasetId: {
|
|
17
|
+
description: "Parent dataset id",
|
|
18
|
+
ref: "ObsDataset",
|
|
19
|
+
required: true,
|
|
20
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
21
|
+
},
|
|
22
|
+
expectedOutput: {
|
|
23
|
+
description: "Human- or machine-authored expected output label",
|
|
24
|
+
type: mongoose.Schema.Types.Mixed,
|
|
25
|
+
},
|
|
26
|
+
input: {
|
|
27
|
+
description: "Dataset item input payload",
|
|
28
|
+
required: true,
|
|
29
|
+
type: mongoose.Schema.Types.Mixed,
|
|
30
|
+
},
|
|
31
|
+
metadata: {
|
|
32
|
+
description: "Additional import metadata",
|
|
33
|
+
type: mongoose.Schema.Types.Mixed,
|
|
34
|
+
},
|
|
35
|
+
origin: {
|
|
36
|
+
description: "How this item entered the dataset",
|
|
37
|
+
enum: ["manual", "synthetic", "trace"],
|
|
38
|
+
required: true,
|
|
39
|
+
type: String,
|
|
40
|
+
},
|
|
41
|
+
outcomeClass: {
|
|
42
|
+
description: "Optional confusion-matrix class for classification features",
|
|
43
|
+
enum: ["fn", "fp", "tn", "tp"],
|
|
44
|
+
type: String,
|
|
45
|
+
},
|
|
46
|
+
proofread: {
|
|
47
|
+
default: false,
|
|
48
|
+
description: "When true, the item is treated as human-annotated",
|
|
49
|
+
type: Boolean,
|
|
50
|
+
},
|
|
51
|
+
sourceTraceId: {
|
|
52
|
+
description: "Trace copied into this item, when origin is trace",
|
|
53
|
+
ref: "ObsTrace",
|
|
54
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
55
|
+
},
|
|
56
|
+
tags: {
|
|
57
|
+
default: [],
|
|
58
|
+
description: "Item-level tags for filtering experiments",
|
|
59
|
+
type: [String],
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
obsDatasetItemSchema.plugin(createdUpdatedPlugin);
|
|
66
|
+
obsDatasetItemSchema.plugin(isDeletedPlugin);
|
|
67
|
+
obsDatasetItemSchema.plugin(findOneOrNone);
|
|
68
|
+
obsDatasetItemSchema.plugin(findExactlyOne);
|
|
69
|
+
obsDatasetItemSchema.index({datasetId: 1, origin: 1, proofread: 1});
|
|
70
|
+
obsDatasetItemSchema.index({created: -1, datasetId: 1});
|
|
71
|
+
|
|
72
|
+
export const registerObsDatasetItem = (): ObsDatasetItemModel => {
|
|
73
|
+
if (mongoose.models.ObsDatasetItem) {
|
|
74
|
+
return mongoose.models.ObsDatasetItem as ObsDatasetItemModel;
|
|
75
|
+
}
|
|
76
|
+
return mongoose.model<ObsDatasetItemDocument, ObsDatasetItemModel>(
|
|
77
|
+
"ObsDatasetItem",
|
|
78
|
+
obsDatasetItemSchema
|
|
79
|
+
);
|
|
80
|
+
};
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsEvaluatorDocument, ObsEvaluatorModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsEvaluatorSchema = new mongoose.Schema<ObsEvaluatorDocument, ObsEvaluatorModel>(
|
|
7
|
+
{
|
|
8
|
+
assertion: {
|
|
9
|
+
description: "json-assert path and constraint when type is json-assert",
|
|
10
|
+
type: mongoose.Schema.Types.Mixed,
|
|
11
|
+
},
|
|
12
|
+
confidenceAlertBelow: {
|
|
13
|
+
default: 0.7,
|
|
14
|
+
description: "Flag low-confidence scores below this threshold (0–1)",
|
|
15
|
+
type: Number,
|
|
16
|
+
},
|
|
17
|
+
description: {description: "Operator-facing evaluator description", type: String},
|
|
18
|
+
dimensions: {
|
|
19
|
+
default: [],
|
|
20
|
+
description: "Typed score dimensions written by this evaluator",
|
|
21
|
+
type: [
|
|
22
|
+
{
|
|
23
|
+
dataType: {
|
|
24
|
+
description: "How the dimension value is interpreted",
|
|
25
|
+
enum: ["boolean", "categorical", "numeric"],
|
|
26
|
+
required: true,
|
|
27
|
+
type: String,
|
|
28
|
+
},
|
|
29
|
+
key: {description: "Dimension key written onto scores", required: true, type: String},
|
|
30
|
+
range: {
|
|
31
|
+
description: "Numeric bounds, categorical values, or a free-text hint",
|
|
32
|
+
type: String,
|
|
33
|
+
},
|
|
34
|
+
required: {
|
|
35
|
+
default: true,
|
|
36
|
+
description: "Whether a review submit must include this dimension",
|
|
37
|
+
type: Boolean,
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
],
|
|
41
|
+
},
|
|
42
|
+
instructions: {
|
|
43
|
+
description: "Human-reviewer instructions shown above the score form",
|
|
44
|
+
type: String,
|
|
45
|
+
},
|
|
46
|
+
judgePromptName: {
|
|
47
|
+
description: "Named judge prompt when type is llm-judge",
|
|
48
|
+
type: String,
|
|
49
|
+
},
|
|
50
|
+
name: {description: "Unique evaluator name", required: true, type: String},
|
|
51
|
+
runModes: {
|
|
52
|
+
description: "Where this evaluator may run",
|
|
53
|
+
required: true,
|
|
54
|
+
type: {
|
|
55
|
+
allowManualRun: {
|
|
56
|
+
default: true,
|
|
57
|
+
description: "When true, operators may run this evaluator on demand",
|
|
58
|
+
type: Boolean,
|
|
59
|
+
},
|
|
60
|
+
availableInExperiments: {
|
|
61
|
+
default: true,
|
|
62
|
+
description: "When true, this evaluator can be attached to an experiment",
|
|
63
|
+
type: Boolean,
|
|
64
|
+
},
|
|
65
|
+
liveSampleRate: {
|
|
66
|
+
default: 0,
|
|
67
|
+
description: "Percent of live traces to sample (0–100). Must be 0 for human evaluators",
|
|
68
|
+
type: Number,
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
target: {
|
|
73
|
+
description: "What the evaluator scores",
|
|
74
|
+
enum: ["dataset item", "full trace", "generation span"],
|
|
75
|
+
required: true,
|
|
76
|
+
type: String,
|
|
77
|
+
},
|
|
78
|
+
type: {
|
|
79
|
+
description: "Evaluator implementation",
|
|
80
|
+
enum: ["human", "json-assert", "llm-judge"],
|
|
81
|
+
required: true,
|
|
82
|
+
type: String,
|
|
83
|
+
},
|
|
84
|
+
},
|
|
85
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
obsEvaluatorSchema.plugin(createdUpdatedPlugin);
|
|
89
|
+
obsEvaluatorSchema.plugin(isDeletedPlugin);
|
|
90
|
+
obsEvaluatorSchema.plugin(findOneOrNone);
|
|
91
|
+
obsEvaluatorSchema.plugin(findExactlyOne);
|
|
92
|
+
obsEvaluatorSchema.index({name: 1}, {unique: true});
|
|
93
|
+
|
|
94
|
+
export const registerObsEvaluator = (): ObsEvaluatorModel => {
|
|
95
|
+
if (mongoose.models.ObsEvaluator) {
|
|
96
|
+
return mongoose.models.ObsEvaluator as ObsEvaluatorModel;
|
|
97
|
+
}
|
|
98
|
+
return mongoose.model<ObsEvaluatorDocument, ObsEvaluatorModel>(
|
|
99
|
+
"ObsEvaluator",
|
|
100
|
+
obsEvaluatorSchema
|
|
101
|
+
);
|
|
102
|
+
};
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsExperimentDocument, ObsExperimentModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsExperimentSchema = new mongoose.Schema<ObsExperimentDocument, ObsExperimentModel>(
|
|
7
|
+
{
|
|
8
|
+
backgroundTaskId: {
|
|
9
|
+
description: "Background task executing this experiment locally",
|
|
10
|
+
ref: "BackgroundTask",
|
|
11
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
12
|
+
},
|
|
13
|
+
datasetId: {
|
|
14
|
+
description: "Dataset compared in this experiment",
|
|
15
|
+
ref: "ObsDataset",
|
|
16
|
+
required: true,
|
|
17
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
18
|
+
},
|
|
19
|
+
estimate: {
|
|
20
|
+
description: "Pre-run estimate shown in the wizard",
|
|
21
|
+
type: mongoose.Schema.Types.Mixed,
|
|
22
|
+
},
|
|
23
|
+
evaluatorIds: {
|
|
24
|
+
default: [],
|
|
25
|
+
description: "Evaluators attached to this experiment",
|
|
26
|
+
type: [{ref: "ObsEvaluator", type: mongoose.Schema.Types.ObjectId}],
|
|
27
|
+
},
|
|
28
|
+
includeUnproofread: {
|
|
29
|
+
default: false,
|
|
30
|
+
description: "When true, synthetic/unproofread items are included",
|
|
31
|
+
type: Boolean,
|
|
32
|
+
},
|
|
33
|
+
modelOverride: {
|
|
34
|
+
description: "Optional model id override for all generations",
|
|
35
|
+
type: String,
|
|
36
|
+
},
|
|
37
|
+
name: {description: "Operator-facing experiment name", required: true, type: String},
|
|
38
|
+
promptName: {
|
|
39
|
+
description: "Prompt whose versions are compared",
|
|
40
|
+
required: true,
|
|
41
|
+
type: String,
|
|
42
|
+
},
|
|
43
|
+
results: {
|
|
44
|
+
description: "Aggregated gate, outlier, and progress results",
|
|
45
|
+
type: mongoose.Schema.Types.Mixed,
|
|
46
|
+
},
|
|
47
|
+
status: {
|
|
48
|
+
default: "pending",
|
|
49
|
+
description: "Experiment lifecycle status",
|
|
50
|
+
enum: ["completed", "failed", "pending", "running"],
|
|
51
|
+
required: true,
|
|
52
|
+
type: String,
|
|
53
|
+
},
|
|
54
|
+
thresholds: {
|
|
55
|
+
default: [],
|
|
56
|
+
description: "Gate thresholds evaluated after the run",
|
|
57
|
+
type: mongoose.Schema.Types.Mixed,
|
|
58
|
+
},
|
|
59
|
+
versions: {
|
|
60
|
+
description: "Two or three prompt versions compared in this run",
|
|
61
|
+
required: true,
|
|
62
|
+
type: [Number],
|
|
63
|
+
validate: {
|
|
64
|
+
message: "experiments must compare 2–3 prompt versions",
|
|
65
|
+
validator: (value: number[]) => {
|
|
66
|
+
return value.length >= 2 && value.length <= 3;
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
obsExperimentSchema.plugin(createdUpdatedPlugin);
|
|
75
|
+
obsExperimentSchema.plugin(isDeletedPlugin);
|
|
76
|
+
obsExperimentSchema.plugin(findOneOrNone);
|
|
77
|
+
obsExperimentSchema.plugin(findExactlyOne);
|
|
78
|
+
obsExperimentSchema.index({created: -1, datasetId: 1});
|
|
79
|
+
obsExperimentSchema.index({created: -1, status: 1});
|
|
80
|
+
// biome-ignore assist/source/useSortedKeys: promptName must be the leading key for prompt-filtered lists
|
|
81
|
+
obsExperimentSchema.index({promptName: 1, created: -1});
|
|
82
|
+
|
|
83
|
+
export const registerObsExperiment = (): ObsExperimentModel => {
|
|
84
|
+
if (mongoose.models.ObsExperiment) {
|
|
85
|
+
return mongoose.models.ObsExperiment as ObsExperimentModel;
|
|
86
|
+
}
|
|
87
|
+
return mongoose.model<ObsExperimentDocument, ObsExperimentModel>(
|
|
88
|
+
"ObsExperiment",
|
|
89
|
+
obsExperimentSchema
|
|
90
|
+
);
|
|
91
|
+
};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsExperimentItemDocument, ObsExperimentItemModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsExperimentItemSchema = new mongoose.Schema<
|
|
7
|
+
ObsExperimentItemDocument,
|
|
8
|
+
ObsExperimentItemModel
|
|
9
|
+
>(
|
|
10
|
+
{
|
|
11
|
+
datasetItemId: {
|
|
12
|
+
description: "Dataset item evaluated in this experiment row",
|
|
13
|
+
ref: "ObsDatasetItem",
|
|
14
|
+
required: true,
|
|
15
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
16
|
+
},
|
|
17
|
+
experimentId: {
|
|
18
|
+
description: "Parent experiment id",
|
|
19
|
+
ref: "ObsExperiment",
|
|
20
|
+
required: true,
|
|
21
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
22
|
+
},
|
|
23
|
+
failed: {
|
|
24
|
+
default: false,
|
|
25
|
+
description: "When true, this row failed a gate or evaluator",
|
|
26
|
+
type: Boolean,
|
|
27
|
+
},
|
|
28
|
+
versionResults: {
|
|
29
|
+
default: {},
|
|
30
|
+
description: "Per-version outputs and evaluator score maps keyed by version number",
|
|
31
|
+
type: mongoose.Schema.Types.Mixed,
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
obsExperimentItemSchema.plugin(createdUpdatedPlugin);
|
|
38
|
+
obsExperimentItemSchema.plugin(isDeletedPlugin);
|
|
39
|
+
obsExperimentItemSchema.plugin(findOneOrNone);
|
|
40
|
+
obsExperimentItemSchema.plugin(findExactlyOne);
|
|
41
|
+
obsExperimentItemSchema.index({experimentId: 1, failed: 1});
|
|
42
|
+
obsExperimentItemSchema.index({datasetItemId: 1, experimentId: 1}, {unique: true});
|
|
43
|
+
|
|
44
|
+
export const registerObsExperimentItem = (): ObsExperimentItemModel => {
|
|
45
|
+
if (mongoose.models.ObsExperimentItem) {
|
|
46
|
+
return mongoose.models.ObsExperimentItem as ObsExperimentItemModel;
|
|
47
|
+
}
|
|
48
|
+
return mongoose.model<ObsExperimentItemDocument, ObsExperimentItemModel>(
|
|
49
|
+
"ObsExperimentItem",
|
|
50
|
+
obsExperimentItemSchema
|
|
51
|
+
);
|
|
52
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsPromptDocument, ObsPromptModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsPromptSchema = new mongoose.Schema<ObsPromptDocument, ObsPromptModel>(
|
|
7
|
+
{
|
|
8
|
+
description: {
|
|
9
|
+
description: "Optional operator-facing summary of what this prompt is for",
|
|
10
|
+
type: String,
|
|
11
|
+
},
|
|
12
|
+
folder: {
|
|
13
|
+
description: "Folder path used to group prompts in the admin list",
|
|
14
|
+
required: true,
|
|
15
|
+
type: String,
|
|
16
|
+
},
|
|
17
|
+
name: {
|
|
18
|
+
description: "Unique prompt name used by PromptRegistry.get",
|
|
19
|
+
required: true,
|
|
20
|
+
type: String,
|
|
21
|
+
},
|
|
22
|
+
tags: {
|
|
23
|
+
default: [],
|
|
24
|
+
description: "Freeform tags for filtering prompts",
|
|
25
|
+
type: [String],
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
obsPromptSchema.plugin(createdUpdatedPlugin);
|
|
32
|
+
obsPromptSchema.plugin(isDeletedPlugin);
|
|
33
|
+
obsPromptSchema.plugin(findOneOrNone);
|
|
34
|
+
obsPromptSchema.plugin(findExactlyOne);
|
|
35
|
+
obsPromptSchema.index({name: 1}, {unique: true});
|
|
36
|
+
obsPromptSchema.index({folder: 1, name: 1});
|
|
37
|
+
|
|
38
|
+
export const registerObsPrompt = (): ObsPromptModel => {
|
|
39
|
+
if (mongoose.models.ObsPrompt) {
|
|
40
|
+
return mongoose.models.ObsPrompt as ObsPromptModel;
|
|
41
|
+
}
|
|
42
|
+
return mongoose.model<ObsPromptDocument, ObsPromptModel>("ObsPrompt", obsPromptSchema);
|
|
43
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsPromptLabelDocument, ObsPromptLabelModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsPromptLabelSchema = new mongoose.Schema<ObsPromptLabelDocument, ObsPromptLabelModel>(
|
|
7
|
+
{
|
|
8
|
+
label: {
|
|
9
|
+
description: "Movable label such as production, latest, or staging",
|
|
10
|
+
required: true,
|
|
11
|
+
type: String,
|
|
12
|
+
},
|
|
13
|
+
promptId: {
|
|
14
|
+
description: "Prompt this label belongs to",
|
|
15
|
+
ref: "ObsPrompt",
|
|
16
|
+
required: true,
|
|
17
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
18
|
+
},
|
|
19
|
+
versionId: {
|
|
20
|
+
description: "Immutable version currently pointed at by this label",
|
|
21
|
+
ref: "ObsPromptVersion",
|
|
22
|
+
required: true,
|
|
23
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
24
|
+
},
|
|
25
|
+
},
|
|
26
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
obsPromptLabelSchema.plugin(createdUpdatedPlugin);
|
|
30
|
+
obsPromptLabelSchema.plugin(isDeletedPlugin);
|
|
31
|
+
obsPromptLabelSchema.plugin(findOneOrNone);
|
|
32
|
+
obsPromptLabelSchema.plugin(findExactlyOne);
|
|
33
|
+
obsPromptLabelSchema.index({label: 1, promptId: 1}, {unique: true});
|
|
34
|
+
|
|
35
|
+
export const registerObsPromptLabel = (): ObsPromptLabelModel => {
|
|
36
|
+
if (mongoose.models.ObsPromptLabel) {
|
|
37
|
+
return mongoose.models.ObsPromptLabel as ObsPromptLabelModel;
|
|
38
|
+
}
|
|
39
|
+
return mongoose.model<ObsPromptLabelDocument, ObsPromptLabelModel>(
|
|
40
|
+
"ObsPromptLabel",
|
|
41
|
+
obsPromptLabelSchema
|
|
42
|
+
);
|
|
43
|
+
};
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import {
|
|
2
|
+
APIError,
|
|
3
|
+
createdUpdatedPlugin,
|
|
4
|
+
findExactlyOne,
|
|
5
|
+
findOneOrNone,
|
|
6
|
+
isDeletedPlugin,
|
|
7
|
+
} from "@terreno/api";
|
|
8
|
+
import mongoose from "mongoose";
|
|
9
|
+
|
|
10
|
+
import type {ObsPromptVersionDocument, ObsPromptVersionModel} from "../../../types/observability";
|
|
11
|
+
|
|
12
|
+
const obsPromptVersionSchema = new mongoose.Schema<ObsPromptVersionDocument, ObsPromptVersionModel>(
|
|
13
|
+
{
|
|
14
|
+
config: {
|
|
15
|
+
description: "Version config such as temperature preset and model hint",
|
|
16
|
+
type: mongoose.Schema.Types.Mixed,
|
|
17
|
+
},
|
|
18
|
+
inputSchema: {
|
|
19
|
+
description: "JSON Schema describing expected playground / dataset input",
|
|
20
|
+
type: mongoose.Schema.Types.Mixed,
|
|
21
|
+
},
|
|
22
|
+
outputFieldNotes: {
|
|
23
|
+
description: "Per-output-field notes shown to human reviewers",
|
|
24
|
+
type: mongoose.Schema.Types.Mixed,
|
|
25
|
+
},
|
|
26
|
+
outputSchema: {
|
|
27
|
+
description: "JSON Schema describing expected model output",
|
|
28
|
+
type: mongoose.Schema.Types.Mixed,
|
|
29
|
+
},
|
|
30
|
+
promptId: {
|
|
31
|
+
description: "Prompt this immutable version belongs to",
|
|
32
|
+
ref: "ObsPrompt",
|
|
33
|
+
required: true,
|
|
34
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
35
|
+
},
|
|
36
|
+
sensitive: {
|
|
37
|
+
default: false,
|
|
38
|
+
description: "When true, traces inherit sensitive unless the generate call overrides it",
|
|
39
|
+
type: Boolean,
|
|
40
|
+
},
|
|
41
|
+
system: {
|
|
42
|
+
description: "System prompt body for this version",
|
|
43
|
+
type: String,
|
|
44
|
+
},
|
|
45
|
+
template: {
|
|
46
|
+
description: "User-facing template body, including {{variable}} placeholders",
|
|
47
|
+
type: String,
|
|
48
|
+
},
|
|
49
|
+
type: {
|
|
50
|
+
description: "Prompt shape: a single text template or a chat transcript",
|
|
51
|
+
enum: ["chat", "text"],
|
|
52
|
+
required: true,
|
|
53
|
+
type: String,
|
|
54
|
+
},
|
|
55
|
+
variables: {
|
|
56
|
+
default: [],
|
|
57
|
+
description: "Named template variables with reviewer labels",
|
|
58
|
+
type: [
|
|
59
|
+
{
|
|
60
|
+
key: {
|
|
61
|
+
description: "Variable key used in {{key}} placeholders",
|
|
62
|
+
required: true,
|
|
63
|
+
type: String,
|
|
64
|
+
},
|
|
65
|
+
label: {description: "Human label shown on review given-panels", type: String},
|
|
66
|
+
required: {
|
|
67
|
+
default: true,
|
|
68
|
+
description: "Whether the playground requires this variable",
|
|
69
|
+
type: Boolean,
|
|
70
|
+
},
|
|
71
|
+
reviewerNote: {
|
|
72
|
+
description: "Guidance shown next to this field in human review",
|
|
73
|
+
type: String,
|
|
74
|
+
},
|
|
75
|
+
},
|
|
76
|
+
],
|
|
77
|
+
},
|
|
78
|
+
version: {
|
|
79
|
+
description: "Monotonic version number starting at 1",
|
|
80
|
+
required: true,
|
|
81
|
+
type: Number,
|
|
82
|
+
},
|
|
83
|
+
},
|
|
84
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
85
|
+
);
|
|
86
|
+
|
|
87
|
+
obsPromptVersionSchema.plugin(createdUpdatedPlugin);
|
|
88
|
+
obsPromptVersionSchema.plugin(isDeletedPlugin);
|
|
89
|
+
obsPromptVersionSchema.plugin(findOneOrNone);
|
|
90
|
+
obsPromptVersionSchema.plugin(findExactlyOne);
|
|
91
|
+
obsPromptVersionSchema.index({promptId: 1, version: 1}, {unique: true});
|
|
92
|
+
|
|
93
|
+
obsPromptVersionSchema.pre("save", function () {
|
|
94
|
+
if (!this.isNew) {
|
|
95
|
+
throw new APIError({status: 400, title: "ObsPromptVersion documents are immutable"});
|
|
96
|
+
}
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
export const registerObsPromptVersion = (): ObsPromptVersionModel => {
|
|
100
|
+
if (mongoose.models.ObsPromptVersion) {
|
|
101
|
+
return mongoose.models.ObsPromptVersion as ObsPromptVersionModel;
|
|
102
|
+
}
|
|
103
|
+
return mongoose.model<ObsPromptVersionDocument, ObsPromptVersionModel>(
|
|
104
|
+
"ObsPromptVersion",
|
|
105
|
+
obsPromptVersionSchema
|
|
106
|
+
);
|
|
107
|
+
};
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsReviewItemDocument, ObsReviewItemModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsReviewItemSchema = new mongoose.Schema<ObsReviewItemDocument, ObsReviewItemModel>(
|
|
7
|
+
{
|
|
8
|
+
assigneeId: {
|
|
9
|
+
description: "User assigned to review this item",
|
|
10
|
+
ref: "User",
|
|
11
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
12
|
+
},
|
|
13
|
+
comment: {description: "Reviewer comment submitted with scores", type: String},
|
|
14
|
+
datasetItemId: {
|
|
15
|
+
description: "Optional dataset item this review is bound to",
|
|
16
|
+
ref: "ObsDatasetItem",
|
|
17
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
18
|
+
},
|
|
19
|
+
enqueuedAt: {
|
|
20
|
+
description: "When the item entered the queue",
|
|
21
|
+
required: true,
|
|
22
|
+
type: Date,
|
|
23
|
+
},
|
|
24
|
+
evaluatorId: {
|
|
25
|
+
description: "Human evaluator that owns this review form",
|
|
26
|
+
ref: "ObsEvaluator",
|
|
27
|
+
required: true,
|
|
28
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
29
|
+
},
|
|
30
|
+
reason: {
|
|
31
|
+
description: "Why the item was enqueued",
|
|
32
|
+
enum: ["dataset_candidate", "eval", "feedback", "manual"],
|
|
33
|
+
required: true,
|
|
34
|
+
type: String,
|
|
35
|
+
},
|
|
36
|
+
scores: {
|
|
37
|
+
description: "Dimension-keyed scores written on submit",
|
|
38
|
+
type: mongoose.Schema.Types.Mixed,
|
|
39
|
+
},
|
|
40
|
+
spanId: {
|
|
41
|
+
description: "Optional span under review",
|
|
42
|
+
ref: "ObsSpan",
|
|
43
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
44
|
+
},
|
|
45
|
+
status: {
|
|
46
|
+
description: "Queue status",
|
|
47
|
+
enum: ["done", "in_progress", "pending", "skipped"],
|
|
48
|
+
required: true,
|
|
49
|
+
type: String,
|
|
50
|
+
},
|
|
51
|
+
traceId: {
|
|
52
|
+
description: "Trace under review",
|
|
53
|
+
ref: "ObsTrace",
|
|
54
|
+
required: true,
|
|
55
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
obsReviewItemSchema.plugin(createdUpdatedPlugin);
|
|
62
|
+
obsReviewItemSchema.plugin(isDeletedPlugin);
|
|
63
|
+
obsReviewItemSchema.plugin(findOneOrNone);
|
|
64
|
+
obsReviewItemSchema.plugin(findExactlyOne);
|
|
65
|
+
const reviewIndexSpec: Array<[string, 1]> = [
|
|
66
|
+
["status", 1],
|
|
67
|
+
["enqueuedAt", 1],
|
|
68
|
+
];
|
|
69
|
+
obsReviewItemSchema.index(Object.fromEntries(reviewIndexSpec));
|
|
70
|
+
|
|
71
|
+
export const registerObsReviewItem = (): ObsReviewItemModel => {
|
|
72
|
+
if (mongoose.models.ObsReviewItem) {
|
|
73
|
+
return mongoose.models.ObsReviewItem as ObsReviewItemModel;
|
|
74
|
+
}
|
|
75
|
+
return mongoose.model<ObsReviewItemDocument, ObsReviewItemModel>(
|
|
76
|
+
"ObsReviewItem",
|
|
77
|
+
obsReviewItemSchema
|
|
78
|
+
);
|
|
79
|
+
};
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import {createdUpdatedPlugin, findExactlyOne, findOneOrNone, isDeletedPlugin} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {ObsScoreDocument, ObsScoreModel} from "../../../types/observability";
|
|
5
|
+
|
|
6
|
+
const obsScoreSchema = new mongoose.Schema<ObsScoreDocument, ObsScoreModel>(
|
|
7
|
+
{
|
|
8
|
+
comment: {description: "Optional human or judge comment", type: String},
|
|
9
|
+
confidence: {description: "Optional confidence in 0–1", type: Number},
|
|
10
|
+
dataType: {
|
|
11
|
+
description: "How to interpret value",
|
|
12
|
+
enum: ["boolean", "categorical", "numeric"],
|
|
13
|
+
required: true,
|
|
14
|
+
type: String,
|
|
15
|
+
},
|
|
16
|
+
evaluatorId: {
|
|
17
|
+
description: "Evaluator that produced this score when applicable",
|
|
18
|
+
ref: "ObsEvaluator",
|
|
19
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
20
|
+
},
|
|
21
|
+
name: {description: "Score dimension or name", required: true, type: String},
|
|
22
|
+
source: {
|
|
23
|
+
description: "Who wrote the score",
|
|
24
|
+
enum: ["code", "human", "llm-judge", "user-feedback"],
|
|
25
|
+
required: true,
|
|
26
|
+
type: String,
|
|
27
|
+
},
|
|
28
|
+
spanId: {
|
|
29
|
+
description: "Optional span this score attaches to",
|
|
30
|
+
ref: "ObsSpan",
|
|
31
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
32
|
+
},
|
|
33
|
+
traceId: {
|
|
34
|
+
description: "Trace this score attaches to",
|
|
35
|
+
ref: "ObsTrace",
|
|
36
|
+
required: true,
|
|
37
|
+
type: mongoose.Schema.Types.ObjectId,
|
|
38
|
+
},
|
|
39
|
+
value: {
|
|
40
|
+
description: "Score value; type matches dataType",
|
|
41
|
+
required: true,
|
|
42
|
+
type: mongoose.Schema.Types.Mixed,
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
{strict: "throw", toJSON: {virtuals: true}, toObject: {virtuals: true}}
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
obsScoreSchema.plugin(createdUpdatedPlugin);
|
|
49
|
+
obsScoreSchema.plugin(isDeletedPlugin);
|
|
50
|
+
obsScoreSchema.plugin(findOneOrNone);
|
|
51
|
+
obsScoreSchema.plugin(findExactlyOne);
|
|
52
|
+
obsScoreSchema.index({traceId: 1});
|
|
53
|
+
|
|
54
|
+
export const registerObsScore = (): ObsScoreModel => {
|
|
55
|
+
if (mongoose.models.ObsScore) {
|
|
56
|
+
return mongoose.models.ObsScore as ObsScoreModel;
|
|
57
|
+
}
|
|
58
|
+
return mongoose.model<ObsScoreDocument, ObsScoreModel>("ObsScore", obsScoreSchema);
|
|
59
|
+
};
|