@terreno/ai 57.9.0 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +133 -0
- package/dist/harness/errors.js +107 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +522 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +191 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +64 -0
- package/dist/harness/runners/inProcessRunner.js +227 -0
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +52 -0
- package/dist/harness/runtime.js +604 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +952 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +28 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +125 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +769 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +778 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +206 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +907 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +530 -0
- package/src/harness/runners/inProcessRunner.ts +260 -0
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +802 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1119 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
+
import {assert} from "chai";
|
|
3
|
+
import {DateTime} from "luxon";
|
|
4
|
+
import mongoose from "mongoose";
|
|
5
|
+
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
6
|
+
import {LocalEvaluatorStore} from "./evaluatorStore";
|
|
7
|
+
import {createLocalObservabilityPlugin} from "./localPlugin";
|
|
8
|
+
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
9
|
+
import {registerObsReviewItem} from "./models/obsReviewItem";
|
|
10
|
+
import {registerObsTrace} from "./models/obsTrace";
|
|
11
|
+
import {LocalReviewStore} from "./reviewStore";
|
|
12
|
+
import {LocalTraceStore, MemoryScoreSink} from "./traceStore";
|
|
13
|
+
|
|
14
|
+
describe("LocalReviewStore", () => {
|
|
15
|
+
let reviewStore: LocalReviewStore;
|
|
16
|
+
let evaluatorId: string;
|
|
17
|
+
let traceId: string;
|
|
18
|
+
let scoreSink: MemoryScoreSink;
|
|
19
|
+
|
|
20
|
+
afterEach(() => {
|
|
21
|
+
resetObservabilityApp();
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
beforeEach(async () => {
|
|
25
|
+
scoreSink = new MemoryScoreSink();
|
|
26
|
+
const plugin = createLocalObservabilityPlugin();
|
|
27
|
+
plugin.scoreSink = scoreSink;
|
|
28
|
+
new ObservabilityApp({plugins: [plugin]});
|
|
29
|
+
reviewStore = new LocalReviewStore();
|
|
30
|
+
await registerObsReviewItem().deleteMany({});
|
|
31
|
+
await registerObsEvaluator().deleteMany({});
|
|
32
|
+
await registerObsTrace().deleteMany({});
|
|
33
|
+
|
|
34
|
+
const evaluator = await new LocalEvaluatorStore().create({
|
|
35
|
+
dimensions: [{dataType: "boolean", key: "correct", required: true}],
|
|
36
|
+
instructions: "Pass if correct.",
|
|
37
|
+
name: "correctness-review",
|
|
38
|
+
target: "full trace",
|
|
39
|
+
type: "human",
|
|
40
|
+
});
|
|
41
|
+
evaluatorId = evaluator.id;
|
|
42
|
+
const exported = await new LocalTraceStore().exportTrace({
|
|
43
|
+
id: "t",
|
|
44
|
+
input: {raw: "prompt text"},
|
|
45
|
+
name: "gen",
|
|
46
|
+
output: {raw: "model text"},
|
|
47
|
+
prompts: [],
|
|
48
|
+
sensitive: false,
|
|
49
|
+
spans: [
|
|
50
|
+
{
|
|
51
|
+
id: "s",
|
|
52
|
+
kind: "LLM",
|
|
53
|
+
name: "llm",
|
|
54
|
+
startedAt: DateTime.utc().toISO() ?? "",
|
|
55
|
+
status: "ok",
|
|
56
|
+
},
|
|
57
|
+
],
|
|
58
|
+
startedAt: DateTime.utc().toISO() ?? "",
|
|
59
|
+
status: "ok",
|
|
60
|
+
});
|
|
61
|
+
traceId = exported.id;
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it("submits scores, hides the item from pending, and falls back panel keys", async () => {
|
|
65
|
+
const [item] = await reviewStore.enqueue({
|
|
66
|
+
evaluatorId,
|
|
67
|
+
reason: "manual",
|
|
68
|
+
traceIds: [traceId],
|
|
69
|
+
});
|
|
70
|
+
const detail = await reviewStore.getDetail(item.id);
|
|
71
|
+
expect(item.traceName).toBe("gen");
|
|
72
|
+
expect(detail.rawInput).toEqual({raw: "prompt text"});
|
|
73
|
+
expect(detail.rawOutput).toEqual({raw: "model text"});
|
|
74
|
+
expect(detail.panels.given[0]?.key).toBe("raw");
|
|
75
|
+
expect(detail.panels.wrote[0]?.key).toBe("raw");
|
|
76
|
+
|
|
77
|
+
await expect(
|
|
78
|
+
reviewStore.submit({
|
|
79
|
+
id: item.id,
|
|
80
|
+
scores: {},
|
|
81
|
+
sinks: [scoreSink],
|
|
82
|
+
})
|
|
83
|
+
).rejects.toMatchObject({status: 400, title: 'Score "correct" is required'});
|
|
84
|
+
|
|
85
|
+
await reviewStore.submit({
|
|
86
|
+
comment: "looks good",
|
|
87
|
+
id: item.id,
|
|
88
|
+
scores: {correct: true},
|
|
89
|
+
sinks: [scoreSink],
|
|
90
|
+
});
|
|
91
|
+
expect(scoreSink.scores[0]?.name).toBe("correct");
|
|
92
|
+
expect(scoreSink.scores[0]?.value).toBe(true);
|
|
93
|
+
|
|
94
|
+
const pending = await reviewStore.list("pending");
|
|
95
|
+
expect(pending.data.map((row) => row.id)).not.toContain(item.id);
|
|
96
|
+
expect(pending.counts.done).toBe(1);
|
|
97
|
+
|
|
98
|
+
const other = (
|
|
99
|
+
await reviewStore.enqueue({evaluatorId, reason: "manual", traceIds: [traceId]})
|
|
100
|
+
)[0];
|
|
101
|
+
await reviewStore.skip(other.id);
|
|
102
|
+
const skipped = await reviewStore.list("skipped");
|
|
103
|
+
expect(skipped.data.map((row) => row.id)).toContain(other.id);
|
|
104
|
+
|
|
105
|
+
const assigned = (
|
|
106
|
+
await reviewStore.enqueue({evaluatorId, reason: "manual", traceIds: [traceId]})
|
|
107
|
+
)[0];
|
|
108
|
+
await reviewStore.assign(assigned.id, new mongoose.Types.ObjectId().toString());
|
|
109
|
+
const inProgress = await reviewStore.list("in_progress");
|
|
110
|
+
expect(inProgress.data.map((row) => row.id)).toContain(assigned.id);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it("rejects automatic evaluators from the human review queue", async () => {
|
|
114
|
+
const automatic = await new LocalEvaluatorStore().create({
|
|
115
|
+
assertion: {constraint: "exists", path: "output"},
|
|
116
|
+
dimensions: [{dataType: "boolean", key: "valid", required: true}],
|
|
117
|
+
name: "automatic-schema-check",
|
|
118
|
+
target: "full trace",
|
|
119
|
+
type: "json-assert",
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
try {
|
|
123
|
+
await reviewStore.enqueue({
|
|
124
|
+
evaluatorId: automatic.id,
|
|
125
|
+
reason: "manual",
|
|
126
|
+
traceIds: [traceId],
|
|
127
|
+
});
|
|
128
|
+
assert.fail("Expected human-review enqueue to reject an automatic evaluator");
|
|
129
|
+
} catch (error) {
|
|
130
|
+
assert.deepInclude(error as object, {
|
|
131
|
+
status: 400,
|
|
132
|
+
title: "Human review requires a human evaluator",
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it("skips blank optional dimensions and stores scores without configured sinks", async () => {
|
|
138
|
+
const optionalEvaluator = await new LocalEvaluatorStore().create({
|
|
139
|
+
dimensions: [
|
|
140
|
+
{dataType: "boolean", key: "correct", required: true},
|
|
141
|
+
{dataType: "categorical", key: "failureMode", required: false},
|
|
142
|
+
],
|
|
143
|
+
name: "optional-human-score",
|
|
144
|
+
target: "full trace",
|
|
145
|
+
type: "human",
|
|
146
|
+
});
|
|
147
|
+
const [item] = await reviewStore.enqueue({
|
|
148
|
+
evaluatorId: optionalEvaluator.id,
|
|
149
|
+
reason: "manual",
|
|
150
|
+
traceIds: [traceId],
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
await reviewStore.submit({
|
|
154
|
+
id: item.id,
|
|
155
|
+
scores: {correct: true},
|
|
156
|
+
sinks: [],
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
const trace = await new LocalTraceStore().getDetail(traceId);
|
|
160
|
+
assert.deepInclude(trace.scores[0], {name: "correct", value: true});
|
|
161
|
+
assert.lengthOf(trace.scores, 1);
|
|
162
|
+
assert.equal((await reviewStore.getDetail(item.id)).status, "done");
|
|
163
|
+
});
|
|
164
|
+
});
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import {APIError} from "@terreno/api";
|
|
2
|
+
import {DateTime} from "luxon";
|
|
3
|
+
import mongoose from "mongoose";
|
|
4
|
+
|
|
5
|
+
import type {EvaluatorDimension} from "../../types/observability";
|
|
6
|
+
import {buildReviewPanels, type ReviewPanels} from "../reviewPanels";
|
|
7
|
+
import type {ScoreRecord, ScoreSink} from "../types";
|
|
8
|
+
import {registerObsEvaluator} from "./models/obsEvaluator";
|
|
9
|
+
import {registerObsPrompt} from "./models/obsPrompt";
|
|
10
|
+
import {registerObsPromptVersion} from "./models/obsPromptVersion";
|
|
11
|
+
import {registerObsReviewItem} from "./models/obsReviewItem";
|
|
12
|
+
import {registerObsTrace} from "./models/obsTrace";
|
|
13
|
+
import {LocalTraceStore} from "./traceStore";
|
|
14
|
+
|
|
15
|
+
export type ReviewStatus = "done" | "in_progress" | "pending" | "skipped";
|
|
16
|
+
|
|
17
|
+
export interface ReviewListItem {
|
|
18
|
+
assigneeId?: string;
|
|
19
|
+
enqueuedAt: string;
|
|
20
|
+
evaluatorId: string;
|
|
21
|
+
id: string;
|
|
22
|
+
promptName?: string;
|
|
23
|
+
reason: string;
|
|
24
|
+
status: ReviewStatus;
|
|
25
|
+
traceId: string;
|
|
26
|
+
traceName: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const objectId = (value: string, label: string): mongoose.Types.ObjectId => {
|
|
30
|
+
if (!mongoose.Types.ObjectId.isValid(value)) {
|
|
31
|
+
throw new APIError({status: 400, title: `${label} must be an object id`});
|
|
32
|
+
}
|
|
33
|
+
return new mongoose.Types.ObjectId(value);
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
const toIso = (value: Date): string => {
|
|
37
|
+
return DateTime.fromJSDate(value).toUTC().toISO() ?? "";
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export class LocalReviewStore {
|
|
41
|
+
async enqueue(params: {
|
|
42
|
+
evaluatorId: string;
|
|
43
|
+
reason?: "dataset_candidate" | "eval" | "feedback" | "manual";
|
|
44
|
+
spanId?: string;
|
|
45
|
+
traceIds: string[];
|
|
46
|
+
}): Promise<ReviewListItem[]> {
|
|
47
|
+
if (params.traceIds.length === 0) {
|
|
48
|
+
throw new APIError({status: 400, title: "traceIds are required"});
|
|
49
|
+
}
|
|
50
|
+
const evaluator = await registerObsEvaluator().findOneOrNone({
|
|
51
|
+
_id: objectId(params.evaluatorId, "evaluatorId"),
|
|
52
|
+
});
|
|
53
|
+
if (!evaluator) {
|
|
54
|
+
throw new APIError({status: 404, title: "Unknown evaluator"});
|
|
55
|
+
}
|
|
56
|
+
if (evaluator.type !== "human") {
|
|
57
|
+
throw new APIError({
|
|
58
|
+
status: 400,
|
|
59
|
+
title: "Human review requires a human evaluator",
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
const created: ReviewListItem[] = [];
|
|
63
|
+
for (const traceId of params.traceIds) {
|
|
64
|
+
const trace = await registerObsTrace().findOneOrNone({_id: objectId(traceId, "traceId")});
|
|
65
|
+
if (!trace) {
|
|
66
|
+
throw new APIError({status: 404, title: `Unknown trace "${traceId}"`});
|
|
67
|
+
}
|
|
68
|
+
const item = await registerObsReviewItem().create({
|
|
69
|
+
enqueuedAt: DateTime.utc().toJSDate(),
|
|
70
|
+
evaluatorId: evaluator._id,
|
|
71
|
+
reason: params.reason ?? "manual",
|
|
72
|
+
status: "pending",
|
|
73
|
+
traceId: trace._id,
|
|
74
|
+
...(params.spanId ? {spanId: objectId(params.spanId, "spanId")} : {}),
|
|
75
|
+
});
|
|
76
|
+
created.push(this.toListItem(item, trace.name, trace.prompts[0]?.name));
|
|
77
|
+
}
|
|
78
|
+
return created;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async list(status?: ReviewStatus): Promise<{
|
|
82
|
+
counts: Record<ReviewStatus, number>;
|
|
83
|
+
data: ReviewListItem[];
|
|
84
|
+
}> {
|
|
85
|
+
const ObsReviewItem = registerObsReviewItem();
|
|
86
|
+
const counts = {
|
|
87
|
+
done: await ObsReviewItem.countDocuments({status: "done"}),
|
|
88
|
+
in_progress: await ObsReviewItem.countDocuments({status: "in_progress"}),
|
|
89
|
+
pending: await ObsReviewItem.countDocuments({status: "pending"}),
|
|
90
|
+
skipped: await ObsReviewItem.countDocuments({status: "skipped"}),
|
|
91
|
+
};
|
|
92
|
+
const filter = status ? {status} : {};
|
|
93
|
+
const rows = await ObsReviewItem.find(filter).sort({enqueuedAt: 1});
|
|
94
|
+
const data: ReviewListItem[] = [];
|
|
95
|
+
for (const row of rows) {
|
|
96
|
+
const trace = await registerObsTrace().findOneOrNone({_id: row.traceId});
|
|
97
|
+
data.push(this.toListItem(row, trace?.name ?? "Unknown trace", trace?.prompts[0]?.name));
|
|
98
|
+
}
|
|
99
|
+
return {counts, data};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async getDetail(id: string): Promise<{
|
|
103
|
+
comment?: string;
|
|
104
|
+
dimensions: EvaluatorDimension[];
|
|
105
|
+
evaluatorId: string;
|
|
106
|
+
id: string;
|
|
107
|
+
instructions?: string;
|
|
108
|
+
panels: ReviewPanels;
|
|
109
|
+
rawInput?: unknown;
|
|
110
|
+
rawOutput?: unknown;
|
|
111
|
+
scores?: Record<string, boolean | number | string>;
|
|
112
|
+
status: ReviewStatus;
|
|
113
|
+
traceId: string;
|
|
114
|
+
}> {
|
|
115
|
+
const item = await registerObsReviewItem().findOneOrNone({_id: objectId(id, "id")});
|
|
116
|
+
if (!item) {
|
|
117
|
+
throw new APIError({status: 404, title: `Unknown review item "${id}"`});
|
|
118
|
+
}
|
|
119
|
+
const evaluator = await registerObsEvaluator().findOneOrNone({_id: item.evaluatorId});
|
|
120
|
+
if (!evaluator) {
|
|
121
|
+
throw new APIError({status: 404, title: "Unknown evaluator"});
|
|
122
|
+
}
|
|
123
|
+
const trace = await registerObsTrace().findOneOrNone({_id: item.traceId});
|
|
124
|
+
const promptRef = trace?.prompts[0];
|
|
125
|
+
let variables:
|
|
126
|
+
| {key: string; label?: string; required: boolean; reviewerNote?: string}[]
|
|
127
|
+
| undefined;
|
|
128
|
+
let outputFieldNotes: Record<string, string> | undefined;
|
|
129
|
+
let outputSchema: Record<string, unknown> | undefined;
|
|
130
|
+
if (promptRef) {
|
|
131
|
+
const prompt = await registerObsPrompt().findOneOrNone({name: promptRef.name});
|
|
132
|
+
if (prompt) {
|
|
133
|
+
const version = await registerObsPromptVersion().findOneOrNone({
|
|
134
|
+
promptId: prompt._id,
|
|
135
|
+
version: promptRef.version,
|
|
136
|
+
});
|
|
137
|
+
variables = version?.variables;
|
|
138
|
+
outputFieldNotes = version?.outputFieldNotes;
|
|
139
|
+
outputSchema = version?.outputSchema;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return {
|
|
143
|
+
comment: item.comment,
|
|
144
|
+
dimensions: evaluator.dimensions,
|
|
145
|
+
evaluatorId: String(evaluator._id),
|
|
146
|
+
id: String(item._id),
|
|
147
|
+
instructions: evaluator.instructions,
|
|
148
|
+
panels: buildReviewPanels({
|
|
149
|
+
input: trace?.input,
|
|
150
|
+
output: trace?.output,
|
|
151
|
+
outputFieldNotes,
|
|
152
|
+
outputSchema,
|
|
153
|
+
variables,
|
|
154
|
+
}),
|
|
155
|
+
rawInput: trace?.input,
|
|
156
|
+
rawOutput: trace?.output,
|
|
157
|
+
scores: item.scores,
|
|
158
|
+
status: item.status,
|
|
159
|
+
traceId: String(item.traceId),
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async submit(params: {
|
|
164
|
+
comment?: string;
|
|
165
|
+
id: string;
|
|
166
|
+
scores: Record<string, boolean | number | string>;
|
|
167
|
+
sinks: ScoreSink[];
|
|
168
|
+
}): Promise<void> {
|
|
169
|
+
const item = await this.requireItem(params.id);
|
|
170
|
+
const evaluator = await registerObsEvaluator().findOneOrNone({_id: item.evaluatorId});
|
|
171
|
+
if (!evaluator) {
|
|
172
|
+
throw new APIError({status: 404, title: "Unknown evaluator"});
|
|
173
|
+
}
|
|
174
|
+
const missing = evaluator.dimensions.find((dimension) => {
|
|
175
|
+
return dimension.required && params.scores[dimension.key] === undefined;
|
|
176
|
+
});
|
|
177
|
+
if (missing) {
|
|
178
|
+
throw new APIError({status: 400, title: `Score "${missing.key}" is required`});
|
|
179
|
+
}
|
|
180
|
+
const records: ScoreRecord[] = [];
|
|
181
|
+
for (const dimension of evaluator.dimensions) {
|
|
182
|
+
const value = params.scores[dimension.key];
|
|
183
|
+
if (value === undefined) {
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
records.push({
|
|
187
|
+
comment: params.comment,
|
|
188
|
+
dataType: dimension.dataType,
|
|
189
|
+
evaluatorId: String(evaluator._id),
|
|
190
|
+
name: dimension.key,
|
|
191
|
+
source: "human" as const,
|
|
192
|
+
traceId: String(item.traceId),
|
|
193
|
+
value,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
for (const record of records) {
|
|
197
|
+
if (params.sinks.length === 0) {
|
|
198
|
+
await new LocalTraceStore().exportScore(record);
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
const results = await Promise.allSettled(
|
|
202
|
+
params.sinks.map((sink) => {
|
|
203
|
+
return sink.export(record);
|
|
204
|
+
})
|
|
205
|
+
);
|
|
206
|
+
for (const result of results) {
|
|
207
|
+
if (result.status === "rejected") {
|
|
208
|
+
throw result.reason;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
item.comment = params.comment;
|
|
213
|
+
item.scores = params.scores;
|
|
214
|
+
item.status = "done";
|
|
215
|
+
await item.save();
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async skip(id: string): Promise<void> {
|
|
219
|
+
const item = await this.requireItem(id);
|
|
220
|
+
item.status = "skipped";
|
|
221
|
+
await item.save();
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async assign(id: string, assigneeId: string): Promise<void> {
|
|
225
|
+
const item = await this.requireItem(id);
|
|
226
|
+
item.assigneeId = objectId(assigneeId, "assigneeId");
|
|
227
|
+
item.status = "in_progress";
|
|
228
|
+
await item.save();
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
private async requireItem(id: string) {
|
|
232
|
+
const item = await registerObsReviewItem().findOneOrNone({_id: objectId(id, "id")});
|
|
233
|
+
if (!item) {
|
|
234
|
+
throw new APIError({status: 404, title: `Unknown review item "${id}"`});
|
|
235
|
+
}
|
|
236
|
+
return item;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
private toListItem(
|
|
240
|
+
row: {
|
|
241
|
+
_id: mongoose.Types.ObjectId;
|
|
242
|
+
assigneeId?: mongoose.Types.ObjectId;
|
|
243
|
+
enqueuedAt: Date;
|
|
244
|
+
evaluatorId: mongoose.Types.ObjectId;
|
|
245
|
+
reason: string;
|
|
246
|
+
status: ReviewStatus;
|
|
247
|
+
traceId: mongoose.Types.ObjectId;
|
|
248
|
+
},
|
|
249
|
+
traceName: string,
|
|
250
|
+
promptName?: string
|
|
251
|
+
): ReviewListItem {
|
|
252
|
+
return {
|
|
253
|
+
assigneeId: row.assigneeId ? String(row.assigneeId) : undefined,
|
|
254
|
+
enqueuedAt: toIso(row.enqueuedAt),
|
|
255
|
+
evaluatorId: String(row.evaluatorId),
|
|
256
|
+
id: String(row._id),
|
|
257
|
+
promptName,
|
|
258
|
+
reason: row.reason,
|
|
259
|
+
status: row.status,
|
|
260
|
+
traceId: String(row.traceId),
|
|
261
|
+
traceName,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
}
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import {afterEach, beforeEach, describe, expect, it} from "bun:test";
|
|
2
|
+
import {DateTime} from "luxon";
|
|
3
|
+
import mongoose from "mongoose";
|
|
4
|
+
|
|
5
|
+
import {ObservabilityApp, resetObservabilityApp} from "../observabilityApp";
|
|
6
|
+
import type {ScoreRecord, TraceRecord} from "../types";
|
|
7
|
+
import {createLocalObservabilityPlugin} from "./localPlugin";
|
|
8
|
+
import {registerObsScore} from "./models/obsScore";
|
|
9
|
+
import {registerObsSpan} from "./models/obsSpan";
|
|
10
|
+
import {registerObsTrace} from "./models/obsTrace";
|
|
11
|
+
import {LocalTraceSink, LocalTraceStore, MemoryScoreSink, MemoryTraceSink} from "./traceStore";
|
|
12
|
+
|
|
13
|
+
const iso = (msAgo: number): string => {
|
|
14
|
+
return DateTime.utc().minus({milliseconds: msAgo}).toISO() ?? "";
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
const baseTrace = (overrides: Partial<TraceRecord> = {}): TraceRecord => {
|
|
18
|
+
return {
|
|
19
|
+
endedAt: iso(0),
|
|
20
|
+
id: "trace-root",
|
|
21
|
+
name: "generate",
|
|
22
|
+
prompts: [{name: "summarize", version: 1}],
|
|
23
|
+
sensitive: false,
|
|
24
|
+
spans: [
|
|
25
|
+
{
|
|
26
|
+
durationMs: 12,
|
|
27
|
+
endedAt: iso(0),
|
|
28
|
+
id: "span-root",
|
|
29
|
+
kind: "LLM",
|
|
30
|
+
name: "generate",
|
|
31
|
+
startedAt: iso(12),
|
|
32
|
+
status: "ok",
|
|
33
|
+
usage: {inputTokens: 5, model: "mock-model", outputTokens: 10},
|
|
34
|
+
},
|
|
35
|
+
],
|
|
36
|
+
startedAt: iso(12),
|
|
37
|
+
status: "ok",
|
|
38
|
+
...overrides,
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
describe("LocalTraceStore", () => {
|
|
43
|
+
let store: LocalTraceStore;
|
|
44
|
+
|
|
45
|
+
afterEach(() => {
|
|
46
|
+
resetObservabilityApp();
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
beforeEach(async () => {
|
|
50
|
+
new ObservabilityApp({plugins: [createLocalObservabilityPlugin()]});
|
|
51
|
+
store = new LocalTraceStore();
|
|
52
|
+
await registerObsTrace().deleteMany({});
|
|
53
|
+
await registerObsSpan().deleteMany({});
|
|
54
|
+
await registerObsScore().deleteMany({});
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it("round-trips nested spans and derives errorSummary from the first error span", async () => {
|
|
58
|
+
const exported = await store.exportTrace(
|
|
59
|
+
baseTrace({
|
|
60
|
+
prompts: [
|
|
61
|
+
{label: "production", name: "summarize", version: 1},
|
|
62
|
+
{name: "title", version: 2},
|
|
63
|
+
],
|
|
64
|
+
spans: [
|
|
65
|
+
{
|
|
66
|
+
durationMs: 40,
|
|
67
|
+
id: "parent",
|
|
68
|
+
kind: "CHAIN",
|
|
69
|
+
name: "agent",
|
|
70
|
+
startedAt: iso(40),
|
|
71
|
+
status: "ok",
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
durationMs: 10,
|
|
75
|
+
error: "tool timed out",
|
|
76
|
+
id: "child",
|
|
77
|
+
kind: "TOOL",
|
|
78
|
+
name: "search",
|
|
79
|
+
parentSpanId: "parent",
|
|
80
|
+
startedAt: iso(20),
|
|
81
|
+
startOffsetMs: 8,
|
|
82
|
+
status: "error",
|
|
83
|
+
},
|
|
84
|
+
],
|
|
85
|
+
status: "error",
|
|
86
|
+
})
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
const detail = await store.getDetail(exported.id);
|
|
90
|
+
expect(detail.prompts).toHaveLength(2);
|
|
91
|
+
expect(detail.errorSummary).toBe("tool timed out");
|
|
92
|
+
expect(detail.spans).toEqual([
|
|
93
|
+
expect.objectContaining({
|
|
94
|
+
children: [
|
|
95
|
+
expect.objectContaining({
|
|
96
|
+
error: "tool timed out",
|
|
97
|
+
kind: "TOOL",
|
|
98
|
+
name: "search",
|
|
99
|
+
startOffsetMs: 8,
|
|
100
|
+
status: "error",
|
|
101
|
+
}),
|
|
102
|
+
],
|
|
103
|
+
kind: "CHAIN",
|
|
104
|
+
name: "agent",
|
|
105
|
+
}),
|
|
106
|
+
]);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it("omits costUsd when the exported usage has no price", async () => {
|
|
110
|
+
const exported = await store.exportTrace(
|
|
111
|
+
baseTrace({
|
|
112
|
+
usage: {inputTokens: 5, model: "unpriced", outputTokens: 10},
|
|
113
|
+
})
|
|
114
|
+
);
|
|
115
|
+
const detail = await store.getDetail(exported.id);
|
|
116
|
+
expect(detail.usage).toEqual({inputTokens: 5, model: "unpriced", outputTokens: 10});
|
|
117
|
+
expect("costUsd" in (detail.usage ?? {})).toBe(false);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("applies list filters, pagination, and hasScore", async () => {
|
|
121
|
+
const userId = new mongoose.Types.ObjectId();
|
|
122
|
+
const older = await store.exportTrace(
|
|
123
|
+
baseTrace({
|
|
124
|
+
flaggedForDataset: true,
|
|
125
|
+
name: "old",
|
|
126
|
+
sessionId: "sess-a",
|
|
127
|
+
startedAt: iso(86_400_000),
|
|
128
|
+
userId: userId.toString(),
|
|
129
|
+
})
|
|
130
|
+
);
|
|
131
|
+
const newer = await store.exportTrace(
|
|
132
|
+
baseTrace({
|
|
133
|
+
name: "new",
|
|
134
|
+
prompts: [{name: "other", version: 1}],
|
|
135
|
+
sensitive: true,
|
|
136
|
+
sessionId: "sess-b",
|
|
137
|
+
startedAt: iso(1_000),
|
|
138
|
+
status: "error",
|
|
139
|
+
userId: new mongoose.Types.ObjectId().toString(),
|
|
140
|
+
})
|
|
141
|
+
);
|
|
142
|
+
await store.exportScore({
|
|
143
|
+
dataType: "boolean",
|
|
144
|
+
name: "correct",
|
|
145
|
+
source: "human",
|
|
146
|
+
traceId: older.id,
|
|
147
|
+
value: true,
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
const byPrompt = await store.list({prompt: "summarize"});
|
|
151
|
+
expect(byPrompt.data.map((row) => row.id)).toEqual([older.id]);
|
|
152
|
+
|
|
153
|
+
const byStatus = await store.list({status: "error"});
|
|
154
|
+
expect(byStatus.data.map((row) => row.id)).toEqual([newer.id]);
|
|
155
|
+
|
|
156
|
+
const byUser = await store.list({userId: userId.toString()});
|
|
157
|
+
expect(byUser.data.map((row) => row.id)).toEqual([older.id]);
|
|
158
|
+
|
|
159
|
+
const bySession = await store.list({sessionId: "sess-b"});
|
|
160
|
+
expect(bySession.data.map((row) => row.id)).toEqual([newer.id]);
|
|
161
|
+
|
|
162
|
+
const bySensitive = await store.list({sensitive: true});
|
|
163
|
+
expect(bySensitive.data.map((row) => row.id)).toEqual([newer.id]);
|
|
164
|
+
|
|
165
|
+
const byFlag = await store.list({flaggedForDataset: true});
|
|
166
|
+
expect(byFlag.data.map((row) => row.id)).toEqual([older.id]);
|
|
167
|
+
|
|
168
|
+
const scored = await store.list({hasScore: true});
|
|
169
|
+
expect(scored.data.map((row) => row.id)).toEqual([older.id]);
|
|
170
|
+
expect(scored.data[0]?.spanCount).toBe(1);
|
|
171
|
+
expect(scored.data[0]?.scoreCount).toBe(1);
|
|
172
|
+
|
|
173
|
+
const unscored = await store.list({hasScore: false});
|
|
174
|
+
expect(unscored.data.map((row) => row.id)).toEqual([newer.id]);
|
|
175
|
+
|
|
176
|
+
const since = DateTime.utc().minus({hours: 1}).toISO() ?? "";
|
|
177
|
+
const recent = await store.list({from: since});
|
|
178
|
+
expect(recent.data.map((row) => row.id)).toEqual([newer.id]);
|
|
179
|
+
|
|
180
|
+
const until = DateTime.utc().minus({hours: 12}).toISO() ?? "";
|
|
181
|
+
const olderWindow = await store.list({to: until});
|
|
182
|
+
expect(olderWindow.data.map((row) => row.id)).toEqual([older.id]);
|
|
183
|
+
|
|
184
|
+
const page = await store.list({limit: 1, page: 2});
|
|
185
|
+
expect(page.meta).toEqual({limit: 1, page: 2, total: 2});
|
|
186
|
+
expect(page.data).toHaveLength(1);
|
|
187
|
+
|
|
188
|
+
const byPromptVersion = await store.list({prompt: "summarize", promptVersion: 2});
|
|
189
|
+
expect(byPromptVersion.data).toHaveLength(0);
|
|
190
|
+
await store.exportTrace(
|
|
191
|
+
baseTrace({
|
|
192
|
+
name: "summarize-v2",
|
|
193
|
+
prompts: [{name: "summarize", version: 2}],
|
|
194
|
+
startedAt: iso(500),
|
|
195
|
+
})
|
|
196
|
+
);
|
|
197
|
+
const v2Only = await store.list({prompt: "summarize", promptVersion: 2});
|
|
198
|
+
expect(v2Only.data.map((row) => row.name)).toEqual(["summarize-v2"]);
|
|
199
|
+
const v1Only = await store.list({prompt: "summarize", promptVersion: 1});
|
|
200
|
+
expect(v1Only.data.map((row) => row.id)).toEqual([older.id]);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it("returns score metadata on trace detail and exports through the sink wrapper", async () => {
|
|
204
|
+
const exported = await store.exportTrace(baseTrace());
|
|
205
|
+
await store.exportScore({
|
|
206
|
+
comment: "looks good",
|
|
207
|
+
confidence: 0.9,
|
|
208
|
+
dataType: "numeric",
|
|
209
|
+
evaluatorId: new mongoose.Types.ObjectId().toString(),
|
|
210
|
+
name: "quality",
|
|
211
|
+
source: "human",
|
|
212
|
+
spanId: "span-root",
|
|
213
|
+
traceId: exported.id,
|
|
214
|
+
value: 5,
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
const detail = await store.getDetail(exported.id);
|
|
218
|
+
expect(detail.scores).toEqual([
|
|
219
|
+
expect.objectContaining({
|
|
220
|
+
comment: "looks good",
|
|
221
|
+
confidence: 0.9,
|
|
222
|
+
dataType: "numeric",
|
|
223
|
+
name: "quality",
|
|
224
|
+
source: "human",
|
|
225
|
+
traceId: exported.id,
|
|
226
|
+
value: 5,
|
|
227
|
+
}),
|
|
228
|
+
]);
|
|
229
|
+
|
|
230
|
+
const sink = new LocalTraceSink(store);
|
|
231
|
+
await sink.export(baseTrace({name: "sink"}));
|
|
232
|
+
expect(sink.store).toBe(store);
|
|
233
|
+
const listed = await store.list({limit: 10});
|
|
234
|
+
const sinkRow = listed.data.find((row) => row.name === "sink");
|
|
235
|
+
expect(sinkRow).toBeDefined();
|
|
236
|
+
const sinkDetail = await store.getDetail(sinkRow!.id);
|
|
237
|
+
expect(sinkDetail.name).toBe("sink");
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
it("throws when loading an unknown trace", async () => {
|
|
241
|
+
await expect(store.getDetail(new mongoose.Types.ObjectId().toString())).rejects.toThrow(
|
|
242
|
+
/Unknown trace/
|
|
243
|
+
);
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
it("buffers exported traces and scores in memory sinks", async () => {
|
|
247
|
+
const trace = baseTrace({id: "memory-trace"});
|
|
248
|
+
const score: ScoreRecord = {
|
|
249
|
+
dataType: "boolean",
|
|
250
|
+
name: "correct",
|
|
251
|
+
source: "human",
|
|
252
|
+
traceId: "memory-trace",
|
|
253
|
+
value: true,
|
|
254
|
+
};
|
|
255
|
+
const traceSink = new MemoryTraceSink();
|
|
256
|
+
const scoreSink = new MemoryScoreSink();
|
|
257
|
+
|
|
258
|
+
await traceSink.export(trace);
|
|
259
|
+
await scoreSink.export(score);
|
|
260
|
+
|
|
261
|
+
expect(traceSink.traces).toEqual([trace]);
|
|
262
|
+
expect(scoreSink.scores).toEqual([score]);
|
|
263
|
+
});
|
|
264
|
+
});
|