@terreno/ai 57.9.0 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +133 -0
- package/dist/harness/errors.js +107 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +522 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +191 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +64 -0
- package/dist/harness/runners/inProcessRunner.js +227 -0
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +52 -0
- package/dist/harness/runtime.js +604 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +952 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +28 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +125 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +769 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +778 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +206 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +907 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +530 -0
- package/src/harness/runners/inProcessRunner.ts +260 -0
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +802 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1119 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,693 @@
|
|
|
1
|
+
import {afterEach, beforeAll, beforeEach, describe, expect, it, spyOn} from "bun:test";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
5
|
+
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
6
|
+
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
7
|
+
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
8
|
+
import type {HarnessTestHooks} from "../types/harness";
|
|
9
|
+
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
10
|
+
import {defineTask, Harness, HarnessCommitConflictError, InProcessRunner} from "./harness";
|
|
11
|
+
import {registerHarnessTask} from "./models/harnessTask";
|
|
12
|
+
|
|
13
|
+
interface IntakeInput {
|
|
14
|
+
patientId: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
interface IntakeState {
|
|
18
|
+
chart?: string;
|
|
19
|
+
summary?: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const intakeTask = defineTask<IntakeInput, IntakeState, {status: string; summary: string}>({
|
|
23
|
+
initial: () => ({phase: "fetch", state: {}}),
|
|
24
|
+
name: "test.intake",
|
|
25
|
+
phases: {
|
|
26
|
+
fetch: {
|
|
27
|
+
replay: "safe",
|
|
28
|
+
run: async (task, rt) => {
|
|
29
|
+
await rt.commit({phase: "summarize", state: {chart: `chart-for-${task.input.patientId}`}});
|
|
30
|
+
},
|
|
31
|
+
},
|
|
32
|
+
summarize: {
|
|
33
|
+
run: async (task, rt) => {
|
|
34
|
+
await rt.commit({
|
|
35
|
+
terminal: {
|
|
36
|
+
result: {status: "filed", summary: `summary of ${task.state.chart}`},
|
|
37
|
+
status: "completed",
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
retry: {maxAttempts: 3},
|
|
44
|
+
version: 1,
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
const TaskModel = registerHarnessTask();
|
|
48
|
+
let SpanModel: ObsSpanModel;
|
|
49
|
+
let TraceModel: ObsTraceModel;
|
|
50
|
+
|
|
51
|
+
const openHarnesses: Harness[] = [];
|
|
52
|
+
|
|
53
|
+
const openHarness = async (
|
|
54
|
+
options: {
|
|
55
|
+
registry?: Parameters<typeof Harness.open>[0]["registry"];
|
|
56
|
+
testHooks?: HarnessTestHooks;
|
|
57
|
+
} = {}
|
|
58
|
+
): Promise<Harness> => {
|
|
59
|
+
const harness = await Harness.open({
|
|
60
|
+
registry: options.registry ?? [intakeTask],
|
|
61
|
+
runner: new InProcessRunner({pollInterval: {milliseconds: 20}}),
|
|
62
|
+
testHooks: options.testHooks,
|
|
63
|
+
});
|
|
64
|
+
openHarnesses.push(harness);
|
|
65
|
+
return harness;
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const deferred = (): {promise: Promise<void>; resolve: () => void} => {
|
|
69
|
+
let resolve: () => void = () => {};
|
|
70
|
+
const promise = new Promise<void>((done) => {
|
|
71
|
+
resolve = done;
|
|
72
|
+
});
|
|
73
|
+
return {promise, resolve};
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
describe("Harness", () => {
|
|
77
|
+
beforeAll(() => {
|
|
78
|
+
createLocalObservabilityPlugin();
|
|
79
|
+
SpanModel = registerObsSpan();
|
|
80
|
+
TraceModel = registerObsTrace();
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
beforeEach(async () => {
|
|
84
|
+
await Promise.all([
|
|
85
|
+
TaskModel.deleteMany({}),
|
|
86
|
+
SpanModel.deleteMany({}),
|
|
87
|
+
TraceModel.deleteMany({}),
|
|
88
|
+
]);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
afterEach(async () => {
|
|
92
|
+
await Promise.all(openHarnesses.splice(0).map((harness) => harness.stop()));
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
describe("two-phase task", () => {
|
|
96
|
+
it("runs every phase to completed and stores the outcome", async () => {
|
|
97
|
+
const harness = await openHarness();
|
|
98
|
+
await harness.start();
|
|
99
|
+
|
|
100
|
+
const created = await harness.createTask(intakeTask, {patientId: "p1"});
|
|
101
|
+
expect(created.status).toBe("pending");
|
|
102
|
+
expect(created.phase).toBe("fetch");
|
|
103
|
+
expect(created.retry?.maxAttempts).toBe(3);
|
|
104
|
+
expect(String(created.rootTaskId)).toBe(String(created._id));
|
|
105
|
+
|
|
106
|
+
const done = await harness.waitForTask(created._id);
|
|
107
|
+
expect(done.status).toBe("completed");
|
|
108
|
+
expect(done.phase).toBe("summarize");
|
|
109
|
+
expect(done.state).toEqual({chart: "chart-for-p1"});
|
|
110
|
+
expect(done.outcome?.status).toBe("completed");
|
|
111
|
+
expect(done.outcome?.result).toEqual({status: "filed", summary: "summary of chart-for-p1"});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it("writes one root trace, one root CHAIN span, and exactly one CHAIN span per phase", async () => {
|
|
115
|
+
const harness = await openHarness();
|
|
116
|
+
await harness.start();
|
|
117
|
+
const userId = new mongoose.Types.ObjectId();
|
|
118
|
+
|
|
119
|
+
const created = await harness.createTask(
|
|
120
|
+
intakeTask,
|
|
121
|
+
{patientId: "p2"},
|
|
122
|
+
{userId: String(userId)}
|
|
123
|
+
);
|
|
124
|
+
const done = await harness.waitForTask(created._id);
|
|
125
|
+
|
|
126
|
+
const traces = await TraceModel.find({});
|
|
127
|
+
expect(traces).toHaveLength(1);
|
|
128
|
+
const [trace] = traces;
|
|
129
|
+
expect(String(trace._id)).toBe(String(done.traceId));
|
|
130
|
+
expect(trace.name).toBe("test.intake@1");
|
|
131
|
+
expect(trace.status).toBe("ok");
|
|
132
|
+
expect(trace.endedAt).toBeInstanceOf(Date);
|
|
133
|
+
expect(trace.input).toEqual({patientId: "p2"});
|
|
134
|
+
expect(trace.output).toEqual({status: "filed", summary: "summary of chart-for-p2"});
|
|
135
|
+
expect(String(trace.userId)).toBe(String(userId));
|
|
136
|
+
expect(String(done.userId)).toBe(String(userId));
|
|
137
|
+
|
|
138
|
+
const spans = await SpanModel.find({traceId: trace._id});
|
|
139
|
+
expect(spans).toHaveLength(3);
|
|
140
|
+
const root = spans.find((span) => String(span._id) === String(done.rootSpanId));
|
|
141
|
+
expect(root?.kind).toBe("CHAIN");
|
|
142
|
+
expect(root?.name).toBe("test.intake@1");
|
|
143
|
+
expect(root?.parentSpanId).toBeUndefined();
|
|
144
|
+
expect(root?.status).toBe("ok");
|
|
145
|
+
expect(root?.endedAt).toBeInstanceOf(Date);
|
|
146
|
+
|
|
147
|
+
// Spans of fast phases can share a millisecond, so order by name, not time.
|
|
148
|
+
const phaseSpans = spans
|
|
149
|
+
.filter((span) => String(span._id) !== String(done.rootSpanId))
|
|
150
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
151
|
+
expect(phaseSpans.map((span) => span.name)).toEqual(["fetch", "summarize"]);
|
|
152
|
+
for (const span of phaseSpans) {
|
|
153
|
+
expect(span.kind).toBe("CHAIN");
|
|
154
|
+
expect(span.status).toBe("ok");
|
|
155
|
+
expect(String(span.parentSpanId)).toBe(String(done.rootSpanId));
|
|
156
|
+
expect(span.endedAt).toBeInstanceOf(Date);
|
|
157
|
+
expect(span.durationMs).toBeGreaterThanOrEqual(0);
|
|
158
|
+
}
|
|
159
|
+
expect(phaseSpans[0].output).toEqual({phase: "summarize", state: {chart: "chart-for-p2"}});
|
|
160
|
+
expect(phaseSpans[1].input).toEqual({attempt: 0, state: {chart: "chart-for-p2"}});
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it("keeps the current state when a phase commit omits state", async () => {
|
|
164
|
+
const keepState = defineTask<unknown, {count: number}, number>({
|
|
165
|
+
initial: () => ({phase: "first", state: {count: 7}}),
|
|
166
|
+
name: "test.keepState",
|
|
167
|
+
phases: {
|
|
168
|
+
first: {run: async (_task, rt) => rt.commit({phase: "second"})},
|
|
169
|
+
second: {
|
|
170
|
+
run: async (task, rt) =>
|
|
171
|
+
rt.commit({terminal: {result: task.state.count, status: "completed"}}),
|
|
172
|
+
},
|
|
173
|
+
},
|
|
174
|
+
version: 1,
|
|
175
|
+
});
|
|
176
|
+
const harness = await openHarness({registry: [keepState]});
|
|
177
|
+
await harness.start();
|
|
178
|
+
|
|
179
|
+
const done = await harness.waitForTask((await harness.createTask(keepState, {}))._id);
|
|
180
|
+
expect(done.state).toEqual({count: 7});
|
|
181
|
+
expect(done.outcome?.result).toBe(7);
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
describe("transactional commit", () => {
|
|
186
|
+
it("leaves neither the task update nor the span when the commit transaction aborts", async () => {
|
|
187
|
+
const aborted = deferred();
|
|
188
|
+
const seenInsideTransaction: {phaseSpan?: boolean; rootEnded?: boolean; status?: string} = {};
|
|
189
|
+
const harness = await openHarness({
|
|
190
|
+
testHooks: {
|
|
191
|
+
beforeCommitEnd: async ({phase, session, taskId}) => {
|
|
192
|
+
if (phase !== "summarize") {
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
// Prove the writes happened inside this transaction before it is aborted.
|
|
196
|
+
const inFlight = await TaskModel.findById(taskId).session(session);
|
|
197
|
+
const phaseSpan = await SpanModel.find({name: "summarize"}).session(session);
|
|
198
|
+
const root = await SpanModel.findById(inFlight?.rootSpanId).session(session);
|
|
199
|
+
seenInsideTransaction.status = inFlight?.status;
|
|
200
|
+
seenInsideTransaction.phaseSpan = phaseSpan.length === 1;
|
|
201
|
+
seenInsideTransaction.rootEnded = root?.endedAt instanceof Date;
|
|
202
|
+
aborted.resolve();
|
|
203
|
+
throw new Error("injected failure inside the commit transaction");
|
|
204
|
+
},
|
|
205
|
+
},
|
|
206
|
+
});
|
|
207
|
+
await harness.start();
|
|
208
|
+
|
|
209
|
+
const created = await harness.createTask(intakeTask, {patientId: "p3"});
|
|
210
|
+
await aborted.promise;
|
|
211
|
+
await harness.stop();
|
|
212
|
+
|
|
213
|
+
expect(seenInsideTransaction).toEqual({
|
|
214
|
+
phaseSpan: true,
|
|
215
|
+
rootEnded: true,
|
|
216
|
+
status: "completed",
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
const task = await TaskModel.findExactlyOne({_id: created._id});
|
|
220
|
+
expect(task.status).toBe("running");
|
|
221
|
+
expect(task.phase).toBe("summarize");
|
|
222
|
+
expect(task.state).toEqual({chart: "chart-for-p3"});
|
|
223
|
+
expect(task.outcome?.status).toBeUndefined();
|
|
224
|
+
|
|
225
|
+
const spans = await SpanModel.find({traceId: task.traceId});
|
|
226
|
+
expect(spans.map((span) => span.name).sort()).toEqual(["fetch", "test.intake@1"]);
|
|
227
|
+
const root = await SpanModel.findExactlyOne({_id: task.rootSpanId});
|
|
228
|
+
expect(root.endedAt).toBeUndefined();
|
|
229
|
+
expect(root.output).toBeUndefined();
|
|
230
|
+
expect(root.status).toBe("ok");
|
|
231
|
+
const trace = await TraceModel.findExactlyOne({_id: task.traceId});
|
|
232
|
+
expect(trace.endedAt).toBeUndefined();
|
|
233
|
+
expect(trace.output).toBeUndefined();
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
it("leaves the task at its checkpoint when a phase swallows a failed commit", async () => {
|
|
237
|
+
const finished = deferred();
|
|
238
|
+
const swallowing = defineTask<unknown, unknown, unknown>({
|
|
239
|
+
initial: () => ({phase: "only"}),
|
|
240
|
+
name: "test.swallowing",
|
|
241
|
+
phases: {
|
|
242
|
+
only: {
|
|
243
|
+
run: async (_task, rt) => {
|
|
244
|
+
try {
|
|
245
|
+
await rt.commit({terminal: {status: "completed"}});
|
|
246
|
+
} catch {
|
|
247
|
+
// The phase ignores the failure; the engine must not treat it as committed.
|
|
248
|
+
}
|
|
249
|
+
finished.resolve();
|
|
250
|
+
},
|
|
251
|
+
},
|
|
252
|
+
},
|
|
253
|
+
version: 1,
|
|
254
|
+
});
|
|
255
|
+
const harness = await openHarness({
|
|
256
|
+
registry: [swallowing],
|
|
257
|
+
testHooks: {
|
|
258
|
+
beforeCommitEnd: () => {
|
|
259
|
+
throw new Error("injected failure inside the commit transaction");
|
|
260
|
+
},
|
|
261
|
+
},
|
|
262
|
+
});
|
|
263
|
+
await harness.start();
|
|
264
|
+
|
|
265
|
+
const created = await harness.createTask(swallowing, {});
|
|
266
|
+
await finished.promise;
|
|
267
|
+
await harness.stop();
|
|
268
|
+
|
|
269
|
+
const task = await TaskModel.findExactlyOne({_id: created._id});
|
|
270
|
+
expect(task.status).toBe("running");
|
|
271
|
+
expect(await SpanModel.countDocuments({name: "only"})).toBe(0);
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
it("rejects a commit whose checkpoint moved underneath it", async () => {
|
|
275
|
+
const finished = deferred();
|
|
276
|
+
let commitError: unknown;
|
|
277
|
+
const raced = defineTask<unknown, unknown, unknown>({
|
|
278
|
+
initial: () => ({phase: "first"}),
|
|
279
|
+
name: "test.raced",
|
|
280
|
+
phases: {
|
|
281
|
+
first: {
|
|
282
|
+
run: async (task, rt) => {
|
|
283
|
+
// Another owner advanced the checkpoint while this phase was working.
|
|
284
|
+
await TaskModel.updateOne({_id: task.id}, {$set: {phase: "second"}});
|
|
285
|
+
try {
|
|
286
|
+
await rt.commit({phase: "second", state: {stale: true}});
|
|
287
|
+
} catch (error: unknown) {
|
|
288
|
+
commitError = error;
|
|
289
|
+
throw error;
|
|
290
|
+
} finally {
|
|
291
|
+
finished.resolve();
|
|
292
|
+
}
|
|
293
|
+
},
|
|
294
|
+
},
|
|
295
|
+
second: {run: async () => {}},
|
|
296
|
+
},
|
|
297
|
+
version: 1,
|
|
298
|
+
});
|
|
299
|
+
const harness = await openHarness({registry: [raced]});
|
|
300
|
+
await harness.start();
|
|
301
|
+
|
|
302
|
+
const created = await harness.createTask(raced, {});
|
|
303
|
+
await finished.promise;
|
|
304
|
+
await harness.stop();
|
|
305
|
+
|
|
306
|
+
expect(commitError).toBeInstanceOf(HarnessCommitConflictError);
|
|
307
|
+
const task = await TaskModel.findExactlyOne({_id: created._id});
|
|
308
|
+
expect(task.phase).toBe("second");
|
|
309
|
+
expect(task.state).toBeUndefined();
|
|
310
|
+
expect(await SpanModel.countDocuments({name: "first"})).toBe(0);
|
|
311
|
+
});
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
describe("phase failures", () => {
|
|
315
|
+
it("fails the task with an error span when a phase throws with no retries left", async () => {
|
|
316
|
+
const throwing = defineTask<unknown, unknown, unknown>({
|
|
317
|
+
initial: () => ({phase: "explode"}),
|
|
318
|
+
name: "test.throwing",
|
|
319
|
+
phases: {
|
|
320
|
+
explode: {
|
|
321
|
+
run: async () => {
|
|
322
|
+
throw new Error("EHR unavailable");
|
|
323
|
+
},
|
|
324
|
+
},
|
|
325
|
+
},
|
|
326
|
+
retry: {maxAttempts: 1},
|
|
327
|
+
version: 1,
|
|
328
|
+
});
|
|
329
|
+
const harness = await openHarness({registry: [throwing]});
|
|
330
|
+
await harness.start();
|
|
331
|
+
|
|
332
|
+
const done = await harness.waitForTask((await harness.createTask(throwing, {}))._id);
|
|
333
|
+
expect(done.status).toBe("failed");
|
|
334
|
+
expect(done.toObject().outcome).toEqual({error: "EHR unavailable", status: "failed"});
|
|
335
|
+
|
|
336
|
+
const phaseSpan = await SpanModel.findExactlyOne({name: "explode"});
|
|
337
|
+
expect(phaseSpan.status).toBe("error");
|
|
338
|
+
expect(phaseSpan.error).toBe("EHR unavailable");
|
|
339
|
+
const trace = await TraceModel.findExactlyOne({_id: done.traceId});
|
|
340
|
+
expect(trace.status).toBe("error");
|
|
341
|
+
expect(trace.errorSummary).toBe("EHR unavailable");
|
|
342
|
+
const root = await SpanModel.findExactlyOne({_id: done.rootSpanId});
|
|
343
|
+
expect(root.status).toBe("error");
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
it("fails the task when a phase returns without committing", async () => {
|
|
347
|
+
const silent = defineTask<unknown, unknown, unknown>({
|
|
348
|
+
initial: () => ({phase: "noop"}),
|
|
349
|
+
name: "test.silent",
|
|
350
|
+
phases: {noop: {run: async () => {}}},
|
|
351
|
+
version: 1,
|
|
352
|
+
});
|
|
353
|
+
const harness = await openHarness({registry: [silent]});
|
|
354
|
+
await harness.start();
|
|
355
|
+
|
|
356
|
+
const done = await harness.waitForTask((await harness.createTask(silent, {}))._id);
|
|
357
|
+
expect(done.status).toBe("failed");
|
|
358
|
+
expect(done.outcome?.error).toBe(
|
|
359
|
+
'test.silent@1: phase "noop" returned without calling rt.commit'
|
|
360
|
+
);
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
it("fails the task when a phase commits to an unknown phase", async () => {
|
|
364
|
+
const lost = defineTask<unknown, unknown, unknown>({
|
|
365
|
+
initial: () => ({phase: "start"}),
|
|
366
|
+
name: "test.lost",
|
|
367
|
+
phases: {start: {run: async (_task, rt) => rt.commit({phase: "nowhere"})}},
|
|
368
|
+
version: 1,
|
|
369
|
+
});
|
|
370
|
+
const harness = await openHarness({registry: [lost]});
|
|
371
|
+
await harness.start();
|
|
372
|
+
|
|
373
|
+
const done = await harness.waitForTask((await harness.createTask(lost, {}))._id);
|
|
374
|
+
expect(done.status).toBe("failed");
|
|
375
|
+
expect(done.outcome?.error).toBe('test.lost@1: rt.commit to unknown phase "nowhere"');
|
|
376
|
+
});
|
|
377
|
+
|
|
378
|
+
it("rejects an invalid terminal status", async () => {
|
|
379
|
+
const badTerminal = defineTask<unknown, unknown, unknown>({
|
|
380
|
+
initial: () => ({phase: "start"}),
|
|
381
|
+
name: "test.badTerminal",
|
|
382
|
+
phases: {
|
|
383
|
+
start: {
|
|
384
|
+
run: async (_task, rt) =>
|
|
385
|
+
rt.commit({terminal: {status: "aborted"}} as unknown as Parameters<
|
|
386
|
+
typeof rt.commit
|
|
387
|
+
>[0]),
|
|
388
|
+
},
|
|
389
|
+
},
|
|
390
|
+
version: 1,
|
|
391
|
+
});
|
|
392
|
+
const harness = await openHarness({registry: [badTerminal]});
|
|
393
|
+
await harness.start();
|
|
394
|
+
|
|
395
|
+
const done = await harness.waitForTask((await harness.createTask(badTerminal, {}))._id);
|
|
396
|
+
expect(done.status).toBe("failed");
|
|
397
|
+
expect(done.outcome?.error).toBe(
|
|
398
|
+
'test.badTerminal@1: terminal status must be "completed" or "failed"'
|
|
399
|
+
);
|
|
400
|
+
});
|
|
401
|
+
|
|
402
|
+
it("keeps the first checkpoint when a phase commits twice", async () => {
|
|
403
|
+
const twice = defineTask<unknown, {step: number}, unknown>({
|
|
404
|
+
initial: () => ({phase: "first", state: {step: 0}}),
|
|
405
|
+
name: "test.twice",
|
|
406
|
+
phases: {
|
|
407
|
+
first: {
|
|
408
|
+
run: async (_task, rt) => {
|
|
409
|
+
await rt.commit({phase: "second", state: {step: 1}});
|
|
410
|
+
await rt.commit({phase: "second", state: {step: 2}});
|
|
411
|
+
},
|
|
412
|
+
},
|
|
413
|
+
second: {
|
|
414
|
+
run: async (task, rt) =>
|
|
415
|
+
rt.commit({terminal: {result: task.state.step, status: "completed"}}),
|
|
416
|
+
},
|
|
417
|
+
},
|
|
418
|
+
version: 1,
|
|
419
|
+
});
|
|
420
|
+
const harness = await openHarness({registry: [twice]});
|
|
421
|
+
await harness.start();
|
|
422
|
+
|
|
423
|
+
const done = await harness.waitForTask((await harness.createTask(twice, {}))._id);
|
|
424
|
+
expect(done.status).toBe("completed");
|
|
425
|
+
expect(done.outcome?.result).toBe(1);
|
|
426
|
+
expect(await SpanModel.countDocuments({name: "first"})).toBe(1);
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
it("fails a task whose stored phase is no longer defined", async () => {
|
|
430
|
+
const harness = await openHarness();
|
|
431
|
+
const created = await harness.createTask(intakeTask, {patientId: "p4"});
|
|
432
|
+
await TaskModel.updateOne({_id: created._id}, {$set: {phase: "removedPhase"}});
|
|
433
|
+
await harness.start();
|
|
434
|
+
|
|
435
|
+
const done = await harness.waitForTask(created._id);
|
|
436
|
+
expect(done.status).toBe("failed");
|
|
437
|
+
expect(done.outcome?.error).toBe('test.intake@1: unknown phase "removedPhase"');
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
it("leaves the task at its checkpoint when recording a failure also fails", async () => {
|
|
441
|
+
const attempted = deferred();
|
|
442
|
+
const throwing = defineTask<unknown, unknown, unknown>({
|
|
443
|
+
initial: () => ({phase: "explode"}),
|
|
444
|
+
name: "test.unrecordable",
|
|
445
|
+
phases: {
|
|
446
|
+
explode: {
|
|
447
|
+
run: async () => {
|
|
448
|
+
throw new Error("boom");
|
|
449
|
+
},
|
|
450
|
+
},
|
|
451
|
+
},
|
|
452
|
+
version: 1,
|
|
453
|
+
});
|
|
454
|
+
const harness = await openHarness({
|
|
455
|
+
registry: [throwing],
|
|
456
|
+
testHooks: {
|
|
457
|
+
beforeCommitEnd: () => {
|
|
458
|
+
attempted.resolve();
|
|
459
|
+
throw new Error("audit store down");
|
|
460
|
+
},
|
|
461
|
+
},
|
|
462
|
+
});
|
|
463
|
+
await harness.start();
|
|
464
|
+
|
|
465
|
+
const created = await harness.createTask(throwing, {});
|
|
466
|
+
await attempted.promise;
|
|
467
|
+
await harness.stop();
|
|
468
|
+
|
|
469
|
+
const task = await TaskModel.findExactlyOne({_id: created._id});
|
|
470
|
+
expect(task.status).toBe("running");
|
|
471
|
+
expect(task.phase).toBe("explode");
|
|
472
|
+
expect(await SpanModel.countDocuments({name: "explode"})).toBe(0);
|
|
473
|
+
});
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
describe("createTask", () => {
|
|
477
|
+
it("returns the existing task for a repeated requestId", async () => {
|
|
478
|
+
const harness = await openHarness();
|
|
479
|
+
|
|
480
|
+
const first = await harness.createTask(
|
|
481
|
+
intakeTask,
|
|
482
|
+
{patientId: "p5"},
|
|
483
|
+
{requestId: "intake-p5"}
|
|
484
|
+
);
|
|
485
|
+
const second = await harness.createTask(
|
|
486
|
+
intakeTask,
|
|
487
|
+
{patientId: "p5"},
|
|
488
|
+
{requestId: "intake-p5"}
|
|
489
|
+
);
|
|
490
|
+
|
|
491
|
+
expect(String(second._id)).toBe(String(first._id));
|
|
492
|
+
expect(await TaskModel.countDocuments({})).toBe(1);
|
|
493
|
+
expect(await TraceModel.countDocuments({})).toBe(1);
|
|
494
|
+
expect(await SpanModel.countDocuments({})).toBe(1);
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
it("creates exactly one task when the same requestId races", async () => {
|
|
498
|
+
const harness = await openHarness();
|
|
499
|
+
|
|
500
|
+
const results = await Promise.all(
|
|
501
|
+
[1, 2, 3].map(() =>
|
|
502
|
+
harness.createTask(intakeTask, {patientId: "p6"}, {requestId: "intake-p6"})
|
|
503
|
+
)
|
|
504
|
+
);
|
|
505
|
+
|
|
506
|
+
expect(new Set(results.map((task) => String(task._id))).size).toBe(1);
|
|
507
|
+
expect(await TaskModel.countDocuments({requestId: "intake-p6"})).toBe(1);
|
|
508
|
+
expect(await TraceModel.countDocuments({})).toBe(1);
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
it("rejects a requestId that belongs to a different task definition", async () => {
|
|
512
|
+
const other = defineTask<unknown, unknown, unknown>({
|
|
513
|
+
initial: () => ({phase: "only"}),
|
|
514
|
+
name: "test.other",
|
|
515
|
+
phases: {only: {run: async (_task, rt) => rt.commit({terminal: {status: "completed"}})}},
|
|
516
|
+
version: 1,
|
|
517
|
+
});
|
|
518
|
+
const harness = await openHarness({registry: [intakeTask, other]});
|
|
519
|
+
await harness.createTask(intakeTask, {patientId: "p7"}, {requestId: "shared-key"});
|
|
520
|
+
|
|
521
|
+
await expect(harness.createTask(other, {}, {requestId: "shared-key"})).rejects.toThrow(
|
|
522
|
+
harnessErrorMatching("requestIdConflict", 'requestId "shared-key" already belongs to task')
|
|
523
|
+
);
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
it("rejects a requestId reused by a different user", async () => {
|
|
527
|
+
const harness = await openHarness();
|
|
528
|
+
const owner = new mongoose.Types.ObjectId();
|
|
529
|
+
await harness.createTask(
|
|
530
|
+
intakeTask,
|
|
531
|
+
{patientId: "p12"},
|
|
532
|
+
{requestId: "user-key", userId: owner}
|
|
533
|
+
);
|
|
534
|
+
|
|
535
|
+
await expect(
|
|
536
|
+
harness.createTask(
|
|
537
|
+
intakeTask,
|
|
538
|
+
{patientId: "p12"},
|
|
539
|
+
{requestId: "user-key", userId: new mongoose.Types.ObjectId()}
|
|
540
|
+
)
|
|
541
|
+
).rejects.toThrow(
|
|
542
|
+
harnessErrorMatching(
|
|
543
|
+
"requestIdConflict",
|
|
544
|
+
'requestId "user-key" already belongs to a task for a different user'
|
|
545
|
+
)
|
|
546
|
+
);
|
|
547
|
+
await expect(
|
|
548
|
+
harness.createTask(intakeTask, {patientId: "p12"}, {requestId: "user-key"})
|
|
549
|
+
).rejects.toThrow(harnessErrorMatching("requestIdConflict", "different user"));
|
|
550
|
+
const same = await harness.createTask(
|
|
551
|
+
intakeTask,
|
|
552
|
+
{patientId: "p12"},
|
|
553
|
+
{requestId: "user-key", userId: String(owner)}
|
|
554
|
+
);
|
|
555
|
+
expect(String(same.userId)).toBe(String(owner));
|
|
556
|
+
});
|
|
557
|
+
|
|
558
|
+
it("lets tasks without a requestId coexist", async () => {
|
|
559
|
+
const harness = await openHarness();
|
|
560
|
+
await harness.createTask(intakeTask, {patientId: "a"});
|
|
561
|
+
await harness.createTask(intakeTask, {patientId: "b"});
|
|
562
|
+
expect(await TaskModel.countDocuments({})).toBe(2);
|
|
563
|
+
});
|
|
564
|
+
|
|
565
|
+
it("rejects a definition that is not registered", async () => {
|
|
566
|
+
const unregistered = defineTask<unknown, unknown, unknown>({
|
|
567
|
+
initial: () => ({phase: "only"}),
|
|
568
|
+
name: "test.unregistered",
|
|
569
|
+
phases: {only: {run: async () => {}}},
|
|
570
|
+
version: 1,
|
|
571
|
+
});
|
|
572
|
+
const harness = await openHarness();
|
|
573
|
+
await expect(harness.createTask(unregistered, {})).rejects.toThrow(
|
|
574
|
+
harnessErrorMatching("notRegistered", "test.unregistered@1 is not in this harness registry")
|
|
575
|
+
);
|
|
576
|
+
});
|
|
577
|
+
|
|
578
|
+
it("rejects an initial phase that is not defined and writes nothing", async () => {
|
|
579
|
+
const badInitial = defineTask<unknown, unknown, unknown>({
|
|
580
|
+
initial: () => ({phase: "missing"}),
|
|
581
|
+
name: "test.badInitial",
|
|
582
|
+
phases: {only: {run: async () => {}}},
|
|
583
|
+
version: 1,
|
|
584
|
+
});
|
|
585
|
+
const harness = await openHarness({registry: [badInitial]});
|
|
586
|
+
await expect(harness.createTask(badInitial, {})).rejects.toThrow(
|
|
587
|
+
harnessErrorMatching(
|
|
588
|
+
"definitionInvalid",
|
|
589
|
+
'test.badInitial@1: initial phase "missing" is not one of only'
|
|
590
|
+
)
|
|
591
|
+
);
|
|
592
|
+
expect(await TaskModel.countDocuments({})).toBe(0);
|
|
593
|
+
expect(await TraceModel.countDocuments({})).toBe(0);
|
|
594
|
+
});
|
|
595
|
+
});
|
|
596
|
+
|
|
597
|
+
describe("open / start / stop", () => {
|
|
598
|
+
it("throws when the local observability models are not registered", async () => {
|
|
599
|
+
mongoose.deleteModel("ObsSpan");
|
|
600
|
+
try {
|
|
601
|
+
await expect(Harness.open({registry: [intakeTask]})).rejects.toThrow(
|
|
602
|
+
harnessErrorMatching(
|
|
603
|
+
"configInvalid",
|
|
604
|
+
"Harness.open requires the local observability plugin"
|
|
605
|
+
)
|
|
606
|
+
);
|
|
607
|
+
} finally {
|
|
608
|
+
SpanModel = registerObsSpan();
|
|
609
|
+
}
|
|
610
|
+
});
|
|
611
|
+
|
|
612
|
+
it("throws when the connection is not a replica set", async () => {
|
|
613
|
+
const db = mongoose.connection.db;
|
|
614
|
+
if (!db) {
|
|
615
|
+
throw new Error("expected a connected database");
|
|
616
|
+
}
|
|
617
|
+
// A standalone mongod answers `hello` without a replica set name.
|
|
618
|
+
const commandSpy = spyOn(db, "command").mockResolvedValueOnce({
|
|
619
|
+
isWritablePrimary: true,
|
|
620
|
+
ok: 1,
|
|
621
|
+
});
|
|
622
|
+
await expect(Harness.open({registry: [intakeTask]})).rejects.toThrow(
|
|
623
|
+
harnessErrorMatching("replicaSetRequired", "Harness.open requires a MongoDB replica set")
|
|
624
|
+
);
|
|
625
|
+
commandSpy.mockRestore();
|
|
626
|
+
});
|
|
627
|
+
|
|
628
|
+
it("accepts a mongos router, which supports transactions", async () => {
|
|
629
|
+
const db = mongoose.connection.db;
|
|
630
|
+
if (!db) {
|
|
631
|
+
throw new Error("expected a connected database");
|
|
632
|
+
}
|
|
633
|
+
spyOn(db, "command").mockResolvedValueOnce({isWritablePrimary: true, msg: "isdbgrid", ok: 1});
|
|
634
|
+
const harness = await Harness.open({registry: [intakeTask]});
|
|
635
|
+
openHarnesses.push(harness);
|
|
636
|
+
expect(harness).toBeInstanceOf(Harness);
|
|
637
|
+
});
|
|
638
|
+
|
|
639
|
+
it("throws when a name@version is registered twice", async () => {
|
|
640
|
+
await expect(Harness.open({registry: [intakeTask, intakeTask]})).rejects.toThrow(
|
|
641
|
+
harnessErrorMatching("configInvalid", "Harness registry lists test.intake@1 more than once")
|
|
642
|
+
);
|
|
643
|
+
});
|
|
644
|
+
|
|
645
|
+
it("throws when started twice and allows stop before start", async () => {
|
|
646
|
+
const harness = await openHarness();
|
|
647
|
+
await harness.stop();
|
|
648
|
+
await harness.start();
|
|
649
|
+
await expect(harness.start()).rejects.toThrow(
|
|
650
|
+
harnessErrorMatching("alreadyStarted", "Harness is already started")
|
|
651
|
+
);
|
|
652
|
+
});
|
|
653
|
+
|
|
654
|
+
it("defaults to an InProcessRunner", async () => {
|
|
655
|
+
const harness = await Harness.open({registry: [intakeTask]});
|
|
656
|
+
openHarnesses.push(harness);
|
|
657
|
+
await harness.start();
|
|
658
|
+
const done = await harness.waitForTask(
|
|
659
|
+
(await harness.createTask(intakeTask, {patientId: "p9"}))._id
|
|
660
|
+
);
|
|
661
|
+
expect(done.status).toBe("completed");
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
it("never claims tasks when the registry is empty", async () => {
|
|
665
|
+
const idle = await openHarness({registry: []});
|
|
666
|
+
await idle.start();
|
|
667
|
+
// Created after start, so the start-time version check does not reject it.
|
|
668
|
+
const writer = await openHarness();
|
|
669
|
+
const created = await writer.createTask(intakeTask, {patientId: "p11"});
|
|
670
|
+
await new Promise((resolve) => setTimeout(resolve, 60));
|
|
671
|
+
await idle.stop();
|
|
672
|
+
|
|
673
|
+
const task = await TaskModel.findExactlyOne({_id: created._id});
|
|
674
|
+
expect(task.status).toBe("pending");
|
|
675
|
+
});
|
|
676
|
+
|
|
677
|
+
it("times out waiting for a task that never finishes", async () => {
|
|
678
|
+
const harness = await openHarness();
|
|
679
|
+
const created = await harness.createTask(intakeTask, {patientId: "p10"});
|
|
680
|
+
await expect(
|
|
681
|
+
harness.waitForTask(created._id, {
|
|
682
|
+
pollInterval: {milliseconds: 5},
|
|
683
|
+
timeout: {milliseconds: 30},
|
|
684
|
+
})
|
|
685
|
+
).rejects.toThrow(
|
|
686
|
+
harnessErrorMatching(
|
|
687
|
+
"waitTimedOut",
|
|
688
|
+
`Timed out waiting for task ${created._id} (test.intake@1) in status pending`
|
|
689
|
+
)
|
|
690
|
+
);
|
|
691
|
+
});
|
|
692
|
+
});
|
|
693
|
+
});
|