@terreno/ai 57.9.0 → 57.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +127 -0
- package/dist/harness/errors.js +101 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +520 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +186 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +57 -0
- package/dist/harness/runners/inProcessRunner.js +196 -0
- package/dist/harness/runtime.d.ts +34 -0
- package/dist/harness/runtime.js +551 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +934 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +17 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +119 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +757 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +767 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +201 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +912 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +402 -0
- package/src/harness/runners/inProcessRunner.ts +220 -0
- package/src/harness/runtime.ts +714 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1095 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
3
|
+
import type {HarnessTaskDefinitionInput} from "../types/harness";
|
|
4
|
+
import {defineTask, taskDefinitionKey} from "./defineTask";
|
|
5
|
+
|
|
6
|
+
const validInput = (): HarnessTaskDefinitionInput<unknown, unknown, unknown> => ({
|
|
7
|
+
initial: () => ({phase: "only"}),
|
|
8
|
+
name: "test.define",
|
|
9
|
+
phases: {only: {run: async () => {}}},
|
|
10
|
+
version: 1,
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
describe("defineTask", () => {
|
|
14
|
+
it("returns a frozen task definition keyed by name@version", () => {
|
|
15
|
+
const definition = defineTask({...validInput(), retry: {maxAttempts: 2}, version: 3});
|
|
16
|
+
expect(definition.kind).toBe("task");
|
|
17
|
+
expect(definition.key).toBe("test.define@3");
|
|
18
|
+
expect(definition.retry).toEqual({maxAttempts: 2});
|
|
19
|
+
expect(Object.isFrozen(definition)).toBe(true);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it("keeps phase replay policies and the abort handler", () => {
|
|
23
|
+
const abort = async (): Promise<void> => {};
|
|
24
|
+
const definition = defineTask({
|
|
25
|
+
...validInput(),
|
|
26
|
+
abort,
|
|
27
|
+
phases: {
|
|
28
|
+
risky: {run: async () => {}},
|
|
29
|
+
safe: {replay: "safe", run: async () => {}},
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
expect(definition.phases.safe.replay).toBe("safe");
|
|
33
|
+
expect(definition.phases.risky.replay).toBeUndefined();
|
|
34
|
+
expect(definition.abort).toBe(abort);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it("formats registry keys", () => {
|
|
38
|
+
expect(taskDefinitionKey({name: "clinic.intake", version: 2})).toBe("clinic.intake@2");
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it.each([
|
|
42
|
+
[{name: " "}, "defineTask: name is required"],
|
|
43
|
+
[{version: 0}, "defineTask(test.define): version must be a positive integer"],
|
|
44
|
+
[{version: 1.5}, "defineTask(test.define): version must be a positive integer"],
|
|
45
|
+
[{phases: {}}, "defineTask(test.define): at least one phase is required"],
|
|
46
|
+
[
|
|
47
|
+
{phases: {broken: {} as unknown as {run: () => Promise<void>}}},
|
|
48
|
+
'defineTask(test.define): phase "broken" needs a run function',
|
|
49
|
+
],
|
|
50
|
+
[
|
|
51
|
+
{phases: {odd: {replay: "sometimes" as "safe", run: async () => {}}}},
|
|
52
|
+
'defineTask(test.define): phase "odd" replay must be "safe" or "never"',
|
|
53
|
+
],
|
|
54
|
+
])("rejects an invalid definition %#", (override, message) => {
|
|
55
|
+
expect(() => defineTask({...validInput(), ...override})).toThrow(
|
|
56
|
+
harnessErrorMatching("definitionInvalid", message)
|
|
57
|
+
);
|
|
58
|
+
});
|
|
59
|
+
});
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type {HarnessTaskDefinition, HarnessTaskDefinitionInput} from "../types/harness";
|
|
2
|
+
import {assertValidApprovalPolicies} from "./approvalPolicy";
|
|
3
|
+
import {HarnessDefinitionError} from "./definitionError";
|
|
4
|
+
import {assertValidRetryPolicy} from "./retryBackoff";
|
|
5
|
+
|
|
6
|
+
/** Registry key that pins a run to one definition version. */
|
|
7
|
+
export const taskDefinitionKey = ({name, version}: {name: string; version: number}): string =>
|
|
8
|
+
`${name}@${version}`;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Declare a durable, multi-phase task. Each phase does work and calls `rt.commit()`
|
|
12
|
+
* with the next phase or a terminal outcome; a committed phase is the resume point.
|
|
13
|
+
*/
|
|
14
|
+
export const defineTask = <In, State, Out>(
|
|
15
|
+
definition: HarnessTaskDefinitionInput<In, State, Out>
|
|
16
|
+
): HarnessTaskDefinition<In, State, Out> => {
|
|
17
|
+
if (!definition.name?.trim()) {
|
|
18
|
+
throw new HarnessDefinitionError("defineTask: name is required");
|
|
19
|
+
}
|
|
20
|
+
if (!Number.isInteger(definition.version) || definition.version < 1) {
|
|
21
|
+
throw new HarnessDefinitionError(
|
|
22
|
+
`defineTask(${definition.name}): version must be a positive integer`
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
const phaseNames = Object.keys(definition.phases ?? {});
|
|
26
|
+
if (phaseNames.length === 0) {
|
|
27
|
+
throw new HarnessDefinitionError(
|
|
28
|
+
`defineTask(${definition.name}): at least one phase is required`
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
for (const phaseName of phaseNames) {
|
|
32
|
+
const phase = definition.phases[phaseName];
|
|
33
|
+
if (typeof phase?.run !== "function") {
|
|
34
|
+
throw new HarnessDefinitionError(
|
|
35
|
+
`defineTask(${definition.name}): phase "${phaseName}" needs a run function`
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
if (phase.replay !== undefined && phase.replay !== "safe" && phase.replay !== "never") {
|
|
39
|
+
throw new HarnessDefinitionError(
|
|
40
|
+
`defineTask(${definition.name}): phase "${phaseName}" replay must be "safe" or "never"`
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (definition.retry !== undefined) {
|
|
45
|
+
assertValidRetryPolicy(`defineTask(${definition.name})`, definition.retry);
|
|
46
|
+
}
|
|
47
|
+
if (
|
|
48
|
+
definition.onInterrupt !== undefined &&
|
|
49
|
+
definition.onInterrupt !== "park" &&
|
|
50
|
+
definition.onInterrupt !== "fail"
|
|
51
|
+
) {
|
|
52
|
+
throw new HarnessDefinitionError(
|
|
53
|
+
`defineTask(${definition.name}): onInterrupt must be "park" or "fail"`
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
if (
|
|
57
|
+
definition.spanKind !== undefined &&
|
|
58
|
+
!["AGENT", "CHAIN", "TOOL"].includes(definition.spanKind)
|
|
59
|
+
) {
|
|
60
|
+
throw new HarnessDefinitionError(
|
|
61
|
+
`defineTask(${definition.name}): spanKind must be AGENT, CHAIN, or TOOL`
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
if (definition.spanName !== undefined && typeof definition.spanName !== "function") {
|
|
65
|
+
throw new HarnessDefinitionError(`defineTask(${definition.name}): spanName must be a function`);
|
|
66
|
+
}
|
|
67
|
+
if (definition.abort !== undefined && typeof definition.abort !== "function") {
|
|
68
|
+
throw new HarnessDefinitionError(`defineTask(${definition.name}): abort must be a function`);
|
|
69
|
+
}
|
|
70
|
+
assertValidApprovalPolicies(`defineTask(${definition.name})`, definition.approvals);
|
|
71
|
+
return Object.freeze({
|
|
72
|
+
...definition,
|
|
73
|
+
key: taskDefinitionKey(definition),
|
|
74
|
+
kind: "task" as const,
|
|
75
|
+
});
|
|
76
|
+
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type {HarnessToolDefinition, HarnessToolDefinitionInput} from "../types/harness";
|
|
2
|
+
import {HarnessDefinitionError} from "./definitionError";
|
|
3
|
+
|
|
4
|
+
/** Provider-safe tool names (the strictest common limit across providers). */
|
|
5
|
+
const TOOL_NAME_PATTERN = /^[A-Za-z0-9_-]{1,64}$/;
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Declare a tool an agent may call. Each call runs as its own child task of the turn;
|
|
9
|
+
* `replay` (default `"never"`) decides whether an interrupted call is re-run or reported
|
|
10
|
+
* to the model as "interrupted, not retried".
|
|
11
|
+
*/
|
|
12
|
+
export const defineTool = <Args, Result>(
|
|
13
|
+
definition: HarnessToolDefinitionInput<Args, Result>
|
|
14
|
+
): HarnessToolDefinition<Args, Result> => {
|
|
15
|
+
if (!TOOL_NAME_PATTERN.test(definition.name ?? "")) {
|
|
16
|
+
throw new HarnessDefinitionError(
|
|
17
|
+
`defineTool(${definition.name}): name must be 1-64 letters, digits, "_" or "-"`
|
|
18
|
+
);
|
|
19
|
+
}
|
|
20
|
+
if (!definition.description?.trim()) {
|
|
21
|
+
throw new HarnessDefinitionError(`defineTool(${definition.name}): description is required`);
|
|
22
|
+
}
|
|
23
|
+
if (typeof definition.parameters?.safeParse !== "function") {
|
|
24
|
+
throw new HarnessDefinitionError(
|
|
25
|
+
`defineTool(${definition.name}): parameters must be a zod schema`
|
|
26
|
+
);
|
|
27
|
+
}
|
|
28
|
+
if (typeof definition.execute !== "function") {
|
|
29
|
+
throw new HarnessDefinitionError(`defineTool(${definition.name}): execute must be a function`);
|
|
30
|
+
}
|
|
31
|
+
if (
|
|
32
|
+
definition.replay !== undefined &&
|
|
33
|
+
definition.replay !== "safe" &&
|
|
34
|
+
definition.replay !== "never"
|
|
35
|
+
) {
|
|
36
|
+
throw new HarnessDefinitionError(
|
|
37
|
+
`defineTool(${definition.name}): replay must be "safe" or "never"`
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
return Object.freeze({
|
|
41
|
+
...definition,
|
|
42
|
+
kind: "tool" as const,
|
|
43
|
+
replay: definition.replay ?? "never",
|
|
44
|
+
});
|
|
45
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import {APIError} from "@terreno/api";
|
|
2
|
+
|
|
3
|
+
import {HARNESS_ERRORS} from "./errors";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A misuse of the task API; retrying the phase cannot fix it, so the task fails at once.
|
|
7
|
+
* An `APIError` (500, `harness-definition-invalid`); the specific problem is in `detail`.
|
|
8
|
+
*/
|
|
9
|
+
export class HarnessDefinitionError extends APIError {
|
|
10
|
+
constructor(detail: string) {
|
|
11
|
+
super({...HARNESS_ERRORS.definitionInvalid, detail});
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import {describe, expect, it} from "bun:test";
|
|
2
|
+
import {APIError, isAPIError} from "@terreno/api";
|
|
3
|
+
|
|
4
|
+
import {HarnessDefinitionError} from "./definitionError";
|
|
5
|
+
import {errorMessage, HARNESS_ERRORS, harnessError} from "./errors";
|
|
6
|
+
import {internalRuntime} from "./internalRuntime";
|
|
7
|
+
|
|
8
|
+
describe("harness errors", () => {
|
|
9
|
+
it("builds an APIError whose message is the kind's stable title", () => {
|
|
10
|
+
const cause = new Error("socket hang up");
|
|
11
|
+
const error = harnessError({
|
|
12
|
+
cause,
|
|
13
|
+
detail: "Task 1 is already completed",
|
|
14
|
+
kind: "taskTerminal",
|
|
15
|
+
});
|
|
16
|
+
expect(isAPIError(error)).toBe(true);
|
|
17
|
+
expect(error).toMatchObject({
|
|
18
|
+
code: "harness-task-terminal",
|
|
19
|
+
detail: "Task 1 is already completed",
|
|
20
|
+
message: "Task already ended",
|
|
21
|
+
name: "HarnessTaskTerminal",
|
|
22
|
+
status: 409,
|
|
23
|
+
});
|
|
24
|
+
expect(error.cause).toBe(cause);
|
|
25
|
+
expect(error.toJSON()).toMatchObject({
|
|
26
|
+
code: "harness-task-terminal",
|
|
27
|
+
detail: "Task 1 is already completed",
|
|
28
|
+
status: 409,
|
|
29
|
+
title: "Task already ended",
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it("gives every kind a harness- code, an HTTP error status, and a title", () => {
|
|
34
|
+
const codes = Object.values(HARNESS_ERRORS).map(({code}) => code);
|
|
35
|
+
expect(new Set(codes).size).toBe(codes.length);
|
|
36
|
+
for (const {code, status, title} of Object.values(HARNESS_ERRORS)) {
|
|
37
|
+
expect(code).toStartWith("harness-");
|
|
38
|
+
expect(status).toBeGreaterThanOrEqual(400);
|
|
39
|
+
expect(status).toBeLessThan(600);
|
|
40
|
+
expect(title.trim()).not.toBe("");
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it("keeps HarnessDefinitionError an instanceof-checkable definition APIError", () => {
|
|
45
|
+
const error = new HarnessDefinitionError("defineTask: name is required");
|
|
46
|
+
expect(error).toBeInstanceOf(HarnessDefinitionError);
|
|
47
|
+
expect(error).toBeInstanceOf(APIError);
|
|
48
|
+
expect(error).toMatchObject({
|
|
49
|
+
code: HARNESS_ERRORS.definitionInvalid.code,
|
|
50
|
+
detail: "defineTask: name is required",
|
|
51
|
+
message: HARNESS_ERRORS.definitionInvalid.title,
|
|
52
|
+
name: "HarnessDefinitionError",
|
|
53
|
+
status: 500,
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it("records a harness error by its detail and any other error with its full text", () => {
|
|
58
|
+
expect(
|
|
59
|
+
errorMessage(harnessError({detail: "memo requires a non-empty key", kind: "invalidRequest"}))
|
|
60
|
+
).toBe("memo requires a non-empty key");
|
|
61
|
+
expect(
|
|
62
|
+
errorMessage(new HarnessDefinitionError("defineTool(x): execute must be a function"))
|
|
63
|
+
).toBe("defineTool(x): execute must be a function");
|
|
64
|
+
// An app's own APIError keeps its title, since its detail may not stand alone.
|
|
65
|
+
expect(
|
|
66
|
+
errorMessage(
|
|
67
|
+
new APIError({
|
|
68
|
+
code: "ehr-down",
|
|
69
|
+
detail: "p-1 timed out",
|
|
70
|
+
status: 502,
|
|
71
|
+
title: "EHR unavailable",
|
|
72
|
+
})
|
|
73
|
+
)
|
|
74
|
+
).toBe("EHR unavailable: p-1 timed out");
|
|
75
|
+
expect(errorMessage(new APIError({status: 404, title: "Chart not found"}))).toBe(
|
|
76
|
+
"Chart not found"
|
|
77
|
+
);
|
|
78
|
+
expect(errorMessage(new Error("boom"))).toBe("boom");
|
|
79
|
+
expect(errorMessage("plain string")).toBe("plain string");
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it("refuses a runtime the engine did not build", () => {
|
|
83
|
+
expect(() => internalRuntime({} as never)).toThrow(
|
|
84
|
+
expect.objectContaining({
|
|
85
|
+
code: HARNESS_ERRORS.internal.code,
|
|
86
|
+
detail: "This runtime was not built by the harness engine",
|
|
87
|
+
})
|
|
88
|
+
);
|
|
89
|
+
});
|
|
90
|
+
});
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import {APIError, errorDetail, isAPIError} from "@terreno/api";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Harness error vocabulary. Each kind is one stable problem type: `code` and `title` never
|
|
5
|
+
* change between occurrences; the per-occurrence sentence goes in `detail`.
|
|
6
|
+
*/
|
|
7
|
+
export interface HarnessErrorKind {
|
|
8
|
+
code: string;
|
|
9
|
+
status: number;
|
|
10
|
+
title: string;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export const HARNESS_ERRORS = {
|
|
14
|
+
/** `start` called twice on a harness or runner. */
|
|
15
|
+
alreadyStarted: {code: "harness-already-started", status: 409, title: "Already started"},
|
|
16
|
+
/** Two writers raced on an approval; the loser can no longer decide it. */
|
|
17
|
+
approvalNotPending: {
|
|
18
|
+
code: "harness-approval-not-pending",
|
|
19
|
+
status: 409,
|
|
20
|
+
title: "Approval can no longer be decided",
|
|
21
|
+
},
|
|
22
|
+
/** A commit lost its checkpoint fence (lease taken over, task aborted, or phase moved). */
|
|
23
|
+
commitConflict: {
|
|
24
|
+
code: "harness-commit-conflict",
|
|
25
|
+
status: 409,
|
|
26
|
+
title: "Harness commit lost its checkpoint fence",
|
|
27
|
+
},
|
|
28
|
+
/** Invalid `Harness.open`, `HarnessApp`, runner, or model-resolver configuration. */
|
|
29
|
+
configInvalid: {
|
|
30
|
+
code: "harness-config-invalid",
|
|
31
|
+
status: 500,
|
|
32
|
+
title: "Invalid harness configuration",
|
|
33
|
+
},
|
|
34
|
+
/** A conversation is running a turn and `submit` cannot start another. */
|
|
35
|
+
conversationBusy: {
|
|
36
|
+
code: "harness-conversation-busy",
|
|
37
|
+
status: 409,
|
|
38
|
+
title: "Conversation is busy",
|
|
39
|
+
},
|
|
40
|
+
/** A subagent conversation: only its owning task's `rt.runAgent` runs turns. */
|
|
41
|
+
conversationOwned: {
|
|
42
|
+
code: "harness-conversation-owned",
|
|
43
|
+
status: 409,
|
|
44
|
+
title: "Conversation is run by its owning task",
|
|
45
|
+
},
|
|
46
|
+
/** Misuse of a definition API (`defineTask`, `rt.*`, ...); retrying cannot fix it. */
|
|
47
|
+
definitionInvalid: {
|
|
48
|
+
code: "harness-definition-invalid",
|
|
49
|
+
status: 500,
|
|
50
|
+
title: "Invalid harness definition",
|
|
51
|
+
},
|
|
52
|
+
/** An internal invariant broke; the work is retried or logged. */
|
|
53
|
+
internal: {code: "harness-internal", status: 500, title: "Harness invariant violated"},
|
|
54
|
+
/** A caller passed an invalid argument to a harness method. */
|
|
55
|
+
invalidRequest: {code: "harness-invalid-request", status: 400, title: "Invalid harness request"},
|
|
56
|
+
/** A harness row the call depends on no longer exists. */
|
|
57
|
+
notFound: {code: "harness-not-found", status: 404, title: "Harness record not found"},
|
|
58
|
+
/** A task definition, agent, or extension is not in this harness registry. */
|
|
59
|
+
notRegistered: {
|
|
60
|
+
code: "harness-not-registered",
|
|
61
|
+
status: 404,
|
|
62
|
+
title: "Not registered in this harness",
|
|
63
|
+
},
|
|
64
|
+
/** `Harness.open` on a MongoDB deployment without transactions (not a replica set or mongos). */
|
|
65
|
+
replicaSetRequired: {
|
|
66
|
+
code: "harness-replica-set-required",
|
|
67
|
+
status: 500,
|
|
68
|
+
title: "MongoDB replica set required",
|
|
69
|
+
},
|
|
70
|
+
/** A `requestId` (task or event) is already used for something else. */
|
|
71
|
+
requestIdConflict: {
|
|
72
|
+
code: "harness-request-id-conflict",
|
|
73
|
+
status: 409,
|
|
74
|
+
title: "requestId is already in use",
|
|
75
|
+
},
|
|
76
|
+
/** `retry` on a task whose abort is still running. */
|
|
77
|
+
taskAborting: {code: "harness-task-aborting", status: 409, title: "Task is being aborted"},
|
|
78
|
+
/** `resolveInterrupted` on a task that is not interrupted. */
|
|
79
|
+
taskNotInterrupted: {
|
|
80
|
+
code: "harness-task-not-interrupted",
|
|
81
|
+
status: 409,
|
|
82
|
+
title: "Task is not interrupted",
|
|
83
|
+
},
|
|
84
|
+
/** The task already ended, so it cannot be aborted or receive events. */
|
|
85
|
+
taskTerminal: {code: "harness-task-terminal", status: 409, title: "Task already ended"},
|
|
86
|
+
/** `Harness.waitFor` gave up before the task reached the status it waited for. */
|
|
87
|
+
waitTimedOut: {
|
|
88
|
+
code: "harness-wait-timed-out",
|
|
89
|
+
status: 504,
|
|
90
|
+
title: "Timed out waiting for task",
|
|
91
|
+
},
|
|
92
|
+
} as const satisfies Record<string, HarnessErrorKind>;
|
|
93
|
+
|
|
94
|
+
export type HarnessErrorKindName = keyof typeof HARNESS_ERRORS;
|
|
95
|
+
|
|
96
|
+
/** Build an `APIError` of one harness kind; `detail` is a self-contained sentence. */
|
|
97
|
+
export const harnessError = ({
|
|
98
|
+
cause,
|
|
99
|
+
detail,
|
|
100
|
+
kind,
|
|
101
|
+
}: {
|
|
102
|
+
cause?: unknown;
|
|
103
|
+
detail: string;
|
|
104
|
+
kind: HarnessErrorKindName;
|
|
105
|
+
}): APIError => {
|
|
106
|
+
return new APIError({...HARNESS_ERRORS[kind], cause, detail});
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The text recorded for a failure (task `error`, tool results the model sees, logs). Harness
|
|
111
|
+
* details are self-contained sentences, so they stand alone; other `APIError`s keep their
|
|
112
|
+
* title too, and plain errors use their message.
|
|
113
|
+
*/
|
|
114
|
+
export const errorMessage = (error: unknown): string => {
|
|
115
|
+
if (isAPIError(error) && error.code?.startsWith("harness-") && error.detail) {
|
|
116
|
+
return error.detail;
|
|
117
|
+
}
|
|
118
|
+
return errorDetail(error);
|
|
119
|
+
};
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import {afterEach, describe, expect, it} from "bun:test";
|
|
2
|
+
import {DateTime} from "luxon";
|
|
3
|
+
import mongoose from "mongoose";
|
|
4
|
+
|
|
5
|
+
import {HarnessEventHub, type HubEvent, type HubSubscriber} from "./eventHub";
|
|
6
|
+
import {registerHarnessEvent} from "./models/harnessEvent";
|
|
7
|
+
|
|
8
|
+
const EventModel = registerHarnessEvent();
|
|
9
|
+
|
|
10
|
+
interface Collector {
|
|
11
|
+
events: HubEvent[];
|
|
12
|
+
failures: unknown[];
|
|
13
|
+
subscriber: HubSubscriber;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const collector = (
|
|
17
|
+
streamId: mongoose.Types.ObjectId,
|
|
18
|
+
taskId?: mongoose.Types.ObjectId
|
|
19
|
+
): Collector => {
|
|
20
|
+
const events: HubEvent[] = [];
|
|
21
|
+
const failures: unknown[] = [];
|
|
22
|
+
return {
|
|
23
|
+
events,
|
|
24
|
+
failures,
|
|
25
|
+
subscriber: {
|
|
26
|
+
deliver: (event) => events.push(event),
|
|
27
|
+
fail: (error) => failures.push(error),
|
|
28
|
+
streamId: String(streamId),
|
|
29
|
+
taskId: taskId === undefined ? undefined : String(taskId),
|
|
30
|
+
},
|
|
31
|
+
};
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
const waitUntil = async (predicate: () => boolean, label: string): Promise<void> => {
|
|
35
|
+
const deadline = DateTime.now().plus({seconds: 10});
|
|
36
|
+
while (!predicate()) {
|
|
37
|
+
if (DateTime.now() > deadline) {
|
|
38
|
+
expect(`timed out waiting for ${label}`).toBe("");
|
|
39
|
+
}
|
|
40
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
const seqs = (events: HubEvent[]): number[] => events.map(({seq}) => seq);
|
|
45
|
+
|
|
46
|
+
const unsubscribes: Array<() => void> = [];
|
|
47
|
+
|
|
48
|
+
afterEach(() => {
|
|
49
|
+
for (const unsubscribe of unsubscribes.splice(0)) {
|
|
50
|
+
unsubscribe();
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe("HarnessEventHub", () => {
|
|
55
|
+
it("delivers a task subscriber only the events on that task's path", async () => {
|
|
56
|
+
const hub = new HarnessEventHub();
|
|
57
|
+
const root = new mongoose.Types.ObjectId();
|
|
58
|
+
const childA = new mongoose.Types.ObjectId();
|
|
59
|
+
const childB = new mongoose.Types.ObjectId();
|
|
60
|
+
const whole = collector(root);
|
|
61
|
+
const onA = collector(root, childA);
|
|
62
|
+
const onB = collector(root, childB);
|
|
63
|
+
const elsewhere = collector(new mongoose.Types.ObjectId());
|
|
64
|
+
for (const {subscriber} of [whole, onA, onB, elsewhere]) {
|
|
65
|
+
unsubscribes.push(await hub.subscribe(subscriber));
|
|
66
|
+
}
|
|
67
|
+
expect(hub.openStreams).toBe(1);
|
|
68
|
+
|
|
69
|
+
const created = DateTime.now().toJSDate();
|
|
70
|
+
await EventModel.create(
|
|
71
|
+
[
|
|
72
|
+
{created, seq: 1, streamId: root, taskId: childA, taskPath: [childA, root], type: "output"},
|
|
73
|
+
{created, seq: 2, streamId: root, taskId: childB, taskPath: [childB, root], type: "output"},
|
|
74
|
+
{created, seq: 3, streamId: root, taskId: root, taskPath: [root], type: "output"},
|
|
75
|
+
],
|
|
76
|
+
{ordered: true}
|
|
77
|
+
);
|
|
78
|
+
|
|
79
|
+
await waitUntil(() => whole.events.length === 3, "every event on the root stream");
|
|
80
|
+
expect(seqs(whole.events)).toEqual([1, 2, 3]);
|
|
81
|
+
expect(seqs(onA.events)).toEqual([1]);
|
|
82
|
+
expect(seqs(onB.events)).toEqual([2]);
|
|
83
|
+
expect(elsewhere.events).toEqual([]);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it("fails every subscriber when its tail closes, even if closing the cursor again fails", async () => {
|
|
87
|
+
const hub = new HarnessEventHub();
|
|
88
|
+
const streamId = new mongoose.Types.ObjectId();
|
|
89
|
+
const first = collector(streamId);
|
|
90
|
+
const second = collector(streamId);
|
|
91
|
+
unsubscribes.push(await hub.subscribe(first.subscriber));
|
|
92
|
+
unsubscribes.push(await hub.subscribe(second.subscriber));
|
|
93
|
+
|
|
94
|
+
// The cursor goes away underneath the tail (a failover), and a second close rejects.
|
|
95
|
+
// Wait until the server has assigned a cursor id: closing before that skips killCursors
|
|
96
|
+
// and leaves the cursor open.
|
|
97
|
+
const tail = (
|
|
98
|
+
hub as unknown as {changes: {close: () => Promise<void>; cursor?: {id?: unknown}}}
|
|
99
|
+
).changes;
|
|
100
|
+
await waitUntil(() => tail.cursor?.id != null, "the tail's server cursor");
|
|
101
|
+
const closeCursor = tail.close.bind(tail);
|
|
102
|
+
let closeAttempts = 0;
|
|
103
|
+
tail.close = () => {
|
|
104
|
+
closeAttempts += 1;
|
|
105
|
+
return Promise.reject(new Error("cursor already gone"));
|
|
106
|
+
};
|
|
107
|
+
await closeCursor();
|
|
108
|
+
|
|
109
|
+
await waitUntil(() => first.failures.length === 1 && second.failures.length === 1, "failures");
|
|
110
|
+
// Either the driver's "closed" error or the hub's own; both end the streams.
|
|
111
|
+
expect(first.failures[0]).toBeInstanceOf(Error);
|
|
112
|
+
expect(second.failures[0]).toBe(first.failures[0]);
|
|
113
|
+
await waitUntil(() => closeAttempts > 0, "the tail's own close attempt");
|
|
114
|
+
expect(hub.openStreams).toBe(0);
|
|
115
|
+
|
|
116
|
+
// The next subscriber opens a fresh tail that delivers.
|
|
117
|
+
const later = collector(streamId);
|
|
118
|
+
unsubscribes.push(await hub.subscribe(later.subscriber));
|
|
119
|
+
expect(hub.openStreams).toBe(1);
|
|
120
|
+
await EventModel.create({created: DateTime.now().toJSDate(), seq: 1, streamId, type: "output"});
|
|
121
|
+
await waitUntil(() => later.events.length === 1, "an event on the new tail");
|
|
122
|
+
expect(first.events).toEqual([]);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import {APIError, logger} from "@terreno/api";
|
|
2
|
+
import mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {HarnessEventDocument} from "../types/harness";
|
|
5
|
+
import {errorMessage, harnessError} from "./errors";
|
|
6
|
+
import {registerHarnessEvent} from "./models/harnessEvent";
|
|
7
|
+
|
|
8
|
+
/** How long one wait on the tail may block; also how soon a stop takes effect. */
|
|
9
|
+
const TAIL_AWAIT_MS = 1000;
|
|
10
|
+
|
|
11
|
+
/** An event as a change stream delivers it (a raw document, not a mongoose model). */
|
|
12
|
+
export type HubEvent = Pick<
|
|
13
|
+
HarnessEventDocument,
|
|
14
|
+
"created" | "payload" | "seq" | "streamId" | "taskId" | "taskPath" | "type"
|
|
15
|
+
>;
|
|
16
|
+
|
|
17
|
+
/** One SSE connection's interest in the log. */
|
|
18
|
+
export interface HubSubscriber {
|
|
19
|
+
deliver: (event: HubEvent) => void;
|
|
20
|
+
/** The shared tail failed; the subscriber is already removed. */
|
|
21
|
+
fail: (error: unknown) => void;
|
|
22
|
+
streamId: string;
|
|
23
|
+
/** Task streams: only events whose `taskPath` includes this task. */
|
|
24
|
+
taskId?: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
type EventChangeStream = mongoose.mongo.ChangeStream<HubEvent>;
|
|
28
|
+
|
|
29
|
+
const matches = (subscriber: HubSubscriber, event: HubEvent): boolean =>
|
|
30
|
+
String(event.streamId) === subscriber.streamId &&
|
|
31
|
+
(subscriber.taskId === undefined ||
|
|
32
|
+
(event.taskPath ?? []).some((id) => String(id) === subscriber.taskId));
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* One change stream on `HarnessEvent` per `HarnessApp`, shared by every SSE connection
|
|
36
|
+
* and fanned out in memory by stream (and task path). It opens with the first subscriber,
|
|
37
|
+
* at a cluster time read after that subscriber registered, and closes with the last. A
|
|
38
|
+
* subscriber registers before it queries its replay, and the tail delivers every insert
|
|
39
|
+
* committed after it opened, so each event committed after a subscriber's replay query
|
|
40
|
+
* reaches it through the tail: replay plus tail, deduplicated by `seq`, is exactly once.
|
|
41
|
+
* One stream holds one pooled connection however many clients watch.
|
|
42
|
+
*/
|
|
43
|
+
export class HarnessEventHub {
|
|
44
|
+
private changes: EventChangeStream | undefined;
|
|
45
|
+
private starting: Promise<void> | undefined;
|
|
46
|
+
private readonly subscribers = new Set<HubSubscriber>();
|
|
47
|
+
|
|
48
|
+
/** Number of open tails (0 or 1); for tests and diagnostics. */
|
|
49
|
+
get openStreams(): number {
|
|
50
|
+
return this.changes ? 1 : 0;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Register `subscriber` and make sure the tail is open. Returns the unsubscribe
|
|
55
|
+
* function. Throws (503) when no cluster time is available to start the tail.
|
|
56
|
+
*/
|
|
57
|
+
async subscribe(subscriber: HubSubscriber): Promise<() => void> {
|
|
58
|
+
this.subscribers.add(subscriber);
|
|
59
|
+
const unsubscribe = (): void => this.unsubscribe(subscriber);
|
|
60
|
+
try {
|
|
61
|
+
await this.ensureStarted();
|
|
62
|
+
} catch (error: unknown) {
|
|
63
|
+
unsubscribe();
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
return unsubscribe;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
private unsubscribe(subscriber: HubSubscriber): void {
|
|
70
|
+
this.subscribers.delete(subscriber);
|
|
71
|
+
if (this.subscribers.size === 0) {
|
|
72
|
+
// The pump finishes its current read (so the server cursor has an id) and closes the tail.
|
|
73
|
+
this.changes = undefined;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
private async ensureStarted(): Promise<void> {
|
|
78
|
+
if (this.changes) {
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
this.starting ??= this.start().finally(() => {
|
|
82
|
+
this.starting = undefined;
|
|
83
|
+
});
|
|
84
|
+
await this.starting;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
private async start(): Promise<void> {
|
|
88
|
+
const ping = await mongoose.connection.db?.command({ping: 1});
|
|
89
|
+
const startAtOperationTime = ping?.operationTime;
|
|
90
|
+
// Without a start time the tail could miss events committed during a replay.
|
|
91
|
+
if (!startAtOperationTime) {
|
|
92
|
+
throw new APIError({status: 503, title: "Event stream unavailable: no cluster time"});
|
|
93
|
+
}
|
|
94
|
+
// Every subscriber left while the cluster time was read.
|
|
95
|
+
if (this.subscribers.size === 0) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
const changes = registerHarnessEvent().collection.watch<HubEvent>(
|
|
99
|
+
[{$match: {operationType: "insert"}}],
|
|
100
|
+
{maxAwaitTimeMS: TAIL_AWAIT_MS, startAtOperationTime}
|
|
101
|
+
) as unknown as EventChangeStream;
|
|
102
|
+
this.changes = changes;
|
|
103
|
+
void this.pump(changes);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Read the tail until it is stopped or replaced (`this.changes` no longer points at it),
|
|
108
|
+
* then close it. The first read always finishes: `close()` before the server has assigned
|
|
109
|
+
* a cursor id skips `killCursors`, and the opening aggregate can still leave that cursor
|
|
110
|
+
* open on the server.
|
|
111
|
+
*/
|
|
112
|
+
private async pump(changes: EventChangeStream): Promise<void> {
|
|
113
|
+
let initialized = false;
|
|
114
|
+
try {
|
|
115
|
+
while (this.changes === changes || !initialized) {
|
|
116
|
+
// An invalidated (collection dropped) or externally closed tail is a failure
|
|
117
|
+
// only while this pump still owns it.
|
|
118
|
+
if (changes.closed) {
|
|
119
|
+
if (this.changes !== changes) {
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
throw harnessError({detail: "The HarnessEvent change stream closed", kind: "internal"});
|
|
123
|
+
}
|
|
124
|
+
const change = (await changes.tryNext()) as {fullDocument?: HubEvent} | null;
|
|
125
|
+
initialized = true;
|
|
126
|
+
if (this.changes !== changes) {
|
|
127
|
+
break;
|
|
128
|
+
}
|
|
129
|
+
const event = change?.fullDocument;
|
|
130
|
+
if (!event) {
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
for (const subscriber of [...this.subscribers]) {
|
|
134
|
+
if (matches(subscriber, event)) {
|
|
135
|
+
subscriber.deliver(event);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
} catch (error: unknown) {
|
|
140
|
+
if (this.changes === changes) {
|
|
141
|
+
this.fail(error);
|
|
142
|
+
}
|
|
143
|
+
} finally {
|
|
144
|
+
// A failed close only means the cursor is already gone.
|
|
145
|
+
await changes.close().catch(() => undefined);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** The tail failed: drop every subscriber (each closes its stream); the next one restarts it. */
|
|
150
|
+
private fail(error: unknown): void {
|
|
151
|
+
logger.warn(`Harness event tail failed; closing its SSE streams: ${errorMessage(error)}`);
|
|
152
|
+
const subscribers = [...this.subscribers];
|
|
153
|
+
this.subscribers.clear();
|
|
154
|
+
this.changes = undefined;
|
|
155
|
+
for (const subscriber of subscribers) {
|
|
156
|
+
subscriber.fail(error);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|