@terreno/ai 57.9.0 → 57.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +127 -0
- package/dist/harness/errors.js +101 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +520 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +186 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +57 -0
- package/dist/harness/runners/inProcessRunner.js +196 -0
- package/dist/harness/runtime.d.ts +34 -0
- package/dist/harness/runtime.js +551 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +934 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +17 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +119 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +757 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +767 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +201 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +912 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +402 -0
- package/src/harness/runners/inProcessRunner.ts +220 -0
- package/src/harness/runtime.ts +714 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1095 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,757 @@
|
|
|
1
|
+
import {afterEach, beforeAll, beforeEach, describe, expect, it, setDefaultTimeout} from "bun:test";
|
|
2
|
+
import net from "node:net";
|
|
3
|
+
import {TerrenoApp} from "@terreno/api";
|
|
4
|
+
import {APICallError, type LanguageModel} from "ai";
|
|
5
|
+
import type express from "express";
|
|
6
|
+
import {DateTime} from "luxon";
|
|
7
|
+
import mongoose from "mongoose";
|
|
8
|
+
import supertest from "supertest";
|
|
9
|
+
|
|
10
|
+
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
11
|
+
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
12
|
+
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
13
|
+
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
14
|
+
import {
|
|
15
|
+
gatedStreamingModel,
|
|
16
|
+
listen,
|
|
17
|
+
openSse,
|
|
18
|
+
type SseClient,
|
|
19
|
+
type SseFrame,
|
|
20
|
+
} from "../tests/harnessStreaming";
|
|
21
|
+
import {ensureTestUsers, UserModel} from "../tests/helpers";
|
|
22
|
+
import type {HarnessStreamingOptions} from "../types/harness";
|
|
23
|
+
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
24
|
+
import type {HarnessModels} from "./commit";
|
|
25
|
+
import {appendEvents, HarnessDeltaWriter, resolveStreamingOptions} from "./events";
|
|
26
|
+
import {
|
|
27
|
+
type AnyHarnessTaskDefinition,
|
|
28
|
+
defineAgent,
|
|
29
|
+
defineTask,
|
|
30
|
+
Harness,
|
|
31
|
+
HarnessApp,
|
|
32
|
+
type HarnessRegistryEntry,
|
|
33
|
+
InProcessRunner,
|
|
34
|
+
} from "./harness";
|
|
35
|
+
import {registerHarnessApproval} from "./models/harnessApproval";
|
|
36
|
+
import {registerHarnessConversation} from "./models/harnessConversation";
|
|
37
|
+
import {registerHarnessEvent, registerHarnessEventStream} from "./models/harnessEvent";
|
|
38
|
+
import {registerHarnessInboxEvent} from "./models/harnessInboxEvent";
|
|
39
|
+
import {registerHarnessMessage} from "./models/harnessMessage";
|
|
40
|
+
import {registerHarnessOwner} from "./models/harnessOwner";
|
|
41
|
+
import {registerHarnessTask} from "./models/harnessTask";
|
|
42
|
+
|
|
43
|
+
// Whole turns over real HTTP on a replica set; the first test also pays for index creation.
|
|
44
|
+
setDefaultTimeout(30_000);
|
|
45
|
+
|
|
46
|
+
const TaskModel = registerHarnessTask();
|
|
47
|
+
const OwnerModel = registerHarnessOwner();
|
|
48
|
+
const ConversationModel = registerHarnessConversation();
|
|
49
|
+
const MessageModel = registerHarnessMessage();
|
|
50
|
+
const ApprovalModel = registerHarnessApproval();
|
|
51
|
+
const InboxModel = registerHarnessInboxEvent();
|
|
52
|
+
const EventModel = registerHarnessEvent();
|
|
53
|
+
const EventStreamModel = registerHarnessEventStream();
|
|
54
|
+
let SpanModel: ObsSpanModel;
|
|
55
|
+
let TraceModel: ObsTraceModel;
|
|
56
|
+
|
|
57
|
+
const OWNER = {admin: false, email: "owner@example.com", name: "Owner", password: "pw-owner"};
|
|
58
|
+
const OTHER = {admin: false, email: "other@example.com", name: "Other", password: "pw-other"};
|
|
59
|
+
const ADMIN = {admin: true, email: "admin@example.com", name: "Admin", password: "pw-admin"};
|
|
60
|
+
|
|
61
|
+
const pause = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
|
62
|
+
|
|
63
|
+
const waitUntil = async (predicate: () => Promise<boolean>, label: string): Promise<void> => {
|
|
64
|
+
for (let i = 0; i < 500; i++) {
|
|
65
|
+
if (await predicate()) {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
await pause(10);
|
|
69
|
+
}
|
|
70
|
+
throw new Error(`waitUntil timed out: ${label} (${await describeChangeStreams()})`);
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
const cleanups: Array<() => Promise<void> | void> = [];
|
|
74
|
+
|
|
75
|
+
const openHarness = async ({
|
|
76
|
+
model,
|
|
77
|
+
registry,
|
|
78
|
+
start,
|
|
79
|
+
streaming = {deltaFlushChars: 1000, deltaFlushInterval: {milliseconds: 20}},
|
|
80
|
+
}: {
|
|
81
|
+
model?: unknown;
|
|
82
|
+
registry: ReadonlyArray<HarnessRegistryEntry>;
|
|
83
|
+
start: boolean;
|
|
84
|
+
streaming?: HarnessStreamingOptions;
|
|
85
|
+
}): Promise<Harness> => {
|
|
86
|
+
const harness = await Harness.open({
|
|
87
|
+
models: () => model as LanguageModel,
|
|
88
|
+
registry,
|
|
89
|
+
runner: new InProcessRunner({pollInterval: {milliseconds: 20}}),
|
|
90
|
+
streaming,
|
|
91
|
+
testHooks: {random: () => 0},
|
|
92
|
+
});
|
|
93
|
+
cleanups.push(() => harness.stop());
|
|
94
|
+
if (start) {
|
|
95
|
+
await harness.start();
|
|
96
|
+
}
|
|
97
|
+
return harness;
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
/** An express app serving `harness`'s HTTP API, listening on a free port. */
|
|
101
|
+
const serve = async (
|
|
102
|
+
harness: Harness
|
|
103
|
+
): Promise<{app: express.Application; plugin: HarnessApp; url: string}> => {
|
|
104
|
+
const plugin = new HarnessApp({harness, heartbeatInterval: {milliseconds: 50}});
|
|
105
|
+
const app = new TerrenoApp({skipListen: true, userModel: UserModel}).register(plugin).build();
|
|
106
|
+
const {stop, url} = await listen(app);
|
|
107
|
+
cleanups.push(stop);
|
|
108
|
+
return {app, plugin, url};
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
/** Open change-stream cursors on the server, idle ones included. */
|
|
112
|
+
const openChangeStreamOps = async (): Promise<Array<Record<string, unknown>>> => {
|
|
113
|
+
const dbName = mongoose.connection.db?.databaseName;
|
|
114
|
+
return mongoose.connection.client
|
|
115
|
+
.db("admin")
|
|
116
|
+
.aggregate([
|
|
117
|
+
{$currentOp: {allUsers: true, idleCursors: true}},
|
|
118
|
+
{
|
|
119
|
+
$match: {
|
|
120
|
+
"cursor.originatingCommand.$db": dbName,
|
|
121
|
+
"cursor.originatingCommand.pipeline.0.$changeStream": {$exists: true},
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
])
|
|
125
|
+
.toArray() as Promise<Array<Record<string, unknown>>>;
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
const openChangeStreams = async (): Promise<number> => (await openChangeStreamOps()).length;
|
|
129
|
+
|
|
130
|
+
const describeChangeStreams = async (): Promise<string> => {
|
|
131
|
+
const ops = await openChangeStreamOps();
|
|
132
|
+
if (ops.length === 0) {
|
|
133
|
+
return "none";
|
|
134
|
+
}
|
|
135
|
+
return ops
|
|
136
|
+
.map((op) => {
|
|
137
|
+
const cursor = op.cursor as
|
|
138
|
+
| {
|
|
139
|
+
createdDate?: string;
|
|
140
|
+
nBatchesReturned?: number;
|
|
141
|
+
originatingCommand?: {aggregate?: string};
|
|
142
|
+
}
|
|
143
|
+
| undefined;
|
|
144
|
+
const name = String(cursor?.originatingCommand?.aggregate ?? op.ns);
|
|
145
|
+
return `${name} type=${String(op.type)} batches=${String(cursor?.nBatchesReturned)} created=${String(cursor?.createdDate)}`;
|
|
146
|
+
})
|
|
147
|
+
.join("; ");
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
const tokenFor = async (app: express.Application, email: string, password: string) => {
|
|
151
|
+
const res = await supertest(app).post("/auth/login").send({email, password}).expect(200);
|
|
152
|
+
return `Bearer ${res.body.data.token as string}`;
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
const userIdOf = async (email: string): Promise<string> =>
|
|
156
|
+
String((await UserModel.find({email}).lean())[0]?._id);
|
|
157
|
+
|
|
158
|
+
const sse = async (
|
|
159
|
+
url: string,
|
|
160
|
+
token: string,
|
|
161
|
+
headers: Record<string, string> = {}
|
|
162
|
+
): Promise<SseClient> => {
|
|
163
|
+
const client = await openSse(url, {headers: {authorization: token, ...headers}});
|
|
164
|
+
cleanups.push(() => client.close());
|
|
165
|
+
return client;
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
/** Status and body of a request that must fail before streaming starts. */
|
|
169
|
+
const refused = async (url: string, headers: Record<string, string> = {}) => {
|
|
170
|
+
const response = await fetch(url, {headers});
|
|
171
|
+
return {body: await response.json(), status: response.status};
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
const storedEvents = (streamId: unknown) => EventModel.find({streamId}).sort({seq: 1}).lean();
|
|
175
|
+
|
|
176
|
+
const deltaText = (frames: ReadonlyArray<SseFrame>): string =>
|
|
177
|
+
frames
|
|
178
|
+
.filter((frame) => frame.event === "delta")
|
|
179
|
+
.map((frame) => (frame.data.payload as {text: string}).text)
|
|
180
|
+
.join("");
|
|
181
|
+
|
|
182
|
+
const agent = defineAgent({
|
|
183
|
+
instructions: "Answer briefly.",
|
|
184
|
+
model: {modelId: "gated-model", provider: "mock"},
|
|
185
|
+
modelRetry: {backoffMs: 1, maxBackoffMs: 2},
|
|
186
|
+
name: "test.streamer",
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
describe("Harness event stream", () => {
|
|
190
|
+
let ownerId: string;
|
|
191
|
+
|
|
192
|
+
beforeAll(async () => {
|
|
193
|
+
createLocalObservabilityPlugin();
|
|
194
|
+
SpanModel = registerObsSpan();
|
|
195
|
+
TraceModel = registerObsTrace();
|
|
196
|
+
await ensureTestUsers([OWNER, OTHER, ADMIN]);
|
|
197
|
+
ownerId = await userIdOf(OWNER.email);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
beforeEach(async () => {
|
|
201
|
+
await Promise.all(
|
|
202
|
+
[
|
|
203
|
+
TaskModel,
|
|
204
|
+
OwnerModel,
|
|
205
|
+
ConversationModel,
|
|
206
|
+
MessageModel,
|
|
207
|
+
ApprovalModel,
|
|
208
|
+
InboxModel,
|
|
209
|
+
EventModel,
|
|
210
|
+
EventStreamModel,
|
|
211
|
+
SpanModel,
|
|
212
|
+
TraceModel,
|
|
213
|
+
].map((model) => (model as {deleteMany: (filter: object) => Promise<unknown>}).deleteMany({}))
|
|
214
|
+
);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
afterEach(async () => {
|
|
218
|
+
for (const cleanup of cleanups.splice(0).reverse()) {
|
|
219
|
+
await cleanup();
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
describe("reconnect across instances (AC9)", () => {
|
|
224
|
+
it("replays every missed event exactly once from another instance, then streams live", async () => {
|
|
225
|
+
const gated = gatedStreamingModel();
|
|
226
|
+
const harnessA = await openHarness({model: gated.model, registry: [agent], start: true});
|
|
227
|
+
// Instance B serves HTTP only; instance A's runner executes the turn.
|
|
228
|
+
const harnessB = await openHarness({model: gated.model, registry: [agent], start: false});
|
|
229
|
+
const a = await serve(harnessA);
|
|
230
|
+
const b = await serve(harnessB);
|
|
231
|
+
const token = await tokenFor(a.app, OWNER.email, OWNER.password);
|
|
232
|
+
const conversation = await harnessA.createConversation({agent, userId: ownerId});
|
|
233
|
+
const path = `/harness/conversations/${conversation.id}/events`;
|
|
234
|
+
|
|
235
|
+
const submitted = await supertest(a.app)
|
|
236
|
+
.post(`/harness/conversations/${conversation.id}/submit`)
|
|
237
|
+
.set("authorization", token)
|
|
238
|
+
.send({content: "Say hello", requestId: "r1"})
|
|
239
|
+
.expect(200);
|
|
240
|
+
expect(submitted.body.data.disposition).toBe("started");
|
|
241
|
+
|
|
242
|
+
const first = await sse(`${a.url}${path}`, token);
|
|
243
|
+
const request = await gated.nextRequest();
|
|
244
|
+
request.text("Hel");
|
|
245
|
+
await first.waitFor((frame) => frame.event === "delta");
|
|
246
|
+
expect(first.frames.map((frame) => frame.event)).toEqual([
|
|
247
|
+
"message.created",
|
|
248
|
+
"turn.started",
|
|
249
|
+
"delta",
|
|
250
|
+
]);
|
|
251
|
+
const lastSeen = first.frames.at(-1)?.id as number;
|
|
252
|
+
first.close();
|
|
253
|
+
|
|
254
|
+
// Missed while disconnected: two deltas.
|
|
255
|
+
request.text("lo ");
|
|
256
|
+
await waitUntil(async () => (await storedEvents(conversation.id)).length === 4, "2nd delta");
|
|
257
|
+
request.text("wor");
|
|
258
|
+
await waitUntil(async () => (await storedEvents(conversation.id)).length === 5, "3rd delta");
|
|
259
|
+
|
|
260
|
+
const second = await sse(`${b.url}${path}`, token, {"Last-Event-ID": String(lastSeen)});
|
|
261
|
+
await second.waitFor((frame) => frame.id === 5);
|
|
262
|
+
request.text("ld");
|
|
263
|
+
request.finish();
|
|
264
|
+
await second.waitFor((frame) => frame.event === "turn.finished");
|
|
265
|
+
|
|
266
|
+
const stored = await storedEvents(conversation.id);
|
|
267
|
+
const replayedAndLive = second.frames.map((frame) => frame.id);
|
|
268
|
+
// Exactly the events after Last-Event-ID, once each, in order.
|
|
269
|
+
expect(replayedAndLive).toEqual(
|
|
270
|
+
stored.filter((event) => event.seq > lastSeen).map((event) => event.seq)
|
|
271
|
+
);
|
|
272
|
+
expect([...first.frames, ...second.frames].map((frame) => frame.id)).toEqual(
|
|
273
|
+
stored.map((_event, index) => index + 1)
|
|
274
|
+
);
|
|
275
|
+
expect(second.frames.map((frame) => frame.event)).toEqual([
|
|
276
|
+
"delta",
|
|
277
|
+
"delta",
|
|
278
|
+
"delta",
|
|
279
|
+
"message.created",
|
|
280
|
+
"turn.finished",
|
|
281
|
+
]);
|
|
282
|
+
expect(deltaText([...first.frames, ...second.frames])).toBe("Hello world");
|
|
283
|
+
const answer = second.frames.find((frame) => frame.event === "message.created");
|
|
284
|
+
expect(answer?.data.payload).toMatchObject({
|
|
285
|
+
message: {parts: [{text: "Hello world", type: "text"}], role: "assistant", seq: 2},
|
|
286
|
+
});
|
|
287
|
+
const finished = second.frames.at(-1);
|
|
288
|
+
expect(finished?.data.payload).toMatchObject({
|
|
289
|
+
outcome: {
|
|
290
|
+
result: {finishReason: "stop", steps: 1, text: "Hello world"},
|
|
291
|
+
status: "completed",
|
|
292
|
+
},
|
|
293
|
+
status: "completed",
|
|
294
|
+
turnTaskId: submitted.body.data.turnTaskId,
|
|
295
|
+
});
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
it("resumes from ?after= when no Last-Event-ID header is sent", async () => {
|
|
299
|
+
const gated = gatedStreamingModel();
|
|
300
|
+
const harness = await openHarness({model: gated.model, registry: [agent], start: true});
|
|
301
|
+
const {app, url} = await serve(harness);
|
|
302
|
+
const token = await tokenFor(app, OWNER.email, OWNER.password);
|
|
303
|
+
const conversation = await harness.createConversation({agent, userId: ownerId});
|
|
304
|
+
await conversation.submit({content: "Hi", requestId: "r1"});
|
|
305
|
+
const request = await gated.nextRequest();
|
|
306
|
+
request.text("Hey");
|
|
307
|
+
request.finish();
|
|
308
|
+
await waitUntil(
|
|
309
|
+
async () =>
|
|
310
|
+
(await ConversationModel.findExactlyOne({_id: conversation.id})).status === "idle",
|
|
311
|
+
"turn finished"
|
|
312
|
+
);
|
|
313
|
+
|
|
314
|
+
const client = await sse(
|
|
315
|
+
`${url}/harness/conversations/${conversation.id}/events?after=2`,
|
|
316
|
+
token
|
|
317
|
+
);
|
|
318
|
+
await client.waitFor((frame) => frame.event === "turn.finished");
|
|
319
|
+
expect(client.frames.map((frame) => [frame.id, frame.event])).toEqual([
|
|
320
|
+
[3, "delta"],
|
|
321
|
+
[4, "message.created"],
|
|
322
|
+
[5, "turn.finished"],
|
|
323
|
+
]);
|
|
324
|
+
// Last-Event-ID wins over ?after= (a browser resends it on reconnect).
|
|
325
|
+
const resumed = await sse(
|
|
326
|
+
`${url}/harness/conversations/${conversation.id}/events?after=0`,
|
|
327
|
+
token,
|
|
328
|
+
{"Last-Event-ID": "4"}
|
|
329
|
+
);
|
|
330
|
+
await resumed.waitFor((frame) => frame.event === "turn.finished");
|
|
331
|
+
expect(resumed.frames.map((frame) => frame.id)).toEqual([5]);
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
it("pages a long replay and delivers writes racing it exactly once", async () => {
|
|
335
|
+
const harness = await openHarness({registry: [agent], start: false});
|
|
336
|
+
const {app, url} = await serve(harness);
|
|
337
|
+
const token = await tokenFor(app, OWNER.email, OWNER.password);
|
|
338
|
+
const conversation = await harness.createConversation({agent, userId: ownerId});
|
|
339
|
+
const models = {event: EventModel, eventStream: EventStreamModel} as unknown as HarnessModels;
|
|
340
|
+
const write = (count: number, label: string) =>
|
|
341
|
+
appendEvents({
|
|
342
|
+
events: Array.from({length: count}, (_value, index) => ({
|
|
343
|
+
payload: {text: `${label}${index}`},
|
|
344
|
+
type: "output" as const,
|
|
345
|
+
})),
|
|
346
|
+
models,
|
|
347
|
+
streamId: conversation.id,
|
|
348
|
+
});
|
|
349
|
+
await write(1200, "old");
|
|
350
|
+
|
|
351
|
+
// A writer keeps appending while the client connects and pages through the replay,
|
|
352
|
+
// so events land in both the replay and the change-stream buffer.
|
|
353
|
+
let isWriting = true;
|
|
354
|
+
let written = 1200;
|
|
355
|
+
const writer = (async () => {
|
|
356
|
+
while (isWriting) {
|
|
357
|
+
await write(1, "race");
|
|
358
|
+
written += 1;
|
|
359
|
+
await pause(2);
|
|
360
|
+
}
|
|
361
|
+
})();
|
|
362
|
+
const client = await sse(`${url}/harness/conversations/${conversation.id}/events`, token);
|
|
363
|
+
await client.waitFor((frame) => frame.id > 1210);
|
|
364
|
+
isWriting = false;
|
|
365
|
+
await writer;
|
|
366
|
+
await client.waitFor((frame) => frame.id === written);
|
|
367
|
+
await pause(50);
|
|
368
|
+
expect(written).toBeGreaterThan(1210);
|
|
369
|
+
expect(client.frames.map((frame) => frame.id)).toEqual(
|
|
370
|
+
Array.from({length: written}, (_value, index) => index + 1)
|
|
371
|
+
);
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
it("rejects a heartbeat interval that is not positive", () => {
|
|
375
|
+
const app = new TerrenoApp({skipListen: true, userModel: UserModel});
|
|
376
|
+
expect(() =>
|
|
377
|
+
app
|
|
378
|
+
.register(new HarnessApp({harness: {} as Harness, heartbeatInterval: {seconds: 0}}))
|
|
379
|
+
.build()
|
|
380
|
+
).toThrow(
|
|
381
|
+
harnessErrorMatching("configInvalid", "HarnessApp heartbeatInterval must be positive")
|
|
382
|
+
);
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
it("sends heartbeat comments while the stream is idle", async () => {
|
|
386
|
+
const harness = await openHarness({registry: [agent], start: false});
|
|
387
|
+
const {app, url} = await serve(harness);
|
|
388
|
+
const token = await tokenFor(app, OWNER.email, OWNER.password);
|
|
389
|
+
const conversation = await harness.createConversation({agent, userId: ownerId});
|
|
390
|
+
const client = await sse(`${url}/harness/conversations/${conversation.id}/events`, token);
|
|
391
|
+
await waitUntil(async () => client.comments.includes("heartbeat"), "heartbeat");
|
|
392
|
+
expect(client.frames).toEqual([]);
|
|
393
|
+
});
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
describe("connections", () => {
|
|
397
|
+
it("share one change stream per app and release it when the last client leaves", async () => {
|
|
398
|
+
const harness = await openHarness({registry: [agent], start: false});
|
|
399
|
+
const {app, plugin, url} = await serve(harness);
|
|
400
|
+
const token = await tokenFor(app, OWNER.email, OWNER.password);
|
|
401
|
+
const conversation = await harness.createConversation({agent, userId: ownerId});
|
|
402
|
+
// Earlier tests' streams close asynchronously; start from none.
|
|
403
|
+
await waitUntil(async () => (await openChangeStreams()) === 0, "no cursors yet");
|
|
404
|
+
const base = 0;
|
|
405
|
+
const clients = await Promise.all(
|
|
406
|
+
Array.from({length: 5}, () =>
|
|
407
|
+
sse(`${url}/harness/conversations/${conversation.id}/events`, token)
|
|
408
|
+
)
|
|
409
|
+
);
|
|
410
|
+
expect(plugin.eventHub.openStreams).toBe(1);
|
|
411
|
+
await waitUntil(async () => (await openChangeStreams()) === base + 1, "one shared cursor");
|
|
412
|
+
for (const client of clients) {
|
|
413
|
+
client.close();
|
|
414
|
+
}
|
|
415
|
+
await waitUntil(async () => plugin.eventHub.openStreams === 0, "hub closed");
|
|
416
|
+
await waitUntil(async () => (await openChangeStreams()) === base, "cursor released");
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
it("leaves nothing open when clients disconnect before the stream starts", async () => {
|
|
420
|
+
const harness = await openHarness({registry: [agent], start: false});
|
|
421
|
+
const {app, plugin, url} = await serve(harness);
|
|
422
|
+
const token = await tokenFor(app, OWNER.email, OWNER.password);
|
|
423
|
+
const conversation = await harness.createConversation({agent, userId: ownerId});
|
|
424
|
+
// Earlier tests' streams close asynchronously; start from none.
|
|
425
|
+
await waitUntil(async () => (await openChangeStreams()) === 0, "no cursors yet");
|
|
426
|
+
const base = 0;
|
|
427
|
+
const port = Number(new URL(url).port);
|
|
428
|
+
for (let i = 0; i < 20; i++) {
|
|
429
|
+
await new Promise<void>((resolve) => {
|
|
430
|
+
const socket = net.connect(port, "127.0.0.1", () => {
|
|
431
|
+
socket.write(
|
|
432
|
+
`GET /harness/conversations/${conversation.id}/events HTTP/1.1\r\nHost: x\r\nAuthorization: ${token}\r\n\r\n`
|
|
433
|
+
);
|
|
434
|
+
// Gone during auth, the lookups, or the replay.
|
|
435
|
+
setTimeout(() => {
|
|
436
|
+
socket.destroy();
|
|
437
|
+
resolve();
|
|
438
|
+
}, i % 4);
|
|
439
|
+
});
|
|
440
|
+
socket.on("error", () => resolve());
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
await pause(500);
|
|
444
|
+
await waitUntil(async () => plugin.eventHub.openStreams === 0, "hub closed");
|
|
445
|
+
await waitUntil(async () => (await openChangeStreams()) === base, "no cursors left");
|
|
446
|
+
// The app still serves streams afterwards.
|
|
447
|
+
const client = await sse(`${url}/harness/conversations/${conversation.id}/events`, token);
|
|
448
|
+
await waitUntil(async () => client.comments.includes("heartbeat"), "heartbeat");
|
|
449
|
+
});
|
|
450
|
+
it("ends every stream when the shared tail fails, and serves again on reconnect", async () => {
|
|
451
|
+
const harness = await openHarness({registry: [agent], start: false});
|
|
452
|
+
const {app, plugin, url} = await serve(harness);
|
|
453
|
+
const token = await tokenFor(app, OWNER.email, OWNER.password);
|
|
454
|
+
const conversation = await harness.createConversation({agent, userId: ownerId});
|
|
455
|
+
const response = await fetch(`${url}/harness/conversations/${conversation.id}/events`, {
|
|
456
|
+
headers: {authorization: token},
|
|
457
|
+
});
|
|
458
|
+
await waitUntil(async () => plugin.eventHub.openStreams === 1, "tail open");
|
|
459
|
+
// Close only after the server cursor id exists. Closing during the opening aggregate
|
|
460
|
+
// skips killCursors and leaves the cursor open; closing after the id exists ends the tail.
|
|
461
|
+
const tail = (
|
|
462
|
+
plugin.eventHub as unknown as {
|
|
463
|
+
changes: {close: () => Promise<void>; cursor?: {id?: unknown}};
|
|
464
|
+
}
|
|
465
|
+
).changes;
|
|
466
|
+
await waitUntil(async () => tail.cursor?.id != null, "tail cursor");
|
|
467
|
+
await tail.close();
|
|
468
|
+
expect(await response.text()).not.toContain("event:");
|
|
469
|
+
expect(plugin.eventHub.openStreams).toBe(0);
|
|
470
|
+
|
|
471
|
+
const client = await sse(`${url}/harness/conversations/${conversation.id}/events`, token);
|
|
472
|
+
await appendEvents({
|
|
473
|
+
events: [{payload: {text: "back"}, type: "output"}],
|
|
474
|
+
models: {event: EventModel, eventStream: EventStreamModel} as unknown as HarnessModels,
|
|
475
|
+
streamId: conversation.id,
|
|
476
|
+
});
|
|
477
|
+
await client.waitFor((frame) => frame.event === "output");
|
|
478
|
+
});
|
|
479
|
+
});
|
|
480
|
+
|
|
481
|
+
describe("access", () => {
|
|
482
|
+
it("requires a login, and the owner or an admin", async () => {
|
|
483
|
+
const harness = await openHarness({registry: [agent], start: false});
|
|
484
|
+
const {app, url} = await serve(harness);
|
|
485
|
+
const conversation = await harness.createConversation({agent, userId: ownerId});
|
|
486
|
+
const path = `${url}/harness/conversations/${conversation.id}/events`;
|
|
487
|
+
|
|
488
|
+
expect((await refused(path)).status).toBe(401);
|
|
489
|
+
const other = await tokenFor(app, OTHER.email, OTHER.password);
|
|
490
|
+
const forbidden = await refused(path, {authorization: other});
|
|
491
|
+
expect(forbidden.status).toBe(403);
|
|
492
|
+
expect(forbidden.body.title).toBe("Only the owner or an admin may read this event stream");
|
|
493
|
+
expect(
|
|
494
|
+
(
|
|
495
|
+
await refused(`${url}/harness/conversations/507f1f77bcf86cd799439011/events`, {
|
|
496
|
+
authorization: other,
|
|
497
|
+
})
|
|
498
|
+
).status
|
|
499
|
+
).toBe(404);
|
|
500
|
+
expect(
|
|
501
|
+
(await refused(`${url}/harness/conversations/not-an-id/events`, {authorization: other}))
|
|
502
|
+
.status
|
|
503
|
+
).toBe(404);
|
|
504
|
+
const owner = await tokenFor(app, OWNER.email, OWNER.password);
|
|
505
|
+
const badId = await refused(path, {authorization: owner, "Last-Event-ID": "abc"});
|
|
506
|
+
expect(badId.status).toBe(400);
|
|
507
|
+
expect((await refused(`${path}?after=-1`, {authorization: owner})).status).toBe(400);
|
|
508
|
+
|
|
509
|
+
// Owner and admin both get a stream.
|
|
510
|
+
await sse(path, owner);
|
|
511
|
+
await sse(path, await tokenFor(app, ADMIN.email, ADMIN.password));
|
|
512
|
+
});
|
|
513
|
+
});
|
|
514
|
+
|
|
515
|
+
describe("task streams", () => {
|
|
516
|
+
it("carry status, output, and approval events of the task and its descendants", async () => {
|
|
517
|
+
const child = defineTask<{n: number}, unknown, string>({
|
|
518
|
+
initial: () => ({phase: "work"}),
|
|
519
|
+
name: "test.child",
|
|
520
|
+
phases: {
|
|
521
|
+
work: {
|
|
522
|
+
replay: "safe",
|
|
523
|
+
run: async (task, rt) => {
|
|
524
|
+
await rt.output(`child ${task.input.n} working`);
|
|
525
|
+
const decision = await rt.approval("go", {title: "Go ahead?"});
|
|
526
|
+
await rt.commit({terminal: {result: String(decision.approved), status: "completed"}});
|
|
527
|
+
},
|
|
528
|
+
},
|
|
529
|
+
},
|
|
530
|
+
version: 1,
|
|
531
|
+
});
|
|
532
|
+
const parent = defineTask<Record<string, never>, unknown, string>({
|
|
533
|
+
initial: () => ({phase: "spawn"}),
|
|
534
|
+
name: "test.parent",
|
|
535
|
+
phases: {
|
|
536
|
+
spawn: {
|
|
537
|
+
replay: "safe",
|
|
538
|
+
run: async (_task, rt) => {
|
|
539
|
+
await rt.output("parent starting");
|
|
540
|
+
const id = await rt.createTask(child, {n: 1});
|
|
541
|
+
const [outcome] = await rt.waitForTasks([id]);
|
|
542
|
+
await rt.commit({terminal: {result: String(outcome?.result), status: "completed"}});
|
|
543
|
+
},
|
|
544
|
+
},
|
|
545
|
+
},
|
|
546
|
+
version: 1,
|
|
547
|
+
});
|
|
548
|
+
const harness = await openHarness({
|
|
549
|
+
registry: [parent, child] as unknown as AnyHarnessTaskDefinition[],
|
|
550
|
+
start: true,
|
|
551
|
+
});
|
|
552
|
+
const {app, url} = await serve(harness);
|
|
553
|
+
const owner = await tokenFor(app, OWNER.email, OWNER.password);
|
|
554
|
+
const created = await harness.createTask(parent, {}, {userId: ownerId});
|
|
555
|
+
const parentStream = await sse(`${url}/harness/tasks/${created._id}/events`, owner);
|
|
556
|
+
await waitUntil(async () => (await ApprovalModel.countDocuments({})) === 1, "approval");
|
|
557
|
+
const approval = (await ApprovalModel.find({}).lean())[0];
|
|
558
|
+
const childId = String(approval?.taskId);
|
|
559
|
+
const childStream = await sse(`${url}/harness/tasks/${childId}/events`, owner);
|
|
560
|
+
await childStream.waitFor((frame) => frame.event === "approval.requested");
|
|
561
|
+
|
|
562
|
+
await harness.decideApproval(approval?._id as never, {approved: true});
|
|
563
|
+
await parentStream.waitFor(
|
|
564
|
+
(frame) =>
|
|
565
|
+
frame.event === "task.status" &&
|
|
566
|
+
frame.data.taskId === String(created._id) &&
|
|
567
|
+
(frame.data.payload as {status: string}).status === "completed"
|
|
568
|
+
);
|
|
569
|
+
await childStream.waitFor(
|
|
570
|
+
(frame) =>
|
|
571
|
+
frame.event === "task.status" &&
|
|
572
|
+
(frame.data.payload as {status: string}).status === "completed"
|
|
573
|
+
);
|
|
574
|
+
|
|
575
|
+
const describe = (frame: SseFrame): string => {
|
|
576
|
+
const payload = frame.data.payload as {status?: string; text?: string};
|
|
577
|
+
const who = frame.data.taskId === String(created._id) ? "parent" : "child";
|
|
578
|
+
return `${who} ${frame.event} ${payload.status ?? payload.text ?? ""}`.trim();
|
|
579
|
+
};
|
|
580
|
+
expect(parentStream.frames.map(describe)).toEqual([
|
|
581
|
+
"parent task.status pending",
|
|
582
|
+
"parent output parent starting",
|
|
583
|
+
"child task.status pending",
|
|
584
|
+
"parent task.status waiting",
|
|
585
|
+
"child output child 1 working",
|
|
586
|
+
"child approval.requested pending",
|
|
587
|
+
"child task.status waiting",
|
|
588
|
+
"child approval.decided approved",
|
|
589
|
+
// Each phase re-runs after its wait, so its output is sent again.
|
|
590
|
+
"child output child 1 working",
|
|
591
|
+
"child task.status completed",
|
|
592
|
+
"parent output parent starting",
|
|
593
|
+
"parent task.status completed",
|
|
594
|
+
]);
|
|
595
|
+
// The child's stream is the same log narrowed to the child: no parent events.
|
|
596
|
+
expect(childStream.frames.map(describe)).toEqual(
|
|
597
|
+
parentStream.frames.map(describe).filter((line) => line.startsWith("child"))
|
|
598
|
+
);
|
|
599
|
+
expect(childStream.frames.map((frame) => frame.id)).toEqual(
|
|
600
|
+
parentStream.frames
|
|
601
|
+
.filter((frame) => frame.data.taskId === childId)
|
|
602
|
+
.map((frame) => frame.id)
|
|
603
|
+
);
|
|
604
|
+
|
|
605
|
+
const other = await tokenFor(app, OTHER.email, OTHER.password);
|
|
606
|
+
expect(
|
|
607
|
+
(await refused(`${url}/harness/tasks/${childId}/events`, {authorization: other})).status
|
|
608
|
+
).toBe(403);
|
|
609
|
+
expect((await refused(`${url}/harness/tasks/${childId}/events`)).status).toBe(401);
|
|
610
|
+
await sse(
|
|
611
|
+
`${url}/harness/tasks/${childId}/events`,
|
|
612
|
+
await tokenFor(app, ADMIN.email, ADMIN.password)
|
|
613
|
+
);
|
|
614
|
+
});
|
|
615
|
+
});
|
|
616
|
+
|
|
617
|
+
describe("deltas", () => {
|
|
618
|
+
const models = (): HarnessModels =>
|
|
619
|
+
({event: EventModel, eventStream: EventStreamModel}) as unknown as HarnessModels;
|
|
620
|
+
const source = {
|
|
621
|
+
conversationId: "507f1f77bcf86cd799439011",
|
|
622
|
+
requestKey: "k1",
|
|
623
|
+
step: 1,
|
|
624
|
+
turnTaskId: "507f1f77bcf86cd799439012",
|
|
625
|
+
};
|
|
626
|
+
|
|
627
|
+
it("coalesce by size and by time, and expire", async () => {
|
|
628
|
+
const writer = new HarnessDeltaWriter({
|
|
629
|
+
models: models(),
|
|
630
|
+
options: resolveStreamingOptions({
|
|
631
|
+
deltaFlushChars: 6,
|
|
632
|
+
deltaFlushInterval: {milliseconds: 500},
|
|
633
|
+
deltaTtl: {minutes: 5},
|
|
634
|
+
}),
|
|
635
|
+
source,
|
|
636
|
+
});
|
|
637
|
+
writer.push("ab");
|
|
638
|
+
writer.push("cd");
|
|
639
|
+
await pause(10);
|
|
640
|
+
expect(await EventModel.countDocuments({})).toBe(0);
|
|
641
|
+
// Reaching deltaFlushChars writes at once.
|
|
642
|
+
writer.push("ef");
|
|
643
|
+
await waitUntil(async () => (await EventModel.countDocuments({})) === 1, "size flush");
|
|
644
|
+
// Below the size, the interval writes it.
|
|
645
|
+
writer.push("g");
|
|
646
|
+
await waitUntil(async () => (await EventModel.countDocuments({})) === 2, "time flush");
|
|
647
|
+
writer.push("h");
|
|
648
|
+
await writer.close();
|
|
649
|
+
|
|
650
|
+
const rows = await storedEvents(source.conversationId);
|
|
651
|
+
expect(rows.map((row) => [row.seq, row.type, (row.payload as {text: string}).text])).toEqual([
|
|
652
|
+
[1, "delta", "abcdef"],
|
|
653
|
+
[2, "delta", "g"],
|
|
654
|
+
[3, "delta", "h"],
|
|
655
|
+
]);
|
|
656
|
+
for (const row of rows) {
|
|
657
|
+
expect(row.payload).toMatchObject({
|
|
658
|
+
requestKey: "k1",
|
|
659
|
+
step: 1,
|
|
660
|
+
turnTaskId: source.turnTaskId,
|
|
661
|
+
});
|
|
662
|
+
const ttl = DateTime.fromJSDate(row.expiresAt as Date).diff(
|
|
663
|
+
DateTime.fromJSDate(row.created)
|
|
664
|
+
);
|
|
665
|
+
expect(Math.round(ttl.as("minutes"))).toBe(5);
|
|
666
|
+
}
|
|
667
|
+
const indexes = await EventModel.collection.indexes();
|
|
668
|
+
expect(indexes.find((index) => index.key.expiresAt === 1)?.expireAfterSeconds).toBe(0);
|
|
669
|
+
});
|
|
670
|
+
|
|
671
|
+
it("keeps a failed attempt's deltas under their own requestKey; the message holds only the retry", async () => {
|
|
672
|
+
const gated = gatedStreamingModel();
|
|
673
|
+
const harness = await openHarness({model: gated.model, registry: [agent], start: true});
|
|
674
|
+
const conversation = await harness.createConversation({agent, userId: ownerId});
|
|
675
|
+
const turn = await conversation.submit({content: "Go", requestId: "r1"});
|
|
676
|
+
const first = await gated.nextRequest();
|
|
677
|
+
first.text("Stale");
|
|
678
|
+
await waitUntil(
|
|
679
|
+
async () => (await EventModel.countDocuments({type: "delta"})) === 1,
|
|
680
|
+
"first delta"
|
|
681
|
+
);
|
|
682
|
+
first.fail(
|
|
683
|
+
new APICallError({
|
|
684
|
+
isRetryable: true,
|
|
685
|
+
message: "HTTP 503",
|
|
686
|
+
requestBodyValues: {},
|
|
687
|
+
statusCode: 503,
|
|
688
|
+
url: "https://mock.invalid/v1/chat",
|
|
689
|
+
})
|
|
690
|
+
);
|
|
691
|
+
const retry = await gated.nextRequest();
|
|
692
|
+
retry.text("Fresh");
|
|
693
|
+
retry.finish();
|
|
694
|
+
await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
695
|
+
|
|
696
|
+
const rows = await storedEvents(conversation.id);
|
|
697
|
+
const deltas = rows
|
|
698
|
+
.filter((row) => row.type === "delta")
|
|
699
|
+
.map((row) => row.payload as {requestKey: string; step: number; text: string});
|
|
700
|
+
expect(deltas.map(({step, text}) => [step, text])).toEqual([
|
|
701
|
+
[1, "Stale"],
|
|
702
|
+
[1, "Fresh"],
|
|
703
|
+
]);
|
|
704
|
+
expect(deltas[0]?.requestKey).not.toBe(deltas[1]?.requestKey);
|
|
705
|
+
const answer = rows.filter((row) => row.type === "message.created").at(-1);
|
|
706
|
+
expect(answer?.payload).toMatchObject({
|
|
707
|
+
message: {parts: [{text: "Fresh", type: "text"}], role: "assistant"},
|
|
708
|
+
});
|
|
709
|
+
});
|
|
710
|
+
|
|
711
|
+
it("persists a streamed turn as coalesced deltas; committed events never expire", async () => {
|
|
712
|
+
const gated = gatedStreamingModel();
|
|
713
|
+
const harness = await openHarness({
|
|
714
|
+
model: gated.model,
|
|
715
|
+
registry: [agent],
|
|
716
|
+
start: true,
|
|
717
|
+
streaming: {deltaFlushChars: 1000, deltaFlushInterval: {seconds: 10}},
|
|
718
|
+
});
|
|
719
|
+
const conversation = await harness.createConversation({agent, userId: ownerId});
|
|
720
|
+
const turn = await conversation.submit({content: "Count", requestId: "r1"});
|
|
721
|
+
const request = await gated.nextRequest();
|
|
722
|
+
for (const chunk of ["one ", "two ", "three"]) {
|
|
723
|
+
request.text(chunk);
|
|
724
|
+
}
|
|
725
|
+
request.finish();
|
|
726
|
+
await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
727
|
+
|
|
728
|
+
const rows = await storedEvents(conversation.id);
|
|
729
|
+
expect(rows.map((row) => row.type)).toEqual([
|
|
730
|
+
"message.created",
|
|
731
|
+
"turn.started",
|
|
732
|
+
"delta",
|
|
733
|
+
"message.created",
|
|
734
|
+
"turn.finished",
|
|
735
|
+
]);
|
|
736
|
+
// One delta: the stream ended before the interval, and close() flushed it.
|
|
737
|
+
expect(rows[2]?.payload).toMatchObject({
|
|
738
|
+
step: 1,
|
|
739
|
+
text: "one two three",
|
|
740
|
+
turnTaskId: String(turn._id),
|
|
741
|
+
});
|
|
742
|
+
expect(rows[2]?.expiresAt).toBeInstanceOf(Date);
|
|
743
|
+
expect(
|
|
744
|
+
rows.filter((row) => row.type !== "delta").every((row) => row.expiresAt === undefined)
|
|
745
|
+
).toBe(true);
|
|
746
|
+
});
|
|
747
|
+
|
|
748
|
+
it("rejects invalid streaming options", () => {
|
|
749
|
+
expect(() => resolveStreamingOptions({deltaFlushChars: 0})).toThrow(
|
|
750
|
+
harnessErrorMatching("configInvalid", "deltaFlushChars must be a positive integer")
|
|
751
|
+
);
|
|
752
|
+
expect(() => resolveStreamingOptions({deltaTtl: {seconds: 0}})).toThrow(
|
|
753
|
+
harnessErrorMatching("configInvalid", "must be positive")
|
|
754
|
+
);
|
|
755
|
+
});
|
|
756
|
+
});
|
|
757
|
+
});
|