@terreno/ai 57.9.0 → 57.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +127 -0
- package/dist/harness/errors.js +101 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +520 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +186 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +57 -0
- package/dist/harness/runners/inProcessRunner.js +196 -0
- package/dist/harness/runtime.d.ts +34 -0
- package/dist/harness/runtime.js +551 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +934 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +17 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +119 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +757 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +767 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +201 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +912 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +402 -0
- package/src/harness/runners/inProcessRunner.ts +220 -0
- package/src/harness/runtime.ts +714 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1095 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,1034 @@
|
|
|
1
|
+
import {
|
|
2
|
+
afterEach,
|
|
3
|
+
beforeAll,
|
|
4
|
+
beforeEach,
|
|
5
|
+
describe,
|
|
6
|
+
expect,
|
|
7
|
+
it,
|
|
8
|
+
mock,
|
|
9
|
+
setDefaultTimeout,
|
|
10
|
+
} from "bun:test";
|
|
11
|
+
import {z} from "@terreno/api";
|
|
12
|
+
import {APICallError, type LanguageModel} from "ai";
|
|
13
|
+
import mongoose from "mongoose";
|
|
14
|
+
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
15
|
+
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
16
|
+
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
17
|
+
import {generateToStream} from "../tests/generateStream";
|
|
18
|
+
import {harnessErrorMatching} from "../tests/harnessErrors";
|
|
19
|
+
import type {HarnessAgentDefinition, HarnessTaskDocument, HarnessTestHooks} from "../types/harness";
|
|
20
|
+
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
21
|
+
import type {ExecutionEnv} from "./executionEnv";
|
|
22
|
+
import {
|
|
23
|
+
AGENT_TOOL_TASK_NAME,
|
|
24
|
+
AGENT_TURN_TASK_NAME,
|
|
25
|
+
defineAgent,
|
|
26
|
+
defineTask,
|
|
27
|
+
defineTool,
|
|
28
|
+
Harness,
|
|
29
|
+
HarnessConversationBusyError,
|
|
30
|
+
type HarnessModelRef,
|
|
31
|
+
InProcessRunner,
|
|
32
|
+
} from "./harness";
|
|
33
|
+
import {registerHarnessConversation} from "./models/harnessConversation";
|
|
34
|
+
import {registerHarnessMessage} from "./models/harnessMessage";
|
|
35
|
+
import {registerHarnessTask} from "./models/harnessTask";
|
|
36
|
+
|
|
37
|
+
interface ScriptedToolCall {
|
|
38
|
+
id: string;
|
|
39
|
+
input: Record<string, unknown>;
|
|
40
|
+
name: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
type ScriptStep = {error: unknown} | {text?: string; toolCalls?: ScriptedToolCall[]};
|
|
44
|
+
|
|
45
|
+
interface GenerateCall {
|
|
46
|
+
prompt: Array<{content: unknown; role: string}>;
|
|
47
|
+
tools?: Array<{name: string}>;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** A LanguageModelV2 mock that answers each request with the next scripted step. */
|
|
51
|
+
const scriptedModel = (modelId: string, steps: ScriptStep[]) => {
|
|
52
|
+
const calls: GenerateCall[] = [];
|
|
53
|
+
const model = {
|
|
54
|
+
doGenerate: mock(async (options: GenerateCall) => {
|
|
55
|
+
calls.push(options);
|
|
56
|
+
const step = steps.shift();
|
|
57
|
+
if (!step) {
|
|
58
|
+
throw new Error(`${modelId} script exhausted`);
|
|
59
|
+
}
|
|
60
|
+
if ("error" in step) {
|
|
61
|
+
throw step.error;
|
|
62
|
+
}
|
|
63
|
+
const toolCalls = step.toolCalls ?? [];
|
|
64
|
+
return {
|
|
65
|
+
content: [
|
|
66
|
+
...(step.text ? [{text: step.text, type: "text" as const}] : []),
|
|
67
|
+
...toolCalls.map((call) => ({
|
|
68
|
+
input: JSON.stringify(call.input),
|
|
69
|
+
toolCallId: call.id,
|
|
70
|
+
toolName: call.name,
|
|
71
|
+
type: "tool-call" as const,
|
|
72
|
+
})),
|
|
73
|
+
],
|
|
74
|
+
finishReason: toolCalls.length > 0 ? ("tool-calls" as const) : ("stop" as const),
|
|
75
|
+
usage: {inputTokens: 100, outputTokens: 20, totalTokens: 120},
|
|
76
|
+
warnings: [],
|
|
77
|
+
};
|
|
78
|
+
}),
|
|
79
|
+
doStream: mock(async (options: GenerateCall) =>
|
|
80
|
+
generateToStream(await model.doGenerate(options))
|
|
81
|
+
),
|
|
82
|
+
modelId,
|
|
83
|
+
provider: "mock",
|
|
84
|
+
specificationVersion: "v2" as const,
|
|
85
|
+
supportedUrls: {},
|
|
86
|
+
};
|
|
87
|
+
return {calls, model};
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
const httpError = (statusCode: number): APICallError =>
|
|
91
|
+
new APICallError({
|
|
92
|
+
isRetryable: statusCode === 429 || statusCode >= 500,
|
|
93
|
+
message: `HTTP ${statusCode}`,
|
|
94
|
+
requestBodyValues: {},
|
|
95
|
+
statusCode,
|
|
96
|
+
url: "https://mock.invalid/v1/chat",
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
/** Plain JSON copy; matchers must never walk mongoose documents (they recurse forever). */
|
|
100
|
+
const plain = (value: unknown): unknown => JSON.parse(JSON.stringify(value));
|
|
101
|
+
|
|
102
|
+
const PRIMARY: HarnessModelRef = {modelId: "primary-model", provider: "mock"};
|
|
103
|
+
const FALLBACK: HarnessModelRef = {modelId: "fallback-model", provider: "mock"};
|
|
104
|
+
|
|
105
|
+
// Each test opens a harness and runs whole turns on a real replica set; the first one also
|
|
106
|
+
// pays for collection and index creation. A timed-out test makes Bun kill mongod.
|
|
107
|
+
setDefaultTimeout(20_000);
|
|
108
|
+
|
|
109
|
+
const TaskModel = registerHarnessTask();
|
|
110
|
+
const ConversationModel = registerHarnessConversation();
|
|
111
|
+
const MessageModel = registerHarnessMessage();
|
|
112
|
+
let SpanModel: ObsSpanModel;
|
|
113
|
+
let TraceModel: ObsTraceModel;
|
|
114
|
+
|
|
115
|
+
const openHarnesses: Harness[] = [];
|
|
116
|
+
|
|
117
|
+
const openHarness = async ({
|
|
118
|
+
beforeCommitEnd,
|
|
119
|
+
env,
|
|
120
|
+
models,
|
|
121
|
+
registry,
|
|
122
|
+
}: {
|
|
123
|
+
beforeCommitEnd?: HarnessTestHooks["beforeCommitEnd"];
|
|
124
|
+
env?: ExecutionEnv;
|
|
125
|
+
models: Record<string, ReturnType<typeof scriptedModel>["model"]>;
|
|
126
|
+
registry: Parameters<typeof Harness.open>[0]["registry"];
|
|
127
|
+
}): Promise<Harness> => {
|
|
128
|
+
const harness = await Harness.open({
|
|
129
|
+
env,
|
|
130
|
+
models: ({modelId}) => {
|
|
131
|
+
const model = models[modelId];
|
|
132
|
+
if (!model) {
|
|
133
|
+
throw new Error(`No mock model ${modelId}`);
|
|
134
|
+
}
|
|
135
|
+
return model as unknown as LanguageModel;
|
|
136
|
+
},
|
|
137
|
+
priceMap: {"primary-model": {inputPerMTok: 3, outputPerMTok: 15}},
|
|
138
|
+
registry,
|
|
139
|
+
runner: new InProcessRunner({pollInterval: {milliseconds: 20}}),
|
|
140
|
+
testHooks: {beforeCommitEnd, random: () => 0},
|
|
141
|
+
});
|
|
142
|
+
openHarnesses.push(harness);
|
|
143
|
+
await harness.start();
|
|
144
|
+
return harness;
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
const lookupTool = defineTool({
|
|
148
|
+
description: "Look up a patient's chart by id",
|
|
149
|
+
execute: async ({patientId}: {patientId: string}, api) => {
|
|
150
|
+
api.output(`fetching ${patientId}`);
|
|
151
|
+
return {chart: `chart-${patientId}`, conversationId: api.conversationId};
|
|
152
|
+
},
|
|
153
|
+
name: "lookup",
|
|
154
|
+
parameters: z.object({patientId: z.string()}),
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
const agentWith = (
|
|
158
|
+
overrides: Partial<Parameters<typeof defineAgent>[0]> = {}
|
|
159
|
+
): HarnessAgentDefinition =>
|
|
160
|
+
defineAgent({
|
|
161
|
+
instructions: "Answer the clinician.",
|
|
162
|
+
model: PRIMARY,
|
|
163
|
+
modelRetry: {backoffMs: 1, maxBackoffMs: 2},
|
|
164
|
+
name: "test.clinician",
|
|
165
|
+
tools: [lookupTool],
|
|
166
|
+
...overrides,
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
const messagesOf = async (
|
|
170
|
+
conversationId: string
|
|
171
|
+
): Promise<Array<{parts: unknown; role: string; seq: number; status?: string}>> => {
|
|
172
|
+
const rows = await MessageModel.find({conversationId}).sort({seq: 1});
|
|
173
|
+
return rows.map(({parts, role, seq, status}) => ({parts, role, seq, status}));
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
const spansOf = (task: HarnessTaskDocument) =>
|
|
177
|
+
SpanModel.find({traceId: task.traceId}).sort({_id: 1, startedAt: 1});
|
|
178
|
+
|
|
179
|
+
describe("Agent turns", () => {
|
|
180
|
+
beforeAll(async () => {
|
|
181
|
+
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
182
|
+
// cold first import can exceed it, so load it here first.
|
|
183
|
+
await import("../langfuseClient");
|
|
184
|
+
createLocalObservabilityPlugin();
|
|
185
|
+
SpanModel = registerObsSpan();
|
|
186
|
+
TraceModel = registerObsTrace();
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
beforeEach(async () => {
|
|
190
|
+
await Promise.all([
|
|
191
|
+
TaskModel.deleteMany({}),
|
|
192
|
+
ConversationModel.deleteMany({}),
|
|
193
|
+
MessageModel.deleteMany({}),
|
|
194
|
+
SpanModel.deleteMany({}),
|
|
195
|
+
TraceModel.deleteMany({}),
|
|
196
|
+
]);
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
afterEach(async () => {
|
|
200
|
+
await Promise.all(openHarnesses.splice(0).map((harness) => harness.stop()));
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it("answers a single text turn and records the transcript and an LLM span with cost", async () => {
|
|
204
|
+
const primary = scriptedModel("primary-model", [{text: "The chart looks normal."}]);
|
|
205
|
+
const agent = agentWith({tools: []});
|
|
206
|
+
const harness = await openHarness({
|
|
207
|
+
models: {"primary-model": primary.model},
|
|
208
|
+
registry: [agent],
|
|
209
|
+
});
|
|
210
|
+
const userId = new mongoose.Types.ObjectId();
|
|
211
|
+
|
|
212
|
+
const conversation = await harness.createConversation({agent, userId});
|
|
213
|
+
expect(plain(conversation.document.agent)).toMatchObject({
|
|
214
|
+
instructions: "Answer the clinician.",
|
|
215
|
+
maxSteps: 10,
|
|
216
|
+
model: PRIMARY,
|
|
217
|
+
name: "test.clinician",
|
|
218
|
+
tools: [],
|
|
219
|
+
});
|
|
220
|
+
const turn = await conversation.submit({content: "How is the chart?", requestId: "r1"});
|
|
221
|
+
expect(turn.name).toBe(AGENT_TURN_TASK_NAME);
|
|
222
|
+
expect(plain(turn.ownership)).toEqual({id: conversation.id, kind: "conversation"});
|
|
223
|
+
expect(String(turn.userId)).toBe(String(userId));
|
|
224
|
+
|
|
225
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
226
|
+
expect(done.status).toBe("completed");
|
|
227
|
+
expect(done.outcome?.result).toEqual({
|
|
228
|
+
finishReason: "stop",
|
|
229
|
+
steps: 1,
|
|
230
|
+
text: "The chart looks normal.",
|
|
231
|
+
});
|
|
232
|
+
expect(await messagesOf(conversation.id)).toEqual([
|
|
233
|
+
{parts: [{text: "How is the chart?", type: "text"}], role: "user", seq: 1, status: undefined},
|
|
234
|
+
{
|
|
235
|
+
parts: [{text: "The chart looks normal.", type: "text"}],
|
|
236
|
+
role: "assistant",
|
|
237
|
+
seq: 2,
|
|
238
|
+
status: undefined,
|
|
239
|
+
},
|
|
240
|
+
]);
|
|
241
|
+
// The model saw the system prompt and the user's message.
|
|
242
|
+
expect(primary.calls[0]?.prompt).toEqual([
|
|
243
|
+
{content: "Answer the clinician.", role: "system"},
|
|
244
|
+
{content: [{text: "How is the chart?", type: "text"}], role: "user"},
|
|
245
|
+
]);
|
|
246
|
+
|
|
247
|
+
const reloaded = await harness.conversation(conversation.id);
|
|
248
|
+
expect(reloaded.document.status).toBe("idle");
|
|
249
|
+
expect(reloaded.document.activeTurnTaskId).toBeUndefined();
|
|
250
|
+
expect(reloaded.document.seq).toBe(2);
|
|
251
|
+
|
|
252
|
+
const llm = await SpanModel.findExactlyOne({kind: "LLM", traceId: done.traceId});
|
|
253
|
+
expect(llm.name).toBe("mock/primary-model");
|
|
254
|
+
expect(String(llm.parentSpanId)).toBe(String(done.rootSpanId));
|
|
255
|
+
expect(llm.status).toBe("ok");
|
|
256
|
+
expect(llm.usage).toEqual({
|
|
257
|
+
costUsd: (3 * 100 + 15 * 20) / 1_000_000,
|
|
258
|
+
inputTokens: 100,
|
|
259
|
+
model: "primary-model",
|
|
260
|
+
outputTokens: 20,
|
|
261
|
+
});
|
|
262
|
+
expect(llm.output).toMatchObject({
|
|
263
|
+
attempts: [{attempt: 1, modelId: "primary-model", provider: "mock"}],
|
|
264
|
+
finishReason: "stop",
|
|
265
|
+
text: "The chart looks normal.",
|
|
266
|
+
toolCalls: [],
|
|
267
|
+
});
|
|
268
|
+
const trace = await TraceModel.findExactlyOne({_id: done.traceId});
|
|
269
|
+
expect(trace.endedAt).toBeDefined();
|
|
270
|
+
expect(trace.status).toBe("ok");
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
it("runs a tool call as a child task, feeds its result back, and finishes with the answer", async () => {
|
|
274
|
+
const primary = scriptedModel("primary-model", [
|
|
275
|
+
{toolCalls: [{id: "call-1", input: {patientId: "p7"}, name: "lookup"}]},
|
|
276
|
+
{text: "Chart p7 is fine."},
|
|
277
|
+
]);
|
|
278
|
+
const env: ExecutionEnv = {
|
|
279
|
+
exec: async () => ({exitCode: 0, stderr: "", stdout: ""}),
|
|
280
|
+
read: async () => "",
|
|
281
|
+
write: async () => {},
|
|
282
|
+
};
|
|
283
|
+
const seen: Array<{args: unknown; envMatches: boolean; hasSignal: boolean; taskId: string}> =
|
|
284
|
+
[];
|
|
285
|
+
const spyTool = defineTool({
|
|
286
|
+
description: "Look up a patient's chart by id",
|
|
287
|
+
execute: async (args: {patientId: string}, api) => {
|
|
288
|
+
seen.push({
|
|
289
|
+
args,
|
|
290
|
+
envMatches: api.env === env,
|
|
291
|
+
hasSignal: api.signal instanceof AbortSignal,
|
|
292
|
+
taskId: api.taskId,
|
|
293
|
+
});
|
|
294
|
+
return lookupTool.execute(args, api);
|
|
295
|
+
},
|
|
296
|
+
name: "lookup",
|
|
297
|
+
parameters: z.object({patientId: z.string()}),
|
|
298
|
+
});
|
|
299
|
+
const agent = agentWith({tools: [spyTool]});
|
|
300
|
+
const harness = await openHarness({
|
|
301
|
+
env,
|
|
302
|
+
models: {"primary-model": primary.model},
|
|
303
|
+
registry: [agent],
|
|
304
|
+
});
|
|
305
|
+
const conversation = await harness.createConversation({agent});
|
|
306
|
+
const turn = await conversation.submit({content: "Check p7", requestId: "r1"});
|
|
307
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
308
|
+
|
|
309
|
+
expect(done.status).toBe("completed");
|
|
310
|
+
expect(done.outcome?.result).toEqual({
|
|
311
|
+
finishReason: "stop",
|
|
312
|
+
steps: 2,
|
|
313
|
+
text: "Chart p7 is fine.",
|
|
314
|
+
});
|
|
315
|
+
expect(seen).toHaveLength(1);
|
|
316
|
+
expect(seen[0]?.args).toEqual({patientId: "p7"});
|
|
317
|
+
expect(seen[0]?.envMatches).toBe(true);
|
|
318
|
+
expect(seen[0]?.hasSignal).toBe(true);
|
|
319
|
+
|
|
320
|
+
const toolTask = await TaskModel.findExactlyOne({name: AGENT_TOOL_TASK_NAME});
|
|
321
|
+
expect(String(toolTask._id)).toBe(seen[0]?.taskId);
|
|
322
|
+
expect(plain(toolTask.ownership)).toEqual({id: String(done._id), kind: "task"});
|
|
323
|
+
expect(toolTask.status).toBe("completed");
|
|
324
|
+
|
|
325
|
+
expect(await messagesOf(conversation.id)).toEqual([
|
|
326
|
+
{parts: [{text: "Check p7", type: "text"}], role: "user", seq: 1, status: undefined},
|
|
327
|
+
{
|
|
328
|
+
parts: [
|
|
329
|
+
{input: {patientId: "p7"}, toolCallId: "call-1", toolName: "lookup", type: "tool-call"},
|
|
330
|
+
],
|
|
331
|
+
role: "assistant",
|
|
332
|
+
seq: 2,
|
|
333
|
+
status: undefined,
|
|
334
|
+
},
|
|
335
|
+
{
|
|
336
|
+
parts: [
|
|
337
|
+
{
|
|
338
|
+
isError: false,
|
|
339
|
+
output: {chart: "chart-p7", conversationId: conversation.id},
|
|
340
|
+
toolCallId: "call-1",
|
|
341
|
+
toolName: "lookup",
|
|
342
|
+
type: "tool-result",
|
|
343
|
+
},
|
|
344
|
+
],
|
|
345
|
+
role: "tool",
|
|
346
|
+
seq: 3,
|
|
347
|
+
status: "ok",
|
|
348
|
+
},
|
|
349
|
+
{
|
|
350
|
+
parts: [{text: "Chart p7 is fine.", type: "text"}],
|
|
351
|
+
role: "assistant",
|
|
352
|
+
seq: 4,
|
|
353
|
+
status: undefined,
|
|
354
|
+
},
|
|
355
|
+
]);
|
|
356
|
+
// The second request carried the tool result back to the model.
|
|
357
|
+
expect(primary.calls[1]?.prompt.at(-1)).toMatchObject({
|
|
358
|
+
content: [
|
|
359
|
+
{
|
|
360
|
+
output: {type: "json", value: {chart: "chart-p7", conversationId: conversation.id}},
|
|
361
|
+
toolCallId: "call-1",
|
|
362
|
+
toolName: "lookup",
|
|
363
|
+
type: "tool-result",
|
|
364
|
+
},
|
|
365
|
+
],
|
|
366
|
+
role: "tool",
|
|
367
|
+
});
|
|
368
|
+
expect(primary.calls[0]?.tools?.map(({name}) => name)).toEqual(["lookup"]);
|
|
369
|
+
|
|
370
|
+
// Span tree: CHAIN(turn) -> LLM, TOOL (-> its phase CHAIN), LLM.
|
|
371
|
+
const spans = await spansOf(done);
|
|
372
|
+
const root = spans.find(({_id}) => String(_id) === String(done.rootSpanId));
|
|
373
|
+
expect(root?.kind).toBe("CHAIN");
|
|
374
|
+
expect(root?.name).toBe(`${AGENT_TURN_TASK_NAME}@1`);
|
|
375
|
+
const underRoot = spans.filter(({parentSpanId}) => String(parentSpanId) === String(root?._id));
|
|
376
|
+
expect(underRoot.filter(({kind}) => kind === "LLM")).toHaveLength(2);
|
|
377
|
+
const toolSpan = spans.find(({kind}) => kind === "TOOL");
|
|
378
|
+
expect(toolSpan?.name).toBe("lookup");
|
|
379
|
+
expect(String(toolSpan?.parentSpanId)).toBe(String(root?._id));
|
|
380
|
+
expect(String(toolSpan?._id)).toBe(String(toolTask.rootSpanId));
|
|
381
|
+
expect(toolSpan?.status).toBe("ok");
|
|
382
|
+
expect(toolSpan?.endedAt).toBeDefined();
|
|
383
|
+
expect(toolSpan?.output).toEqual({
|
|
384
|
+
streamedOutput: "fetching p7",
|
|
385
|
+
value: {chart: "chart-p7", conversationId: conversation.id},
|
|
386
|
+
});
|
|
387
|
+
const toolPhase = spans.find(
|
|
388
|
+
({parentSpanId}) => String(parentSpanId) === String(toolSpan?._id)
|
|
389
|
+
);
|
|
390
|
+
expect(toolPhase?.kind).toBe("CHAIN");
|
|
391
|
+
expect(toolPhase?.name).toBe("execute");
|
|
392
|
+
expect(underRoot.filter(({kind, name}) => kind === "CHAIN" && name === "request")).toHaveLength(
|
|
393
|
+
2
|
|
394
|
+
);
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
it("stops after maxSteps model requests, answering the last tool calls first", async () => {
|
|
398
|
+
// The same toolCallId in every step: child keys are scoped to the phase visit.
|
|
399
|
+
const call = (patientId: string): ScriptStep => ({
|
|
400
|
+
toolCalls: [{id: "same-id", input: {patientId}, name: "lookup"}],
|
|
401
|
+
});
|
|
402
|
+
const primary = scriptedModel("primary-model", [call("a"), call("b"), call("c")]);
|
|
403
|
+
const agent = agentWith({maxSteps: 2});
|
|
404
|
+
const harness = await openHarness({
|
|
405
|
+
models: {"primary-model": primary.model},
|
|
406
|
+
registry: [agent],
|
|
407
|
+
});
|
|
408
|
+
const conversation = await harness.createConversation({agent});
|
|
409
|
+
const turn = await conversation.submit({content: "Loop", requestId: "r1"});
|
|
410
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
411
|
+
|
|
412
|
+
expect(done.status).toBe("completed");
|
|
413
|
+
expect(done.outcome?.result).toEqual({finishReason: "max-steps", steps: 2, text: ""});
|
|
414
|
+
expect(primary.model.doGenerate).toHaveBeenCalledTimes(2);
|
|
415
|
+
const roles = (await messagesOf(conversation.id)).map(({role}) => role);
|
|
416
|
+
expect(roles).toEqual(["user", "assistant", "tool", "assistant", "tool"]);
|
|
417
|
+
expect(await TaskModel.countDocuments({name: AGENT_TOOL_TASK_NAME})).toBe(2);
|
|
418
|
+
const outputs = (await messagesOf(conversation.id))
|
|
419
|
+
.filter(({role}) => role === "tool")
|
|
420
|
+
.map(({parts}) => (parts as Array<{output: {chart: string}}>)[0]?.output.chart);
|
|
421
|
+
expect(outputs).toEqual(["chart-a", "chart-b"]);
|
|
422
|
+
expect((await harness.conversation(conversation.id)).document.status).toBe("idle");
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
it("reports failing, invalid, and unknown tool calls to the model as error results", async () => {
|
|
426
|
+
const failing = defineTool({
|
|
427
|
+
description: "Always fails",
|
|
428
|
+
execute: async () => {
|
|
429
|
+
throw new Error("EHR unavailable");
|
|
430
|
+
},
|
|
431
|
+
name: "failing",
|
|
432
|
+
parameters: z.object({}),
|
|
433
|
+
});
|
|
434
|
+
const primary = scriptedModel("primary-model", [
|
|
435
|
+
{
|
|
436
|
+
toolCalls: [
|
|
437
|
+
{id: "c1", input: {}, name: "failing"},
|
|
438
|
+
{id: "c2", input: {patientId: 42}, name: "lookup"},
|
|
439
|
+
{id: "c3", input: {}, name: "missing"},
|
|
440
|
+
],
|
|
441
|
+
},
|
|
442
|
+
{text: "I could not reach the EHR."},
|
|
443
|
+
]);
|
|
444
|
+
const agent = agentWith({tools: [lookupTool, failing]});
|
|
445
|
+
const harness = await openHarness({
|
|
446
|
+
models: {"primary-model": primary.model},
|
|
447
|
+
registry: [agent],
|
|
448
|
+
});
|
|
449
|
+
const conversation = await harness.createConversation({agent});
|
|
450
|
+
const turn = await conversation.submit({content: "Try", requestId: "r1"});
|
|
451
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
452
|
+
|
|
453
|
+
expect(done.status).toBe("completed");
|
|
454
|
+
expect(done.outcome?.result).toMatchObject({text: "I could not reach the EHR."});
|
|
455
|
+
const tools = (await messagesOf(conversation.id)).filter(({role}) => role === "tool");
|
|
456
|
+
expect(tools.map(({status}) => status)).toEqual(["error", "error", "error"]);
|
|
457
|
+
const outputs = tools.map(({parts}) => (parts as Array<{output: string}>)[0]?.output);
|
|
458
|
+
expect(outputs[0]).toBe('Tool "failing" failed: EHR unavailable');
|
|
459
|
+
expect(outputs[1]).toStartWith('Invalid arguments for tool "lookup":');
|
|
460
|
+
expect(outputs[2]).toBe('Unknown tool "missing"');
|
|
461
|
+
expect(primary.calls[1]?.prompt.at(-1)).toMatchObject({
|
|
462
|
+
content: [
|
|
463
|
+
{output: {type: "error-text", value: 'Tool "failing" failed: EHR unavailable'}},
|
|
464
|
+
{output: {type: "error-text"}},
|
|
465
|
+
{output: {type: "error-text", value: 'Unknown tool "missing"'}},
|
|
466
|
+
],
|
|
467
|
+
role: "tool",
|
|
468
|
+
});
|
|
469
|
+
const toolSpans = await SpanModel.find({kind: "TOOL", traceId: done.traceId});
|
|
470
|
+
expect(toolSpans.map(({status}) => status)).toEqual(["error", "error", "error"]);
|
|
471
|
+
});
|
|
472
|
+
|
|
473
|
+
it("parses structured output when the agent declares an output schema", async () => {
|
|
474
|
+
const primary = scriptedModel("primary-model", [
|
|
475
|
+
{text: '```json\n{"summary": "stable", "risk": 2}\n```'},
|
|
476
|
+
{text: "not json at all"},
|
|
477
|
+
]);
|
|
478
|
+
const agent = agentWith({
|
|
479
|
+
name: "test.structured",
|
|
480
|
+
output: z.object({risk: z.number(), summary: z.string()}),
|
|
481
|
+
tools: [],
|
|
482
|
+
});
|
|
483
|
+
const harness = await openHarness({
|
|
484
|
+
models: {"primary-model": primary.model},
|
|
485
|
+
registry: [agent],
|
|
486
|
+
});
|
|
487
|
+
const conversation = await harness.createConversation({agent});
|
|
488
|
+
const first = await harness.waitForTask(
|
|
489
|
+
(await conversation.submit({content: "Summarize", requestId: "r1"}))._id,
|
|
490
|
+
{timeout: {seconds: 10}}
|
|
491
|
+
);
|
|
492
|
+
expect(first.outcome?.result).toMatchObject({output: {risk: 2, summary: "stable"}});
|
|
493
|
+
|
|
494
|
+
const second = await harness.waitForTask(
|
|
495
|
+
(await conversation.submit({content: "Again", requestId: "r2"}))._id,
|
|
496
|
+
{timeout: {seconds: 10}}
|
|
497
|
+
);
|
|
498
|
+
expect(second.status).toBe("failed");
|
|
499
|
+
expect(second.outcome?.error).toStartWith('Agent "test.structured" output is not JSON');
|
|
500
|
+
// The answer and its LLM span are still recorded.
|
|
501
|
+
expect((await messagesOf(conversation.id)).map(({role}) => role)).toEqual([
|
|
502
|
+
"user",
|
|
503
|
+
"assistant",
|
|
504
|
+
"user",
|
|
505
|
+
"assistant",
|
|
506
|
+
]);
|
|
507
|
+
expect(await SpanModel.countDocuments({kind: "LLM", traceId: second.traceId})).toBe(1);
|
|
508
|
+
expect((await harness.conversation(conversation.id)).document.status).toBe("idle");
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
describe("model-call resilience", () => {
|
|
512
|
+
it("retries a 503 twice and succeeds on the third attempt (AC4)", async () => {
|
|
513
|
+
const primary = scriptedModel("primary-model", [
|
|
514
|
+
{error: httpError(503)},
|
|
515
|
+
{error: httpError(503)},
|
|
516
|
+
{text: "Recovered."},
|
|
517
|
+
]);
|
|
518
|
+
const agent = agentWith({tools: []});
|
|
519
|
+
const harness = await openHarness({
|
|
520
|
+
models: {"primary-model": primary.model},
|
|
521
|
+
registry: [agent],
|
|
522
|
+
});
|
|
523
|
+
const conversation = await harness.createConversation({agent});
|
|
524
|
+
const turn = await conversation.submit({content: "Hi", requestId: "r1"});
|
|
525
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
526
|
+
|
|
527
|
+
expect(done.status).toBe("completed");
|
|
528
|
+
expect(done.outcome?.result).toMatchObject({text: "Recovered."});
|
|
529
|
+
expect(primary.model.doGenerate).toHaveBeenCalledTimes(3);
|
|
530
|
+
const llm = await SpanModel.findExactlyOne({kind: "LLM", traceId: done.traceId});
|
|
531
|
+
expect(llm.name).toBe("mock/primary-model");
|
|
532
|
+
expect((llm.output as {attempts: unknown[]}).attempts).toEqual([
|
|
533
|
+
{
|
|
534
|
+
attempt: 1,
|
|
535
|
+
error: "HTTP 503",
|
|
536
|
+
modelId: "primary-model",
|
|
537
|
+
provider: "mock",
|
|
538
|
+
retryable: true,
|
|
539
|
+
statusCode: 503,
|
|
540
|
+
},
|
|
541
|
+
{
|
|
542
|
+
attempt: 2,
|
|
543
|
+
error: "HTTP 503",
|
|
544
|
+
modelId: "primary-model",
|
|
545
|
+
provider: "mock",
|
|
546
|
+
retryable: true,
|
|
547
|
+
statusCode: 503,
|
|
548
|
+
},
|
|
549
|
+
{attempt: 3, modelId: "primary-model", provider: "mock"},
|
|
550
|
+
]);
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
it("falls back to the next model after three 503s (AC4)", async () => {
|
|
554
|
+
const primary = scriptedModel("primary-model", [
|
|
555
|
+
{error: httpError(503)},
|
|
556
|
+
{error: httpError(503)},
|
|
557
|
+
{error: httpError(503)},
|
|
558
|
+
]);
|
|
559
|
+
const fallback = scriptedModel("fallback-model", [{text: "From the fallback."}]);
|
|
560
|
+
const agent = agentWith({fallbackModels: [FALLBACK], tools: []});
|
|
561
|
+
const harness = await openHarness({
|
|
562
|
+
models: {"fallback-model": fallback.model, "primary-model": primary.model},
|
|
563
|
+
registry: [agent],
|
|
564
|
+
});
|
|
565
|
+
const conversation = await harness.createConversation({agent});
|
|
566
|
+
const turn = await conversation.submit({content: "Hi", requestId: "r1"});
|
|
567
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
568
|
+
|
|
569
|
+
expect(done.status).toBe("completed");
|
|
570
|
+
expect(done.outcome?.result).toMatchObject({text: "From the fallback."});
|
|
571
|
+
expect(primary.model.doGenerate).toHaveBeenCalledTimes(3);
|
|
572
|
+
expect(fallback.model.doGenerate).toHaveBeenCalledTimes(1);
|
|
573
|
+
const llm = await SpanModel.findExactlyOne({kind: "LLM", traceId: done.traceId});
|
|
574
|
+
expect(llm.name).toBe("mock/fallback-model");
|
|
575
|
+
// No price for the fallback model, so no cost.
|
|
576
|
+
expect(llm.usage).toEqual({inputTokens: 100, model: "fallback-model", outputTokens: 20});
|
|
577
|
+
expect(
|
|
578
|
+
(llm.output as {attempts: Array<{modelId: string}>}).attempts.map(({modelId}) => modelId)
|
|
579
|
+
).toEqual(["primary-model", "primary-model", "primary-model", "fallback-model"]);
|
|
580
|
+
});
|
|
581
|
+
|
|
582
|
+
it("fails the turn at once on a 400 without retrying or falling back (AC4)", async () => {
|
|
583
|
+
const primary = scriptedModel("primary-model", [{error: httpError(400)}]);
|
|
584
|
+
const fallback = scriptedModel("fallback-model", [{text: "never"}]);
|
|
585
|
+
const agent = agentWith({fallbackModels: [FALLBACK], tools: []});
|
|
586
|
+
const harness = await openHarness({
|
|
587
|
+
models: {"fallback-model": fallback.model, "primary-model": primary.model},
|
|
588
|
+
registry: [agent],
|
|
589
|
+
});
|
|
590
|
+
const conversation = await harness.createConversation({agent});
|
|
591
|
+
const turn = await conversation.submit({content: "Hi", requestId: "r1"});
|
|
592
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
593
|
+
|
|
594
|
+
expect(done.status).toBe("failed");
|
|
595
|
+
expect(done.outcome?.error).toBe(
|
|
596
|
+
"Model mock/primary-model failed with a non-retryable error: HTTP 400"
|
|
597
|
+
);
|
|
598
|
+
expect(primary.model.doGenerate).toHaveBeenCalledTimes(1);
|
|
599
|
+
expect(fallback.model.doGenerate).toHaveBeenCalledTimes(0);
|
|
600
|
+
// The failed attempt is still audited, on an error LLM span.
|
|
601
|
+
const llm = await SpanModel.findExactlyOne({kind: "LLM", traceId: done.traceId});
|
|
602
|
+
expect(llm.status).toBe("error");
|
|
603
|
+
expect(llm.name).toBe("mock/primary-model");
|
|
604
|
+
expect(llm.error).toBe(done.outcome?.error);
|
|
605
|
+
expect((llm.output as {attempts: Array<{statusCode: number}>}).attempts).toMatchObject([
|
|
606
|
+
{retryable: false, statusCode: 400},
|
|
607
|
+
]);
|
|
608
|
+
expect((await harness.conversation(conversation.id)).document.status).toBe("idle");
|
|
609
|
+
expect((await messagesOf(conversation.id)).map(({role}) => role)).toEqual(["user"]);
|
|
610
|
+
});
|
|
611
|
+
|
|
612
|
+
it("retries a dropped connection like a 5xx", async () => {
|
|
613
|
+
const primary = scriptedModel("primary-model", [
|
|
614
|
+
{error: new TypeError("fetch failed")},
|
|
615
|
+
{error: Object.assign(new Error("socket hang up"), {code: "ECONNRESET"})},
|
|
616
|
+
{text: "Back online."},
|
|
617
|
+
]);
|
|
618
|
+
const agent = agentWith({tools: []});
|
|
619
|
+
const harness = await openHarness({
|
|
620
|
+
models: {"primary-model": primary.model},
|
|
621
|
+
registry: [agent],
|
|
622
|
+
});
|
|
623
|
+
const conversation = await harness.createConversation({agent});
|
|
624
|
+
const turn = await conversation.submit({content: "Hi", requestId: "r1"});
|
|
625
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
626
|
+
expect(done.status).toBe("completed");
|
|
627
|
+
expect(primary.model.doGenerate).toHaveBeenCalledTimes(3);
|
|
628
|
+
const llm = await SpanModel.findExactlyOne({kind: "LLM", traceId: done.traceId});
|
|
629
|
+
expect(
|
|
630
|
+
(llm.output as {attempts: Array<{retryable?: boolean}>}).attempts.map(
|
|
631
|
+
({retryable}) => retryable
|
|
632
|
+
)
|
|
633
|
+
).toEqual([true, true, undefined]);
|
|
634
|
+
});
|
|
635
|
+
|
|
636
|
+
it("fails the turn once every model exhausts its retries", async () => {
|
|
637
|
+
const primary = scriptedModel("primary-model", [
|
|
638
|
+
{error: httpError(429)},
|
|
639
|
+
{error: httpError(500)},
|
|
640
|
+
]);
|
|
641
|
+
const agent = agentWith({modelRetry: {backoffMs: 0, maxAttempts: 2}, tools: []});
|
|
642
|
+
const harness = await openHarness({
|
|
643
|
+
models: {"primary-model": primary.model},
|
|
644
|
+
registry: [agent],
|
|
645
|
+
});
|
|
646
|
+
const conversation = await harness.createConversation({agent});
|
|
647
|
+
const turn = await conversation.submit({content: "Hi", requestId: "r1"});
|
|
648
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
649
|
+
|
|
650
|
+
expect(done.status).toBe("failed");
|
|
651
|
+
expect(done.outcome?.error).toBe(
|
|
652
|
+
"Model call failed after 2 attempts across mock/primary-model: HTTP 500"
|
|
653
|
+
);
|
|
654
|
+
const llm = await SpanModel.findExactlyOne({kind: "LLM", traceId: done.traceId});
|
|
655
|
+
expect(
|
|
656
|
+
(llm.output as {attempts: Array<{statusCode: number}>}).attempts.map(
|
|
657
|
+
({statusCode}) => statusCode
|
|
658
|
+
)
|
|
659
|
+
).toEqual([429, 500]);
|
|
660
|
+
expect((await harness.conversation(conversation.id)).document.status).toBe("idle");
|
|
661
|
+
});
|
|
662
|
+
|
|
663
|
+
it("fails without calling anything when the models resolver throws", async () => {
|
|
664
|
+
const agent = agentWith({tools: []});
|
|
665
|
+
const harness = await openHarness({models: {}, registry: [agent]});
|
|
666
|
+
const conversation = await harness.createConversation({agent});
|
|
667
|
+
const turn = await conversation.submit({content: "Hi", requestId: "r1"});
|
|
668
|
+
const done = await harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
669
|
+
expect(done.status).toBe("failed");
|
|
670
|
+
expect(done.outcome?.error).toBe(
|
|
671
|
+
"Model mock/primary-model could not be resolved: No mock model primary-model"
|
|
672
|
+
);
|
|
673
|
+
});
|
|
674
|
+
});
|
|
675
|
+
|
|
676
|
+
describe("conversations", () => {
|
|
677
|
+
it("is idempotent on requestId, rejects a submit while busy, and keeps seq increasing", async () => {
|
|
678
|
+
let releaseTool: () => void = () => {};
|
|
679
|
+
const toolGate = new Promise<void>((resolve) => {
|
|
680
|
+
releaseTool = resolve;
|
|
681
|
+
});
|
|
682
|
+
let toolStarted: () => void = () => {};
|
|
683
|
+
const started = new Promise<void>((resolve) => {
|
|
684
|
+
toolStarted = resolve;
|
|
685
|
+
});
|
|
686
|
+
const slow = defineTool({
|
|
687
|
+
description: "Waits for the test",
|
|
688
|
+
execute: async () => {
|
|
689
|
+
toolStarted();
|
|
690
|
+
await toolGate;
|
|
691
|
+
return "done";
|
|
692
|
+
},
|
|
693
|
+
name: "slow",
|
|
694
|
+
parameters: z.object({}),
|
|
695
|
+
replay: "safe",
|
|
696
|
+
});
|
|
697
|
+
const primary = scriptedModel("primary-model", [
|
|
698
|
+
{toolCalls: [{id: "s1", input: {}, name: "slow"}]},
|
|
699
|
+
{text: "First answer."},
|
|
700
|
+
{text: "Second answer."},
|
|
701
|
+
]);
|
|
702
|
+
const agent = agentWith({tools: [slow]});
|
|
703
|
+
const harness = await openHarness({
|
|
704
|
+
models: {"primary-model": primary.model},
|
|
705
|
+
registry: [agent],
|
|
706
|
+
});
|
|
707
|
+
const conversation = await harness.createConversation({agent});
|
|
708
|
+
|
|
709
|
+
const first = await conversation.submit({content: "One", requestId: "r1"});
|
|
710
|
+
const repeated = await conversation.submit({content: "One", requestId: "r1"});
|
|
711
|
+
expect(String(repeated._id)).toBe(String(first._id));
|
|
712
|
+
await started;
|
|
713
|
+
|
|
714
|
+
const busy = await conversation
|
|
715
|
+
.submit({content: "Two", requestId: "r2"})
|
|
716
|
+
.catch((error: unknown) => error);
|
|
717
|
+
expect(busy).toBeInstanceOf(HarnessConversationBusyError);
|
|
718
|
+
expect((busy as HarnessConversationBusyError).activeTurnTaskId).toBe(String(first._id));
|
|
719
|
+
expect((await harness.conversation(conversation.id)).document.status).toBe("busy");
|
|
720
|
+
|
|
721
|
+
releaseTool();
|
|
722
|
+
await harness.waitForTask(first._id, {timeout: {seconds: 10}});
|
|
723
|
+
const second = await conversation.submit({content: "Two", requestId: "r2"});
|
|
724
|
+
const done = await harness.waitForTask(second._id, {timeout: {seconds: 10}});
|
|
725
|
+
expect(done.outcome?.result).toMatchObject({text: "Second answer."});
|
|
726
|
+
|
|
727
|
+
const messages = await messagesOf(conversation.id);
|
|
728
|
+
expect(messages.map(({seq}) => seq)).toEqual([1, 2, 3, 4, 5, 6]);
|
|
729
|
+
expect(messages.map(({role}) => role)).toEqual([
|
|
730
|
+
"user",
|
|
731
|
+
"assistant",
|
|
732
|
+
"tool",
|
|
733
|
+
"assistant",
|
|
734
|
+
"user",
|
|
735
|
+
"assistant",
|
|
736
|
+
]);
|
|
737
|
+
// The second turn's request carried the whole transcript.
|
|
738
|
+
expect(primary.calls[2]?.prompt.map(({role}) => role)).toEqual([
|
|
739
|
+
"system",
|
|
740
|
+
"user",
|
|
741
|
+
"assistant",
|
|
742
|
+
"tool",
|
|
743
|
+
"assistant",
|
|
744
|
+
"user",
|
|
745
|
+
]);
|
|
746
|
+
});
|
|
747
|
+
|
|
748
|
+
it("rejects a racing submit with the running turn's id while that turn is open", async () => {
|
|
749
|
+
let releaseModel: () => void = () => {};
|
|
750
|
+
const modelGate = new Promise<void>((resolve) => {
|
|
751
|
+
releaseModel = resolve;
|
|
752
|
+
});
|
|
753
|
+
const primary = scriptedModel("primary-model", [{text: "held"}]);
|
|
754
|
+
const answer = primary.model.doGenerate.getMockImplementation();
|
|
755
|
+
primary.model.doGenerate.mockImplementation(async (options: GenerateCall) => {
|
|
756
|
+
await modelGate;
|
|
757
|
+
return (answer as (options: GenerateCall) => ReturnType<typeof primary.model.doGenerate>)(
|
|
758
|
+
options
|
|
759
|
+
);
|
|
760
|
+
});
|
|
761
|
+
const agent = agentWith({tools: []});
|
|
762
|
+
const harness = await openHarness({
|
|
763
|
+
models: {"primary-model": primary.model},
|
|
764
|
+
registry: [agent],
|
|
765
|
+
});
|
|
766
|
+
const conversation = await harness.createConversation({agent});
|
|
767
|
+
const settled = await Promise.allSettled([
|
|
768
|
+
conversation.submit({content: "One", requestId: "r1"}),
|
|
769
|
+
conversation.submit({content: "Two", requestId: "r2"}),
|
|
770
|
+
]);
|
|
771
|
+
const won = settled.filter((result) => result.status === "fulfilled");
|
|
772
|
+
const lost = settled.filter((result) => result.status === "rejected");
|
|
773
|
+
expect(won).toHaveLength(1);
|
|
774
|
+
expect(lost).toHaveLength(1);
|
|
775
|
+
const winner = (won[0] as PromiseFulfilledResult<HarnessTaskDocument>).value;
|
|
776
|
+
const error = (lost[0] as PromiseRejectedResult).reason;
|
|
777
|
+
expect(error).toBeInstanceOf(HarnessConversationBusyError);
|
|
778
|
+
expect((error as HarnessConversationBusyError).activeTurnTaskId).toBe(String(winner._id));
|
|
779
|
+
|
|
780
|
+
releaseModel();
|
|
781
|
+
await harness.waitForTask(winner._id, {timeout: {seconds: 10}});
|
|
782
|
+
expect((await messagesOf(conversation.id)).map(({role}) => role)).toEqual([
|
|
783
|
+
"user",
|
|
784
|
+
"assistant",
|
|
785
|
+
]);
|
|
786
|
+
expect(await TaskModel.countDocuments({name: AGENT_TURN_TASK_NAME})).toBe(1);
|
|
787
|
+
});
|
|
788
|
+
|
|
789
|
+
it("lets a racing submit proceed once the winning turn has already finished", async () => {
|
|
790
|
+
// An instant model: the loser may find the conversation busy (and must name the
|
|
791
|
+
// running turn) or idle again (and must start its own turn). Repeat to hit both.
|
|
792
|
+
for (let round = 0; round < 5; round++) {
|
|
793
|
+
const primary = scriptedModel("primary-model", [{text: "a"}, {text: "b"}]);
|
|
794
|
+
const agent = agentWith({name: `test.race${round}`, tools: []});
|
|
795
|
+
const harness = await openHarness({
|
|
796
|
+
models: {"primary-model": primary.model},
|
|
797
|
+
registry: [agent],
|
|
798
|
+
});
|
|
799
|
+
const conversation = await harness.createConversation({agent});
|
|
800
|
+
const settled = await Promise.allSettled([
|
|
801
|
+
conversation.submit({content: "One", requestId: "r1"}),
|
|
802
|
+
conversation.submit({content: "Two", requestId: "r2"}),
|
|
803
|
+
]);
|
|
804
|
+
const turns = settled
|
|
805
|
+
.filter((result) => result.status === "fulfilled")
|
|
806
|
+
.map((result) => (result as PromiseFulfilledResult<HarnessTaskDocument>).value);
|
|
807
|
+
for (const result of settled) {
|
|
808
|
+
if (result.status === "rejected") {
|
|
809
|
+
expect(result.reason).toBeInstanceOf(HarnessConversationBusyError);
|
|
810
|
+
expect((result.reason as HarnessConversationBusyError).activeTurnTaskId).toBe(
|
|
811
|
+
String(turns[0]?._id)
|
|
812
|
+
);
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
for (const turn of turns) {
|
|
816
|
+
expect((await harness.waitForTask(turn._id, {timeout: {seconds: 10}})).status).toBe(
|
|
817
|
+
"completed"
|
|
818
|
+
);
|
|
819
|
+
}
|
|
820
|
+
const messages = await messagesOf(conversation.id);
|
|
821
|
+
expect(messages.map(({seq}) => seq)).toEqual(
|
|
822
|
+
Array.from({length: turns.length * 2}, (_value, index) => index + 1)
|
|
823
|
+
);
|
|
824
|
+
expect(messages.map(({role}) => role)).toEqual(turns.flatMap(() => ["user", "assistant"]));
|
|
825
|
+
await harness.stop();
|
|
826
|
+
await Promise.all([TaskModel.deleteMany({}), MessageModel.deleteMany({})]);
|
|
827
|
+
}
|
|
828
|
+
});
|
|
829
|
+
|
|
830
|
+
it("returns the same turn to two concurrent submits with one requestId", async () => {
|
|
831
|
+
const primary = scriptedModel("primary-model", [{text: "only once"}]);
|
|
832
|
+
const agent = agentWith({tools: []});
|
|
833
|
+
const harness = await openHarness({
|
|
834
|
+
models: {"primary-model": primary.model},
|
|
835
|
+
registry: [agent],
|
|
836
|
+
});
|
|
837
|
+
const conversation = await harness.createConversation({agent});
|
|
838
|
+
const [first, second] = await Promise.all([
|
|
839
|
+
conversation.submit({content: "One", requestId: "r1"}),
|
|
840
|
+
conversation.submit({content: "One", requestId: "r1"}),
|
|
841
|
+
]);
|
|
842
|
+
expect(String(second._id)).toBe(String(first._id));
|
|
843
|
+
await harness.waitForTask(first._id, {timeout: {seconds: 10}});
|
|
844
|
+
expect((await messagesOf(conversation.id)).map(({seq}) => seq)).toEqual([1, 2]);
|
|
845
|
+
});
|
|
846
|
+
|
|
847
|
+
it("writes the transcript, LLM span, and checkpoint atomically", async () => {
|
|
848
|
+
const primary = scriptedModel("primary-model", [{text: "never stored"}]);
|
|
849
|
+
const agent = agentWith({tools: []});
|
|
850
|
+
let failedCommits = 0;
|
|
851
|
+
const harness = await openHarness({
|
|
852
|
+
beforeCommitEnd: ({phase}) => {
|
|
853
|
+
if (phase === "request") {
|
|
854
|
+
failedCommits += 1;
|
|
855
|
+
throw new Error("injected commit failure");
|
|
856
|
+
}
|
|
857
|
+
},
|
|
858
|
+
models: {"primary-model": primary.model},
|
|
859
|
+
registry: [agent],
|
|
860
|
+
});
|
|
861
|
+
const conversation = await harness.createConversation({agent});
|
|
862
|
+
const turn = await conversation.submit({content: "Hi", requestId: "r1"});
|
|
863
|
+
for (let i = 0; i < 300 && failedCommits === 0; i++) {
|
|
864
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
865
|
+
}
|
|
866
|
+
await harness.stop();
|
|
867
|
+
expect(failedCommits).toBeGreaterThan(0);
|
|
868
|
+
|
|
869
|
+
const row = await TaskModel.findExactlyOne({_id: turn._id});
|
|
870
|
+
expect(row.status).toBe("running");
|
|
871
|
+
expect(row.phase).toBe("request");
|
|
872
|
+
expect((await messagesOf(conversation.id)).map(({role}) => role)).toEqual(["user"]);
|
|
873
|
+
expect((await harness.conversation(conversation.id)).document.seq).toBe(1);
|
|
874
|
+
expect(await SpanModel.countDocuments({kind: "LLM", traceId: turn.traceId})).toBe(0);
|
|
875
|
+
});
|
|
876
|
+
|
|
877
|
+
it("frees a conversation whose turn finished without releasing it", async () => {
|
|
878
|
+
const primary = scriptedModel("primary-model", [{text: "a"}, {text: "b"}]);
|
|
879
|
+
const agent = agentWith({tools: []});
|
|
880
|
+
const harness = await openHarness({
|
|
881
|
+
models: {"primary-model": primary.model},
|
|
882
|
+
registry: [agent],
|
|
883
|
+
});
|
|
884
|
+
const conversation = await harness.createConversation({agent});
|
|
885
|
+
const first = await conversation.submit({content: "One", requestId: "r1"});
|
|
886
|
+
await harness.waitForTask(first._id, {timeout: {seconds: 10}});
|
|
887
|
+
// As if the process died between the turn's terminal commit and the release.
|
|
888
|
+
await ConversationModel.updateOne(
|
|
889
|
+
{_id: conversation.document._id},
|
|
890
|
+
{$set: {activeTurnTaskId: first._id, status: "busy"}}
|
|
891
|
+
);
|
|
892
|
+
const second = await conversation.submit({content: "Two", requestId: "r2"});
|
|
893
|
+
expect((await harness.waitForTask(second._id, {timeout: {seconds: 10}})).status).toBe(
|
|
894
|
+
"completed"
|
|
895
|
+
);
|
|
896
|
+
});
|
|
897
|
+
|
|
898
|
+
it("aborting a turn aborts its running tool and frees the conversation", async () => {
|
|
899
|
+
let toolStarted: () => void = () => {};
|
|
900
|
+
const started = new Promise<void>((resolve) => {
|
|
901
|
+
toolStarted = resolve;
|
|
902
|
+
});
|
|
903
|
+
const hanging = defineTool({
|
|
904
|
+
description: "Never finishes on its own",
|
|
905
|
+
execute: async (_args, api) => {
|
|
906
|
+
toolStarted();
|
|
907
|
+
await new Promise((_resolve, reject) => {
|
|
908
|
+
api.signal.addEventListener("abort", () => reject(api.signal.reason));
|
|
909
|
+
});
|
|
910
|
+
},
|
|
911
|
+
name: "hanging",
|
|
912
|
+
parameters: z.object({}),
|
|
913
|
+
});
|
|
914
|
+
const primary = scriptedModel("primary-model", [
|
|
915
|
+
{toolCalls: [{id: "h1", input: {}, name: "hanging"}]},
|
|
916
|
+
{text: "Cancelled"},
|
|
917
|
+
]);
|
|
918
|
+
const agent = agentWith({tools: [hanging]});
|
|
919
|
+
const harness = await openHarness({
|
|
920
|
+
models: {"primary-model": primary.model},
|
|
921
|
+
registry: [agent],
|
|
922
|
+
});
|
|
923
|
+
const conversation = await harness.createConversation({agent});
|
|
924
|
+
const turn = await conversation.submit({content: "Go", requestId: "r1"});
|
|
925
|
+
await started;
|
|
926
|
+
|
|
927
|
+
const aborted = await harness.abort(turn._id, {reason: "Clinician cancelled"});
|
|
928
|
+
expect(aborted.status).toBe("aborted");
|
|
929
|
+
const tool = await TaskModel.findExactlyOne({name: AGENT_TOOL_TASK_NAME});
|
|
930
|
+
expect(tool.status).toBe("aborted");
|
|
931
|
+
const freed = await harness.conversation(conversation.id);
|
|
932
|
+
expect(freed.document.status).toBe("idle");
|
|
933
|
+
expect(freed.document.activeTurnTaskId).toBeUndefined();
|
|
934
|
+
// An aborted tool writes no result. The assistant message that asked for it is
|
|
935
|
+
// marked aborted so the next request does not replay an unanswered tool call.
|
|
936
|
+
const stored = await messagesOf(conversation.id);
|
|
937
|
+
expect(stored.map(({role}) => role)).toEqual(["user", "assistant"]);
|
|
938
|
+
const assistant = await MessageModel.findExactlyOne({
|
|
939
|
+
conversationId: conversation.id,
|
|
940
|
+
role: "assistant",
|
|
941
|
+
});
|
|
942
|
+
expect(assistant.aborted).toBe(true);
|
|
943
|
+
|
|
944
|
+
const again = await conversation.submit({content: "Again", requestId: "r2"});
|
|
945
|
+
expect((await harness.waitForTask(again._id, {timeout: {seconds: 10}})).status).toBe(
|
|
946
|
+
"completed"
|
|
947
|
+
);
|
|
948
|
+
const replay = JSON.stringify(primary.calls[1]?.prompt);
|
|
949
|
+
expect(replay).toContain("Again");
|
|
950
|
+
expect(replay).not.toContain("h1");
|
|
951
|
+
expect(replay).not.toContain("tool-call");
|
|
952
|
+
});
|
|
953
|
+
|
|
954
|
+
it("validates submit input and conversation lookups", async () => {
|
|
955
|
+
const primary = scriptedModel("primary-model", []);
|
|
956
|
+
const agent = agentWith({tools: []});
|
|
957
|
+
const stranger = agentWith({name: "test.stranger"});
|
|
958
|
+
const harness = await openHarness({
|
|
959
|
+
models: {"primary-model": primary.model},
|
|
960
|
+
registry: [agent],
|
|
961
|
+
});
|
|
962
|
+
const conversation = await harness.createConversation({agent});
|
|
963
|
+
|
|
964
|
+
await expect(conversation.submit({content: " ", requestId: "r1"})).rejects.toThrow(
|
|
965
|
+
harnessErrorMatching("invalidRequest", "submit requires non-empty content")
|
|
966
|
+
);
|
|
967
|
+
await expect(conversation.submit({content: "Hi", requestId: ""})).rejects.toThrow(
|
|
968
|
+
harnessErrorMatching("invalidRequest", "submit requires a requestId")
|
|
969
|
+
);
|
|
970
|
+
await expect(harness.createConversation({agent: stranger})).rejects.toThrow(
|
|
971
|
+
harnessErrorMatching(
|
|
972
|
+
"notRegistered",
|
|
973
|
+
'Agent "test.stranger" is not in this harness registry'
|
|
974
|
+
)
|
|
975
|
+
);
|
|
976
|
+
await expect(harness.conversation(new mongoose.Types.ObjectId())).rejects.toMatchObject({
|
|
977
|
+
status: 404,
|
|
978
|
+
});
|
|
979
|
+
expect(await conversation.messages()).toEqual([]);
|
|
980
|
+
});
|
|
981
|
+
});
|
|
982
|
+
|
|
983
|
+
describe("Harness.open", () => {
|
|
984
|
+
it("requires a models resolver when the registry lists agents", async () => {
|
|
985
|
+
await expect(Harness.open({registry: [agentWith()]})).rejects.toThrow(
|
|
986
|
+
harnessErrorMatching(
|
|
987
|
+
"configInvalid",
|
|
988
|
+
"Harness.open: the registry lists agents; pass `models` to resolve them"
|
|
989
|
+
)
|
|
990
|
+
);
|
|
991
|
+
});
|
|
992
|
+
|
|
993
|
+
it("rejects an agent name listed twice", async () => {
|
|
994
|
+
await expect(
|
|
995
|
+
Harness.open({
|
|
996
|
+
models: () => {
|
|
997
|
+
throw new Error("unused");
|
|
998
|
+
},
|
|
999
|
+
registry: [agentWith(), agentWith()],
|
|
1000
|
+
})
|
|
1001
|
+
).rejects.toThrow(
|
|
1002
|
+
harnessErrorMatching(
|
|
1003
|
+
"configInvalid",
|
|
1004
|
+
'Harness registry lists agent "test.clinician" more than once'
|
|
1005
|
+
)
|
|
1006
|
+
);
|
|
1007
|
+
});
|
|
1008
|
+
|
|
1009
|
+
it("hands the execution env to task phases as rt.env", async () => {
|
|
1010
|
+
const env: ExecutionEnv = {
|
|
1011
|
+
exec: async () => ({exitCode: 0, stderr: "", stdout: "ok"}),
|
|
1012
|
+
read: async (path) => `contents of ${path}`,
|
|
1013
|
+
write: async () => {},
|
|
1014
|
+
};
|
|
1015
|
+
const reader = defineTask<unknown, unknown, string>({
|
|
1016
|
+
initial: () => ({phase: "read"}),
|
|
1017
|
+
name: "test.envReader",
|
|
1018
|
+
phases: {
|
|
1019
|
+
read: {
|
|
1020
|
+
run: async (_task, rt) => {
|
|
1021
|
+
const text = (await rt.env?.read("notes.txt")) ?? "no env";
|
|
1022
|
+
await rt.commit({terminal: {result: text, status: "completed"}});
|
|
1023
|
+
},
|
|
1024
|
+
},
|
|
1025
|
+
},
|
|
1026
|
+
version: 1,
|
|
1027
|
+
});
|
|
1028
|
+
const harness = await openHarness({env, models: {}, registry: [reader]});
|
|
1029
|
+
const task = await harness.createTask(reader, {});
|
|
1030
|
+
const done = await harness.waitForTask(task._id, {timeout: {seconds: 10}});
|
|
1031
|
+
expect(done.outcome?.result).toBe("contents of notes.txt");
|
|
1032
|
+
});
|
|
1033
|
+
});
|
|
1034
|
+
});
|