@terreno/ai 57.9.0 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +133 -0
- package/dist/harness/errors.js +107 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +522 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +191 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +64 -0
- package/dist/harness/runners/inProcessRunner.js +227 -0
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +52 -0
- package/dist/harness/runtime.js +604 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +952 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +28 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +125 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +769 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +778 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +206 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +907 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +530 -0
- package/src/harness/runners/inProcessRunner.ts +260 -0
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +802 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1119 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,618 @@
|
|
|
1
|
+
import {logger} from "@terreno/api";
|
|
2
|
+
import {Duration} from "luxon";
|
|
3
|
+
import type mongoose from "mongoose";
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
HarnessAgentDefinition,
|
|
7
|
+
HarnessChildOutcome,
|
|
8
|
+
HarnessExtensionDefinition,
|
|
9
|
+
HarnessTaskDefinition,
|
|
10
|
+
HarnessTaskDocument,
|
|
11
|
+
HarnessTaskView,
|
|
12
|
+
HarnessTerminalStatus,
|
|
13
|
+
HarnessTestHooks,
|
|
14
|
+
HarnessWaitPolicy,
|
|
15
|
+
} from "../types/harness";
|
|
16
|
+
import {
|
|
17
|
+
HARNESS_TASK_STATUSES,
|
|
18
|
+
HARNESS_TERMINAL_STATUSES,
|
|
19
|
+
HARNESS_WAIT_KINDS,
|
|
20
|
+
HARNESS_WAIT_POLICIES,
|
|
21
|
+
} from "../types/harness";
|
|
22
|
+
import {AGENT_TURN_TASK_NAME} from "./agentTaskNames";
|
|
23
|
+
import {
|
|
24
|
+
claimAbortHandler,
|
|
25
|
+
commitAbort,
|
|
26
|
+
fenceForAbort,
|
|
27
|
+
HarnessCommitConflictError,
|
|
28
|
+
type HarnessModels,
|
|
29
|
+
wakeWaitingTask,
|
|
30
|
+
} from "./commit";
|
|
31
|
+
import {startNextQueuedTurn} from "./conversation";
|
|
32
|
+
import {taskDefinitionKey} from "./defineTask";
|
|
33
|
+
import {errorMessage, harnessError} from "./errors";
|
|
34
|
+
import type {ExecutionEnv} from "./executionEnv";
|
|
35
|
+
|
|
36
|
+
/** Most waiting tasks one sweep re-checks; the next sweep picks up the rest. */
|
|
37
|
+
const WAIT_SWEEP_BATCH_SIZE = 100;
|
|
38
|
+
|
|
39
|
+
const NON_TERMINAL_STATUSES = Object.values(HARNESS_TASK_STATUSES).filter(
|
|
40
|
+
(status) => !HARNESS_TERMINAL_STATUSES.has(status)
|
|
41
|
+
);
|
|
42
|
+
|
|
43
|
+
/** Everything the engine's task, ownership, and wait logic shares within one harness. */
|
|
44
|
+
export interface HarnessEngine {
|
|
45
|
+
/** Registered agents, by name; `rt.runAgent` only runs these. */
|
|
46
|
+
agents: Map<string, HarnessAgentDefinition>;
|
|
47
|
+
/** Abort controllers of tasks running in this process, by task id. */
|
|
48
|
+
controllers: Map<string, AbortController>;
|
|
49
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
50
|
+
/** Handed to phases as `rt.env` and to tools as `api.env`. */
|
|
51
|
+
env?: ExecutionEnv;
|
|
52
|
+
/** Registered extensions, by name. */
|
|
53
|
+
extensions: Map<string, HarnessExtensionDefinition>;
|
|
54
|
+
models: HarnessModels;
|
|
55
|
+
testHooks?: HarnessTestHooks;
|
|
56
|
+
/** Tell the runner new work may be runnable. */
|
|
57
|
+
wake: () => void;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export const toTaskView = (task: HarnessTaskDocument): HarnessTaskView<unknown, unknown> => {
|
|
61
|
+
return {
|
|
62
|
+
attempt: task.attempt,
|
|
63
|
+
id: String(task._id),
|
|
64
|
+
input: task.input,
|
|
65
|
+
name: task.name,
|
|
66
|
+
phase: task.phase,
|
|
67
|
+
state: task.state,
|
|
68
|
+
userId: task.userId ? String(task.userId) : undefined,
|
|
69
|
+
version: task.version,
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
const isTerminal = (task: HarnessTaskDocument): boolean =>
|
|
74
|
+
HARNESS_TERMINAL_STATUSES.has(task.status);
|
|
75
|
+
|
|
76
|
+
const isFailure = (task: HarnessTaskDocument): boolean =>
|
|
77
|
+
task.status === HARNESS_TASK_STATUSES.failed || task.status === HARNESS_TASK_STATUSES.aborted;
|
|
78
|
+
|
|
79
|
+
/** Tasks owned by any of `ownerIds`, oldest first. */
|
|
80
|
+
const findOwnedTasks = (
|
|
81
|
+
models: HarnessModels,
|
|
82
|
+
ownerIds: mongoose.Types.ObjectId[]
|
|
83
|
+
): Promise<HarnessTaskDocument[]> => {
|
|
84
|
+
return models.task
|
|
85
|
+
.find({"ownership.id": {$in: ownerIds}, "ownership.kind": "task"})
|
|
86
|
+
.sort({created: 1});
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
/** Abort the in-process run of `task`, if this process is running it. */
|
|
90
|
+
const signalAbort = (engine: HarnessEngine, task: HarnessTaskDocument, reason: string): void => {
|
|
91
|
+
const controller = engine.controllers.get(String(task._id));
|
|
92
|
+
if (controller && !controller.signal.aborted) {
|
|
93
|
+
controller.abort(new Error(`Aborted: ${reason}`));
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
/** How often an abort re-checks a task whose handler another aborter is running. */
|
|
98
|
+
const ABORT_CLAIM_POLL = Duration.fromObject({milliseconds: 50});
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Wait until this call may run `task`'s abort handler. Returns null when the claim is
|
|
102
|
+
* ours, or the task once it is terminal (aborted by another aborter, or finished on its
|
|
103
|
+
* own).
|
|
104
|
+
*/
|
|
105
|
+
const awaitAbortClaim = async (
|
|
106
|
+
engine: HarnessEngine,
|
|
107
|
+
task: HarnessTaskDocument
|
|
108
|
+
): Promise<HarnessTaskDocument | null> => {
|
|
109
|
+
for (;;) {
|
|
110
|
+
if (await claimAbortHandler({models: engine.models, taskId: task._id})) {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
const current = await engine.models.task.findExactlyOne({_id: task._id});
|
|
114
|
+
if (isTerminal(current)) {
|
|
115
|
+
return current;
|
|
116
|
+
}
|
|
117
|
+
await new Promise((resolve) => setTimeout(resolve, ABORT_CLAIM_POLL.toMillis()));
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Fence every non-terminal task below `top` for abort and return them deepest first (and
|
|
123
|
+
* newest first within a level), so each is aborted only after everything it owns. Each
|
|
124
|
+
* level is fenced before its children are read, so no task in the tree can create a
|
|
125
|
+
* child the scan misses.
|
|
126
|
+
*/
|
|
127
|
+
const fenceOwnedForAbort = async ({
|
|
128
|
+
engine,
|
|
129
|
+
reason,
|
|
130
|
+
top,
|
|
131
|
+
userId,
|
|
132
|
+
}: {
|
|
133
|
+
engine: HarnessEngine;
|
|
134
|
+
reason: string;
|
|
135
|
+
top: HarnessTaskDocument;
|
|
136
|
+
userId?: string;
|
|
137
|
+
}): Promise<HarnessTaskDocument[]> => {
|
|
138
|
+
const levels: HarnessTaskDocument[][] = [];
|
|
139
|
+
const seen = new Set<string>([String(top._id)]);
|
|
140
|
+
let frontier = [top._id];
|
|
141
|
+
while (frontier.length > 0) {
|
|
142
|
+
const found = (await findOwnedTasks(engine.models, frontier)).filter(
|
|
143
|
+
(task) => !seen.has(String(task._id))
|
|
144
|
+
);
|
|
145
|
+
const fenced: HarnessTaskDocument[] = [];
|
|
146
|
+
for (const task of found) {
|
|
147
|
+
seen.add(String(task._id));
|
|
148
|
+
if (isTerminal(task)) {
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
const current = await fenceForAbort({
|
|
152
|
+
models: engine.models,
|
|
153
|
+
reason,
|
|
154
|
+
taskId: task._id,
|
|
155
|
+
userId,
|
|
156
|
+
});
|
|
157
|
+
if (current) {
|
|
158
|
+
signalAbort(engine, current, reason);
|
|
159
|
+
fenced.push(current);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
levels.push(fenced);
|
|
163
|
+
frontier = found.map((task) => task._id);
|
|
164
|
+
}
|
|
165
|
+
return levels.reverse().flatMap((level) => [...level].reverse());
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
interface AbortHandlerResult {
|
|
169
|
+
error?: string;
|
|
170
|
+
status: "error" | "none" | "ok" | "unregistered";
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** Run `task`'s abort handler; a throw is captured, never rethrown. */
|
|
174
|
+
const runAbortHandler = async ({
|
|
175
|
+
engine,
|
|
176
|
+
reason,
|
|
177
|
+
task,
|
|
178
|
+
userId,
|
|
179
|
+
}: {
|
|
180
|
+
engine: HarnessEngine;
|
|
181
|
+
reason: string;
|
|
182
|
+
task: HarnessTaskDocument;
|
|
183
|
+
userId?: string;
|
|
184
|
+
}): Promise<AbortHandlerResult> => {
|
|
185
|
+
const definition = engine.definitions.get(taskDefinitionKey(task));
|
|
186
|
+
if (!definition) {
|
|
187
|
+
return {
|
|
188
|
+
error: `${taskDefinitionKey(task)} is not in this harness registry; no abort handler ran`,
|
|
189
|
+
status: "unregistered",
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
if (!definition.abort) {
|
|
193
|
+
return {status: "none"};
|
|
194
|
+
}
|
|
195
|
+
try {
|
|
196
|
+
await definition.abort(toTaskView(task), {reason, taskId: String(task._id), userId});
|
|
197
|
+
return {status: "ok"};
|
|
198
|
+
} catch (error: unknown) {
|
|
199
|
+
logger.error(
|
|
200
|
+
`Harness abort handler for task ${task._id} (${definition.key}) failed; aborting anyway: ${errorMessage(error)}`
|
|
201
|
+
);
|
|
202
|
+
return {error: errorMessage(error), status: "error"};
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
/** How the top task of an abort is fenced and audited, when not a plain `abort`. */
|
|
207
|
+
export interface AbortTopOverride {
|
|
208
|
+
error: string;
|
|
209
|
+
filter: Record<string, unknown>;
|
|
210
|
+
output: Record<string, unknown>;
|
|
211
|
+
spanName: string;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const abortOne = async ({
|
|
215
|
+
engine,
|
|
216
|
+
override,
|
|
217
|
+
reason,
|
|
218
|
+
requestedOn,
|
|
219
|
+
task,
|
|
220
|
+
userId,
|
|
221
|
+
}: {
|
|
222
|
+
engine: HarnessEngine;
|
|
223
|
+
override?: AbortTopOverride;
|
|
224
|
+
reason: string;
|
|
225
|
+
requestedOn: string;
|
|
226
|
+
task: HarnessTaskDocument;
|
|
227
|
+
userId?: string;
|
|
228
|
+
}): Promise<HarnessTaskDocument> => {
|
|
229
|
+
const settledElsewhere = await awaitAbortClaim(engine, task);
|
|
230
|
+
if (settledElsewhere?.status === HARNESS_TASK_STATUSES.aborted) {
|
|
231
|
+
return settledElsewhere;
|
|
232
|
+
}
|
|
233
|
+
// It completed or failed on its own before this abort could claim it.
|
|
234
|
+
if (settledElsewhere) {
|
|
235
|
+
throw new HarnessCommitConflictError(String(task._id), task.phase);
|
|
236
|
+
}
|
|
237
|
+
const handler = await runAbortHandler({engine, reason, task, userId});
|
|
238
|
+
const handlerFailed = handler.status === "error";
|
|
239
|
+
const handlerOutput = handler.error ? handler : {status: handler.status};
|
|
240
|
+
return commitAbort({
|
|
241
|
+
error: override?.error ?? `Aborted: ${reason}`,
|
|
242
|
+
filter: override?.filter ?? {status: {$in: NON_TERMINAL_STATUSES}},
|
|
243
|
+
models: engine.models,
|
|
244
|
+
span: {
|
|
245
|
+
error: handlerFailed ? `Abort handler failed: ${handler.error}` : undefined,
|
|
246
|
+
name: override?.spanName ?? "abort",
|
|
247
|
+
output: override
|
|
248
|
+
? {...override.output, abortHandler: handlerOutput}
|
|
249
|
+
: {abortedBy: userId, abortHandler: handlerOutput, reason, requestedOn},
|
|
250
|
+
status: handlerFailed ? "error" : "ok",
|
|
251
|
+
},
|
|
252
|
+
task,
|
|
253
|
+
testHooks: engine.testHooks,
|
|
254
|
+
});
|
|
255
|
+
};
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Abort `top` and every non-terminal task it owns. First, top-down, each task is fenced
|
|
259
|
+
* (no new claims; a running phase loses its lease token) and its in-process run is
|
|
260
|
+
* signalled. Then, bottom-up, each task's `abort` handler runs and the task is committed
|
|
261
|
+
* `aborted` with an audit span. A descendant that settled on its own before its fence is
|
|
262
|
+
* skipped. Afterwards `top`'s owner is re-checked, since an aborted child can settle a
|
|
263
|
+
* parent's wait. Throws `HarnessCommitConflictError` when `top` is already terminal.
|
|
264
|
+
*/
|
|
265
|
+
export const abortTaskTree = async ({
|
|
266
|
+
engine,
|
|
267
|
+
override,
|
|
268
|
+
reason,
|
|
269
|
+
settleOwner = true,
|
|
270
|
+
top,
|
|
271
|
+
userId,
|
|
272
|
+
}: {
|
|
273
|
+
engine: HarnessEngine;
|
|
274
|
+
override?: AbortTopOverride;
|
|
275
|
+
reason: string;
|
|
276
|
+
/** False when the caller already handles the owner (a failFast sibling abort). */
|
|
277
|
+
settleOwner?: boolean;
|
|
278
|
+
top: HarnessTaskDocument;
|
|
279
|
+
userId?: string;
|
|
280
|
+
}): Promise<HarnessTaskDocument> => {
|
|
281
|
+
const requestedOn = String(top._id);
|
|
282
|
+
const fencedTop = await fenceForAbort({models: engine.models, reason, taskId: top._id, userId});
|
|
283
|
+
if (!fencedTop) {
|
|
284
|
+
throw new HarnessCommitConflictError(requestedOn, top.phase);
|
|
285
|
+
}
|
|
286
|
+
signalAbort(engine, fencedTop, reason);
|
|
287
|
+
for (const task of await fenceOwnedForAbort({engine, reason, top: fencedTop, userId})) {
|
|
288
|
+
try {
|
|
289
|
+
// Owners are aborted too, so only a subagent turn's conversation needs settling.
|
|
290
|
+
await releaseConversation({
|
|
291
|
+
engine,
|
|
292
|
+
task: await abortOne({engine, reason, requestedOn, task, userId}),
|
|
293
|
+
});
|
|
294
|
+
} catch (error: unknown) {
|
|
295
|
+
// It finished on its own between the scan and the abort commit.
|
|
296
|
+
if (!(error instanceof HarnessCommitConflictError)) {
|
|
297
|
+
throw error;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
const aborted = await abortOne({
|
|
302
|
+
engine,
|
|
303
|
+
override,
|
|
304
|
+
reason,
|
|
305
|
+
requestedOn,
|
|
306
|
+
task: fencedTop,
|
|
307
|
+
userId,
|
|
308
|
+
});
|
|
309
|
+
if (settleOwner) {
|
|
310
|
+
await settleTaskOwner({engine, task: aborted});
|
|
311
|
+
}
|
|
312
|
+
return aborted;
|
|
313
|
+
};
|
|
314
|
+
|
|
315
|
+
/** Settled children of a `waitForTasks` call, in the order their ids were passed. */
|
|
316
|
+
export const childOutcomes = (
|
|
317
|
+
ids: ReadonlyArray<string>,
|
|
318
|
+
children: ReadonlyArray<HarnessTaskDocument>
|
|
319
|
+
): HarnessChildOutcome[] => {
|
|
320
|
+
const byId = new Map(children.map((child) => [String(child._id), child]));
|
|
321
|
+
return ids.map((id) => {
|
|
322
|
+
const child = byId.get(id);
|
|
323
|
+
if (!child) {
|
|
324
|
+
throw harnessError({
|
|
325
|
+
detail: `Child task ${id} disappeared while its owner waited on it`,
|
|
326
|
+
kind: "notFound",
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
const outcome: HarnessChildOutcome = {
|
|
330
|
+
id,
|
|
331
|
+
name: child.name,
|
|
332
|
+
status: child.status as HarnessTerminalStatus,
|
|
333
|
+
};
|
|
334
|
+
if (child.outcome?.error !== undefined) {
|
|
335
|
+
outcome.error = child.outcome.error;
|
|
336
|
+
}
|
|
337
|
+
if (child.outcome?.result !== undefined) {
|
|
338
|
+
outcome.result = child.outcome.result;
|
|
339
|
+
}
|
|
340
|
+
return outcome;
|
|
341
|
+
});
|
|
342
|
+
};
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Settle a child wait under `policy`. Under `failFast`, the first failed or aborted child
|
|
346
|
+
* aborts every sibling still in flight (with the sibling's subtree) before the wait
|
|
347
|
+
* resolves. Returns the children as they now stand and whether the wait is satisfied.
|
|
348
|
+
*/
|
|
349
|
+
export const settleChildren = async ({
|
|
350
|
+
engine,
|
|
351
|
+
ids,
|
|
352
|
+
ownerId,
|
|
353
|
+
policy,
|
|
354
|
+
}: {
|
|
355
|
+
engine: HarnessEngine;
|
|
356
|
+
ids: ReadonlyArray<mongoose.Types.ObjectId>;
|
|
357
|
+
ownerId: mongoose.Types.ObjectId;
|
|
358
|
+
policy: HarnessWaitPolicy;
|
|
359
|
+
}): Promise<{children: HarnessTaskDocument[]; isSettled: boolean}> => {
|
|
360
|
+
let children = await engine.models.task.find({_id: {$in: ids}});
|
|
361
|
+
if (children.every(isTerminal)) {
|
|
362
|
+
return {children, isSettled: true};
|
|
363
|
+
}
|
|
364
|
+
const failed = children.find(isFailure);
|
|
365
|
+
if (policy !== HARNESS_WAIT_POLICIES.failFast || !failed) {
|
|
366
|
+
return {children, isSettled: false};
|
|
367
|
+
}
|
|
368
|
+
for (const sibling of children.filter((child) => !isTerminal(child))) {
|
|
369
|
+
try {
|
|
370
|
+
await abortTaskTree({
|
|
371
|
+
engine,
|
|
372
|
+
reason: `Sibling task ${failed._id} ${failed.status} (failFast wait of ${ownerId})`,
|
|
373
|
+
settleOwner: false,
|
|
374
|
+
top: sibling,
|
|
375
|
+
});
|
|
376
|
+
} catch (error: unknown) {
|
|
377
|
+
if (!(error instanceof HarnessCommitConflictError)) {
|
|
378
|
+
throw error;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
children = await engine.models.task.find({_id: {$in: ids}});
|
|
383
|
+
return {children, isSettled: children.every(isTerminal)};
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* Wake `taskId` when it is `waiting` on child tasks that have now settled. Safe to call
|
|
388
|
+
* any number of times from anywhere: the wake itself is fenced on the wait.
|
|
389
|
+
*/
|
|
390
|
+
export const checkTaskWait = async ({
|
|
391
|
+
engine,
|
|
392
|
+
taskId,
|
|
393
|
+
}: {
|
|
394
|
+
engine: HarnessEngine;
|
|
395
|
+
taskId: mongoose.Types.ObjectId;
|
|
396
|
+
}): Promise<boolean> => {
|
|
397
|
+
const task = await engine.models.task.findOneOrNone({_id: taskId});
|
|
398
|
+
const waiting = task?.waiting;
|
|
399
|
+
if (
|
|
400
|
+
!task ||
|
|
401
|
+
task.status !== HARNESS_TASK_STATUSES.waiting ||
|
|
402
|
+
waiting?.kind !== HARNESS_WAIT_KINDS.tasks
|
|
403
|
+
) {
|
|
404
|
+
return false;
|
|
405
|
+
}
|
|
406
|
+
const {isSettled} = await settleChildren({
|
|
407
|
+
engine,
|
|
408
|
+
ids: waiting.taskIds ?? [],
|
|
409
|
+
ownerId: task._id,
|
|
410
|
+
policy: waiting.policy ?? HARNESS_WAIT_POLICIES.all,
|
|
411
|
+
});
|
|
412
|
+
if (!isSettled) {
|
|
413
|
+
return false;
|
|
414
|
+
}
|
|
415
|
+
const isWoken = await wakeWaitingTask({
|
|
416
|
+
kind: HARNESS_WAIT_KINDS.tasks,
|
|
417
|
+
models: engine.models,
|
|
418
|
+
taskId: task._id,
|
|
419
|
+
});
|
|
420
|
+
if (isWoken) {
|
|
421
|
+
engine.wake();
|
|
422
|
+
}
|
|
423
|
+
return isWoken;
|
|
424
|
+
};
|
|
425
|
+
|
|
426
|
+
/**
|
|
427
|
+
* Mark a conversation `idle` once its active turn task is terminal. Fenced on the turn,
|
|
428
|
+
* so a later turn is never released by an earlier one. `submit` also releases a
|
|
429
|
+
* conversation whose active turn is terminal, should this write be lost to a crash.
|
|
430
|
+
*/
|
|
431
|
+
const releaseConversation = async ({
|
|
432
|
+
engine,
|
|
433
|
+
task,
|
|
434
|
+
}: {
|
|
435
|
+
engine: HarnessEngine;
|
|
436
|
+
task: HarnessTaskDocument;
|
|
437
|
+
}): Promise<void> => {
|
|
438
|
+
const conversationId = turnConversationId(task);
|
|
439
|
+
if (!conversationId) {
|
|
440
|
+
return;
|
|
441
|
+
}
|
|
442
|
+
try {
|
|
443
|
+
await engine.models.conversation.updateOne(
|
|
444
|
+
{_id: conversationId, activeTurnTaskId: task._id, status: "busy"},
|
|
445
|
+
{$set: {status: "idle"}, $unset: {activeTurnTaskId: 1}}
|
|
446
|
+
);
|
|
447
|
+
} catch (error: unknown) {
|
|
448
|
+
logger.error(
|
|
449
|
+
`Harness could not release conversation ${conversationId} after turn ${task._id}: ${errorMessage(error)}`
|
|
450
|
+
);
|
|
451
|
+
}
|
|
452
|
+
};
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* The conversation a turn task runs on: its owner for a root conversation's turn, its
|
|
456
|
+
* input for a subagent turn (owned by the task that called `rt.runAgent`).
|
|
457
|
+
*/
|
|
458
|
+
const turnConversationId = (task: HarnessTaskDocument): string | undefined => {
|
|
459
|
+
if (task.ownership?.kind === "conversation" && task.ownership.id) {
|
|
460
|
+
return String(task.ownership.id);
|
|
461
|
+
}
|
|
462
|
+
if (task.name !== AGENT_TURN_TASK_NAME) {
|
|
463
|
+
return undefined;
|
|
464
|
+
}
|
|
465
|
+
const conversationId = (task.input as {conversationId?: unknown} | undefined)?.conversationId;
|
|
466
|
+
return typeof conversationId === "string" ? conversationId : undefined;
|
|
467
|
+
};
|
|
468
|
+
|
|
469
|
+
/** Registry key of the built-in agent turn. */
|
|
470
|
+
const AGENT_TURN_KEY = taskDefinitionKey({name: AGENT_TURN_TASK_NAME, version: 1});
|
|
471
|
+
|
|
472
|
+
/**
|
|
473
|
+
* Run the conversation's next queued submission now that its turn ended. A failure here
|
|
474
|
+
* leaves the submission queued; the owner's queue sweep (or the next `send`) starts it.
|
|
475
|
+
*/
|
|
476
|
+
const startQueuedTurn = async ({
|
|
477
|
+
conversationId,
|
|
478
|
+
engine,
|
|
479
|
+
}: {
|
|
480
|
+
conversationId: mongoose.Types.ObjectId;
|
|
481
|
+
engine: HarnessEngine;
|
|
482
|
+
}): Promise<boolean> => {
|
|
483
|
+
const turn = engine.definitions.get(AGENT_TURN_KEY);
|
|
484
|
+
if (!turn) {
|
|
485
|
+
return false;
|
|
486
|
+
}
|
|
487
|
+
try {
|
|
488
|
+
const started = await startNextQueuedTurn({
|
|
489
|
+
context: {models: engine.models, turn, wake: engine.wake},
|
|
490
|
+
conversationId,
|
|
491
|
+
});
|
|
492
|
+
return started !== null;
|
|
493
|
+
} catch (error: unknown) {
|
|
494
|
+
logger.error(
|
|
495
|
+
`Harness could not start the next queued turn of conversation ${conversationId}: ${errorMessage(error)}`
|
|
496
|
+
);
|
|
497
|
+
return false;
|
|
498
|
+
}
|
|
499
|
+
};
|
|
500
|
+
|
|
501
|
+
/**
|
|
502
|
+
* Free `busy` conversations whose active turn is terminal or gone. Every terminal turn
|
|
503
|
+
* commit frees its conversation in the same transaction; this repairs conversations left
|
|
504
|
+
* busy by data written before that, or by a write that failed. Fenced on the stale turn,
|
|
505
|
+
* so a conversation that moved on is untouched. Returns how many were freed.
|
|
506
|
+
*/
|
|
507
|
+
const releaseStaleConversations = async (engine: HarnessEngine): Promise<number> => {
|
|
508
|
+
const {conversation, task} = engine.models;
|
|
509
|
+
const stale = await conversation.aggregate<{
|
|
510
|
+
_id: mongoose.Types.ObjectId;
|
|
511
|
+
activeTurnTaskId?: mongoose.Types.ObjectId;
|
|
512
|
+
}>([
|
|
513
|
+
{$match: {status: "busy"}},
|
|
514
|
+
{
|
|
515
|
+
$lookup: {
|
|
516
|
+
as: "turn",
|
|
517
|
+
foreignField: "_id",
|
|
518
|
+
from: task.collection.name,
|
|
519
|
+
localField: "activeTurnTaskId",
|
|
520
|
+
pipeline: [{$project: {status: 1}}],
|
|
521
|
+
},
|
|
522
|
+
},
|
|
523
|
+
{
|
|
524
|
+
$match: {
|
|
525
|
+
$or: [{"turn.0": {$exists: false}}, {"turn.status": {$in: [...HARNESS_TERMINAL_STATUSES]}}],
|
|
526
|
+
},
|
|
527
|
+
},
|
|
528
|
+
{$limit: QUEUE_SWEEP_BATCH_SIZE},
|
|
529
|
+
{$project: {activeTurnTaskId: 1}},
|
|
530
|
+
]);
|
|
531
|
+
let released = 0;
|
|
532
|
+
for (const {_id, activeTurnTaskId} of stale) {
|
|
533
|
+
const result = await conversation.updateOne(
|
|
534
|
+
{_id, activeTurnTaskId: activeTurnTaskId ?? null, status: "busy"},
|
|
535
|
+
{$set: {status: "idle"}, $unset: {activeTurnTaskId: 1}}
|
|
536
|
+
);
|
|
537
|
+
released += result.modifiedCount;
|
|
538
|
+
}
|
|
539
|
+
return released;
|
|
540
|
+
};
|
|
541
|
+
|
|
542
|
+
/** Most idle conversations with a backlog one sweep starts. */
|
|
543
|
+
const QUEUE_SWEEP_BATCH_SIZE = 100;
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* Start the next queued submission of every conversation that still has one, after
|
|
547
|
+
* freeing conversations stuck `busy` on a finished turn. A turn frees its conversation in
|
|
548
|
+
* its terminal commit and starts the next queued turn right after; this covers a process
|
|
549
|
+
* that died between the two. Returns how many turns started.
|
|
550
|
+
*/
|
|
551
|
+
export const sweepQueuedConversations = async (engine: HarnessEngine): Promise<number> => {
|
|
552
|
+
await releaseStaleConversations(engine);
|
|
553
|
+
const stranded = await engine.models.conversation
|
|
554
|
+
.find({"queued.requestId": {$exists: true}, status: "idle"})
|
|
555
|
+
.select({_id: 1})
|
|
556
|
+
.limit(QUEUE_SWEEP_BATCH_SIZE)
|
|
557
|
+
.lean();
|
|
558
|
+
let started = 0;
|
|
559
|
+
for (const {_id} of stranded) {
|
|
560
|
+
if (await startQueuedTurn({conversationId: _id, engine})) {
|
|
561
|
+
started += 1;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
return started;
|
|
565
|
+
};
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* Wake whatever owns a task that just settled: a waiting parent task re-checks its wait;
|
|
569
|
+
* a conversation whose turn ended goes back to `idle` and starts its next queued submission.
|
|
570
|
+
*/
|
|
571
|
+
export const settleTaskOwner = async ({
|
|
572
|
+
engine,
|
|
573
|
+
task,
|
|
574
|
+
}: {
|
|
575
|
+
engine: HarnessEngine;
|
|
576
|
+
task: HarnessTaskDocument;
|
|
577
|
+
}): Promise<void> => {
|
|
578
|
+
if (task.ownership?.kind === "conversation" && task.ownership.id) {
|
|
579
|
+
await releaseConversation({engine, task});
|
|
580
|
+
await startQueuedTurn({conversationId: task.ownership.id, engine});
|
|
581
|
+
return;
|
|
582
|
+
}
|
|
583
|
+
// A subagent turn frees its conversation, then wakes the task waiting on it.
|
|
584
|
+
await releaseConversation({engine, task});
|
|
585
|
+
if (task.ownership?.kind !== "task" || !task.ownership.id) {
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
try {
|
|
589
|
+
await checkTaskWait({engine, taskId: task.ownership.id});
|
|
590
|
+
} catch (error: unknown) {
|
|
591
|
+
// The periodic wait sweep retries; the child's own outcome is already committed.
|
|
592
|
+
logger.error(
|
|
593
|
+
`Harness could not wake owner ${task.ownership.id} of task ${task._id}: ${errorMessage(error)}`
|
|
594
|
+
);
|
|
595
|
+
}
|
|
596
|
+
};
|
|
597
|
+
|
|
598
|
+
/**
|
|
599
|
+
* Re-check tasks waiting on children. Covers wakes lost to a crash between a child's
|
|
600
|
+
* terminal commit and its owner check. Returns how many tasks were woken.
|
|
601
|
+
*/
|
|
602
|
+
export const sweepWaitingTasks = async (engine: HarnessEngine): Promise<number> => {
|
|
603
|
+
const waiting = await engine.models.task
|
|
604
|
+
.find({status: HARNESS_TASK_STATUSES.waiting, "waiting.kind": HARNESS_WAIT_KINDS.tasks})
|
|
605
|
+
.sort({updated: 1})
|
|
606
|
+
.limit(WAIT_SWEEP_BATCH_SIZE);
|
|
607
|
+
let woken = 0;
|
|
608
|
+
for (const task of waiting) {
|
|
609
|
+
try {
|
|
610
|
+
if (await checkTaskWait({engine, taskId: task._id})) {
|
|
611
|
+
woken += 1;
|
|
612
|
+
}
|
|
613
|
+
} catch (error: unknown) {
|
|
614
|
+
logger.error(`Harness could not re-check waiting task ${task._id}: ${errorMessage(error)}`);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
return woken;
|
|
618
|
+
};
|