@terreno/ai 57.9.0 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +133 -0
- package/dist/harness/errors.js +107 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +522 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +191 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +64 -0
- package/dist/harness/runners/inProcessRunner.js +227 -0
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +52 -0
- package/dist/harness/runtime.js +604 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +952 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +28 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +125 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +769 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +778 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +206 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +907 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +530 -0
- package/src/harness/runners/inProcessRunner.ts +260 -0
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +802 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1119 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,802 @@
|
|
|
1
|
+
import {logger} from "@terreno/api";
|
|
2
|
+
import {DateTime, type DurationLike} from "luxon";
|
|
3
|
+
|
|
4
|
+
import mongoose from "mongoose";
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
HarnessAgentDefinition,
|
|
8
|
+
HarnessApprovalOptions,
|
|
9
|
+
HarnessApprovalResult,
|
|
10
|
+
HarnessChildOutcome,
|
|
11
|
+
HarnessChildTaskOptions,
|
|
12
|
+
HarnessCommit,
|
|
13
|
+
HarnessLeaseSettings,
|
|
14
|
+
HarnessRunAgentOptions,
|
|
15
|
+
HarnessRunnableTask,
|
|
16
|
+
HarnessTaskDefinition,
|
|
17
|
+
HarnessTaskDocument,
|
|
18
|
+
HarnessTaskRuntime,
|
|
19
|
+
HarnessWaitForOptions,
|
|
20
|
+
HarnessWaitForTasksOptions,
|
|
21
|
+
} from "../types/harness";
|
|
22
|
+
import {
|
|
23
|
+
HARNESS_EVENT_TYPES,
|
|
24
|
+
HARNESS_TASK_STATUSES,
|
|
25
|
+
HARNESS_TERMINAL_STATUSES,
|
|
26
|
+
HARNESS_WAIT_KINDS,
|
|
27
|
+
HARNESS_WAIT_POLICIES,
|
|
28
|
+
} from "../types/harness";
|
|
29
|
+
import {AGENT_TURN_TASK_NAME} from "./agentTaskNames";
|
|
30
|
+
import {type ApprovalWaitPlan, approvalWaitPlan, parseApprovalRequest} from "./approvals";
|
|
31
|
+
import {
|
|
32
|
+
commitInterruption,
|
|
33
|
+
commitPhase,
|
|
34
|
+
commitRetry,
|
|
35
|
+
commitWaiting,
|
|
36
|
+
createChildTaskRecords,
|
|
37
|
+
expiredLeaseFilter,
|
|
38
|
+
HarnessCommitConflictError,
|
|
39
|
+
type HarnessCommitWrites,
|
|
40
|
+
type HarnessInterruptionAction,
|
|
41
|
+
type HarnessModels,
|
|
42
|
+
newTaskLease,
|
|
43
|
+
} from "./commit";
|
|
44
|
+
import {taskDefinitionKey} from "./defineTask";
|
|
45
|
+
import {HarnessDefinitionError} from "./definitionError";
|
|
46
|
+
import {errorMessage, harnessError} from "./errors";
|
|
47
|
+
import {appendTaskEvents} from "./events";
|
|
48
|
+
import {HARNESS_INTERNAL_RUNTIME} from "./internalRuntime";
|
|
49
|
+
import {startTaskHeartbeat} from "./leases";
|
|
50
|
+
import {createMemo} from "./memo";
|
|
51
|
+
import {
|
|
52
|
+
checkTaskWait,
|
|
53
|
+
childOutcomes,
|
|
54
|
+
type HarnessEngine,
|
|
55
|
+
settleChildren,
|
|
56
|
+
settleTaskOwner,
|
|
57
|
+
sweepQueuedConversations,
|
|
58
|
+
sweepWaitingTasks,
|
|
59
|
+
toTaskView,
|
|
60
|
+
} from "./ownership";
|
|
61
|
+
import {nextRetryAt, resolveRetryPolicy} from "./retryBackoff";
|
|
62
|
+
import {
|
|
63
|
+
serializeOutputSchema,
|
|
64
|
+
startSubagentTurn,
|
|
65
|
+
subagentContent,
|
|
66
|
+
subagentResult,
|
|
67
|
+
} from "./subagent";
|
|
68
|
+
import {HarnessSuspendSignal} from "./suspend";
|
|
69
|
+
import {parseWaitDuration, resolveWaitCall, type WaitCallPlan, waitCallKey} from "./waits";
|
|
70
|
+
|
|
71
|
+
/** Registry key of the built-in agent turn every subagent runs. */
|
|
72
|
+
const AGENT_TURN_KEY = taskDefinitionKey({name: AGENT_TURN_TASK_NAME, version: 1});
|
|
73
|
+
|
|
74
|
+
/** Most expired tasks one recovery pass handles; the next pass picks up the rest. */
|
|
75
|
+
const RECOVERY_BATCH_SIZE = 100;
|
|
76
|
+
|
|
77
|
+
const assertValidNext = (
|
|
78
|
+
definition: HarnessTaskDefinition,
|
|
79
|
+
next: HarnessCommit<unknown, unknown>
|
|
80
|
+
): void => {
|
|
81
|
+
if ("terminal" in next) {
|
|
82
|
+
const {status} = next.terminal;
|
|
83
|
+
if (status !== "completed" && status !== "failed") {
|
|
84
|
+
throw new HarnessDefinitionError(
|
|
85
|
+
`${definition.key}: terminal status must be "completed" or "failed"`
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
if (!definition.phases[next.phase]) {
|
|
91
|
+
throw new HarnessDefinitionError(
|
|
92
|
+
`${definition.key}: rt.commit to unknown phase "${next.phase}"`
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const registeredFilter = (
|
|
98
|
+
definitions: Map<string, HarnessTaskDefinition>
|
|
99
|
+
): Array<{name: string; version: number}> => {
|
|
100
|
+
return [...definitions.values()].map(({name, version}) => ({name, version}));
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Mongo filter for a registered task a runner may claim now: `pending` and due, or
|
|
105
|
+
* `waiting` on an event or sleep whose `timeoutAt` passed, and not being aborted.
|
|
106
|
+
*/
|
|
107
|
+
const runnableTaskFilter = (
|
|
108
|
+
definitions: Map<string, HarnessTaskDefinition>,
|
|
109
|
+
now: DateTime
|
|
110
|
+
): Record<string, unknown> => ({
|
|
111
|
+
$and: [
|
|
112
|
+
{$or: registeredFilter(definitions)},
|
|
113
|
+
{
|
|
114
|
+
$or: [
|
|
115
|
+
{
|
|
116
|
+
$or: [{runAt: {$exists: false}}, {runAt: null}, {runAt: {$lte: now.toJSDate()}}],
|
|
117
|
+
status: HARNESS_TASK_STATUSES.pending,
|
|
118
|
+
},
|
|
119
|
+
// An event wait or sleep whose timeout passed resumes like a due retry.
|
|
120
|
+
{
|
|
121
|
+
status: HARNESS_TASK_STATUSES.waiting,
|
|
122
|
+
"waiting.kind": {$in: [HARNESS_WAIT_KINDS.event, HARNESS_WAIT_KINDS.sleep]},
|
|
123
|
+
"waiting.timeoutAt": {$lte: now.toJSDate()},
|
|
124
|
+
},
|
|
125
|
+
],
|
|
126
|
+
},
|
|
127
|
+
],
|
|
128
|
+
// A task being aborted never starts another run.
|
|
129
|
+
"abortRequested.at": {$exists: false},
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
/** Move a runnable task to `running` under a fresh lease (owner, fencing token, expiry). */
|
|
133
|
+
const claimUpdate = (lease: HarnessLeaseSettings): Record<string, unknown> => ({
|
|
134
|
+
$inc: {claims: 1},
|
|
135
|
+
$set: {lease: newTaskLease(lease), status: HARNESS_TASK_STATUSES.running},
|
|
136
|
+
$unset: {waiting: 1},
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Atomically move the oldest runnable, registered task (`pending` and due, or `waiting` on
|
|
141
|
+
* an event or sleep whose `timeoutAt` passed) to `running` and give it a fresh lease (owner, fencing token, expiry) for the phase about to start.
|
|
142
|
+
*/
|
|
143
|
+
export const claimNextTask = async ({
|
|
144
|
+
definitions,
|
|
145
|
+
lease,
|
|
146
|
+
models,
|
|
147
|
+
}: {
|
|
148
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
149
|
+
lease: HarnessLeaseSettings;
|
|
150
|
+
models: HarnessModels;
|
|
151
|
+
}): Promise<HarnessTaskDocument | null> => {
|
|
152
|
+
if (definitions.size === 0) {
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
return models.task.findOneAndUpdate(
|
|
156
|
+
runnableTaskFilter(definitions, DateTime.now()),
|
|
157
|
+
claimUpdate(lease),
|
|
158
|
+
{returnDocument: "after", sort: {created: 1}}
|
|
159
|
+
);
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Claim one task by id under the same rules as `claimNextTask`. Null when the task is not
|
|
164
|
+
* runnable now: unknown, not due, already claimed by another runner, terminal, or aborting.
|
|
165
|
+
*/
|
|
166
|
+
export const claimTaskById = async ({
|
|
167
|
+
definitions,
|
|
168
|
+
lease,
|
|
169
|
+
models,
|
|
170
|
+
taskId,
|
|
171
|
+
}: {
|
|
172
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
173
|
+
lease: HarnessLeaseSettings;
|
|
174
|
+
models: HarnessModels;
|
|
175
|
+
taskId: string;
|
|
176
|
+
}): Promise<HarnessTaskDocument | null> => {
|
|
177
|
+
if (definitions.size === 0 || !mongoose.isValidObjectId(taskId)) {
|
|
178
|
+
return null;
|
|
179
|
+
}
|
|
180
|
+
return models.task.findOneAndUpdate(
|
|
181
|
+
{...runnableTaskFilter(definitions, DateTime.now()), _id: toObjectId(taskId)},
|
|
182
|
+
claimUpdate(lease),
|
|
183
|
+
{returnDocument: "after"}
|
|
184
|
+
);
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
/** Oldest runnable, registered tasks first, at most `limit`, without claiming them. */
|
|
188
|
+
export const listRunnableTasks = async ({
|
|
189
|
+
definitions,
|
|
190
|
+
limit,
|
|
191
|
+
models,
|
|
192
|
+
}: {
|
|
193
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
194
|
+
limit: number;
|
|
195
|
+
models: HarnessModels;
|
|
196
|
+
}): Promise<HarnessRunnableTask[]> => {
|
|
197
|
+
if (definitions.size === 0) {
|
|
198
|
+
return [];
|
|
199
|
+
}
|
|
200
|
+
const tasks = await models.task
|
|
201
|
+
.find(runnableTaskFilter(definitions, DateTime.now()))
|
|
202
|
+
.select({attempt: 1, claims: 1, phase: 1})
|
|
203
|
+
.sort({created: 1})
|
|
204
|
+
.limit(limit);
|
|
205
|
+
return tasks.map((task) => ({
|
|
206
|
+
attempt: task.attempt ?? 0,
|
|
207
|
+
claims: task.claims ?? 0,
|
|
208
|
+
phase: task.phase,
|
|
209
|
+
taskId: String(task._id),
|
|
210
|
+
}));
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Give a task that is still `running` under `task`'s lease back to the runners as
|
|
215
|
+
* `pending` at its current checkpoint. False when the lease moved on (aborted, recovered).
|
|
216
|
+
*/
|
|
217
|
+
const releaseTask = async (models: HarnessModels, task: HarnessTaskDocument): Promise<boolean> => {
|
|
218
|
+
const result = await models.task.updateOne(
|
|
219
|
+
{
|
|
220
|
+
_id: task._id,
|
|
221
|
+
"lease.token": task.lease?.token ?? null,
|
|
222
|
+
status: HARNESS_TASK_STATUSES.running,
|
|
223
|
+
},
|
|
224
|
+
{$set: {status: HARNESS_TASK_STATUSES.pending}, $unset: {lease: 1}}
|
|
225
|
+
);
|
|
226
|
+
return result.modifiedCount > 0;
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Find `running` tasks whose lease expired (their runner died or froze mid-phase) and
|
|
231
|
+
* resolve each by its current phase's `replay`: `safe` goes back to `pending` at the same
|
|
232
|
+
* checkpoint; anything else is parked `interrupted`, or failed (owner woken) when the
|
|
233
|
+
* definition sets `onInterrupt: "fail"`. Tasks of unregistered versions are
|
|
234
|
+
* left for a runner that registers them. Then wake tasks whose child wait settled
|
|
235
|
+
* without a wake (a crash between a child's outcome and its owner check). Returns how
|
|
236
|
+
* many tasks became runnable.
|
|
237
|
+
*/
|
|
238
|
+
export const recoverExpiredTasks = async (engine: HarnessEngine): Promise<number> => {
|
|
239
|
+
const {definitions, models, testHooks} = engine;
|
|
240
|
+
if (definitions.size === 0) {
|
|
241
|
+
return 0;
|
|
242
|
+
}
|
|
243
|
+
const expired = await models.task
|
|
244
|
+
.find({...expiredLeaseFilter(DateTime.now()), $and: [{$or: registeredFilter(definitions)}]})
|
|
245
|
+
.sort({created: 1})
|
|
246
|
+
.limit(RECOVERY_BATCH_SIZE);
|
|
247
|
+
|
|
248
|
+
let runnable = 0;
|
|
249
|
+
for (const task of expired) {
|
|
250
|
+
const definition = definitions.get(taskDefinitionKey(task));
|
|
251
|
+
const action = interruptionAction(definition, task.phase);
|
|
252
|
+
try {
|
|
253
|
+
const recovered = await commitInterruption({action, models, task, testHooks});
|
|
254
|
+
logger.warn(
|
|
255
|
+
`Harness task ${task._id} (${taskDefinitionKey(task)}) was interrupted in phase "${task.phase}"; ${INTERRUPTION_LOG[action]}`
|
|
256
|
+
);
|
|
257
|
+
if (action === "replay") {
|
|
258
|
+
runnable += 1;
|
|
259
|
+
}
|
|
260
|
+
if (action === "fail") {
|
|
261
|
+
await settleTaskOwner({engine, task: recovered});
|
|
262
|
+
}
|
|
263
|
+
} catch (error: unknown) {
|
|
264
|
+
// Another runner recovered it first, or its lease was renewed after the scan.
|
|
265
|
+
if (error instanceof HarnessCommitConflictError) {
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
logger.error(`Harness could not recover task ${task._id}: ${errorMessage(error)}`);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
return runnable + (await sweepWaitingTasks(engine)) + (await sweepQueuedConversations(engine));
|
|
272
|
+
};
|
|
273
|
+
|
|
274
|
+
const INTERRUPTION_LOG: Record<HarnessInterruptionAction, string> = {
|
|
275
|
+
fail: "failed it (not retried)",
|
|
276
|
+
park: "parked as interrupted",
|
|
277
|
+
replay: "re-running it",
|
|
278
|
+
};
|
|
279
|
+
|
|
280
|
+
/** What recovery does with an expired task in `phase`, by its definition's policies. */
|
|
281
|
+
const interruptionAction = (
|
|
282
|
+
definition: HarnessTaskDefinition | undefined,
|
|
283
|
+
phase: string
|
|
284
|
+
): HarnessInterruptionAction => {
|
|
285
|
+
if (definition?.phases[phase]?.replay === "safe") {
|
|
286
|
+
return "replay";
|
|
287
|
+
}
|
|
288
|
+
return definition?.onInterrupt === "fail" ? "fail" : "park";
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Run a claimed task's phases in order until it stops: terminal, `pending` for a retry,
|
|
293
|
+
* `waiting`, or aborted. A phase that throws is retried under the task's `retry` policy
|
|
294
|
+
* and fails the task once attempts run out. When a commit transaction fails, or the lease
|
|
295
|
+
* was lost, the task is left at its last checkpoint and expired-lease recovery decides
|
|
296
|
+
* what happens next. A terminal task re-checks the task that owns it.
|
|
297
|
+
*/
|
|
298
|
+
export const runClaimedTask = async ({
|
|
299
|
+
engine,
|
|
300
|
+
lease,
|
|
301
|
+
maxPhases,
|
|
302
|
+
task,
|
|
303
|
+
}: {
|
|
304
|
+
engine: HarnessEngine;
|
|
305
|
+
lease: HarnessLeaseSettings;
|
|
306
|
+
/** Hand the task back as `pending` once this many phases committed and it still runs. */
|
|
307
|
+
maxPhases?: number;
|
|
308
|
+
task: HarnessTaskDocument;
|
|
309
|
+
}): Promise<void> => {
|
|
310
|
+
const definition = engine.definitions.get(taskDefinitionKey(task));
|
|
311
|
+
if (!definition) {
|
|
312
|
+
throw harnessError({
|
|
313
|
+
detail: `No registered task definition for ${taskDefinitionKey(task)}`,
|
|
314
|
+
kind: "notRegistered",
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const taskId = String(task._id);
|
|
319
|
+
const controller = new AbortController();
|
|
320
|
+
engine.controllers.set(taskId, controller);
|
|
321
|
+
try {
|
|
322
|
+
let current = task;
|
|
323
|
+
let phasesRun = 0;
|
|
324
|
+
while (current.status === HARNESS_TASK_STATUSES.running && !controller.signal.aborted) {
|
|
325
|
+
if (maxPhases !== undefined && phasesRun >= maxPhases) {
|
|
326
|
+
if (!(await releaseTask(engine.models, current))) {
|
|
327
|
+
logger.info(`Harness task ${taskId} lost its lease before hand-off; stopping`);
|
|
328
|
+
}
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
// An abort from another process may have fenced the task since the last commit.
|
|
332
|
+
if (phasesRun > 0 && !(await holdsLease(engine.models, current))) {
|
|
333
|
+
logger.info(`Harness task ${taskId} lost its lease between phases; stopping`);
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
phasesRun += 1;
|
|
337
|
+
const result = await runPhase({controller, definition, engine, lease, task: current});
|
|
338
|
+
if (!result) {
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
current = result;
|
|
342
|
+
}
|
|
343
|
+
if (HARNESS_TERMINAL_STATUSES.has(current.status)) {
|
|
344
|
+
await settleTaskOwner({engine, task: current});
|
|
345
|
+
}
|
|
346
|
+
} finally {
|
|
347
|
+
engine.controllers.delete(taskId);
|
|
348
|
+
}
|
|
349
|
+
};
|
|
350
|
+
|
|
351
|
+
/** Whether `task`'s current lease token is still the one stored. */
|
|
352
|
+
const holdsLease = async (models: HarnessModels, task: HarnessTaskDocument): Promise<boolean> => {
|
|
353
|
+
const held = await models.task.countDocuments({
|
|
354
|
+
_id: task._id,
|
|
355
|
+
"lease.token": task.lease?.token ?? null,
|
|
356
|
+
status: HARNESS_TASK_STATUSES.running,
|
|
357
|
+
});
|
|
358
|
+
return held > 0;
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
const toObjectId = (id: mongoose.Types.ObjectId | string): mongoose.Types.ObjectId => {
|
|
362
|
+
return typeof id === "string" ? new mongoose.Types.ObjectId(id) : id;
|
|
363
|
+
};
|
|
364
|
+
|
|
365
|
+
const runPhase = async ({
|
|
366
|
+
controller,
|
|
367
|
+
definition,
|
|
368
|
+
engine,
|
|
369
|
+
lease,
|
|
370
|
+
task,
|
|
371
|
+
}: {
|
|
372
|
+
controller: AbortController;
|
|
373
|
+
definition: HarnessTaskDefinition;
|
|
374
|
+
engine: HarnessEngine;
|
|
375
|
+
lease: HarnessLeaseSettings;
|
|
376
|
+
task: HarnessTaskDocument;
|
|
377
|
+
}): Promise<HarnessTaskDocument | null> => {
|
|
378
|
+
const {models, testHooks} = engine;
|
|
379
|
+
const taskId = String(task._id);
|
|
380
|
+
const phaseStartedAt = DateTime.now();
|
|
381
|
+
let committed: HarnessTaskDocument | null = null;
|
|
382
|
+
let commitStarted = false;
|
|
383
|
+
let commitFailure: unknown;
|
|
384
|
+
let childIndex = 0;
|
|
385
|
+
let agentIndex = 0;
|
|
386
|
+
let waitIndex = 0;
|
|
387
|
+
const heartbeat = startTaskHeartbeat({
|
|
388
|
+
lease,
|
|
389
|
+
models,
|
|
390
|
+
onLost: () => controller.abort(new Error("Task lease lost")),
|
|
391
|
+
taskId: task._id,
|
|
392
|
+
testHooks,
|
|
393
|
+
token: task.lease?.token,
|
|
394
|
+
});
|
|
395
|
+
|
|
396
|
+
const assertOpen = (method: string): void => {
|
|
397
|
+
if (commitStarted) {
|
|
398
|
+
throw new HarnessDefinitionError(
|
|
399
|
+
`${definition.key}: ${method} called after the phase "${task.phase}" committed or started waiting`
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
};
|
|
403
|
+
|
|
404
|
+
/** Run one fenced write that ends the phase; record its failure for the caller. */
|
|
405
|
+
const settlePhase = async (write: () => Promise<HarnessTaskDocument>): Promise<void> => {
|
|
406
|
+
commitStarted = true;
|
|
407
|
+
// The commit decides the lease's fate: renewed under a new token, cleared, or
|
|
408
|
+
// (on failure) left to expire for recovery.
|
|
409
|
+
await heartbeat.stop();
|
|
410
|
+
try {
|
|
411
|
+
committed = await write();
|
|
412
|
+
} catch (error: unknown) {
|
|
413
|
+
commitFailure = error;
|
|
414
|
+
throw error;
|
|
415
|
+
}
|
|
416
|
+
};
|
|
417
|
+
|
|
418
|
+
const commitWithWrites = async (
|
|
419
|
+
next: HarnessCommit<unknown, unknown>,
|
|
420
|
+
writes?: HarnessCommitWrites
|
|
421
|
+
): Promise<void> => {
|
|
422
|
+
if (commitStarted) {
|
|
423
|
+
throw new HarnessDefinitionError(
|
|
424
|
+
`${definition.key}: rt.commit called more than once in phase "${task.phase}"`
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
assertValidNext(definition, next);
|
|
428
|
+
await settlePhase(() =>
|
|
429
|
+
commitPhase({lease, models, next, phaseStartedAt, task, testHooks, writes})
|
|
430
|
+
);
|
|
431
|
+
};
|
|
432
|
+
|
|
433
|
+
const commit = (next: HarnessCommit<unknown, unknown>): Promise<void> => commitWithWrites(next);
|
|
434
|
+
|
|
435
|
+
const createTask = async (
|
|
436
|
+
childDefinition: HarnessTaskDefinition<never, unknown, unknown>,
|
|
437
|
+
input: unknown,
|
|
438
|
+
options: HarnessChildTaskOptions = {}
|
|
439
|
+
): Promise<string> => {
|
|
440
|
+
assertOpen("rt.createTask");
|
|
441
|
+
const registered = engine.definitions.get(childDefinition.key);
|
|
442
|
+
if (!registered) {
|
|
443
|
+
throw new HarnessDefinitionError(`${childDefinition.key} is not in this harness registry`);
|
|
444
|
+
}
|
|
445
|
+
const key = options.key ?? String(childIndex);
|
|
446
|
+
childIndex += 1;
|
|
447
|
+
const child = await createChildTaskRecords({
|
|
448
|
+
definition: registered,
|
|
449
|
+
input,
|
|
450
|
+
key,
|
|
451
|
+
lease,
|
|
452
|
+
models,
|
|
453
|
+
options,
|
|
454
|
+
parent: task,
|
|
455
|
+
});
|
|
456
|
+
engine.wake();
|
|
457
|
+
return String(child._id);
|
|
458
|
+
};
|
|
459
|
+
|
|
460
|
+
const waitForTasks = async (
|
|
461
|
+
ids: ReadonlyArray<mongoose.Types.ObjectId | string>,
|
|
462
|
+
options: HarnessWaitForTasksOptions = {}
|
|
463
|
+
): Promise<HarnessChildOutcome[]> => {
|
|
464
|
+
assertOpen("rt.waitForTasks");
|
|
465
|
+
const policy = options.policy ?? HARNESS_WAIT_POLICIES.all;
|
|
466
|
+
if (!Object.values(HARNESS_WAIT_POLICIES).includes(policy)) {
|
|
467
|
+
throw new HarnessDefinitionError(
|
|
468
|
+
`rt.waitForTasks policy must be one of ${Object.values(HARNESS_WAIT_POLICIES).join(", ")}`
|
|
469
|
+
);
|
|
470
|
+
}
|
|
471
|
+
const childIds = [...new Set(ids.map(String))].map(toObjectId);
|
|
472
|
+
const owned = await models.task.countDocuments({
|
|
473
|
+
_id: {$in: childIds},
|
|
474
|
+
"ownership.id": task._id,
|
|
475
|
+
"ownership.kind": "task",
|
|
476
|
+
});
|
|
477
|
+
if (owned !== childIds.length) {
|
|
478
|
+
throw new HarnessDefinitionError(
|
|
479
|
+
`${definition.key}: rt.waitForTasks only waits on tasks this task created with rt.createTask`
|
|
480
|
+
);
|
|
481
|
+
}
|
|
482
|
+
const {children, isSettled} = await settleChildren({
|
|
483
|
+
engine,
|
|
484
|
+
ids: childIds,
|
|
485
|
+
ownerId: task._id,
|
|
486
|
+
policy,
|
|
487
|
+
});
|
|
488
|
+
if (isSettled) {
|
|
489
|
+
return childOutcomes(ids.map(String), children);
|
|
490
|
+
}
|
|
491
|
+
await settlePhase(async () => {
|
|
492
|
+
const parked = await commitWaiting({
|
|
493
|
+
models,
|
|
494
|
+
phaseStartedAt,
|
|
495
|
+
task,
|
|
496
|
+
testHooks,
|
|
497
|
+
waiting: {kind: HARNESS_WAIT_KINDS.tasks, policy, taskIds: childIds},
|
|
498
|
+
});
|
|
499
|
+
return parked.task;
|
|
500
|
+
});
|
|
501
|
+
// A child may have settled between the check above and the waiting commit.
|
|
502
|
+
await checkTaskWait({engine, taskId: task._id});
|
|
503
|
+
throw new HarnessSuspendSignal(taskId);
|
|
504
|
+
};
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* Shared by `rt.waitFor`, `rt.sleep`, and `rt.approval`: resolve the call now, or park the
|
|
508
|
+
* task on it. `plan` builds the request from the call's key.
|
|
509
|
+
*/
|
|
510
|
+
const waitCall = async (
|
|
511
|
+
method: string,
|
|
512
|
+
plan: (callKey: string) => WaitCallPlan
|
|
513
|
+
): Promise<unknown> => {
|
|
514
|
+
assertOpen(method);
|
|
515
|
+
// Taken before any await, so the call's key depends only on its order in the phase.
|
|
516
|
+
const callKey = waitCallKey({index: waitIndex, task});
|
|
517
|
+
waitIndex += 1;
|
|
518
|
+
const {afterPark, onPark, request} = plan(callKey);
|
|
519
|
+
let result: Awaited<ReturnType<typeof resolveWaitCall>>;
|
|
520
|
+
try {
|
|
521
|
+
result = await resolveWaitCall({
|
|
522
|
+
callKey,
|
|
523
|
+
label: definition.key,
|
|
524
|
+
lease,
|
|
525
|
+
models,
|
|
526
|
+
request,
|
|
527
|
+
task,
|
|
528
|
+
testHooks,
|
|
529
|
+
});
|
|
530
|
+
} catch (error: unknown) {
|
|
531
|
+
// A lost lease ends the run like a failed commit: nothing more may be written.
|
|
532
|
+
if (error instanceof HarnessCommitConflictError) {
|
|
533
|
+
commitStarted = true;
|
|
534
|
+
commitFailure = error;
|
|
535
|
+
}
|
|
536
|
+
throw error;
|
|
537
|
+
}
|
|
538
|
+
if (result.isResolved) {
|
|
539
|
+
return result.payload;
|
|
540
|
+
}
|
|
541
|
+
const {entry} = result;
|
|
542
|
+
let isWoken = false;
|
|
543
|
+
await settlePhase(async () => {
|
|
544
|
+
const parked = await commitWaiting({
|
|
545
|
+
models,
|
|
546
|
+
phaseStartedAt,
|
|
547
|
+
task,
|
|
548
|
+
testHooks,
|
|
549
|
+
waitCall: {entry, key: callKey},
|
|
550
|
+
waiting: {key: entry.key, kind: entry.kind, timeoutAt: entry.timeoutAt},
|
|
551
|
+
writes: onPark,
|
|
552
|
+
});
|
|
553
|
+
isWoken = parked.isWoken;
|
|
554
|
+
return parked.task;
|
|
555
|
+
});
|
|
556
|
+
// An event buffered before the wait committed sent the task straight back to pending.
|
|
557
|
+
if (isWoken) {
|
|
558
|
+
engine.wake();
|
|
559
|
+
}
|
|
560
|
+
await afterPark?.();
|
|
561
|
+
throw new HarnessSuspendSignal(taskId);
|
|
562
|
+
};
|
|
563
|
+
|
|
564
|
+
const requestApproval = async (
|
|
565
|
+
key: string,
|
|
566
|
+
options: HarnessApprovalOptions,
|
|
567
|
+
extension?: string
|
|
568
|
+
): Promise<HarnessApprovalResult> => {
|
|
569
|
+
assertOpen("rt.approval");
|
|
570
|
+
const timeout = parseApprovalRequest({key, label: definition.key, options});
|
|
571
|
+
let approval: ApprovalWaitPlan | undefined;
|
|
572
|
+
await waitCall("rt.approval", (callKey) => {
|
|
573
|
+
approval = approvalWaitPlan({callKey, extension, key, models, options, task, timeout});
|
|
574
|
+
return approval;
|
|
575
|
+
});
|
|
576
|
+
return (approval as ApprovalWaitPlan).result();
|
|
577
|
+
};
|
|
578
|
+
|
|
579
|
+
const waitFor = async (event: string, options: HarnessWaitForOptions = {}): Promise<unknown> => {
|
|
580
|
+
if (typeof event !== "string" || !event.trim()) {
|
|
581
|
+
assertOpen("rt.waitFor");
|
|
582
|
+
throw new HarnessDefinitionError(`${definition.key}: rt.waitFor requires an event name`);
|
|
583
|
+
}
|
|
584
|
+
const timeout =
|
|
585
|
+
options?.timeout === undefined
|
|
586
|
+
? undefined
|
|
587
|
+
: parseWaitDuration(options.timeout, `${definition.key}: rt.waitFor timeout`);
|
|
588
|
+
return waitCall("rt.waitFor", () => ({
|
|
589
|
+
request: {duration: timeout, event, kind: HARNESS_WAIT_KINDS.event},
|
|
590
|
+
}));
|
|
591
|
+
};
|
|
592
|
+
|
|
593
|
+
const sleep = async (duration: DurationLike): Promise<void> => {
|
|
594
|
+
const length = parseWaitDuration(duration, `${definition.key}: rt.sleep duration`);
|
|
595
|
+
await waitCall("rt.sleep", () => ({
|
|
596
|
+
request: {duration: length, kind: HARNESS_WAIT_KINDS.sleep},
|
|
597
|
+
}));
|
|
598
|
+
};
|
|
599
|
+
|
|
600
|
+
const runAgent = async <Result>(
|
|
601
|
+
agent: HarnessAgentDefinition,
|
|
602
|
+
options: HarnessRunAgentOptions<Result>
|
|
603
|
+
): Promise<Result> => {
|
|
604
|
+
assertOpen("rt.runAgent");
|
|
605
|
+
if (!options || typeof options !== "object") {
|
|
606
|
+
throw new HarnessDefinitionError(`${definition.key}: rt.runAgent requires {input}`);
|
|
607
|
+
}
|
|
608
|
+
// Taken before any await, so the call's key depends only on its order in the phase.
|
|
609
|
+
const callIndex = agentIndex;
|
|
610
|
+
agentIndex += 1;
|
|
611
|
+
if (!agent?.name || engine.agents.get(agent.name) !== agent) {
|
|
612
|
+
throw new HarnessDefinitionError(
|
|
613
|
+
`${definition.key}: rt.runAgent agent "${agent?.name}" is not in this harness registry`
|
|
614
|
+
);
|
|
615
|
+
}
|
|
616
|
+
const content = subagentContent(options.input);
|
|
617
|
+
if (content === undefined) {
|
|
618
|
+
throw new HarnessDefinitionError(`${definition.key}: rt.runAgent requires non-empty input`);
|
|
619
|
+
}
|
|
620
|
+
const turn = engine.definitions.get(AGENT_TURN_KEY);
|
|
621
|
+
if (!turn) {
|
|
622
|
+
throw harnessError({detail: `${AGENT_TURN_KEY} is not registered`, kind: "notRegistered"});
|
|
623
|
+
}
|
|
624
|
+
const schema = (options.output ?? agent.output) as HarnessRunAgentOptions<Result>["output"];
|
|
625
|
+
let outputSchema: string | undefined;
|
|
626
|
+
if (schema) {
|
|
627
|
+
try {
|
|
628
|
+
outputSchema = await serializeOutputSchema(schema);
|
|
629
|
+
} catch (error: unknown) {
|
|
630
|
+
throw new HarnessDefinitionError(
|
|
631
|
+
`${definition.key}: rt.runAgent output for "${agent.name}" cannot be expressed as JSON Schema: ${errorMessage(error)}`
|
|
632
|
+
);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
const {conversationId, turnTask} = await startSubagentTurn({
|
|
636
|
+
agent,
|
|
637
|
+
callIndex,
|
|
638
|
+
content,
|
|
639
|
+
extensions: engine.extensions,
|
|
640
|
+
instructions: options.instructions,
|
|
641
|
+
lease,
|
|
642
|
+
models,
|
|
643
|
+
outputSchema,
|
|
644
|
+
parent: task,
|
|
645
|
+
turn,
|
|
646
|
+
});
|
|
647
|
+
engine.wake();
|
|
648
|
+
const [outcome] = await waitForTasks([turnTask._id]);
|
|
649
|
+
return subagentResult({
|
|
650
|
+
agent,
|
|
651
|
+
conversationId,
|
|
652
|
+
outcome: outcome as HarnessChildOutcome,
|
|
653
|
+
schema,
|
|
654
|
+
});
|
|
655
|
+
};
|
|
656
|
+
|
|
657
|
+
const failOrRetry = async ({
|
|
658
|
+
error,
|
|
659
|
+
isRetryable,
|
|
660
|
+
}: {
|
|
661
|
+
error: string;
|
|
662
|
+
isRetryable: boolean;
|
|
663
|
+
}): Promise<HarnessTaskDocument | null> => {
|
|
664
|
+
await heartbeat.stop();
|
|
665
|
+
const policy = resolveRetryPolicy(task.retry);
|
|
666
|
+
const failedAttempts = task.attempt + 1;
|
|
667
|
+
try {
|
|
668
|
+
if (isRetryable && failedAttempts < policy.maxAttempts) {
|
|
669
|
+
const runAt = nextRetryAt({failedAttempts, policy, random: testHooks?.random});
|
|
670
|
+
const retried = await commitRetry({
|
|
671
|
+
error,
|
|
672
|
+
failedAttempts,
|
|
673
|
+
maxAttempts: policy.maxAttempts,
|
|
674
|
+
models,
|
|
675
|
+
phaseStartedAt,
|
|
676
|
+
runAt,
|
|
677
|
+
task,
|
|
678
|
+
testHooks,
|
|
679
|
+
});
|
|
680
|
+
logger.warn(
|
|
681
|
+
`Harness task ${taskId} phase "${task.phase}" failed (attempt ${failedAttempts} of ${policy.maxAttempts}); retrying at ${runAt.toISO()}: ${error}`
|
|
682
|
+
);
|
|
683
|
+
return retried;
|
|
684
|
+
}
|
|
685
|
+
return await commitPhase({
|
|
686
|
+
failedAttempts: isRetryable ? failedAttempts : undefined,
|
|
687
|
+
lease,
|
|
688
|
+
models,
|
|
689
|
+
next: {terminal: {error, status: "failed"}},
|
|
690
|
+
phaseStartedAt,
|
|
691
|
+
task,
|
|
692
|
+
testHooks,
|
|
693
|
+
});
|
|
694
|
+
} catch (commitError: unknown) {
|
|
695
|
+
logger.error(
|
|
696
|
+
`Harness task ${taskId} could not record failure of phase "${task.phase}": ${errorMessage(commitError)}`
|
|
697
|
+
);
|
|
698
|
+
return null;
|
|
699
|
+
}
|
|
700
|
+
};
|
|
701
|
+
|
|
702
|
+
const memoFor = (scopeTaskId: mongoose.Types.ObjectId | string) =>
|
|
703
|
+
createMemo({
|
|
704
|
+
assertWritable: () => assertOpen("rt.memo"),
|
|
705
|
+
fence: {phase: task.phase, taskId: task._id, token: task.lease?.token},
|
|
706
|
+
lease,
|
|
707
|
+
models,
|
|
708
|
+
scopeTaskId: toObjectId(scopeTaskId),
|
|
709
|
+
});
|
|
710
|
+
|
|
711
|
+
const output = async (text: string): Promise<void> => {
|
|
712
|
+
assertOpen("rt.output");
|
|
713
|
+
if (typeof text !== "string") {
|
|
714
|
+
throw new HarnessDefinitionError(`${definition.key}: rt.output takes a string`);
|
|
715
|
+
}
|
|
716
|
+
if (!text) {
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
await appendTaskEvents({
|
|
720
|
+
events: [
|
|
721
|
+
{
|
|
722
|
+
payload: {attempt: task.attempt, phase: task.phase, text},
|
|
723
|
+
type: HARNESS_EVENT_TYPES.output,
|
|
724
|
+
},
|
|
725
|
+
],
|
|
726
|
+
models,
|
|
727
|
+
task,
|
|
728
|
+
});
|
|
729
|
+
};
|
|
730
|
+
|
|
731
|
+
const phase = definition.phases[task.phase];
|
|
732
|
+
if (!phase) {
|
|
733
|
+
return failOrRetry({
|
|
734
|
+
error: `${definition.key}: unknown phase "${task.phase}"`,
|
|
735
|
+
isRetryable: false,
|
|
736
|
+
});
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
const rt: HarnessTaskRuntime<unknown, unknown> & {[HARNESS_INTERNAL_RUNTIME]: unknown} = {
|
|
740
|
+
[HARNESS_INTERNAL_RUNTIME]: {
|
|
741
|
+
approvalFor: (extension: string) => (key, options) =>
|
|
742
|
+
requestApproval(key, options, extension),
|
|
743
|
+
commitWithWrites,
|
|
744
|
+
memoFor,
|
|
745
|
+
},
|
|
746
|
+
approval: (key, options) => requestApproval(key, options),
|
|
747
|
+
commit,
|
|
748
|
+
createTask: createTask as HarnessTaskRuntime<unknown, unknown>["createTask"],
|
|
749
|
+
env: engine.env,
|
|
750
|
+
memo: memoFor(task._id),
|
|
751
|
+
output,
|
|
752
|
+
runAgent: runAgent as HarnessTaskRuntime<unknown, unknown>["runAgent"],
|
|
753
|
+
signal: controller.signal,
|
|
754
|
+
sleep,
|
|
755
|
+
taskId,
|
|
756
|
+
waitFor: waitFor as HarnessTaskRuntime<unknown, unknown>["waitFor"],
|
|
757
|
+
waitForTasks,
|
|
758
|
+
};
|
|
759
|
+
let thrown: {error: unknown} | undefined;
|
|
760
|
+
try {
|
|
761
|
+
await phase.run(toTaskView(task), rt);
|
|
762
|
+
} catch (error: unknown) {
|
|
763
|
+
thrown = {error};
|
|
764
|
+
}
|
|
765
|
+
await heartbeat.stop();
|
|
766
|
+
|
|
767
|
+
if (commitFailure !== undefined && controller.signal.aborted) {
|
|
768
|
+
logger.info(
|
|
769
|
+
`Harness task ${taskId} phase "${task.phase}" was aborted; its commit was discarded`
|
|
770
|
+
);
|
|
771
|
+
return null;
|
|
772
|
+
}
|
|
773
|
+
if (commitFailure !== undefined) {
|
|
774
|
+
logger.error(
|
|
775
|
+
`Harness task ${taskId} phase "${task.phase}" commit failed; task stays at its last checkpoint: ${errorMessage(commitFailure)}`
|
|
776
|
+
);
|
|
777
|
+
return null;
|
|
778
|
+
}
|
|
779
|
+
if (committed) {
|
|
780
|
+
if (thrown && !(thrown.error instanceof HarnessSuspendSignal)) {
|
|
781
|
+
logger.warn(
|
|
782
|
+
`Harness task ${taskId} phase "${task.phase}" threw after committing; keeping the checkpoint: ${errorMessage(thrown.error)}`
|
|
783
|
+
);
|
|
784
|
+
}
|
|
785
|
+
return committed;
|
|
786
|
+
}
|
|
787
|
+
// Aborted (or taken over) mid-phase: the abort already settled the task; record nothing.
|
|
788
|
+
if (controller.signal.aborted) {
|
|
789
|
+
logger.info(`Harness task ${taskId} phase "${task.phase}" stopped after its abort signal`);
|
|
790
|
+
return null;
|
|
791
|
+
}
|
|
792
|
+
if (thrown) {
|
|
793
|
+
return failOrRetry({
|
|
794
|
+
error: errorMessage(thrown.error),
|
|
795
|
+
isRetryable: !(thrown.error instanceof HarnessDefinitionError),
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
return failOrRetry({
|
|
799
|
+
error: `${definition.key}: phase "${task.phase}" returned without calling rt.commit`,
|
|
800
|
+
isRetryable: false,
|
|
801
|
+
});
|
|
802
|
+
};
|