@terreno/ai 57.9.0 → 57.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +127 -0
- package/dist/harness/errors.js +101 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +520 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +186 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +57 -0
- package/dist/harness/runners/inProcessRunner.js +196 -0
- package/dist/harness/runtime.d.ts +34 -0
- package/dist/harness/runtime.js +551 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +934 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +17 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +119 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +757 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +767 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +201 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +912 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +402 -0
- package/src/harness/runners/inProcessRunner.ts +220 -0
- package/src/harness/runtime.ts +714 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1095 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,678 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.commitResolution = exports.commitInterruption = exports.expiredLeaseFilter = exports.commitAbort = exports.claimAbortHandler = exports.fenceForAbort = exports.wakeWaitingTask = exports.commitWaitResolution = exports.commitWaiting = exports.commitRetry = exports.commitPhase = exports.createChildTaskRecords = exports.createTaskRecords = exports.newTaskLease = exports.isDuplicateKeyError = exports.HarnessCommitConflictError = void 0;
|
|
7
|
+
const node_crypto_1 = require("node:crypto");
|
|
8
|
+
const api_1 = require("@terreno/api");
|
|
9
|
+
const luxon_1 = require("luxon");
|
|
10
|
+
const mongoose_1 = __importDefault(require("mongoose"));
|
|
11
|
+
const harness_1 = require("../types/harness");
|
|
12
|
+
const agentTaskNames_1 = require("./agentTaskNames");
|
|
13
|
+
const definitionError_1 = require("./definitionError");
|
|
14
|
+
const errors_1 = require("./errors");
|
|
15
|
+
const events_1 = require("./events");
|
|
16
|
+
const transaction_1 = require("./transaction");
|
|
17
|
+
/**
|
|
18
|
+
* The task changed underneath this commit (checkpoint moved, status changed, or another
|
|
19
|
+
* runner holds the lease); nothing was written and the result is discarded.
|
|
20
|
+
*/
|
|
21
|
+
class HarnessCommitConflictError extends api_1.APIError {
|
|
22
|
+
constructor(taskId, phase) {
|
|
23
|
+
super({
|
|
24
|
+
...errors_1.HARNESS_ERRORS.commitConflict,
|
|
25
|
+
detail: `Harness commit for task ${taskId} phase "${phase}" lost its checkpoint fence`,
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
exports.HarnessCommitConflictError = HarnessCommitConflictError;
|
|
30
|
+
const DUPLICATE_KEY_CODE = 11000;
|
|
31
|
+
const isDuplicateKeyError = (error) => {
|
|
32
|
+
return (typeof error === "object" &&
|
|
33
|
+
error !== null &&
|
|
34
|
+
"code" in error &&
|
|
35
|
+
error.code === DUPLICATE_KEY_CODE);
|
|
36
|
+
};
|
|
37
|
+
exports.isDuplicateKeyError = isDuplicateKeyError;
|
|
38
|
+
/** How the task's own audit span is named and typed. */
|
|
39
|
+
const taskSpanIdentity = (definition, input) => ({
|
|
40
|
+
kind: definition.spanKind ?? "CHAIN",
|
|
41
|
+
name: definition.spanName ? definition.spanName(input) : definition.key,
|
|
42
|
+
});
|
|
43
|
+
/** Fresh lease fields for a phase that starts now under `lease.owner`. */
|
|
44
|
+
const newTaskLease = (lease) => {
|
|
45
|
+
const now = luxon_1.DateTime.now();
|
|
46
|
+
return {
|
|
47
|
+
acquiredAt: now.toJSDate(),
|
|
48
|
+
expiresAt: now.plus(lease.duration).toJSDate(),
|
|
49
|
+
owner: lease.owner,
|
|
50
|
+
token: (0, node_crypto_1.randomUUID)(),
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
exports.newTaskLease = newTaskLease;
|
|
54
|
+
const toObjectId = (value) => {
|
|
55
|
+
if (value === undefined) {
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
return typeof value === "string" ? new mongoose_1.default.Types.ObjectId(value) : value;
|
|
59
|
+
};
|
|
60
|
+
const findByRequestId = async ({ definition, models, requestId, userId, }) => {
|
|
61
|
+
// Soft-deleted owners still hold the unique index entry, so look past the deleted filter.
|
|
62
|
+
const existing = await models.task.findExactlyOne({ deleted: { $in: [false, true] }, requestId });
|
|
63
|
+
// Never hand one user's task (input, state) to another caller reusing the key.
|
|
64
|
+
if (String(existing.userId ?? "") !== String(userId ?? "")) {
|
|
65
|
+
throw (0, errors_1.harnessError)({
|
|
66
|
+
detail: `requestId "${requestId}" already belongs to a task for a different user`,
|
|
67
|
+
kind: "requestIdConflict",
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
if (existing.name !== definition.name) {
|
|
71
|
+
throw (0, errors_1.harnessError)({
|
|
72
|
+
detail: `requestId "${requestId}" already belongs to task ${existing._id} (${existing.name}), not ${definition.name}`,
|
|
73
|
+
kind: "requestIdConflict",
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
return existing;
|
|
77
|
+
};
|
|
78
|
+
/**
|
|
79
|
+
* Insert a root task with its `ObsTrace` and root `CHAIN` span in one transaction.
|
|
80
|
+
* A repeated `requestId` hits the unique index, rolls the whole transaction back (so no
|
|
81
|
+
* orphan trace survives), and returns the task that already owns it.
|
|
82
|
+
*/
|
|
83
|
+
const createTaskRecords = async ({ definition, input, models, options, ownership = { kind: "root" }, writes, }) => {
|
|
84
|
+
const initial = definition.initial(input);
|
|
85
|
+
if (!definition.phases[initial.phase]) {
|
|
86
|
+
throw new definitionError_1.HarnessDefinitionError(`${definition.key}: initial phase "${initial.phase}" is not one of ${Object.keys(definition.phases).join(", ")}`);
|
|
87
|
+
}
|
|
88
|
+
const taskId = new mongoose_1.default.Types.ObjectId();
|
|
89
|
+
const traceId = new mongoose_1.default.Types.ObjectId();
|
|
90
|
+
const rootSpanId = new mongoose_1.default.Types.ObjectId();
|
|
91
|
+
const traceStartedAt = luxon_1.DateTime.now();
|
|
92
|
+
const startedAt = traceStartedAt.toJSDate();
|
|
93
|
+
const userId = toObjectId(options.userId);
|
|
94
|
+
const spanIdentity = taskSpanIdentity(definition, input);
|
|
95
|
+
try {
|
|
96
|
+
return await (0, transaction_1.inTransaction)(async (session) => {
|
|
97
|
+
await models.trace.create([{ _id: traceId, input, name: definition.key, startedAt, status: "ok", userId }], { session });
|
|
98
|
+
await models.span.create([
|
|
99
|
+
{
|
|
100
|
+
_id: rootSpanId,
|
|
101
|
+
input,
|
|
102
|
+
kind: spanIdentity.kind,
|
|
103
|
+
name: spanIdentity.name,
|
|
104
|
+
startedAt,
|
|
105
|
+
startOffsetMs: 0,
|
|
106
|
+
status: "ok",
|
|
107
|
+
traceId,
|
|
108
|
+
},
|
|
109
|
+
], { session });
|
|
110
|
+
const [task] = await models.task.create([
|
|
111
|
+
{
|
|
112
|
+
_id: taskId,
|
|
113
|
+
input,
|
|
114
|
+
name: definition.name,
|
|
115
|
+
ownership,
|
|
116
|
+
phase: initial.phase,
|
|
117
|
+
requestId: options.requestId,
|
|
118
|
+
retry: definition.retry,
|
|
119
|
+
rootSpanId,
|
|
120
|
+
rootTaskId: taskId,
|
|
121
|
+
state: initial.state,
|
|
122
|
+
status: harness_1.HARNESS_TASK_STATUSES.pending,
|
|
123
|
+
traceId,
|
|
124
|
+
userId,
|
|
125
|
+
version: definition.version,
|
|
126
|
+
},
|
|
127
|
+
], { session });
|
|
128
|
+
await writes?.({ session, task, traceStartedAt });
|
|
129
|
+
await (0, events_1.appendTaskTransitionEvents)({ after: task, models, session });
|
|
130
|
+
return task;
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
// An earlier or concurrent create with the same requestId owns the unique index entry.
|
|
135
|
+
if (options.requestId && (0, exports.isDuplicateKeyError)(error)) {
|
|
136
|
+
return findByRequestId({ definition, models, requestId: options.requestId, userId });
|
|
137
|
+
}
|
|
138
|
+
throw error;
|
|
139
|
+
}
|
|
140
|
+
};
|
|
141
|
+
exports.createTaskRecords = createTaskRecords;
|
|
142
|
+
/** Whether `task` is the top of its ownership tree, and so owns the `ObsTrace`. */
|
|
143
|
+
const isRootTask = (task) => String(task._id) === String(task.rootTaskId);
|
|
144
|
+
/**
|
|
145
|
+
* Idempotency key that names one child of one attempt of one phase visit of `parent`. A
|
|
146
|
+
* re-run of the same attempt (wake, crash replay) finds the child again; a retry (next
|
|
147
|
+
* attempt) or a later visit of the phase creates a fresh one.
|
|
148
|
+
*/
|
|
149
|
+
const childRequestId = ({ key, parent }) => `harness-child:${parent._id}:${parent.step ?? 0}:${parent.attempt ?? 0}:${key}`;
|
|
150
|
+
/**
|
|
151
|
+
* Insert a child task owned by `parent`, in `parent`'s trace, with its own `CHAIN` span
|
|
152
|
+
* nested under `parent`'s span. Fenced on `parent`'s lease token (renewing the lease in
|
|
153
|
+
* the same transaction), so a runner that lost the parent cannot create children. A
|
|
154
|
+
* repeated `key` within the same attempt of a phase visit returns the child created first.
|
|
155
|
+
*/
|
|
156
|
+
const createChildTaskRecords = async ({ definition, input, key, lease, models, options, parent, span, writes, }) => {
|
|
157
|
+
const initial = definition.initial(input);
|
|
158
|
+
if (!definition.phases[initial.phase]) {
|
|
159
|
+
throw new definitionError_1.HarnessDefinitionError(`${definition.key}: initial phase "${initial.phase}" is not one of ${Object.keys(definition.phases).join(", ")}`);
|
|
160
|
+
}
|
|
161
|
+
const requestId = childRequestId({ key, parent });
|
|
162
|
+
const existing = await models.task.findOneOrNone({ deleted: { $in: [false, true] }, requestId });
|
|
163
|
+
if (existing) {
|
|
164
|
+
if (existing.name !== definition.name) {
|
|
165
|
+
throw (0, errors_1.harnessError)({
|
|
166
|
+
detail: `Child key "${key}" already belongs to task ${existing._id} (${existing.name}), not ${definition.name}`,
|
|
167
|
+
kind: "requestIdConflict",
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
return existing;
|
|
171
|
+
}
|
|
172
|
+
const trace = await models.trace.findExactlyOne({ _id: parent.traceId });
|
|
173
|
+
const taskId = new mongoose_1.default.Types.ObjectId();
|
|
174
|
+
const spanId = new mongoose_1.default.Types.ObjectId();
|
|
175
|
+
const startedAt = luxon_1.DateTime.now();
|
|
176
|
+
const spanIdentity = span ?? taskSpanIdentity(definition, input);
|
|
177
|
+
const spanInput = span && "input" in span ? span.input : input;
|
|
178
|
+
try {
|
|
179
|
+
return await (0, transaction_1.inTransaction)(async (session) => {
|
|
180
|
+
const fenced = await models.task.updateOne({
|
|
181
|
+
_id: parent._id,
|
|
182
|
+
"lease.token": parent.lease?.token ?? null,
|
|
183
|
+
phase: parent.phase,
|
|
184
|
+
status: harness_1.HARNESS_TASK_STATUSES.running,
|
|
185
|
+
}, { $set: { "lease.expiresAt": startedAt.plus(lease.duration).toJSDate() } }, { session });
|
|
186
|
+
if (fenced.matchedCount === 0) {
|
|
187
|
+
throw new HarnessCommitConflictError(String(parent._id), parent.phase);
|
|
188
|
+
}
|
|
189
|
+
await models.span.create([
|
|
190
|
+
{
|
|
191
|
+
_id: spanId,
|
|
192
|
+
input: spanInput,
|
|
193
|
+
kind: spanIdentity.kind,
|
|
194
|
+
name: spanIdentity.name,
|
|
195
|
+
parentSpanId: parent.rootSpanId,
|
|
196
|
+
startedAt: startedAt.toJSDate(),
|
|
197
|
+
startOffsetMs: startedAt.diff(luxon_1.DateTime.fromJSDate(trace.startedAt)).toMillis(),
|
|
198
|
+
status: "ok",
|
|
199
|
+
traceId: parent.traceId,
|
|
200
|
+
},
|
|
201
|
+
], { session });
|
|
202
|
+
const [task] = await models.task.create([
|
|
203
|
+
{
|
|
204
|
+
_id: taskId,
|
|
205
|
+
ancestorIds: [...(parent.ancestorIds ?? []), parent._id],
|
|
206
|
+
background: options.background ?? false,
|
|
207
|
+
input,
|
|
208
|
+
name: definition.name,
|
|
209
|
+
ownership: { id: parent._id, kind: "task" },
|
|
210
|
+
phase: initial.phase,
|
|
211
|
+
requestId,
|
|
212
|
+
retry: definition.retry,
|
|
213
|
+
rootSpanId: spanId,
|
|
214
|
+
rootTaskId: parent.rootTaskId,
|
|
215
|
+
state: initial.state,
|
|
216
|
+
status: harness_1.HARNESS_TASK_STATUSES.pending,
|
|
217
|
+
traceId: parent.traceId,
|
|
218
|
+
userId: parent.userId,
|
|
219
|
+
version: definition.version,
|
|
220
|
+
},
|
|
221
|
+
], { session });
|
|
222
|
+
await writes?.({ session, task, traceStartedAt: luxon_1.DateTime.fromJSDate(trace.startedAt) });
|
|
223
|
+
await (0, events_1.appendTaskTransitionEvents)({ after: task, models, session });
|
|
224
|
+
return task;
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
// A concurrent run of the same phase visit created this child first.
|
|
229
|
+
if ((0, exports.isDuplicateKeyError)(error)) {
|
|
230
|
+
return models.task.findExactlyOne({ deleted: { $in: [false, true] }, requestId });
|
|
231
|
+
}
|
|
232
|
+
throw error;
|
|
233
|
+
}
|
|
234
|
+
};
|
|
235
|
+
exports.createChildTaskRecords = createChildTaskRecords;
|
|
236
|
+
const describeNext = (next) => {
|
|
237
|
+
if ("terminal" in next) {
|
|
238
|
+
return { terminal: next.terminal };
|
|
239
|
+
}
|
|
240
|
+
return { phase: next.phase, state: next.state };
|
|
241
|
+
};
|
|
242
|
+
/**
|
|
243
|
+
* Apply one fenced task update and insert its `CHAIN` span (closing the root span and
|
|
244
|
+
* `ObsTrace` when `close` is set) in a single transaction. Throws
|
|
245
|
+
* `HarnessCommitConflictError`, writing nothing, when `filter` no longer matches.
|
|
246
|
+
*/
|
|
247
|
+
const commitTransition = async ({ close, filter, models, span, task, testHooks, update, writes, }) => {
|
|
248
|
+
const taskId = String(task._id);
|
|
249
|
+
const trace = await models.trace.findExactlyOne({ _id: task.traceId });
|
|
250
|
+
const traceStartedAt = luxon_1.DateTime.fromJSDate(trace.startedAt);
|
|
251
|
+
// A root task's span starts with the trace; a child's span starts when it was created.
|
|
252
|
+
const taskSpanStartedAt = close && !isRootTask(task)
|
|
253
|
+
? luxon_1.DateTime.fromJSDate((await models.span.findExactlyOne({ _id: task.rootSpanId })).startedAt)
|
|
254
|
+
: traceStartedAt;
|
|
255
|
+
const endedAt = luxon_1.DateTime.now();
|
|
256
|
+
return (0, transaction_1.inTransaction)(async (session) => {
|
|
257
|
+
const updated = await models.task.findOneAndUpdate({ _id: task._id, ...filter }, update, {
|
|
258
|
+
returnDocument: "after",
|
|
259
|
+
session,
|
|
260
|
+
});
|
|
261
|
+
if (!updated) {
|
|
262
|
+
throw new HarnessCommitConflictError(taskId, task.phase);
|
|
263
|
+
}
|
|
264
|
+
await models.span.create([
|
|
265
|
+
{
|
|
266
|
+
durationMs: endedAt.diff(span.startedAt).toMillis(),
|
|
267
|
+
endedAt: endedAt.toJSDate(),
|
|
268
|
+
error: span.error,
|
|
269
|
+
input: span.input,
|
|
270
|
+
kind: "CHAIN",
|
|
271
|
+
name: span.name,
|
|
272
|
+
output: span.output,
|
|
273
|
+
parentSpanId: task.rootSpanId,
|
|
274
|
+
startedAt: span.startedAt.toJSDate(),
|
|
275
|
+
startOffsetMs: span.startedAt.diff(traceStartedAt).toMillis(),
|
|
276
|
+
status: span.status,
|
|
277
|
+
traceId: task.traceId,
|
|
278
|
+
},
|
|
279
|
+
], { session });
|
|
280
|
+
if (close) {
|
|
281
|
+
// A child closes its own span; only the root task closes the shared trace.
|
|
282
|
+
await models.span.updateOne({ _id: task.rootSpanId }, {
|
|
283
|
+
$set: {
|
|
284
|
+
durationMs: endedAt.diff(taskSpanStartedAt).toMillis(),
|
|
285
|
+
endedAt: endedAt.toJSDate(),
|
|
286
|
+
error: close.error,
|
|
287
|
+
output: close.output,
|
|
288
|
+
status: close.status,
|
|
289
|
+
},
|
|
290
|
+
}, { session });
|
|
291
|
+
}
|
|
292
|
+
if (close && isRootTask(task)) {
|
|
293
|
+
await models.trace.updateOne({ _id: task.traceId }, {
|
|
294
|
+
$set: {
|
|
295
|
+
endedAt: endedAt.toJSDate(),
|
|
296
|
+
errorSummary: close.error,
|
|
297
|
+
output: close.output,
|
|
298
|
+
status: close.status,
|
|
299
|
+
},
|
|
300
|
+
}, { session });
|
|
301
|
+
}
|
|
302
|
+
await writes?.({ session, task: updated, traceStartedAt });
|
|
303
|
+
await (0, events_1.appendTaskTransitionEvents)({ after: updated, before: task, models, session });
|
|
304
|
+
await releaseTurnConversation({ models, session, task: updated });
|
|
305
|
+
await testHooks?.beforeCommitEnd?.({ phase: task.phase, session, taskId });
|
|
306
|
+
return updated;
|
|
307
|
+
});
|
|
308
|
+
};
|
|
309
|
+
/**
|
|
310
|
+
* Free the conversation of a turn that just became terminal, in the same transaction, on
|
|
311
|
+
* every terminal path (answer, failure, abort, interruption, resolution). Fenced on the
|
|
312
|
+
* turn, so a conversation that already moved on is untouched.
|
|
313
|
+
*/
|
|
314
|
+
const releaseTurnConversation = async ({ models, session, task, }) => {
|
|
315
|
+
if (task.name !== agentTaskNames_1.AGENT_TURN_TASK_NAME || !harness_1.HARNESS_TERMINAL_STATUSES.has(task.status)) {
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
const conversationId = task.input?.conversationId;
|
|
319
|
+
if (typeof conversationId !== "string") {
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
await models.conversation.updateOne({ _id: conversationId, activeTurnTaskId: task._id, status: "busy" }, { $set: { status: "idle" }, $unset: { activeTurnTaskId: 1 } }, { session });
|
|
323
|
+
};
|
|
324
|
+
/** Fence for every commit made by the run that holds `task`'s current lease. */
|
|
325
|
+
const runFence = (task) => ({
|
|
326
|
+
"lease.token": task.lease?.token ?? null,
|
|
327
|
+
phase: task.phase,
|
|
328
|
+
status: harness_1.HARNESS_TASK_STATUSES.running,
|
|
329
|
+
});
|
|
330
|
+
/**
|
|
331
|
+
* Commit one phase: update the task checkpoint (or terminal outcome) and insert the
|
|
332
|
+
* phase's `CHAIN` span in a single transaction. The commit is fenced by the task's
|
|
333
|
+
* status, phase, and lease token, so a runner that lost its lease writes nothing. A
|
|
334
|
+
* phase commit hands the next phase a fresh lease token; a terminal commit clears the
|
|
335
|
+
* lease and closes the root span and `ObsTrace`.
|
|
336
|
+
*/
|
|
337
|
+
const commitPhase = async ({ failedAttempts, lease, models, next, phaseStartedAt, task, testHooks, writes, }) => {
|
|
338
|
+
const isTerminal = "terminal" in next;
|
|
339
|
+
const error = isTerminal && next.terminal.status === "failed" ? next.terminal.error : undefined;
|
|
340
|
+
const isFailure = error !== undefined;
|
|
341
|
+
const update = isTerminal
|
|
342
|
+
? {
|
|
343
|
+
$set: {
|
|
344
|
+
outcome: next.terminal,
|
|
345
|
+
status: next.terminal.status,
|
|
346
|
+
...(failedAttempts === undefined ? {} : { attempt: failedAttempts }),
|
|
347
|
+
},
|
|
348
|
+
$unset: { lease: 1, runAt: 1, waits: 1 },
|
|
349
|
+
}
|
|
350
|
+
: {
|
|
351
|
+
$inc: { step: 1 },
|
|
352
|
+
$set: {
|
|
353
|
+
attempt: 0,
|
|
354
|
+
lease: (0, exports.newTaskLease)(lease),
|
|
355
|
+
phase: next.phase,
|
|
356
|
+
state: next.state === undefined ? task.state : next.state,
|
|
357
|
+
},
|
|
358
|
+
// Wait records belong to one phase visit; the next visit starts with none.
|
|
359
|
+
$unset: { runAt: 1, waits: 1 },
|
|
360
|
+
};
|
|
361
|
+
return commitTransition({
|
|
362
|
+
close: isTerminal
|
|
363
|
+
? {
|
|
364
|
+
error,
|
|
365
|
+
output: next.terminal.status === "failed" ? { error } : next.terminal.result,
|
|
366
|
+
status: isFailure ? "error" : "ok",
|
|
367
|
+
}
|
|
368
|
+
: undefined,
|
|
369
|
+
// A task without a lease (written before leases existed) is fenced on "no token".
|
|
370
|
+
filter: runFence(task),
|
|
371
|
+
models,
|
|
372
|
+
span: {
|
|
373
|
+
error,
|
|
374
|
+
input: { attempt: task.attempt, state: task.state },
|
|
375
|
+
name: task.phase,
|
|
376
|
+
output: describeNext(next),
|
|
377
|
+
startedAt: phaseStartedAt,
|
|
378
|
+
status: isFailure ? "error" : "ok",
|
|
379
|
+
},
|
|
380
|
+
task,
|
|
381
|
+
testHooks,
|
|
382
|
+
update,
|
|
383
|
+
writes,
|
|
384
|
+
});
|
|
385
|
+
};
|
|
386
|
+
exports.commitPhase = commitPhase;
|
|
387
|
+
/**
|
|
388
|
+
* Record a thrown phase that has attempts left: back to `pending` at the same checkpoint
|
|
389
|
+
* with `attempt` raised and `runAt` set to the backoff time, plus an error `CHAIN` span,
|
|
390
|
+
* in one transaction fenced on the run's lease token.
|
|
391
|
+
*/
|
|
392
|
+
const commitRetry = async ({ error, failedAttempts, maxAttempts, models, phaseStartedAt, runAt, task, testHooks, }) => {
|
|
393
|
+
return commitTransition({
|
|
394
|
+
filter: runFence(task),
|
|
395
|
+
models,
|
|
396
|
+
span: {
|
|
397
|
+
error,
|
|
398
|
+
input: { attempt: task.attempt, state: task.state },
|
|
399
|
+
name: task.phase,
|
|
400
|
+
output: { retry: { attempt: failedAttempts, maxAttempts, runAt: runAt.toISO() } },
|
|
401
|
+
startedAt: phaseStartedAt,
|
|
402
|
+
status: "error",
|
|
403
|
+
},
|
|
404
|
+
task,
|
|
405
|
+
testHooks,
|
|
406
|
+
update: {
|
|
407
|
+
$set: {
|
|
408
|
+
attempt: failedAttempts,
|
|
409
|
+
runAt: runAt.toJSDate(),
|
|
410
|
+
status: harness_1.HARNESS_TASK_STATUSES.pending,
|
|
411
|
+
},
|
|
412
|
+
$unset: { lease: 1 },
|
|
413
|
+
},
|
|
414
|
+
});
|
|
415
|
+
};
|
|
416
|
+
exports.commitRetry = commitRetry;
|
|
417
|
+
/**
|
|
418
|
+
* Park a running task as `waiting` on `waiting` (child tasks, an event, or a sleep). Gives
|
|
419
|
+
* up the lease so no runner holds the task while it waits; the phase re-runs from its
|
|
420
|
+
* checkpoint once the wait is satisfied. `waitCall` records the `rt.waitFor` / `rt.sleep`
|
|
421
|
+
* call that started the wait. An event wait checks the inbox inside the transaction and,
|
|
422
|
+
* when a matching event is already buffered, returns the task straight to `pending`:
|
|
423
|
+
* `sendEvent` writes the task in its own transaction, so the two can never miss each other.
|
|
424
|
+
*/
|
|
425
|
+
const commitWaiting = async ({ models, phaseStartedAt, task, testHooks, waitCall, waiting, writes, }) => {
|
|
426
|
+
let isWoken = false;
|
|
427
|
+
const committed = await commitTransition({
|
|
428
|
+
filter: runFence(task),
|
|
429
|
+
models,
|
|
430
|
+
span: {
|
|
431
|
+
input: { attempt: task.attempt, state: task.state },
|
|
432
|
+
name: task.phase,
|
|
433
|
+
output: {
|
|
434
|
+
waiting: {
|
|
435
|
+
...waiting,
|
|
436
|
+
taskIds: waiting.taskIds?.map(String),
|
|
437
|
+
timeoutAt: waiting.timeoutAt?.toISOString(),
|
|
438
|
+
},
|
|
439
|
+
},
|
|
440
|
+
startedAt: phaseStartedAt,
|
|
441
|
+
status: "ok",
|
|
442
|
+
},
|
|
443
|
+
task,
|
|
444
|
+
testHooks,
|
|
445
|
+
update: {
|
|
446
|
+
$set: {
|
|
447
|
+
status: harness_1.HARNESS_TASK_STATUSES.waiting,
|
|
448
|
+
waiting,
|
|
449
|
+
...(waitCall ? { [`waits.${waitCall.key}`]: waitCall.entry } : {}),
|
|
450
|
+
},
|
|
451
|
+
$unset: { lease: 1 },
|
|
452
|
+
},
|
|
453
|
+
writes: async (context) => {
|
|
454
|
+
const { session } = context;
|
|
455
|
+
isWoken = false;
|
|
456
|
+
await writes?.(context);
|
|
457
|
+
if (waiting.kind !== harness_1.HARNESS_WAIT_KINDS.event || !waiting.key) {
|
|
458
|
+
return;
|
|
459
|
+
}
|
|
460
|
+
const buffered = await models.inbox.countDocuments({ consumedKey: { $exists: false }, name: waiting.key, taskId: task._id }, { session });
|
|
461
|
+
if (buffered === 0) {
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
const woken = await models.task.findOneAndUpdate({ _id: task._id }, { $set: { status: harness_1.HARNESS_TASK_STATUSES.pending }, $unset: { waiting: 1 } }, { returnDocument: "after", session });
|
|
465
|
+
isWoken = true;
|
|
466
|
+
if (woken) {
|
|
467
|
+
await (0, events_1.appendTaskTransitionEvents)({ after: woken, before: context.task, models, session });
|
|
468
|
+
}
|
|
469
|
+
},
|
|
470
|
+
});
|
|
471
|
+
return { isWoken, task: committed };
|
|
472
|
+
};
|
|
473
|
+
exports.commitWaiting = commitWaiting;
|
|
474
|
+
/**
|
|
475
|
+
* Record how one `rt.waitFor` / `rt.sleep` call resolved, plus its resume span, in one
|
|
476
|
+
* transaction fenced on the run's lease. `writes` claims the delivered event in the same
|
|
477
|
+
* transaction. The phase keeps running under its lease.
|
|
478
|
+
*/
|
|
479
|
+
const commitWaitResolution = async ({ callKey, entry, lease, models, span, task, testHooks, writes, }) => {
|
|
480
|
+
await commitTransition({
|
|
481
|
+
filter: runFence(task),
|
|
482
|
+
models,
|
|
483
|
+
span: { ...span, startedAt: luxon_1.DateTime.fromJSDate(entry.startedAt), status: "ok" },
|
|
484
|
+
task,
|
|
485
|
+
testHooks,
|
|
486
|
+
update: {
|
|
487
|
+
$set: {
|
|
488
|
+
"lease.expiresAt": luxon_1.DateTime.now().plus(lease.duration).toJSDate(),
|
|
489
|
+
[`waits.${callKey}`]: entry,
|
|
490
|
+
},
|
|
491
|
+
},
|
|
492
|
+
writes,
|
|
493
|
+
});
|
|
494
|
+
};
|
|
495
|
+
exports.commitWaitResolution = commitWaitResolution;
|
|
496
|
+
/**
|
|
497
|
+
* Return a `waiting` task to `pending` at the same checkpoint. Fenced on the wait it was
|
|
498
|
+
* parked on, so only one waker wins and a task that left `waiting` is untouched. Returns
|
|
499
|
+
* whether this call woke it.
|
|
500
|
+
*/
|
|
501
|
+
const wakeWaitingTask = async ({ kind, models, taskId, }) => {
|
|
502
|
+
const result = await models.task.updateOne({ _id: taskId, status: harness_1.HARNESS_TASK_STATUSES.waiting, "waiting.kind": kind }, { $set: { status: harness_1.HARNESS_TASK_STATUSES.pending }, $unset: { waiting: 1 } });
|
|
503
|
+
return result.modifiedCount > 0;
|
|
504
|
+
};
|
|
505
|
+
exports.wakeWaitingTask = wakeWaitingTask;
|
|
506
|
+
/**
|
|
507
|
+
* First step of an abort: record the request (which stops runners from claiming the
|
|
508
|
+
* task) and, when a phase is running, rotate its lease token so the run's commits,
|
|
509
|
+
* child creation, and lease renewals all fail from here on. Returns the task as it now
|
|
510
|
+
* stands, or null when it is already terminal.
|
|
511
|
+
*/
|
|
512
|
+
const fenceForAbort = async ({ models, reason, taskId, userId, }) => {
|
|
513
|
+
const nonTerminal = { $nin: [...harness_1.HARNESS_TERMINAL_STATUSES] };
|
|
514
|
+
// The first request wins; a repeated abort keeps its record (and any handler claim).
|
|
515
|
+
await models.task.updateOne({ _id: taskId, "abortRequested.at": { $exists: false }, status: nonTerminal }, { $set: { abortRequested: { at: luxon_1.DateTime.now().toJSDate(), reason, userId: toObjectId(userId) } } });
|
|
516
|
+
const requested = await models.task.findOneOrNone({ _id: taskId, status: nonTerminal });
|
|
517
|
+
if (!requested || requested.status !== harness_1.HARNESS_TASK_STATUSES.running) {
|
|
518
|
+
return requested;
|
|
519
|
+
}
|
|
520
|
+
// Not fenced on the old token: a phase that just committed holds a newer one.
|
|
521
|
+
const revoked = await models.task.findOneAndUpdate({ _id: taskId, status: harness_1.HARNESS_TASK_STATUSES.running }, { $set: { "lease.token": `abort:${(0, node_crypto_1.randomUUID)()}` } }, { returnDocument: "after" });
|
|
522
|
+
return revoked ?? models.task.findOneOrNone({ _id: taskId, status: nonTerminal });
|
|
523
|
+
};
|
|
524
|
+
exports.fenceForAbort = fenceForAbort;
|
|
525
|
+
/**
|
|
526
|
+
* How long one aborter holds a task's abort-handler claim. Another abort of the same task
|
|
527
|
+
* waits for the claim holder to finish, and takes over only after this lapses (the holder
|
|
528
|
+
* died mid-handler).
|
|
529
|
+
*/
|
|
530
|
+
const ABORT_HANDLER_CLAIM = luxon_1.Duration.fromObject({ minutes: 1 });
|
|
531
|
+
/**
|
|
532
|
+
* Claim the right to run `taskId`'s abort handler and commit its abort, so concurrent
|
|
533
|
+
* aborts (an operator and a failFast sibling, say) never run one handler twice. Returns
|
|
534
|
+
* false while another live aborter holds the claim, or once the task is terminal.
|
|
535
|
+
*/
|
|
536
|
+
const claimAbortHandler = async ({ models, taskId, }) => {
|
|
537
|
+
const now = luxon_1.DateTime.now();
|
|
538
|
+
const claimed = await models.task.updateOne({
|
|
539
|
+
_id: taskId,
|
|
540
|
+
$or: [
|
|
541
|
+
{ "abortRequested.handlerClaimExpiresAt": { $exists: false } },
|
|
542
|
+
{ "abortRequested.handlerClaimExpiresAt": { $lte: now.toJSDate() } },
|
|
543
|
+
],
|
|
544
|
+
status: { $nin: [...harness_1.HARNESS_TERMINAL_STATUSES] },
|
|
545
|
+
}, { $set: { "abortRequested.handlerClaimExpiresAt": now.plus(ABORT_HANDLER_CLAIM).toJSDate() } });
|
|
546
|
+
return claimed.modifiedCount > 0;
|
|
547
|
+
};
|
|
548
|
+
exports.claimAbortHandler = claimAbortHandler;
|
|
549
|
+
/**
|
|
550
|
+
* Mark a task `aborted` with `outcome.error`, clear its lease and wait, close its span
|
|
551
|
+
* (and the trace for a root task), and write one audit span, in one transaction. Fenced
|
|
552
|
+
* on `filter` (status only, not the lease token: an abort deliberately overrides the
|
|
553
|
+
* run in flight, whose later commit is then rejected).
|
|
554
|
+
*/
|
|
555
|
+
const commitAbort = async ({ error, filter, models, span, task, testHooks, }) => {
|
|
556
|
+
return commitTransition({
|
|
557
|
+
close: { error, output: { error }, status: "error" },
|
|
558
|
+
filter,
|
|
559
|
+
models,
|
|
560
|
+
span: {
|
|
561
|
+
...span,
|
|
562
|
+
input: { phase: task.phase, state: task.state, status: task.status },
|
|
563
|
+
startedAt: luxon_1.DateTime.now(),
|
|
564
|
+
},
|
|
565
|
+
task,
|
|
566
|
+
testHooks,
|
|
567
|
+
update: {
|
|
568
|
+
$set: {
|
|
569
|
+
outcome: { error, status: harness_1.HARNESS_TASK_STATUSES.aborted },
|
|
570
|
+
status: harness_1.HARNESS_TASK_STATUSES.aborted,
|
|
571
|
+
},
|
|
572
|
+
$unset: { lease: 1, runAt: 1, waiting: 1, waits: 1 },
|
|
573
|
+
},
|
|
574
|
+
});
|
|
575
|
+
};
|
|
576
|
+
exports.commitAbort = commitAbort;
|
|
577
|
+
/** Mongo filter for a `running` task whose lease lapsed (or that never had one). */
|
|
578
|
+
const expiredLeaseFilter = (now) => ({
|
|
579
|
+
$or: [{ "lease.expiresAt": { $lte: now.toJSDate() } }, { "lease.expiresAt": { $exists: false } }],
|
|
580
|
+
status: harness_1.HARNESS_TASK_STATUSES.running,
|
|
581
|
+
});
|
|
582
|
+
exports.expiredLeaseFilter = expiredLeaseFilter;
|
|
583
|
+
const interruptionOutcomes = (leaseOwner) => ({
|
|
584
|
+
fail: {
|
|
585
|
+
error: `Interrupted, not retried: lease of ${leaseOwner} expired mid-phase (replay: never)`,
|
|
586
|
+
replay: "never",
|
|
587
|
+
status: harness_1.HARNESS_TASK_STATUSES.failed,
|
|
588
|
+
},
|
|
589
|
+
park: {
|
|
590
|
+
error: `Interrupted: lease of ${leaseOwner} expired mid-phase; awaiting resolveInterrupted (replay: never)`,
|
|
591
|
+
replay: "never",
|
|
592
|
+
status: harness_1.HARNESS_TASK_STATUSES.interrupted,
|
|
593
|
+
},
|
|
594
|
+
replay: {
|
|
595
|
+
error: `Interrupted: lease of ${leaseOwner} expired mid-phase; re-running (replay: safe)`,
|
|
596
|
+
replay: "safe",
|
|
597
|
+
status: harness_1.HARNESS_TASK_STATUSES.pending,
|
|
598
|
+
},
|
|
599
|
+
});
|
|
600
|
+
/**
|
|
601
|
+
* Record that a task's lease expired mid-phase. `replay` returns it to `pending` at the
|
|
602
|
+
* same checkpoint so a runner re-runs the phase; `park` sets `interrupted` for
|
|
603
|
+
* `resolveInterrupted`; `fail` (a `replay: "never"` phase of an `onInterrupt: "fail"`
|
|
604
|
+
* task) ends it `failed` with an "Interrupted, not retried" outcome and closes its span.
|
|
605
|
+
* Each writes an error `CHAIN` span in the same transaction. Fenced on the expired lease,
|
|
606
|
+
* so a renewed lease is left alone.
|
|
607
|
+
*/
|
|
608
|
+
const commitInterruption = async ({ action, models, task, testHooks, }) => {
|
|
609
|
+
const now = luxon_1.DateTime.now();
|
|
610
|
+
const leaseOwner = task.lease?.owner ?? "unknown";
|
|
611
|
+
const { error, replay, status } = interruptionOutcomes(leaseOwner)[action];
|
|
612
|
+
const isFailure = action === "fail";
|
|
613
|
+
return commitTransition({
|
|
614
|
+
close: isFailure ? { error, output: { error }, status: "error" } : undefined,
|
|
615
|
+
filter: {
|
|
616
|
+
...(0, exports.expiredLeaseFilter)(now),
|
|
617
|
+
phase: task.phase,
|
|
618
|
+
// Fence on the token from the snapshot so the span names the lease that actually lapsed
|
|
619
|
+
...(task.lease?.token ? { "lease.token": task.lease.token } : {}),
|
|
620
|
+
},
|
|
621
|
+
models,
|
|
622
|
+
span: {
|
|
623
|
+
error,
|
|
624
|
+
input: { attempt: task.attempt, state: task.state },
|
|
625
|
+
name: task.phase,
|
|
626
|
+
output: { interrupted: true, leaseOwner, replay, status },
|
|
627
|
+
startedAt: task.lease?.acquiredAt ? luxon_1.DateTime.fromJSDate(task.lease.acquiredAt) : now,
|
|
628
|
+
status: "error",
|
|
629
|
+
},
|
|
630
|
+
task,
|
|
631
|
+
testHooks,
|
|
632
|
+
update: isFailure
|
|
633
|
+
? { $set: { outcome: { error, status }, status }, $unset: { lease: 1, runAt: 1, waits: 1 } }
|
|
634
|
+
: { $set: { status }, $unset: { lease: 1 } },
|
|
635
|
+
});
|
|
636
|
+
};
|
|
637
|
+
exports.commitInterruption = commitInterruption;
|
|
638
|
+
/**
|
|
639
|
+
* Apply an operator's `retry` or `complete` decision to an `interrupted` task and audit
|
|
640
|
+
* it with a `resolveInterrupted` span recording action, reason, and `decidedBy`. An
|
|
641
|
+
* `abort` decision goes through the abort path so the task's abort handler runs.
|
|
642
|
+
*/
|
|
643
|
+
const commitResolution = async ({ models, options, task, testHooks, }) => {
|
|
644
|
+
const { action, reason, result } = options;
|
|
645
|
+
const decidedBy = options.userId === undefined ? undefined : String(options.userId);
|
|
646
|
+
const resolutions = {
|
|
647
|
+
[harness_1.HARNESS_RESOLVE_ACTIONS.complete]: {
|
|
648
|
+
close: { output: result, status: "ok" },
|
|
649
|
+
update: {
|
|
650
|
+
$set: {
|
|
651
|
+
outcome: { result, status: harness_1.HARNESS_TASK_STATUSES.completed },
|
|
652
|
+
status: harness_1.HARNESS_TASK_STATUSES.completed,
|
|
653
|
+
},
|
|
654
|
+
},
|
|
655
|
+
},
|
|
656
|
+
[harness_1.HARNESS_RESOLVE_ACTIONS.retry]: {
|
|
657
|
+
close: undefined,
|
|
658
|
+
update: { $set: { status: harness_1.HARNESS_TASK_STATUSES.pending } },
|
|
659
|
+
},
|
|
660
|
+
};
|
|
661
|
+
const resolution = resolutions[action];
|
|
662
|
+
return commitTransition({
|
|
663
|
+
close: resolution.close,
|
|
664
|
+
filter: { phase: task.phase, status: harness_1.HARNESS_TASK_STATUSES.interrupted },
|
|
665
|
+
models,
|
|
666
|
+
span: {
|
|
667
|
+
input: { phase: task.phase, state: task.state },
|
|
668
|
+
name: "resolveInterrupted",
|
|
669
|
+
output: { action, decidedBy, phase: task.phase, reason, result },
|
|
670
|
+
startedAt: luxon_1.DateTime.now(),
|
|
671
|
+
status: "ok",
|
|
672
|
+
},
|
|
673
|
+
task,
|
|
674
|
+
testHooks,
|
|
675
|
+
update: resolution.update,
|
|
676
|
+
});
|
|
677
|
+
};
|
|
678
|
+
exports.commitResolution = commitResolution;
|