@terreno/ai 57.9.0 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +133 -0
- package/dist/harness/errors.js +107 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +522 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +191 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +64 -0
- package/dist/harness/runners/inProcessRunner.js +227 -0
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +52 -0
- package/dist/harness/runtime.js +604 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +952 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +28 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +125 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +769 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +778 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +206 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +907 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +530 -0
- package/src/harness/runners/inProcessRunner.ts +260 -0
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +802 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1119 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.InProcessRunner = exports.IN_PROCESS_RUNNER_ROLES = void 0;
|
|
4
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
|
+
const node_os_1 = require("node:os");
|
|
6
|
+
const api_1 = require("@terreno/api");
|
|
7
|
+
const luxon_1 = require("luxon");
|
|
8
|
+
const errors_1 = require("../errors");
|
|
9
|
+
/** `owner` drains tasks; `standby` waits for the owner lease; `stopped` does neither. */
|
|
10
|
+
exports.IN_PROCESS_RUNNER_ROLES = {
|
|
11
|
+
owner: "owner",
|
|
12
|
+
standby: "standby",
|
|
13
|
+
stopped: "stopped",
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Executes runnable tasks inside this process, up to `concurrency` at a time, each until it
|
|
17
|
+
* stops (terminal, retry, waiting, or aborted).
|
|
18
|
+
* Only the process holding the singleton `HarnessOwner` lease drains; every other
|
|
19
|
+
* process stays on standby and takes over once that lease expires. On becoming owner,
|
|
20
|
+
* and on every heartbeat while owner, it recovers tasks whose lease expired.
|
|
21
|
+
*/
|
|
22
|
+
class InProcessRunner {
|
|
23
|
+
concurrency;
|
|
24
|
+
context;
|
|
25
|
+
heartbeatInFlight = Promise.resolve();
|
|
26
|
+
heartbeatTimer;
|
|
27
|
+
hasPendingWake = false;
|
|
28
|
+
inFlight = new Set();
|
|
29
|
+
isHeartbeatStopped = false;
|
|
30
|
+
isStopping = false;
|
|
31
|
+
lease;
|
|
32
|
+
loop;
|
|
33
|
+
pollIntervalMs;
|
|
34
|
+
currentRole = exports.IN_PROCESS_RUNNER_ROLES.stopped;
|
|
35
|
+
wakeSleeper;
|
|
36
|
+
constructor(options = {}) {
|
|
37
|
+
const duration = luxon_1.Duration.fromDurationLike(options.leaseDuration ?? { seconds: 30 });
|
|
38
|
+
const heartbeat = luxon_1.Duration.fromDurationLike(options.heartbeatInterval ?? { seconds: 10 });
|
|
39
|
+
if (heartbeat.toMillis() <= 0 || heartbeat.toMillis() >= duration.toMillis()) {
|
|
40
|
+
throw (0, errors_1.harnessError)({
|
|
41
|
+
detail: "InProcessRunner heartbeatInterval must be positive and shorter than leaseDuration",
|
|
42
|
+
kind: "configInvalid",
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
const concurrency = options.concurrency ?? 8;
|
|
46
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
47
|
+
throw (0, errors_1.harnessError)({
|
|
48
|
+
detail: "InProcessRunner concurrency must be a positive integer",
|
|
49
|
+
kind: "configInvalid",
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
this.concurrency = concurrency;
|
|
53
|
+
this.lease = {
|
|
54
|
+
duration,
|
|
55
|
+
heartbeat,
|
|
56
|
+
owner: options.ownerId ?? `${(0, node_os_1.hostname)()}:${process.pid}:${(0, node_crypto_1.randomUUID)()}`,
|
|
57
|
+
};
|
|
58
|
+
this.pollIntervalMs = luxon_1.Duration.fromDurationLike(options.pollInterval ?? { milliseconds: 250 }).toMillis();
|
|
59
|
+
}
|
|
60
|
+
/** Whether this runner currently drains tasks, waits for the lease, or is stopped. */
|
|
61
|
+
get role() {
|
|
62
|
+
return this.currentRole;
|
|
63
|
+
}
|
|
64
|
+
/** Id written into every lease this runner holds. */
|
|
65
|
+
get ownerId() {
|
|
66
|
+
return this.lease.owner;
|
|
67
|
+
}
|
|
68
|
+
async start(context) {
|
|
69
|
+
if (this.loop !== undefined) {
|
|
70
|
+
throw (0, errors_1.harnessError)({ detail: "InProcessRunner is already started", kind: "alreadyStarted" });
|
|
71
|
+
}
|
|
72
|
+
this.context = context;
|
|
73
|
+
this.isStopping = false;
|
|
74
|
+
this.isHeartbeatStopped = false;
|
|
75
|
+
this.currentRole = exports.IN_PROCESS_RUNNER_ROLES.standby;
|
|
76
|
+
this.loop = this.drain();
|
|
77
|
+
this.scheduleHeartbeat();
|
|
78
|
+
}
|
|
79
|
+
/** Stop claiming new work, wait for every task in flight, then release the owner lease. */
|
|
80
|
+
async stop() {
|
|
81
|
+
if (this.loop === undefined) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
this.isStopping = true;
|
|
85
|
+
this.wake();
|
|
86
|
+
// Keep renewing the owner lease until the tasks in flight settle, so a standby does
|
|
87
|
+
// not take over while this process is still working.
|
|
88
|
+
await this.loop;
|
|
89
|
+
this.isHeartbeatStopped = true;
|
|
90
|
+
clearTimeout(this.heartbeatTimer);
|
|
91
|
+
await this.heartbeatInFlight;
|
|
92
|
+
const context = this.context;
|
|
93
|
+
if (context && this.currentRole === exports.IN_PROCESS_RUNNER_ROLES.owner) {
|
|
94
|
+
try {
|
|
95
|
+
await context.releaseOwnerLease(this.lease);
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
api_1.logger.warn(`InProcessRunner could not release the owner lease: ${(0, errors_1.errorMessage)(error)}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
this.currentRole = exports.IN_PROCESS_RUNNER_ROLES.stopped;
|
|
102
|
+
this.loop = undefined;
|
|
103
|
+
this.context = undefined;
|
|
104
|
+
}
|
|
105
|
+
/** Skip the rest of the idle sleep, e.g. right after a task is created. */
|
|
106
|
+
wake() {
|
|
107
|
+
if (this.wakeSleeper) {
|
|
108
|
+
this.wakeSleeper();
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
// Remember a wake that lands between a claim and the next sleep.
|
|
112
|
+
this.hasPendingWake = true;
|
|
113
|
+
}
|
|
114
|
+
async drain() {
|
|
115
|
+
while (!this.isStopping && this.context) {
|
|
116
|
+
if (this.currentRole !== exports.IN_PROCESS_RUNNER_ROLES.owner) {
|
|
117
|
+
const isOwner = await this.tryTakeOwnership(this.context);
|
|
118
|
+
if (!isOwner) {
|
|
119
|
+
await this.sleep(this.lease.heartbeat.toMillis());
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
await this.fillSlots(this.context);
|
|
124
|
+
if (!this.isStopping) {
|
|
125
|
+
// Woken early by new work or by a task that frees a slot.
|
|
126
|
+
await this.sleep(this.pollIntervalMs);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
await Promise.allSettled([...this.inFlight]);
|
|
130
|
+
}
|
|
131
|
+
/** Become owner when the lease is free or expired, then recover abandoned tasks. */
|
|
132
|
+
async tryTakeOwnership(context) {
|
|
133
|
+
try {
|
|
134
|
+
if (!(await context.acquireOwnerLease(this.lease))) {
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
this.currentRole = exports.IN_PROCESS_RUNNER_ROLES.owner;
|
|
138
|
+
api_1.logger.info(`InProcessRunner ${this.lease.owner} now owns harness execution`);
|
|
139
|
+
await context.recoverExpired();
|
|
140
|
+
return true;
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
api_1.logger.error(`InProcessRunner could not take ownership: ${(0, errors_1.errorMessage)(error)}`);
|
|
144
|
+
return this.currentRole === exports.IN_PROCESS_RUNNER_ROLES.owner;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/** Claim and start tasks until every slot is busy or nothing is runnable. */
|
|
148
|
+
async fillSlots(context) {
|
|
149
|
+
while (this.inFlight.size < this.concurrency &&
|
|
150
|
+
!this.isStopping &&
|
|
151
|
+
this.currentRole === exports.IN_PROCESS_RUNNER_ROLES.owner) {
|
|
152
|
+
let task;
|
|
153
|
+
try {
|
|
154
|
+
task = await context.claimNext(this.lease);
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
api_1.logger.error(`InProcessRunner iteration failed: ${(0, errors_1.errorMessage)(error)}`);
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
if (!task) {
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
const run = this.runOne(context, task);
|
|
164
|
+
this.inFlight.add(run);
|
|
165
|
+
void run.finally(() => {
|
|
166
|
+
this.inFlight.delete(run);
|
|
167
|
+
this.wake();
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/** Run one claimed task to a stop; its own task lease and heartbeat guard its commits. */
|
|
172
|
+
async runOne(context, task) {
|
|
173
|
+
try {
|
|
174
|
+
await context.runTask(task, this.lease);
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
api_1.logger.error(`InProcessRunner task ${task._id} failed to run: ${(0, errors_1.errorMessage)(error)}`);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
scheduleHeartbeat() {
|
|
181
|
+
if (this.isHeartbeatStopped) {
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
this.heartbeatTimer = setTimeout(() => {
|
|
185
|
+
this.heartbeatInFlight = this.heartbeat().finally(() => this.scheduleHeartbeat());
|
|
186
|
+
}, this.lease.heartbeat.toMillis());
|
|
187
|
+
}
|
|
188
|
+
/** Renew the owner lease and sweep for expired task leases; demote on a lost lease. */
|
|
189
|
+
async heartbeat() {
|
|
190
|
+
const context = this.context;
|
|
191
|
+
if (!context || this.currentRole !== exports.IN_PROCESS_RUNNER_ROLES.owner) {
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
try {
|
|
195
|
+
if (!(await context.acquireOwnerLease(this.lease))) {
|
|
196
|
+
this.currentRole = exports.IN_PROCESS_RUNNER_ROLES.standby;
|
|
197
|
+
api_1.logger.warn(`InProcessRunner ${this.lease.owner} lost the owner lease; now standby`);
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
// A stopping owner only holds the lease for its last task; recovery is the next owner's job.
|
|
201
|
+
if (!this.isStopping && (await context.recoverExpired()) > 0) {
|
|
202
|
+
this.wake();
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
api_1.logger.warn(`InProcessRunner heartbeat failed: ${(0, errors_1.errorMessage)(error)}`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
sleep(ms) {
|
|
210
|
+
if (this.hasPendingWake) {
|
|
211
|
+
this.hasPendingWake = false;
|
|
212
|
+
return Promise.resolve();
|
|
213
|
+
}
|
|
214
|
+
return new Promise((resolve) => {
|
|
215
|
+
const timer = setTimeout(() => {
|
|
216
|
+
this.wakeSleeper = undefined;
|
|
217
|
+
resolve();
|
|
218
|
+
}, ms);
|
|
219
|
+
this.wakeSleeper = () => {
|
|
220
|
+
clearTimeout(timer);
|
|
221
|
+
this.wakeSleeper = undefined;
|
|
222
|
+
resolve();
|
|
223
|
+
};
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
exports.InProcessRunner = InProcessRunner;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { type JobsApp } from "@terreno/jobs";
|
|
2
|
+
import { type DurationLike } from "luxon";
|
|
3
|
+
import type { HarnessRunner, HarnessRunnerContext } from "../../types/harness";
|
|
4
|
+
/** Job that runs one phase of one harness task. */
|
|
5
|
+
export declare const HARNESS_PHASE_JOB_NAME = "terreno.harness.phase";
|
|
6
|
+
export interface JobsRunnerOptions {
|
|
7
|
+
/** How many runnable tasks one dispatch tick enqueues at most. */
|
|
8
|
+
dispatchBatchSize?: number;
|
|
9
|
+
/** How often a running phase renews its task lease, and how often expired leases are swept. */
|
|
10
|
+
heartbeatInterval?: DurationLike;
|
|
11
|
+
/** The registered `JobsApp`; the runner defines `terreno.harness.phase` on it. */
|
|
12
|
+
jobs: JobsApp;
|
|
13
|
+
/** How long a task lease lives without a heartbeat. */
|
|
14
|
+
leaseDuration?: DurationLike;
|
|
15
|
+
/** Runner instance id written into task leases. Defaults to `hostname:pid:uuid`. */
|
|
16
|
+
ownerId?: string;
|
|
17
|
+
/** How long the dispatcher sleeps between scans for runnable tasks. */
|
|
18
|
+
pollInterval?: DurationLike;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Executes harness phases as `@terreno/jobs` jobs so any number of instances share the
|
|
22
|
+
* work. Every instance runs a dispatcher that enqueues one `terreno.harness.phase` job per
|
|
23
|
+
* runnable task (deduplicated by idempotency key) and sweeps expired task leases. The job
|
|
24
|
+
* handler claims its task under a fenced lease, runs one phase while heartbeating, and
|
|
25
|
+
* hands a still-running task back as `pending` for the next job. The task lease, not the
|
|
26
|
+
* job lock, decides who may commit: a duplicate or late job finds the task claimed and
|
|
27
|
+
* does nothing, and a stale lease holder's commit is rejected.
|
|
28
|
+
*/
|
|
29
|
+
export declare class JobsRunner implements HarnessRunner {
|
|
30
|
+
private context;
|
|
31
|
+
private readonly dispatchBatchSize;
|
|
32
|
+
private hasPendingWake;
|
|
33
|
+
private readonly inFlight;
|
|
34
|
+
private isStopping;
|
|
35
|
+
private readonly lease;
|
|
36
|
+
private loop;
|
|
37
|
+
private nextSweepAt;
|
|
38
|
+
private readonly pollIntervalMs;
|
|
39
|
+
private wakeSleeper;
|
|
40
|
+
constructor(options: JobsRunnerOptions);
|
|
41
|
+
/** Id written into every task lease this runner holds. */
|
|
42
|
+
get ownerId(): string;
|
|
43
|
+
start(context: HarnessRunnerContext): Promise<void>;
|
|
44
|
+
/** Stop dispatching, then wait for every phase this instance is running. */
|
|
45
|
+
stop(): Promise<void>;
|
|
46
|
+
/** Dispatch now instead of after the rest of the idle sleep. */
|
|
47
|
+
wake(): void;
|
|
48
|
+
/** Job handler: claim the task, run one phase, and hand it back if it still runs. */
|
|
49
|
+
private runPhaseJob;
|
|
50
|
+
private claimAndRunPhase;
|
|
51
|
+
private dispatchLoop;
|
|
52
|
+
/** Recover expired leases once per heartbeat, then enqueue a job per runnable task. */
|
|
53
|
+
private tick;
|
|
54
|
+
/**
|
|
55
|
+
* Enqueue the job for this runnable visit. When the job with that key already ended
|
|
56
|
+
* without claiming the task (its handler kept failing, or it ran where the task was not
|
|
57
|
+
* runnable), follow or extend the chain `<key>:<endedJobId>` until a job is in flight, so
|
|
58
|
+
* the task is not stranded. A newly created job is always in flight, so this ends.
|
|
59
|
+
*/
|
|
60
|
+
private dispatch;
|
|
61
|
+
private sleep;
|
|
62
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.JobsRunner = exports.HARNESS_PHASE_JOB_NAME = void 0;
|
|
4
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
|
+
const node_os_1 = require("node:os");
|
|
6
|
+
const api_1 = require("@terreno/api");
|
|
7
|
+
const jobs_1 = require("@terreno/jobs");
|
|
8
|
+
const luxon_1 = require("luxon");
|
|
9
|
+
const errors_1 = require("../errors");
|
|
10
|
+
/** Job that runs one phase of one harness task. */
|
|
11
|
+
exports.HARNESS_PHASE_JOB_NAME = "terreno.harness.phase";
|
|
12
|
+
/** Job statuses that still lead to a handler run; any other status will never claim. */
|
|
13
|
+
const IN_FLIGHT_JOB_STATUSES = new Set(["pending", "running", "scheduled"]);
|
|
14
|
+
const parsePayload = (payload) => {
|
|
15
|
+
const taskId = payload?.taskId;
|
|
16
|
+
if (typeof taskId !== "string" || !taskId) {
|
|
17
|
+
throw (0, errors_1.harnessError)({
|
|
18
|
+
detail: `${exports.HARNESS_PHASE_JOB_NAME} payload needs a taskId string`,
|
|
19
|
+
kind: "invalidRequest",
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
return { taskId };
|
|
23
|
+
};
|
|
24
|
+
/** `taskId:phase:attempt:claims` — one key per runnable visit of a phase. */
|
|
25
|
+
const phaseJobKey = (task) => `${task.taskId}:${task.phase}:${task.attempt}:${task.claims}`;
|
|
26
|
+
/**
|
|
27
|
+
* Executes harness phases as `@terreno/jobs` jobs so any number of instances share the
|
|
28
|
+
* work. Every instance runs a dispatcher that enqueues one `terreno.harness.phase` job per
|
|
29
|
+
* runnable task (deduplicated by idempotency key) and sweeps expired task leases. The job
|
|
30
|
+
* handler claims its task under a fenced lease, runs one phase while heartbeating, and
|
|
31
|
+
* hands a still-running task back as `pending` for the next job. The task lease, not the
|
|
32
|
+
* job lock, decides who may commit: a duplicate or late job finds the task claimed and
|
|
33
|
+
* does nothing, and a stale lease holder's commit is rejected.
|
|
34
|
+
*/
|
|
35
|
+
class JobsRunner {
|
|
36
|
+
context;
|
|
37
|
+
dispatchBatchSize;
|
|
38
|
+
hasPendingWake = false;
|
|
39
|
+
inFlight = new Set();
|
|
40
|
+
isStopping = false;
|
|
41
|
+
lease;
|
|
42
|
+
loop;
|
|
43
|
+
nextSweepAt = luxon_1.DateTime.fromMillis(0);
|
|
44
|
+
pollIntervalMs;
|
|
45
|
+
wakeSleeper;
|
|
46
|
+
constructor(options) {
|
|
47
|
+
const duration = luxon_1.Duration.fromDurationLike(options.leaseDuration ?? { seconds: 30 });
|
|
48
|
+
const heartbeat = luxon_1.Duration.fromDurationLike(options.heartbeatInterval ?? { seconds: 10 });
|
|
49
|
+
if (heartbeat.toMillis() <= 0 || heartbeat.toMillis() >= duration.toMillis()) {
|
|
50
|
+
throw (0, errors_1.harnessError)({
|
|
51
|
+
detail: "JobsRunner heartbeatInterval must be positive and shorter than leaseDuration",
|
|
52
|
+
kind: "configInvalid",
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
this.lease = {
|
|
56
|
+
duration,
|
|
57
|
+
heartbeat,
|
|
58
|
+
owner: options.ownerId ?? `${(0, node_os_1.hostname)()}:${process.pid}:${(0, node_crypto_1.randomUUID)()}`,
|
|
59
|
+
};
|
|
60
|
+
this.pollIntervalMs = luxon_1.Duration.fromDurationLike(options.pollInterval ?? { milliseconds: 500 }).toMillis();
|
|
61
|
+
const dispatchBatchSize = options.dispatchBatchSize ?? 100;
|
|
62
|
+
if (!Number.isInteger(dispatchBatchSize) || dispatchBatchSize < 1) {
|
|
63
|
+
throw (0, errors_1.harnessError)({
|
|
64
|
+
detail: "JobsRunner dispatchBatchSize must be a positive integer",
|
|
65
|
+
kind: "configInvalid",
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
this.dispatchBatchSize = dispatchBatchSize;
|
|
69
|
+
options.jobs.define(exports.HARNESS_PHASE_JOB_NAME, {
|
|
70
|
+
handler: (payload) => this.runPhaseJob(payload),
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
/** Id written into every task lease this runner holds. */
|
|
74
|
+
get ownerId() {
|
|
75
|
+
return this.lease.owner;
|
|
76
|
+
}
|
|
77
|
+
async start(context) {
|
|
78
|
+
if (this.loop !== undefined) {
|
|
79
|
+
throw (0, errors_1.harnessError)({ detail: "JobsRunner is already started", kind: "alreadyStarted" });
|
|
80
|
+
}
|
|
81
|
+
if (!(0, jobs_1.tryGetJobsService)()) {
|
|
82
|
+
throw (0, errors_1.harnessError)({
|
|
83
|
+
detail: "JobsRunner needs a registered JobsApp: register it and build the app before harness.start()",
|
|
84
|
+
kind: "configInvalid",
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
this.context = context;
|
|
88
|
+
this.isStopping = false;
|
|
89
|
+
this.nextSweepAt = luxon_1.DateTime.fromMillis(0);
|
|
90
|
+
this.loop = this.dispatchLoop();
|
|
91
|
+
}
|
|
92
|
+
/** Stop dispatching, then wait for every phase this instance is running. */
|
|
93
|
+
async stop() {
|
|
94
|
+
if (this.loop === undefined) {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
this.isStopping = true;
|
|
98
|
+
this.wake();
|
|
99
|
+
await this.loop;
|
|
100
|
+
await Promise.allSettled([...this.inFlight]);
|
|
101
|
+
this.loop = undefined;
|
|
102
|
+
this.context = undefined;
|
|
103
|
+
}
|
|
104
|
+
/** Dispatch now instead of after the rest of the idle sleep. */
|
|
105
|
+
wake() {
|
|
106
|
+
if (this.wakeSleeper) {
|
|
107
|
+
this.wakeSleeper();
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
this.hasPendingWake = true;
|
|
111
|
+
}
|
|
112
|
+
/** Job handler: claim the task, run one phase, and hand it back if it still runs. */
|
|
113
|
+
async runPhaseJob(payload) {
|
|
114
|
+
const { taskId } = parsePayload(payload);
|
|
115
|
+
// Tracked from before the claim, so stop() also waits for a claim in flight.
|
|
116
|
+
const job = this.claimAndRunPhase(taskId);
|
|
117
|
+
this.inFlight.add(job);
|
|
118
|
+
try {
|
|
119
|
+
await job;
|
|
120
|
+
}
|
|
121
|
+
finally {
|
|
122
|
+
this.inFlight.delete(job);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
async claimAndRunPhase(taskId) {
|
|
126
|
+
const context = this.context;
|
|
127
|
+
if (!context || this.isStopping) {
|
|
128
|
+
// Throwing makes the jobs worker retry later, on this instance or another one.
|
|
129
|
+
throw (0, errors_1.harnessError)({
|
|
130
|
+
detail: `JobsRunner is not running; task ${taskId} stays runnable`,
|
|
131
|
+
kind: "runnerStopped",
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
const task = await context.claimTask(taskId, this.lease);
|
|
135
|
+
if (!task) {
|
|
136
|
+
// Another job claimed it first, or it stopped being runnable since dispatch.
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
await context.runTask(task, this.lease, { maxPhases: 1 });
|
|
140
|
+
// The next phase (or a retry that is already due) can be dispatched right away.
|
|
141
|
+
this.wake();
|
|
142
|
+
}
|
|
143
|
+
async dispatchLoop() {
|
|
144
|
+
while (!this.isStopping && this.context) {
|
|
145
|
+
await this.tick(this.context);
|
|
146
|
+
if (!this.isStopping) {
|
|
147
|
+
await this.sleep(this.pollIntervalMs);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/** Recover expired leases once per heartbeat, then enqueue a job per runnable task. */
|
|
152
|
+
async tick(context) {
|
|
153
|
+
try {
|
|
154
|
+
if (luxon_1.DateTime.now() >= this.nextSweepAt) {
|
|
155
|
+
this.nextSweepAt = luxon_1.DateTime.now().plus(this.lease.heartbeat);
|
|
156
|
+
await context.recoverExpired();
|
|
157
|
+
}
|
|
158
|
+
const runnable = await context.listRunnable(this.dispatchBatchSize);
|
|
159
|
+
for (const task of runnable) {
|
|
160
|
+
if (this.isStopping) {
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
await this.dispatch(task);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
catch (error) {
|
|
167
|
+
api_1.logger.error(`JobsRunner dispatch failed: ${(0, errors_1.errorMessage)(error)}`);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Enqueue the job for this runnable visit. When the job with that key already ended
|
|
172
|
+
* without claiming the task (its handler kept failing, or it ran where the task was not
|
|
173
|
+
* runnable), follow or extend the chain `<key>:<endedJobId>` until a job is in flight, so
|
|
174
|
+
* the task is not stranded. A newly created job is always in flight, so this ends.
|
|
175
|
+
*/
|
|
176
|
+
async dispatch(task) {
|
|
177
|
+
const jobs = (0, jobs_1.tryGetJobsService)();
|
|
178
|
+
if (!jobs) {
|
|
179
|
+
throw (0, errors_1.harnessError)({ detail: "JobsApp is no longer registered", kind: "configInvalid" });
|
|
180
|
+
}
|
|
181
|
+
let idempotencyKey = phaseJobKey(task);
|
|
182
|
+
for (;;) {
|
|
183
|
+
const job = await jobs.enqueue({
|
|
184
|
+
idempotencyKey,
|
|
185
|
+
name: exports.HARNESS_PHASE_JOB_NAME,
|
|
186
|
+
payload: { taskId: task.taskId },
|
|
187
|
+
});
|
|
188
|
+
if (IN_FLIGHT_JOB_STATUSES.has(job.status)) {
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
idempotencyKey = `${phaseJobKey(task)}:${job._id}`;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
sleep(ms) {
|
|
195
|
+
if (this.hasPendingWake) {
|
|
196
|
+
this.hasPendingWake = false;
|
|
197
|
+
return Promise.resolve();
|
|
198
|
+
}
|
|
199
|
+
return new Promise((resolve) => {
|
|
200
|
+
const timer = setTimeout(() => {
|
|
201
|
+
this.wakeSleeper = undefined;
|
|
202
|
+
resolve();
|
|
203
|
+
}, ms);
|
|
204
|
+
this.wakeSleeper = () => {
|
|
205
|
+
clearTimeout(timer);
|
|
206
|
+
this.wakeSleeper = undefined;
|
|
207
|
+
resolve();
|
|
208
|
+
};
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
exports.JobsRunner = JobsRunner;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { HarnessLeaseSettings, HarnessRunnableTask, HarnessTaskDefinition, HarnessTaskDocument } from "../types/harness";
|
|
2
|
+
import { type HarnessModels } from "./commit";
|
|
3
|
+
import { type HarnessEngine } from "./ownership";
|
|
4
|
+
/**
|
|
5
|
+
* Atomically move the oldest runnable, registered task (`pending` and due, or `waiting` on
|
|
6
|
+
* an event or sleep whose `timeoutAt` passed) to `running` and give it a fresh lease (owner, fencing token, expiry) for the phase about to start.
|
|
7
|
+
*/
|
|
8
|
+
export declare const claimNextTask: ({ definitions, lease, models, }: {
|
|
9
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
10
|
+
lease: HarnessLeaseSettings;
|
|
11
|
+
models: HarnessModels;
|
|
12
|
+
}) => Promise<HarnessTaskDocument | null>;
|
|
13
|
+
/**
|
|
14
|
+
* Claim one task by id under the same rules as `claimNextTask`. Null when the task is not
|
|
15
|
+
* runnable now: unknown, not due, already claimed by another runner, terminal, or aborting.
|
|
16
|
+
*/
|
|
17
|
+
export declare const claimTaskById: ({ definitions, lease, models, taskId, }: {
|
|
18
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
19
|
+
lease: HarnessLeaseSettings;
|
|
20
|
+
models: HarnessModels;
|
|
21
|
+
taskId: string;
|
|
22
|
+
}) => Promise<HarnessTaskDocument | null>;
|
|
23
|
+
/** Oldest runnable, registered tasks first, at most `limit`, without claiming them. */
|
|
24
|
+
export declare const listRunnableTasks: ({ definitions, limit, models, }: {
|
|
25
|
+
definitions: Map<string, HarnessTaskDefinition>;
|
|
26
|
+
limit: number;
|
|
27
|
+
models: HarnessModels;
|
|
28
|
+
}) => Promise<HarnessRunnableTask[]>;
|
|
29
|
+
/**
|
|
30
|
+
* Find `running` tasks whose lease expired (their runner died or froze mid-phase) and
|
|
31
|
+
* resolve each by its current phase's `replay`: `safe` goes back to `pending` at the same
|
|
32
|
+
* checkpoint; anything else is parked `interrupted`, or failed (owner woken) when the
|
|
33
|
+
* definition sets `onInterrupt: "fail"`. Tasks of unregistered versions are
|
|
34
|
+
* left for a runner that registers them. Then wake tasks whose child wait settled
|
|
35
|
+
* without a wake (a crash between a child's outcome and its owner check). Returns how
|
|
36
|
+
* many tasks became runnable.
|
|
37
|
+
*/
|
|
38
|
+
export declare const recoverExpiredTasks: (engine: HarnessEngine) => Promise<number>;
|
|
39
|
+
/**
|
|
40
|
+
* Run a claimed task's phases in order until it stops: terminal, `pending` for a retry,
|
|
41
|
+
* `waiting`, or aborted. A phase that throws is retried under the task's `retry` policy
|
|
42
|
+
* and fails the task once attempts run out. When a commit transaction fails, or the lease
|
|
43
|
+
* was lost, the task is left at its last checkpoint and expired-lease recovery decides
|
|
44
|
+
* what happens next. A terminal task re-checks the task that owns it.
|
|
45
|
+
*/
|
|
46
|
+
export declare const runClaimedTask: ({ engine, lease, maxPhases, task, }: {
|
|
47
|
+
engine: HarnessEngine;
|
|
48
|
+
lease: HarnessLeaseSettings;
|
|
49
|
+
/** Hand the task back as `pending` once this many phases committed and it still runs. */
|
|
50
|
+
maxPhases?: number;
|
|
51
|
+
task: HarnessTaskDocument;
|
|
52
|
+
}) => Promise<void>;
|