@terreno/ai 57.9.0 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +133 -0
- package/dist/harness/errors.js +107 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +522 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +191 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +64 -0
- package/dist/harness/runners/inProcessRunner.js +227 -0
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +52 -0
- package/dist/harness/runtime.js +604 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +952 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +28 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +125 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +769 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +778 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +206 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +907 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +530 -0
- package/src/harness/runners/inProcessRunner.ts +260 -0
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +802 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1119 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
import {randomUUID} from "node:crypto";
|
|
2
|
+
import {hostname} from "node:os";
|
|
3
|
+
import {logger} from "@terreno/api";
|
|
4
|
+
import {Duration, type DurationLike} from "luxon";
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
HarnessLeaseSettings,
|
|
8
|
+
HarnessRunner,
|
|
9
|
+
HarnessRunnerContext,
|
|
10
|
+
HarnessTaskDocument,
|
|
11
|
+
} from "../../types/harness";
|
|
12
|
+
import {errorMessage, harnessError} from "../errors";
|
|
13
|
+
|
|
14
|
+
export interface InProcessRunnerOptions {
|
|
15
|
+
/** Most claimed tasks this runner executes at once. Default 8. */
|
|
16
|
+
concurrency?: number;
|
|
17
|
+
/** How often the runner renews its owner lease and its running task's lease. */
|
|
18
|
+
heartbeatInterval?: DurationLike;
|
|
19
|
+
/** How long the owner lease and each task lease live without a heartbeat. */
|
|
20
|
+
leaseDuration?: DurationLike;
|
|
21
|
+
/** Runner instance id written into leases. Defaults to `hostname:pid:uuid`. */
|
|
22
|
+
ownerId?: string;
|
|
23
|
+
/** How long an idle owner sleeps before polling for runnable tasks again. */
|
|
24
|
+
pollInterval?: DurationLike;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** `owner` drains tasks; `standby` waits for the owner lease; `stopped` does neither. */
|
|
28
|
+
export const IN_PROCESS_RUNNER_ROLES = {
|
|
29
|
+
owner: "owner",
|
|
30
|
+
standby: "standby",
|
|
31
|
+
stopped: "stopped",
|
|
32
|
+
} as const;
|
|
33
|
+
|
|
34
|
+
export type InProcessRunnerRole =
|
|
35
|
+
(typeof IN_PROCESS_RUNNER_ROLES)[keyof typeof IN_PROCESS_RUNNER_ROLES];
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Executes runnable tasks inside this process, up to `concurrency` at a time, each until it
|
|
39
|
+
* stops (terminal, retry, waiting, or aborted).
|
|
40
|
+
* Only the process holding the singleton `HarnessOwner` lease drains; every other
|
|
41
|
+
* process stays on standby and takes over once that lease expires. On becoming owner,
|
|
42
|
+
* and on every heartbeat while owner, it recovers tasks whose lease expired.
|
|
43
|
+
*/
|
|
44
|
+
export class InProcessRunner implements HarnessRunner {
|
|
45
|
+
private readonly concurrency: number;
|
|
46
|
+
private context: HarnessRunnerContext | undefined;
|
|
47
|
+
private heartbeatInFlight: Promise<void> = Promise.resolve();
|
|
48
|
+
private heartbeatTimer: ReturnType<typeof setTimeout> | undefined;
|
|
49
|
+
private hasPendingWake = false;
|
|
50
|
+
private readonly inFlight = new Set<Promise<void>>();
|
|
51
|
+
private isHeartbeatStopped = false;
|
|
52
|
+
private isStopping = false;
|
|
53
|
+
private readonly lease: HarnessLeaseSettings;
|
|
54
|
+
private loop: Promise<void> | undefined;
|
|
55
|
+
private readonly pollIntervalMs: number;
|
|
56
|
+
private currentRole: InProcessRunnerRole = IN_PROCESS_RUNNER_ROLES.stopped;
|
|
57
|
+
private wakeSleeper: (() => void) | undefined;
|
|
58
|
+
|
|
59
|
+
constructor(options: InProcessRunnerOptions = {}) {
|
|
60
|
+
const duration = Duration.fromDurationLike(options.leaseDuration ?? {seconds: 30});
|
|
61
|
+
const heartbeat = Duration.fromDurationLike(options.heartbeatInterval ?? {seconds: 10});
|
|
62
|
+
if (heartbeat.toMillis() <= 0 || heartbeat.toMillis() >= duration.toMillis()) {
|
|
63
|
+
throw harnessError({
|
|
64
|
+
detail: "InProcessRunner heartbeatInterval must be positive and shorter than leaseDuration",
|
|
65
|
+
kind: "configInvalid",
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
const concurrency = options.concurrency ?? 8;
|
|
69
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
70
|
+
throw harnessError({
|
|
71
|
+
detail: "InProcessRunner concurrency must be a positive integer",
|
|
72
|
+
kind: "configInvalid",
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
this.concurrency = concurrency;
|
|
76
|
+
this.lease = {
|
|
77
|
+
duration,
|
|
78
|
+
heartbeat,
|
|
79
|
+
owner: options.ownerId ?? `${hostname()}:${process.pid}:${randomUUID()}`,
|
|
80
|
+
};
|
|
81
|
+
this.pollIntervalMs = Duration.fromDurationLike(
|
|
82
|
+
options.pollInterval ?? {milliseconds: 250}
|
|
83
|
+
).toMillis();
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Whether this runner currently drains tasks, waits for the lease, or is stopped. */
|
|
87
|
+
get role(): InProcessRunnerRole {
|
|
88
|
+
return this.currentRole;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Id written into every lease this runner holds. */
|
|
92
|
+
get ownerId(): string {
|
|
93
|
+
return this.lease.owner;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async start(context: HarnessRunnerContext): Promise<void> {
|
|
97
|
+
if (this.loop !== undefined) {
|
|
98
|
+
throw harnessError({detail: "InProcessRunner is already started", kind: "alreadyStarted"});
|
|
99
|
+
}
|
|
100
|
+
this.context = context;
|
|
101
|
+
this.isStopping = false;
|
|
102
|
+
this.isHeartbeatStopped = false;
|
|
103
|
+
this.currentRole = IN_PROCESS_RUNNER_ROLES.standby;
|
|
104
|
+
this.loop = this.drain();
|
|
105
|
+
this.scheduleHeartbeat();
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Stop claiming new work, wait for every task in flight, then release the owner lease. */
|
|
109
|
+
async stop(): Promise<void> {
|
|
110
|
+
if (this.loop === undefined) {
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
this.isStopping = true;
|
|
114
|
+
this.wake();
|
|
115
|
+
// Keep renewing the owner lease until the tasks in flight settle, so a standby does
|
|
116
|
+
// not take over while this process is still working.
|
|
117
|
+
await this.loop;
|
|
118
|
+
this.isHeartbeatStopped = true;
|
|
119
|
+
clearTimeout(this.heartbeatTimer);
|
|
120
|
+
await this.heartbeatInFlight;
|
|
121
|
+
const context = this.context;
|
|
122
|
+
if (context && this.currentRole === IN_PROCESS_RUNNER_ROLES.owner) {
|
|
123
|
+
try {
|
|
124
|
+
await context.releaseOwnerLease(this.lease);
|
|
125
|
+
} catch (error: unknown) {
|
|
126
|
+
logger.warn(`InProcessRunner could not release the owner lease: ${errorMessage(error)}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
this.currentRole = IN_PROCESS_RUNNER_ROLES.stopped;
|
|
130
|
+
this.loop = undefined;
|
|
131
|
+
this.context = undefined;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Skip the rest of the idle sleep, e.g. right after a task is created. */
|
|
135
|
+
wake(): void {
|
|
136
|
+
if (this.wakeSleeper) {
|
|
137
|
+
this.wakeSleeper();
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
// Remember a wake that lands between a claim and the next sleep.
|
|
141
|
+
this.hasPendingWake = true;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
private async drain(): Promise<void> {
|
|
145
|
+
while (!this.isStopping && this.context) {
|
|
146
|
+
if (this.currentRole !== IN_PROCESS_RUNNER_ROLES.owner) {
|
|
147
|
+
const isOwner = await this.tryTakeOwnership(this.context);
|
|
148
|
+
if (!isOwner) {
|
|
149
|
+
await this.sleep(this.lease.heartbeat.toMillis());
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
await this.fillSlots(this.context);
|
|
154
|
+
if (!this.isStopping) {
|
|
155
|
+
// Woken early by new work or by a task that frees a slot.
|
|
156
|
+
await this.sleep(this.pollIntervalMs);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
await Promise.allSettled([...this.inFlight]);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** Become owner when the lease is free or expired, then recover abandoned tasks. */
|
|
163
|
+
private async tryTakeOwnership(context: HarnessRunnerContext): Promise<boolean> {
|
|
164
|
+
try {
|
|
165
|
+
if (!(await context.acquireOwnerLease(this.lease))) {
|
|
166
|
+
return false;
|
|
167
|
+
}
|
|
168
|
+
this.currentRole = IN_PROCESS_RUNNER_ROLES.owner;
|
|
169
|
+
logger.info(`InProcessRunner ${this.lease.owner} now owns harness execution`);
|
|
170
|
+
await context.recoverExpired();
|
|
171
|
+
return true;
|
|
172
|
+
} catch (error: unknown) {
|
|
173
|
+
logger.error(`InProcessRunner could not take ownership: ${errorMessage(error)}`);
|
|
174
|
+
return this.currentRole === IN_PROCESS_RUNNER_ROLES.owner;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Claim and start tasks until every slot is busy or nothing is runnable. */
|
|
179
|
+
private async fillSlots(context: HarnessRunnerContext): Promise<void> {
|
|
180
|
+
while (
|
|
181
|
+
this.inFlight.size < this.concurrency &&
|
|
182
|
+
!this.isStopping &&
|
|
183
|
+
this.currentRole === IN_PROCESS_RUNNER_ROLES.owner
|
|
184
|
+
) {
|
|
185
|
+
let task: HarnessTaskDocument | null;
|
|
186
|
+
try {
|
|
187
|
+
task = await context.claimNext(this.lease);
|
|
188
|
+
} catch (error: unknown) {
|
|
189
|
+
logger.error(`InProcessRunner iteration failed: ${errorMessage(error)}`);
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
if (!task) {
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
const run = this.runOne(context, task);
|
|
196
|
+
this.inFlight.add(run);
|
|
197
|
+
void run.finally(() => {
|
|
198
|
+
this.inFlight.delete(run);
|
|
199
|
+
this.wake();
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Run one claimed task to a stop; its own task lease and heartbeat guard its commits. */
|
|
205
|
+
private async runOne(context: HarnessRunnerContext, task: HarnessTaskDocument): Promise<void> {
|
|
206
|
+
try {
|
|
207
|
+
await context.runTask(task, this.lease);
|
|
208
|
+
} catch (error: unknown) {
|
|
209
|
+
logger.error(`InProcessRunner task ${task._id} failed to run: ${errorMessage(error)}`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
private scheduleHeartbeat(): void {
|
|
214
|
+
if (this.isHeartbeatStopped) {
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
this.heartbeatTimer = setTimeout(() => {
|
|
218
|
+
this.heartbeatInFlight = this.heartbeat().finally(() => this.scheduleHeartbeat());
|
|
219
|
+
}, this.lease.heartbeat.toMillis());
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** Renew the owner lease and sweep for expired task leases; demote on a lost lease. */
|
|
223
|
+
private async heartbeat(): Promise<void> {
|
|
224
|
+
const context = this.context;
|
|
225
|
+
if (!context || this.currentRole !== IN_PROCESS_RUNNER_ROLES.owner) {
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
try {
|
|
229
|
+
if (!(await context.acquireOwnerLease(this.lease))) {
|
|
230
|
+
this.currentRole = IN_PROCESS_RUNNER_ROLES.standby;
|
|
231
|
+
logger.warn(`InProcessRunner ${this.lease.owner} lost the owner lease; now standby`);
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
// A stopping owner only holds the lease for its last task; recovery is the next owner's job.
|
|
235
|
+
if (!this.isStopping && (await context.recoverExpired()) > 0) {
|
|
236
|
+
this.wake();
|
|
237
|
+
}
|
|
238
|
+
} catch (error: unknown) {
|
|
239
|
+
logger.warn(`InProcessRunner heartbeat failed: ${errorMessage(error)}`);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
private sleep(ms: number): Promise<void> {
|
|
244
|
+
if (this.hasPendingWake) {
|
|
245
|
+
this.hasPendingWake = false;
|
|
246
|
+
return Promise.resolve();
|
|
247
|
+
}
|
|
248
|
+
return new Promise((resolve) => {
|
|
249
|
+
const timer = setTimeout(() => {
|
|
250
|
+
this.wakeSleeper = undefined;
|
|
251
|
+
resolve();
|
|
252
|
+
}, ms);
|
|
253
|
+
this.wakeSleeper = () => {
|
|
254
|
+
clearTimeout(timer);
|
|
255
|
+
this.wakeSleeper = undefined;
|
|
256
|
+
resolve();
|
|
257
|
+
};
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
}
|