@terreno/ai 57.9.0 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +133 -0
- package/dist/harness/errors.js +107 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +522 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +191 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +64 -0
- package/dist/harness/runners/inProcessRunner.js +227 -0
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +52 -0
- package/dist/harness/runtime.js +604 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +952 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +28 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +125 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +769 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +778 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +206 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +907 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +530 -0
- package/src/harness/runners/inProcessRunner.ts +260 -0
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +802 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1119 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import {randomUUID} from "node:crypto";
|
|
2
|
+
import {hostname} from "node:os";
|
|
3
|
+
import {logger} from "@terreno/api";
|
|
4
|
+
import {type JobDocument, type JobsApp, tryGetJobsService} from "@terreno/jobs";
|
|
5
|
+
import {DateTime, Duration, type DurationLike} from "luxon";
|
|
6
|
+
|
|
7
|
+
import type {
|
|
8
|
+
HarnessLeaseSettings,
|
|
9
|
+
HarnessRunnableTask,
|
|
10
|
+
HarnessRunner,
|
|
11
|
+
HarnessRunnerContext,
|
|
12
|
+
} from "../../types/harness";
|
|
13
|
+
import {errorMessage, harnessError} from "../errors";
|
|
14
|
+
|
|
15
|
+
/** Job that runs one phase of one harness task. */
|
|
16
|
+
export const HARNESS_PHASE_JOB_NAME = "terreno.harness.phase";
|
|
17
|
+
|
|
18
|
+
/** Job statuses that still lead to a handler run; any other status will never claim. */
|
|
19
|
+
const IN_FLIGHT_JOB_STATUSES = new Set<JobDocument["status"]>(["pending", "running", "scheduled"]);
|
|
20
|
+
|
|
21
|
+
export interface JobsRunnerOptions {
|
|
22
|
+
/** How many runnable tasks one dispatch tick enqueues at most. */
|
|
23
|
+
dispatchBatchSize?: number;
|
|
24
|
+
/** How often a running phase renews its task lease, and how often expired leases are swept. */
|
|
25
|
+
heartbeatInterval?: DurationLike;
|
|
26
|
+
/** The registered `JobsApp`; the runner defines `terreno.harness.phase` on it. */
|
|
27
|
+
jobs: JobsApp;
|
|
28
|
+
/** How long a task lease lives without a heartbeat. */
|
|
29
|
+
leaseDuration?: DurationLike;
|
|
30
|
+
/** Runner instance id written into task leases. Defaults to `hostname:pid:uuid`. */
|
|
31
|
+
ownerId?: string;
|
|
32
|
+
/** How long the dispatcher sleeps between scans for runnable tasks. */
|
|
33
|
+
pollInterval?: DurationLike;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
interface PhaseJobPayload {
|
|
37
|
+
taskId: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const parsePayload = (payload: unknown): PhaseJobPayload => {
|
|
41
|
+
const taskId = (payload as Partial<PhaseJobPayload> | null)?.taskId;
|
|
42
|
+
if (typeof taskId !== "string" || !taskId) {
|
|
43
|
+
throw harnessError({
|
|
44
|
+
detail: `${HARNESS_PHASE_JOB_NAME} payload needs a taskId string`,
|
|
45
|
+
kind: "invalidRequest",
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
return {taskId};
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/** `taskId:phase:attempt:claims` — one key per runnable visit of a phase. */
|
|
52
|
+
const phaseJobKey = (task: HarnessRunnableTask): string =>
|
|
53
|
+
`${task.taskId}:${task.phase}:${task.attempt}:${task.claims}`;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Executes harness phases as `@terreno/jobs` jobs so any number of instances share the
|
|
57
|
+
* work. Every instance runs a dispatcher that enqueues one `terreno.harness.phase` job per
|
|
58
|
+
* runnable task (deduplicated by idempotency key) and sweeps expired task leases. The job
|
|
59
|
+
* handler claims its task under a fenced lease, runs one phase while heartbeating, and
|
|
60
|
+
* hands a still-running task back as `pending` for the next job. The task lease, not the
|
|
61
|
+
* job lock, decides who may commit: a duplicate or late job finds the task claimed and
|
|
62
|
+
* does nothing, and a stale lease holder's commit is rejected.
|
|
63
|
+
*/
|
|
64
|
+
export class JobsRunner implements HarnessRunner {
|
|
65
|
+
private context: HarnessRunnerContext | undefined;
|
|
66
|
+
private readonly dispatchBatchSize: number;
|
|
67
|
+
private hasPendingWake = false;
|
|
68
|
+
private readonly inFlight = new Set<Promise<void>>();
|
|
69
|
+
private isStopping = false;
|
|
70
|
+
private readonly lease: HarnessLeaseSettings;
|
|
71
|
+
private loop: Promise<void> | undefined;
|
|
72
|
+
private nextSweepAt = DateTime.fromMillis(0);
|
|
73
|
+
private readonly pollIntervalMs: number;
|
|
74
|
+
private wakeSleeper: (() => void) | undefined;
|
|
75
|
+
|
|
76
|
+
constructor(options: JobsRunnerOptions) {
|
|
77
|
+
const duration = Duration.fromDurationLike(options.leaseDuration ?? {seconds: 30});
|
|
78
|
+
const heartbeat = Duration.fromDurationLike(options.heartbeatInterval ?? {seconds: 10});
|
|
79
|
+
if (heartbeat.toMillis() <= 0 || heartbeat.toMillis() >= duration.toMillis()) {
|
|
80
|
+
throw harnessError({
|
|
81
|
+
detail: "JobsRunner heartbeatInterval must be positive and shorter than leaseDuration",
|
|
82
|
+
kind: "configInvalid",
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
this.lease = {
|
|
86
|
+
duration,
|
|
87
|
+
heartbeat,
|
|
88
|
+
owner: options.ownerId ?? `${hostname()}:${process.pid}:${randomUUID()}`,
|
|
89
|
+
};
|
|
90
|
+
this.pollIntervalMs = Duration.fromDurationLike(
|
|
91
|
+
options.pollInterval ?? {milliseconds: 500}
|
|
92
|
+
).toMillis();
|
|
93
|
+
const dispatchBatchSize = options.dispatchBatchSize ?? 100;
|
|
94
|
+
if (!Number.isInteger(dispatchBatchSize) || dispatchBatchSize < 1) {
|
|
95
|
+
throw harnessError({
|
|
96
|
+
detail: "JobsRunner dispatchBatchSize must be a positive integer",
|
|
97
|
+
kind: "configInvalid",
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
this.dispatchBatchSize = dispatchBatchSize;
|
|
101
|
+
options.jobs.define(HARNESS_PHASE_JOB_NAME, {
|
|
102
|
+
handler: (payload) => this.runPhaseJob(payload),
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Id written into every task lease this runner holds. */
|
|
107
|
+
get ownerId(): string {
|
|
108
|
+
return this.lease.owner;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async start(context: HarnessRunnerContext): Promise<void> {
|
|
112
|
+
if (this.loop !== undefined) {
|
|
113
|
+
throw harnessError({detail: "JobsRunner is already started", kind: "alreadyStarted"});
|
|
114
|
+
}
|
|
115
|
+
if (!tryGetJobsService()) {
|
|
116
|
+
throw harnessError({
|
|
117
|
+
detail:
|
|
118
|
+
"JobsRunner needs a registered JobsApp: register it and build the app before harness.start()",
|
|
119
|
+
kind: "configInvalid",
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
this.context = context;
|
|
123
|
+
this.isStopping = false;
|
|
124
|
+
this.nextSweepAt = DateTime.fromMillis(0);
|
|
125
|
+
this.loop = this.dispatchLoop();
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Stop dispatching, then wait for every phase this instance is running. */
|
|
129
|
+
async stop(): Promise<void> {
|
|
130
|
+
if (this.loop === undefined) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
this.isStopping = true;
|
|
134
|
+
this.wake();
|
|
135
|
+
await this.loop;
|
|
136
|
+
await Promise.allSettled([...this.inFlight]);
|
|
137
|
+
this.loop = undefined;
|
|
138
|
+
this.context = undefined;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Dispatch now instead of after the rest of the idle sleep. */
|
|
142
|
+
wake(): void {
|
|
143
|
+
if (this.wakeSleeper) {
|
|
144
|
+
this.wakeSleeper();
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
this.hasPendingWake = true;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Job handler: claim the task, run one phase, and hand it back if it still runs. */
|
|
151
|
+
private async runPhaseJob(payload: unknown): Promise<void> {
|
|
152
|
+
const {taskId} = parsePayload(payload);
|
|
153
|
+
// Tracked from before the claim, so stop() also waits for a claim in flight.
|
|
154
|
+
const job = this.claimAndRunPhase(taskId);
|
|
155
|
+
this.inFlight.add(job);
|
|
156
|
+
try {
|
|
157
|
+
await job;
|
|
158
|
+
} finally {
|
|
159
|
+
this.inFlight.delete(job);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
private async claimAndRunPhase(taskId: string): Promise<void> {
|
|
164
|
+
const context = this.context;
|
|
165
|
+
if (!context || this.isStopping) {
|
|
166
|
+
// Throwing makes the jobs worker retry later, on this instance or another one.
|
|
167
|
+
throw harnessError({
|
|
168
|
+
detail: `JobsRunner is not running; task ${taskId} stays runnable`,
|
|
169
|
+
kind: "runnerStopped",
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
const task = await context.claimTask(taskId, this.lease);
|
|
173
|
+
if (!task) {
|
|
174
|
+
// Another job claimed it first, or it stopped being runnable since dispatch.
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
await context.runTask(task, this.lease, {maxPhases: 1});
|
|
178
|
+
// The next phase (or a retry that is already due) can be dispatched right away.
|
|
179
|
+
this.wake();
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
private async dispatchLoop(): Promise<void> {
|
|
183
|
+
while (!this.isStopping && this.context) {
|
|
184
|
+
await this.tick(this.context);
|
|
185
|
+
if (!this.isStopping) {
|
|
186
|
+
await this.sleep(this.pollIntervalMs);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** Recover expired leases once per heartbeat, then enqueue a job per runnable task. */
|
|
192
|
+
private async tick(context: HarnessRunnerContext): Promise<void> {
|
|
193
|
+
try {
|
|
194
|
+
if (DateTime.now() >= this.nextSweepAt) {
|
|
195
|
+
this.nextSweepAt = DateTime.now().plus(this.lease.heartbeat);
|
|
196
|
+
await context.recoverExpired();
|
|
197
|
+
}
|
|
198
|
+
const runnable = await context.listRunnable(this.dispatchBatchSize);
|
|
199
|
+
for (const task of runnable) {
|
|
200
|
+
if (this.isStopping) {
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
await this.dispatch(task);
|
|
204
|
+
}
|
|
205
|
+
} catch (error: unknown) {
|
|
206
|
+
logger.error(`JobsRunner dispatch failed: ${errorMessage(error)}`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Enqueue the job for this runnable visit. When the job with that key already ended
|
|
212
|
+
* without claiming the task (its handler kept failing, or it ran where the task was not
|
|
213
|
+
* runnable), follow or extend the chain `<key>:<endedJobId>` until a job is in flight, so
|
|
214
|
+
* the task is not stranded. A newly created job is always in flight, so this ends.
|
|
215
|
+
*/
|
|
216
|
+
private async dispatch(task: HarnessRunnableTask): Promise<void> {
|
|
217
|
+
const jobs = tryGetJobsService();
|
|
218
|
+
if (!jobs) {
|
|
219
|
+
throw harnessError({detail: "JobsApp is no longer registered", kind: "configInvalid"});
|
|
220
|
+
}
|
|
221
|
+
let idempotencyKey = phaseJobKey(task);
|
|
222
|
+
for (;;) {
|
|
223
|
+
const job = await jobs.enqueue({
|
|
224
|
+
idempotencyKey,
|
|
225
|
+
name: HARNESS_PHASE_JOB_NAME,
|
|
226
|
+
payload: {taskId: task.taskId} satisfies PhaseJobPayload,
|
|
227
|
+
});
|
|
228
|
+
if (IN_FLIGHT_JOB_STATUSES.has(job.status)) {
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
idempotencyKey = `${phaseJobKey(task)}:${job._id}`;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private sleep(ms: number): Promise<void> {
|
|
236
|
+
if (this.hasPendingWake) {
|
|
237
|
+
this.hasPendingWake = false;
|
|
238
|
+
return Promise.resolve();
|
|
239
|
+
}
|
|
240
|
+
return new Promise((resolve) => {
|
|
241
|
+
const timer = setTimeout(() => {
|
|
242
|
+
this.wakeSleeper = undefined;
|
|
243
|
+
resolve();
|
|
244
|
+
}, ms);
|
|
245
|
+
this.wakeSleeper = () => {
|
|
246
|
+
clearTimeout(timer);
|
|
247
|
+
this.wakeSleeper = undefined;
|
|
248
|
+
resolve();
|
|
249
|
+
};
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
}
|