@terreno/ai 57.8.0 → 57.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursorrules +2 -2
- package/README.md +2 -1
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +16 -5
- package/dist/aiApp.js +17 -7
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +127 -0
- package/dist/harness/errors.js +101 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +520 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +186 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +57 -0
- package/dist/harness/runners/inProcessRunner.js +196 -0
- package/dist/harness/runtime.d.ts +34 -0
- package/dist/harness/runtime.js +551 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +88 -2
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +20 -7
- package/dist/routes/gpt.js +179 -488
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +431 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fileUploadsGate.d.ts +8 -0
- package/dist/service/fileUploadsGate.js +18 -0
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/gemini.d.ts +3 -3
- package/dist/service/gemini.js +3 -3
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/service/vertex.d.ts +6 -6
- package/dist/service/vertex.js +5 -5
- package/dist/types/harness.d.ts +934 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +219 -13
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +17 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +31 -8
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +119 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +757 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +767 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +201 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +912 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +402 -0
- package/src/harness/runners/inProcessRunner.ts +220 -0
- package/src/harness/runtime.ts +714 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.test.ts +2 -2
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +106 -4
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +84 -9
- package/src/routes/files.ts +23 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +214 -552
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +676 -5
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +614 -0
- package/src/service/aiService.ts +575 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fileUploadsGate.ts +23 -0
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/gemini.test.ts +16 -10
- package/src/service/gemini.ts +3 -3
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/service/vertex.test.ts +49 -49
- package/src/service/vertex.ts +7 -8
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1095 -0
- package/src/types/index.ts +237 -12
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type {AdminContribution, TerrenoPlugin} from "@terreno/api";
|
|
2
|
+
import type express from "express";
|
|
3
|
+
import type {DurationLike} from "luxon";
|
|
4
|
+
|
|
5
|
+
import {harnessAdminScreens} from "./adminScreens";
|
|
6
|
+
import {harnessError} from "./errors";
|
|
7
|
+
import {HarnessEventHub} from "./eventHub";
|
|
8
|
+
import type {Harness} from "./harness";
|
|
9
|
+
import {addHarnessApprovalRoutes, assertValidBasePath} from "./routes/approvals";
|
|
10
|
+
import {addHarnessConversationRoutes} from "./routes/conversations";
|
|
11
|
+
import {addHarnessEventRoutes} from "./routes/events";
|
|
12
|
+
import {addHarnessTaskRoutes} from "./routes/tasks";
|
|
13
|
+
|
|
14
|
+
export interface HarnessAppOptions {
|
|
15
|
+
/** Prefix for every harness route. Default `/harness`. */
|
|
16
|
+
basePath?: string;
|
|
17
|
+
/** The opened harness whose registry resolves approvers and which wakes tasks. */
|
|
18
|
+
harness: Harness;
|
|
19
|
+
/** How often an idle SSE stream sends a heartbeat comment. Default 15 seconds. */
|
|
20
|
+
heartbeatInterval?: DurationLike;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* TerrenoPlugin that mounts the harness HTTP API under `basePath`:
|
|
25
|
+
*
|
|
26
|
+
* - `GET /conversations`, `GET /conversations/:id`, `POST /conversations/:id/submit`
|
|
27
|
+
* - `GET /conversations/:id/events` and `GET /tasks/:id/events` (SSE)
|
|
28
|
+
* - `GET /tasks/:id`, `POST /tasks/:id/abort`, `POST /tasks/:id/resolveInterrupted`
|
|
29
|
+
* - `GET /approvals`, `GET /approvals/:id`, `POST /approvals/:id/approve` and `/reject`
|
|
30
|
+
*
|
|
31
|
+
* The SSE routes only read Mongo, so any instance serves any stream. All of an app's SSE
|
|
32
|
+
* connections share one change stream (`eventHub`), so viewers cost no pooled connections.
|
|
33
|
+
*
|
|
34
|
+
* With `AdminApp` registered on the same app, it adds the approvals inbox custom screen
|
|
35
|
+
* (`harness-approvals`, group "AI Harness") to the admin sidebar.
|
|
36
|
+
*
|
|
37
|
+
* @example
|
|
38
|
+
* ```typescript
|
|
39
|
+
* const harness = await Harness.open({registry: [intakeSummary]});
|
|
40
|
+
* await harness.start();
|
|
41
|
+
* new TerrenoApp({userModel: User}).register(new HarnessApp({harness})).start();
|
|
42
|
+
* ```
|
|
43
|
+
*/
|
|
44
|
+
export class HarnessApp implements TerrenoPlugin {
|
|
45
|
+
private readonly basePath: string;
|
|
46
|
+
private readonly harness: Harness;
|
|
47
|
+
private readonly heartbeatInterval: DurationLike | undefined;
|
|
48
|
+
/** The one change stream every SSE connection of this app shares. */
|
|
49
|
+
readonly eventHub = new HarnessEventHub();
|
|
50
|
+
|
|
51
|
+
constructor({basePath = "/harness", harness, heartbeatInterval}: HarnessAppOptions) {
|
|
52
|
+
if (!harness) {
|
|
53
|
+
throw harnessError({detail: "HarnessApp requires an opened Harness", kind: "configInvalid"});
|
|
54
|
+
}
|
|
55
|
+
this.basePath = assertValidBasePath(basePath);
|
|
56
|
+
this.harness = harness;
|
|
57
|
+
this.heartbeatInterval = heartbeatInterval;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
register(app: express.Application, openApi?: unknown): void {
|
|
61
|
+
const {basePath, harness} = this;
|
|
62
|
+
// Before the routers, so `/:id/events` is never taken for a document route.
|
|
63
|
+
addHarnessEventRoutes(app, {
|
|
64
|
+
basePath,
|
|
65
|
+
heartbeatInterval: this.heartbeatInterval,
|
|
66
|
+
hub: this.eventHub,
|
|
67
|
+
});
|
|
68
|
+
addHarnessConversationRoutes(app, {basePath, harness, openApi});
|
|
69
|
+
addHarnessTaskRoutes(app, {basePath, harness, openApi});
|
|
70
|
+
addHarnessApprovalRoutes(app, {basePath, harness, openApi});
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
adminContribution(): AdminContribution {
|
|
74
|
+
return {customScreens: harnessAdminScreens()};
|
|
75
|
+
}
|
|
76
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type mongoose from "mongoose";
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
HarnessApprovalRequest,
|
|
5
|
+
HarnessCommit,
|
|
6
|
+
HarnessMemo,
|
|
7
|
+
HarnessTaskRuntime,
|
|
8
|
+
} from "../types/harness";
|
|
9
|
+
import type {HarnessCommitWrites} from "./commit";
|
|
10
|
+
import {harnessError} from "./errors";
|
|
11
|
+
|
|
12
|
+
/** Key of the engine-only runtime surface; never part of the public `rt`. */
|
|
13
|
+
export const HARNESS_INTERNAL_RUNTIME = Symbol("terreno.harness.internalRuntime");
|
|
14
|
+
|
|
15
|
+
/** Engine-only extras built-in tasks (the agent turn) need beyond the public runtime. */
|
|
16
|
+
export interface HarnessInternalRuntime {
|
|
17
|
+
/** `rt.approval` whose approvers come from `extension`'s `approvals` (tool hooks). */
|
|
18
|
+
approvalFor: (extension: string) => HarnessApprovalRequest;
|
|
19
|
+
/** `rt.commit` plus extra rows written in the same transaction. */
|
|
20
|
+
commitWithWrites: (
|
|
21
|
+
next: HarnessCommit<unknown, unknown>,
|
|
22
|
+
writes: HarnessCommitWrites
|
|
23
|
+
) => Promise<void>;
|
|
24
|
+
/** A memo scoped to another task (the turn, for tool hooks), fenced on this run's lease. */
|
|
25
|
+
memoFor: (scopeTaskId: mongoose.Types.ObjectId | string) => HarnessMemo;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
type RuntimeWithInternals = HarnessTaskRuntime<unknown, unknown> & {
|
|
29
|
+
[HARNESS_INTERNAL_RUNTIME]?: HarnessInternalRuntime;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/** The internal surface of a runtime built by the engine. Throws for any other object. */
|
|
33
|
+
export const internalRuntime = (
|
|
34
|
+
rt: HarnessTaskRuntime<unknown, unknown>
|
|
35
|
+
): HarnessInternalRuntime => {
|
|
36
|
+
const internals = (rt as RuntimeWithInternals)[HARNESS_INTERNAL_RUNTIME];
|
|
37
|
+
if (!internals) {
|
|
38
|
+
throw harnessError({
|
|
39
|
+
detail: "This runtime was not built by the harness engine",
|
|
40
|
+
kind: "internal",
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
return internals;
|
|
44
|
+
};
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import {logger} from "@terreno/api";
|
|
2
|
+
import {DateTime} from "luxon";
|
|
3
|
+
import type mongoose from "mongoose";
|
|
4
|
+
|
|
5
|
+
import type {HarnessLeaseSettings, HarnessTestHooks} from "../types/harness";
|
|
6
|
+
import {HARNESS_TASK_STATUSES} from "../types/harness";
|
|
7
|
+
import {type HarnessModels, isDuplicateKeyError} from "./commit";
|
|
8
|
+
import {errorMessage} from "./errors";
|
|
9
|
+
|
|
10
|
+
/** `HarnessOwner.key` of the lease every `InProcessRunner` competes for. */
|
|
11
|
+
const DEFAULT_OWNER_LEASE_KEY = "default";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Take the owner lease when it is free or expired, or renew it when `lease.owner`
|
|
15
|
+
* already holds it. Returns false when another live owner holds it.
|
|
16
|
+
*/
|
|
17
|
+
export const acquireOwnerLease = async ({
|
|
18
|
+
lease,
|
|
19
|
+
models,
|
|
20
|
+
testHooks,
|
|
21
|
+
}: {
|
|
22
|
+
lease: HarnessLeaseSettings;
|
|
23
|
+
models: HarnessModels;
|
|
24
|
+
testHooks?: HarnessTestHooks;
|
|
25
|
+
}): Promise<boolean> => {
|
|
26
|
+
if (testHooks?.isHeartbeatSuspended?.()) {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
const now = DateTime.now();
|
|
30
|
+
try {
|
|
31
|
+
const held = await models.owner.findOneAndUpdate(
|
|
32
|
+
{
|
|
33
|
+
$or: [{owner: lease.owner}, {expiresAt: {$lte: now.toJSDate()}}],
|
|
34
|
+
key: DEFAULT_OWNER_LEASE_KEY,
|
|
35
|
+
},
|
|
36
|
+
{$set: {expiresAt: now.plus(lease.duration).toJSDate(), owner: lease.owner}},
|
|
37
|
+
{returnDocument: "after", upsert: true}
|
|
38
|
+
);
|
|
39
|
+
return held?.owner === lease.owner;
|
|
40
|
+
} catch (error: unknown) {
|
|
41
|
+
// A live owner's row exists, so the upsert collided with the unique key.
|
|
42
|
+
if (isDuplicateKeyError(error)) {
|
|
43
|
+
return false;
|
|
44
|
+
}
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/** Expire the owner lease now so a standby takes over without waiting it out. */
|
|
50
|
+
export const releaseOwnerLease = async ({
|
|
51
|
+
lease,
|
|
52
|
+
models,
|
|
53
|
+
}: {
|
|
54
|
+
lease: HarnessLeaseSettings;
|
|
55
|
+
models: HarnessModels;
|
|
56
|
+
}): Promise<void> => {
|
|
57
|
+
await models.owner.updateOne(
|
|
58
|
+
{key: DEFAULT_OWNER_LEASE_KEY, owner: lease.owner},
|
|
59
|
+
{$set: {expiresAt: DateTime.now().toJSDate()}}
|
|
60
|
+
);
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
export interface TaskHeartbeat {
|
|
64
|
+
/** Stop renewing and wait for a renewal in flight. */
|
|
65
|
+
stop: () => Promise<void>;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Renew one task lease every `lease.heartbeat` while a phase runs. Renewal is fenced on
|
|
70
|
+
* the lease token; once it no longer matches (another runner took the task) renewal
|
|
71
|
+
* stops, `onLost` fires, and the phase's eventual commit is rejected.
|
|
72
|
+
*/
|
|
73
|
+
export const startTaskHeartbeat = ({
|
|
74
|
+
lease,
|
|
75
|
+
models,
|
|
76
|
+
onLost,
|
|
77
|
+
taskId,
|
|
78
|
+
testHooks,
|
|
79
|
+
token,
|
|
80
|
+
}: {
|
|
81
|
+
lease: HarnessLeaseSettings;
|
|
82
|
+
models: HarnessModels;
|
|
83
|
+
/** Called once when renewal finds the lease gone (taken over, aborted, or settled). */
|
|
84
|
+
onLost?: () => void;
|
|
85
|
+
taskId: mongoose.Types.ObjectId;
|
|
86
|
+
testHooks?: HarnessTestHooks;
|
|
87
|
+
token: string | undefined;
|
|
88
|
+
}): TaskHeartbeat => {
|
|
89
|
+
let isStopped = false;
|
|
90
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
91
|
+
let inFlight: Promise<void> = Promise.resolve();
|
|
92
|
+
|
|
93
|
+
const renew = async (): Promise<void> => {
|
|
94
|
+
if (testHooks?.isHeartbeatSuspended?.()) {
|
|
95
|
+
schedule();
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
const result = await models.task.updateOne(
|
|
100
|
+
{_id: taskId, "lease.token": token ?? null, status: HARNESS_TASK_STATUSES.running},
|
|
101
|
+
{$set: {"lease.expiresAt": DateTime.now().plus(lease.duration).toJSDate()}}
|
|
102
|
+
);
|
|
103
|
+
if (result.matchedCount === 0) {
|
|
104
|
+
logger.warn(`Harness task ${taskId} lost its lease; the running phase cannot commit`);
|
|
105
|
+
isStopped = true;
|
|
106
|
+
onLost?.();
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
} catch (error: unknown) {
|
|
110
|
+
logger.warn(`Harness task ${taskId} lease renewal failed: ${errorMessage(error)}`);
|
|
111
|
+
}
|
|
112
|
+
schedule();
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
const schedule = (): void => {
|
|
116
|
+
if (isStopped) {
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
timer = setTimeout(() => {
|
|
120
|
+
inFlight = renew();
|
|
121
|
+
}, lease.heartbeat.toMillis());
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
schedule();
|
|
125
|
+
return {
|
|
126
|
+
stop: async () => {
|
|
127
|
+
isStopped = true;
|
|
128
|
+
clearTimeout(timer);
|
|
129
|
+
await inFlight;
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
};
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import {DateTime} from "luxon";
|
|
2
|
+
import type mongoose from "mongoose";
|
|
3
|
+
|
|
4
|
+
import type {HarnessLeaseSettings, HarnessMemo} from "../types/harness";
|
|
5
|
+
import {HARNESS_TASK_STATUSES} from "../types/harness";
|
|
6
|
+
import {HarnessCommitConflictError, type HarnessModels, isDuplicateKeyError} from "./commit";
|
|
7
|
+
import {errorMessage, harnessError} from "./errors";
|
|
8
|
+
import {inTransaction} from "./transaction";
|
|
9
|
+
|
|
10
|
+
/** The run allowed to write: its task, phase, and lease token. */
|
|
11
|
+
export interface MemoFence {
|
|
12
|
+
phase: string;
|
|
13
|
+
taskId: mongoose.Types.ObjectId;
|
|
14
|
+
token: string | undefined;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** JSON copy of a memo value; throws for values JSON cannot represent. */
|
|
18
|
+
const toMemoValue = (key: string, value: unknown): unknown => {
|
|
19
|
+
let serialized: string | undefined;
|
|
20
|
+
try {
|
|
21
|
+
serialized = JSON.stringify(value);
|
|
22
|
+
} catch (error: unknown) {
|
|
23
|
+
throw harnessError({
|
|
24
|
+
cause: error,
|
|
25
|
+
detail: `memo "${key}": value is not JSON-serializable (${errorMessage(error)})`,
|
|
26
|
+
kind: "invalidRequest",
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
if (serialized === undefined) {
|
|
30
|
+
throw harnessError({
|
|
31
|
+
detail: `memo "${key}": value is not JSON-serializable`,
|
|
32
|
+
kind: "invalidRequest",
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
return JSON.parse(serialized);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Build a memo function scoped to `scopeTaskId`. Reads return the stored value. A write
|
|
40
|
+
* runs in its own transaction: it renews the fenced run's lease (so a run that lost its
|
|
41
|
+
* lease writes nothing and conflicts with a concurrent commit or takeover), then inserts
|
|
42
|
+
* the row only when the key is unset. The stored value is returned, so the first write
|
|
43
|
+
* wins across concurrent calls, replays, and processes.
|
|
44
|
+
*/
|
|
45
|
+
export const createMemo = ({
|
|
46
|
+
assertWritable,
|
|
47
|
+
fence,
|
|
48
|
+
lease,
|
|
49
|
+
models,
|
|
50
|
+
scopeTaskId,
|
|
51
|
+
}: {
|
|
52
|
+
/** Throws when the phase can no longer write (it already committed). */
|
|
53
|
+
assertWritable: () => void;
|
|
54
|
+
fence: MemoFence;
|
|
55
|
+
lease: HarnessLeaseSettings;
|
|
56
|
+
models: HarnessModels;
|
|
57
|
+
scopeTaskId: mongoose.Types.ObjectId;
|
|
58
|
+
}): HarnessMemo => {
|
|
59
|
+
const read = async (key: string): Promise<unknown> => {
|
|
60
|
+
const row = await models.memo.findOneOrNone({key, taskId: scopeTaskId});
|
|
61
|
+
return row?.value;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const write = async (key: string, value: unknown): Promise<unknown> => {
|
|
65
|
+
assertWritable();
|
|
66
|
+
const stored = toMemoValue(key, value);
|
|
67
|
+
try {
|
|
68
|
+
await inTransaction(async (session) => {
|
|
69
|
+
const renewed = await models.task.updateOne(
|
|
70
|
+
{
|
|
71
|
+
_id: fence.taskId,
|
|
72
|
+
"lease.token": fence.token ?? null,
|
|
73
|
+
phase: fence.phase,
|
|
74
|
+
status: HARNESS_TASK_STATUSES.running,
|
|
75
|
+
},
|
|
76
|
+
{$set: {"lease.expiresAt": DateTime.now().plus(lease.duration).toJSDate()}},
|
|
77
|
+
{session}
|
|
78
|
+
);
|
|
79
|
+
if (renewed.matchedCount === 0) {
|
|
80
|
+
throw new HarnessCommitConflictError(String(fence.taskId), fence.phase);
|
|
81
|
+
}
|
|
82
|
+
await models.memo.updateOne(
|
|
83
|
+
{key, taskId: scopeTaskId},
|
|
84
|
+
{$setOnInsert: {key, taskId: scopeTaskId, value: stored}},
|
|
85
|
+
{session, upsert: true}
|
|
86
|
+
);
|
|
87
|
+
});
|
|
88
|
+
} catch (error: unknown) {
|
|
89
|
+
// A concurrent first write inserted the row between our read and upsert.
|
|
90
|
+
if (!isDuplicateKeyError(error)) {
|
|
91
|
+
throw error;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return read(key);
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const memo = async (key: string, ...rest: unknown[]): Promise<unknown> => {
|
|
98
|
+
if (typeof key !== "string" || !key.trim()) {
|
|
99
|
+
throw harnessError({detail: "memo requires a non-empty key", kind: "invalidRequest"});
|
|
100
|
+
}
|
|
101
|
+
if (rest.length === 0 || rest[0] === undefined) {
|
|
102
|
+
return read(key);
|
|
103
|
+
}
|
|
104
|
+
return write(key, rest[0]);
|
|
105
|
+
};
|
|
106
|
+
return memo as HarnessMemo;
|
|
107
|
+
};
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import {describe, expect, it, mock} from "bun:test";
|
|
2
|
+
import {APICallError, type LanguageModel} from "ai";
|
|
3
|
+
|
|
4
|
+
import {callModelWithFallback, HarnessModelCallError, isRetryableModelError} from "./modelCall";
|
|
5
|
+
|
|
6
|
+
const httpError = (statusCode?: number, isRetryable = false): APICallError =>
|
|
7
|
+
new APICallError({
|
|
8
|
+
isRetryable,
|
|
9
|
+
message: `HTTP ${statusCode ?? "none"}`,
|
|
10
|
+
requestBodyValues: {},
|
|
11
|
+
statusCode,
|
|
12
|
+
url: "https://mock.invalid",
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
const fakeModel = (id: string): LanguageModel => id as unknown as LanguageModel;
|
|
16
|
+
const resolveModel = ({modelId}: {modelId: string}): LanguageModel => fakeModel(modelId);
|
|
17
|
+
const A = {modelId: "a", provider: "p"};
|
|
18
|
+
const B = {modelId: "b", provider: "p"};
|
|
19
|
+
|
|
20
|
+
describe("isRetryableModelError", () => {
|
|
21
|
+
it.each([
|
|
22
|
+
[429, true],
|
|
23
|
+
[500, true],
|
|
24
|
+
[503, true],
|
|
25
|
+
[400, false],
|
|
26
|
+
[401, false],
|
|
27
|
+
[404, false],
|
|
28
|
+
// 408 is often retried elsewhere; here every 4xx but 429 means "this request is wrong".
|
|
29
|
+
[408, false],
|
|
30
|
+
])("HTTP %d -> %p", (status, expected) => {
|
|
31
|
+
expect(isRetryableModelError(httpError(status, true))).toBe(expected);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it("uses the SDK's retryable flag when there is no status code", () => {
|
|
35
|
+
expect(isRetryableModelError(httpError(undefined, true))).toBe(true);
|
|
36
|
+
expect(isRetryableModelError(httpError(undefined, false))).toBe(false);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it("retries network failures and nothing else", () => {
|
|
40
|
+
expect(isRetryableModelError(new TypeError("fetch failed"))).toBe(true);
|
|
41
|
+
expect(isRetryableModelError(Object.assign(new Error("reset"), {code: "ECONNRESET"}))).toBe(
|
|
42
|
+
true
|
|
43
|
+
);
|
|
44
|
+
expect(isRetryableModelError(new Error("wrapped", {cause: {code: "ETIMEDOUT"}}))).toBe(true);
|
|
45
|
+
expect(isRetryableModelError(new Error("bad prompt"))).toBe(false);
|
|
46
|
+
expect(isRetryableModelError("boom")).toBe(false);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
describe("callModelWithFallback", () => {
|
|
51
|
+
it("returns the first success with every attempt recorded", async () => {
|
|
52
|
+
let tries = 0;
|
|
53
|
+
const call = mock(async (model: LanguageModel) => {
|
|
54
|
+
tries += 1;
|
|
55
|
+
if (tries <= 2) {
|
|
56
|
+
throw httpError(502, true);
|
|
57
|
+
}
|
|
58
|
+
return `answer from ${String(model)}`;
|
|
59
|
+
});
|
|
60
|
+
const result = await callModelWithFallback({
|
|
61
|
+
call,
|
|
62
|
+
models: [A, B],
|
|
63
|
+
resolveModel,
|
|
64
|
+
retry: {backoffMs: 0},
|
|
65
|
+
});
|
|
66
|
+
expect(result.result).toBe("answer from a");
|
|
67
|
+
expect(result.model).toEqual(A);
|
|
68
|
+
expect(result.attempts.map(({attempt, statusCode}) => [attempt, statusCode])).toEqual([
|
|
69
|
+
[1, 502],
|
|
70
|
+
[2, 502],
|
|
71
|
+
[3, undefined],
|
|
72
|
+
]);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it("throws HarnessModelCallError once every model is exhausted", async () => {
|
|
76
|
+
const error = await callModelWithFallback({
|
|
77
|
+
call: async () => {
|
|
78
|
+
throw httpError(503, true);
|
|
79
|
+
},
|
|
80
|
+
models: [A, B],
|
|
81
|
+
resolveModel,
|
|
82
|
+
retry: {backoffMs: 0, maxAttempts: 2},
|
|
83
|
+
}).catch((caught: unknown) => caught);
|
|
84
|
+
expect(error).toBeInstanceOf(HarnessModelCallError);
|
|
85
|
+
expect((error as HarnessModelCallError).message).toBe(
|
|
86
|
+
"Model call failed after 4 attempts across p/a, p/b: HTTP 503"
|
|
87
|
+
);
|
|
88
|
+
expect((error as HarnessModelCallError).attempts).toHaveLength(4);
|
|
89
|
+
expect((error as HarnessModelCallError).cause).toBeInstanceOf(APICallError);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it("stops waiting out a backoff as soon as the signal aborts", async () => {
|
|
93
|
+
const controller = new AbortController();
|
|
94
|
+
const reason = new Error("turn aborted");
|
|
95
|
+
const call = mock(async () => {
|
|
96
|
+
setTimeout(() => controller.abort(reason), 10);
|
|
97
|
+
throw httpError(503, true);
|
|
98
|
+
});
|
|
99
|
+
const error = await callModelWithFallback({
|
|
100
|
+
call,
|
|
101
|
+
models: [A],
|
|
102
|
+
resolveModel,
|
|
103
|
+
retry: {backoffMs: 60_000, maxBackoffMs: 60_000},
|
|
104
|
+
signal: controller.signal,
|
|
105
|
+
}).catch((caught: unknown) => caught);
|
|
106
|
+
expect(error).toBe(reason);
|
|
107
|
+
expect(call).toHaveBeenCalledTimes(1);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it("rethrows the abort reason when the call fails because the signal aborted", async () => {
|
|
111
|
+
const controller = new AbortController();
|
|
112
|
+
const reason = new Error("aborted mid-call");
|
|
113
|
+
const error = await callModelWithFallback({
|
|
114
|
+
call: async () => {
|
|
115
|
+
controller.abort(reason);
|
|
116
|
+
throw new Error("The operation was aborted");
|
|
117
|
+
},
|
|
118
|
+
models: [A],
|
|
119
|
+
resolveModel,
|
|
120
|
+
signal: controller.signal,
|
|
121
|
+
}).catch((caught: unknown) => caught);
|
|
122
|
+
expect(error).toBe(reason);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it("does not call a model when the signal is already aborted", async () => {
|
|
126
|
+
const controller = new AbortController();
|
|
127
|
+
controller.abort(new Error("already"));
|
|
128
|
+
const call = mock(async () => "never");
|
|
129
|
+
await expect(
|
|
130
|
+
callModelWithFallback({call, models: [A], resolveModel, signal: controller.signal})
|
|
131
|
+
).rejects.toThrow("already");
|
|
132
|
+
expect(call).toHaveBeenCalledTimes(0);
|
|
133
|
+
});
|
|
134
|
+
});
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import {APICallError, type LanguageModel} from "ai";
|
|
2
|
+
|
|
3
|
+
import type {HarnessModelRef, HarnessModelResolver, HarnessRetryPolicy} from "../types/harness";
|
|
4
|
+
import {HARNESS_MODEL_RETRY_DEFAULTS} from "../types/harness";
|
|
5
|
+
import {retryDelayMs} from "./retryBackoff";
|
|
6
|
+
|
|
7
|
+
/** One try of one model, as recorded on the LLM span. */
|
|
8
|
+
export interface ModelCallAttempt {
|
|
9
|
+
attempt: number;
|
|
10
|
+
error?: string;
|
|
11
|
+
modelId: string;
|
|
12
|
+
provider: string;
|
|
13
|
+
/** Whether the failure allowed another try. */
|
|
14
|
+
retryable?: boolean;
|
|
15
|
+
statusCode?: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface ModelCallResult<T> {
|
|
19
|
+
attempts: ModelCallAttempt[];
|
|
20
|
+
/** The model that answered. */
|
|
21
|
+
model: HarnessModelRef;
|
|
22
|
+
result: T;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Every model (primary, then fallbacks) failed, or a failure was not retryable. */
|
|
26
|
+
export class HarnessModelCallError extends Error {
|
|
27
|
+
readonly attempts: ModelCallAttempt[];
|
|
28
|
+
|
|
29
|
+
constructor(message: string, attempts: ModelCallAttempt[], cause: unknown) {
|
|
30
|
+
super(message, {cause});
|
|
31
|
+
this.name = "HarnessModelCallError";
|
|
32
|
+
this.attempts = attempts;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Socket-level failures worth another try; providers wrap most of these in APICallError. */
|
|
37
|
+
const NETWORK_ERROR_CODES = new Set([
|
|
38
|
+
"ECONNREFUSED",
|
|
39
|
+
"ECONNRESET",
|
|
40
|
+
"EAI_AGAIN",
|
|
41
|
+
"ENOTFOUND",
|
|
42
|
+
"EPIPE",
|
|
43
|
+
"ETIMEDOUT",
|
|
44
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
45
|
+
"UND_ERR_SOCKET",
|
|
46
|
+
]);
|
|
47
|
+
|
|
48
|
+
const errorCode = (error: unknown): string | undefined => {
|
|
49
|
+
if (typeof error !== "object" || error === null) {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
const code = (error as {code?: unknown}).code;
|
|
53
|
+
if (typeof code === "string") {
|
|
54
|
+
return code;
|
|
55
|
+
}
|
|
56
|
+
return errorCode((error as {cause?: unknown}).cause);
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
const isNetworkError = (error: unknown): boolean => {
|
|
60
|
+
const code = errorCode(error);
|
|
61
|
+
if (code && NETWORK_ERROR_CODES.has(code)) {
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
// fetch() reports a dropped connection as `TypeError: fetch failed`.
|
|
65
|
+
return error instanceof TypeError && /fetch failed|network/i.test(error.message);
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Whether a model call failure may succeed on another try: HTTP 429 and 5xx, and network
|
|
70
|
+
* failures. Every other HTTP status (400, 401, 404, ...) and any non-HTTP error fails at
|
|
71
|
+
* once, because repeating the same request cannot fix it.
|
|
72
|
+
*/
|
|
73
|
+
export const isRetryableModelError = (error: unknown): boolean => {
|
|
74
|
+
if (APICallError.isInstance(error)) {
|
|
75
|
+
if (error.statusCode !== undefined) {
|
|
76
|
+
return error.statusCode === 429 || error.statusCode >= 500;
|
|
77
|
+
}
|
|
78
|
+
// No status: the request never got a response (the SDK marks these retryable).
|
|
79
|
+
return error.isRetryable;
|
|
80
|
+
}
|
|
81
|
+
return isNetworkError(error);
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
const describeError = (error: unknown): string =>
|
|
85
|
+
error instanceof Error ? error.message : String(error);
|
|
86
|
+
|
|
87
|
+
const statusCodeOf = (error: unknown): number | undefined =>
|
|
88
|
+
APICallError.isInstance(error) ? error.statusCode : undefined;
|
|
89
|
+
|
|
90
|
+
/** Resolve after `ms`, or reject with the signal's reason as soon as it aborts. */
|
|
91
|
+
const abortableDelay = (ms: number, signal?: AbortSignal): Promise<void> => {
|
|
92
|
+
if (signal?.aborted) {
|
|
93
|
+
return Promise.reject(signal.reason);
|
|
94
|
+
}
|
|
95
|
+
return new Promise((resolve, reject) => {
|
|
96
|
+
const onAbort = (): void => {
|
|
97
|
+
clearTimeout(timer);
|
|
98
|
+
reject(signal?.reason);
|
|
99
|
+
};
|
|
100
|
+
const timer = setTimeout(() => {
|
|
101
|
+
signal?.removeEventListener("abort", onAbort);
|
|
102
|
+
resolve();
|
|
103
|
+
}, ms);
|
|
104
|
+
signal?.addEventListener("abort", onAbort, {once: true});
|
|
105
|
+
});
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const label = (ref: HarnessModelRef): string => `${ref.provider}/${ref.modelId}`;
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Call `call` with the primary model, retrying 429 / 5xx / network failures with
|
|
112
|
+
* exponential backoff and equal jitter (`retry`, default 3 tries per model), then each
|
|
113
|
+
* fallback model in order with the same budget. A non-retryable failure, or an aborted
|
|
114
|
+
* `signal`, throws at once. Throws `HarnessModelCallError` (with every attempt) when no
|
|
115
|
+
* model answered.
|
|
116
|
+
*/
|
|
117
|
+
export const callModelWithFallback = async <T>({
|
|
118
|
+
call,
|
|
119
|
+
models,
|
|
120
|
+
random,
|
|
121
|
+
resolveModel,
|
|
122
|
+
retry,
|
|
123
|
+
signal,
|
|
124
|
+
}: {
|
|
125
|
+
call: (model: LanguageModel) => Promise<T>;
|
|
126
|
+
/** Primary first, then fallbacks. */
|
|
127
|
+
models: ReadonlyArray<HarnessModelRef>;
|
|
128
|
+
random?: () => number;
|
|
129
|
+
resolveModel: HarnessModelResolver;
|
|
130
|
+
retry?: HarnessRetryPolicy;
|
|
131
|
+
signal?: AbortSignal;
|
|
132
|
+
}): Promise<ModelCallResult<T>> => {
|
|
133
|
+
const policy = {
|
|
134
|
+
backoffMs: retry?.backoffMs ?? HARNESS_MODEL_RETRY_DEFAULTS.backoffMs,
|
|
135
|
+
maxAttempts: retry?.maxAttempts ?? HARNESS_MODEL_RETRY_DEFAULTS.maxAttempts,
|
|
136
|
+
maxBackoffMs: retry?.maxBackoffMs ?? HARNESS_MODEL_RETRY_DEFAULTS.maxBackoffMs,
|
|
137
|
+
};
|
|
138
|
+
const attempts: ModelCallAttempt[] = [];
|
|
139
|
+
let lastError: unknown;
|
|
140
|
+
|
|
141
|
+
for (const ref of models) {
|
|
142
|
+
let model: LanguageModel;
|
|
143
|
+
try {
|
|
144
|
+
model = resolveModel(ref);
|
|
145
|
+
} catch (error: unknown) {
|
|
146
|
+
// A model the resolver cannot build will not appear on retry.
|
|
147
|
+
attempts.push({
|
|
148
|
+
attempt: 1,
|
|
149
|
+
error: describeError(error),
|
|
150
|
+
modelId: ref.modelId,
|
|
151
|
+
provider: ref.provider,
|
|
152
|
+
retryable: false,
|
|
153
|
+
});
|
|
154
|
+
throw new HarnessModelCallError(
|
|
155
|
+
`Model ${label(ref)} could not be resolved: ${describeError(error)}`,
|
|
156
|
+
attempts,
|
|
157
|
+
error
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
for (let attempt = 1; attempt <= policy.maxAttempts; attempt++) {
|
|
161
|
+
if (signal?.aborted) {
|
|
162
|
+
throw signal.reason;
|
|
163
|
+
}
|
|
164
|
+
try {
|
|
165
|
+
const result = await call(model);
|
|
166
|
+
attempts.push({attempt, modelId: ref.modelId, provider: ref.provider});
|
|
167
|
+
return {attempts, model: ref, result};
|
|
168
|
+
} catch (error: unknown) {
|
|
169
|
+
if (signal?.aborted) {
|
|
170
|
+
throw signal.reason ?? error;
|
|
171
|
+
}
|
|
172
|
+
lastError = error;
|
|
173
|
+
const retryable = isRetryableModelError(error);
|
|
174
|
+
attempts.push({
|
|
175
|
+
attempt,
|
|
176
|
+
error: describeError(error),
|
|
177
|
+
modelId: ref.modelId,
|
|
178
|
+
provider: ref.provider,
|
|
179
|
+
retryable,
|
|
180
|
+
statusCode: statusCodeOf(error),
|
|
181
|
+
});
|
|
182
|
+
if (!retryable) {
|
|
183
|
+
throw new HarnessModelCallError(
|
|
184
|
+
`Model ${label(ref)} failed with a non-retryable error: ${describeError(error)}`,
|
|
185
|
+
attempts,
|
|
186
|
+
error
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
if (attempt < policy.maxAttempts) {
|
|
190
|
+
await abortableDelay(retryDelayMs({failedAttempts: attempt, policy, random}), signal);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
throw new HarnessModelCallError(
|
|
196
|
+
`Model call failed after ${attempts.length} attempts across ${models.map(label).join(", ")}: ${describeError(lastError)}`,
|
|
197
|
+
attempts,
|
|
198
|
+
lastError
|
|
199
|
+
);
|
|
200
|
+
};
|