@terreno/ai 57.9.0 → 57.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +133 -0
- package/dist/harness/errors.js +107 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +522 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +191 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +64 -0
- package/dist/harness/runners/inProcessRunner.js +227 -0
- package/dist/harness/runners/jobsRunner.d.ts +62 -0
- package/dist/harness/runners/jobsRunner.js +212 -0
- package/dist/harness/runtime.d.ts +52 -0
- package/dist/harness/runtime.js +604 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +952 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +28 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/concurrency.test.ts +155 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +125 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +769 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +778 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +206 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +907 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +530 -0
- package/src/harness/runners/inProcessRunner.ts +260 -0
- package/src/harness/runners/jobsRunner.test.ts +709 -0
- package/src/harness/runners/jobsRunner.ts +252 -0
- package/src/harness/runtime.ts +802 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1119 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
import {afterEach, beforeAll, beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
|
+
import {z} from "@terreno/api";
|
|
3
|
+
import type {LanguageModel} from "ai";
|
|
4
|
+
import {createLocalObservabilityPlugin} from "../observability/local/localPlugin";
|
|
5
|
+
import {registerObsSpan} from "../observability/local/models/obsSpan";
|
|
6
|
+
import {registerObsTrace} from "../observability/local/models/obsTrace";
|
|
7
|
+
import {withGenerateStreaming} from "../tests/generateStream";
|
|
8
|
+
import type {HarnessAgentDefinition, HarnessReplayPolicy} from "../types/harness";
|
|
9
|
+
import type {ObsSpanModel, ObsTraceModel} from "../types/observability";
|
|
10
|
+
import {AGENT_TOOL_TASK_NAME, defineAgent, defineTool, Harness, InProcessRunner} from "./harness";
|
|
11
|
+
import {registerHarnessConversation} from "./models/harnessConversation";
|
|
12
|
+
import {registerHarnessMessage} from "./models/harnessMessage";
|
|
13
|
+
import {registerHarnessOwner} from "./models/harnessOwner";
|
|
14
|
+
import {registerHarnessTask} from "./models/harnessTask";
|
|
15
|
+
|
|
16
|
+
const LEASE_MS = 300;
|
|
17
|
+
const HEARTBEAT_MS = 50;
|
|
18
|
+
|
|
19
|
+
const TaskModel = registerHarnessTask();
|
|
20
|
+
const OwnerModel = registerHarnessOwner();
|
|
21
|
+
const ConversationModel = registerHarnessConversation();
|
|
22
|
+
const MessageModel = registerHarnessMessage();
|
|
23
|
+
let SpanModel: ObsSpanModel;
|
|
24
|
+
let TraceModel: ObsTraceModel;
|
|
25
|
+
|
|
26
|
+
interface Gate {
|
|
27
|
+
promise: Promise<void>;
|
|
28
|
+
release: () => void;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const gate = (): Gate => {
|
|
32
|
+
let release: () => void = () => {};
|
|
33
|
+
const promise = new Promise<void>((resolve) => {
|
|
34
|
+
release = resolve;
|
|
35
|
+
});
|
|
36
|
+
return {promise, release};
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
type Reply = {
|
|
40
|
+
hang?: Gate;
|
|
41
|
+
started?: Gate;
|
|
42
|
+
text?: string;
|
|
43
|
+
toolCall?: {id: string; name: string};
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/** LanguageModelV2 mock answering each request with the next reply; `hang` blocks it first. */
|
|
47
|
+
const scriptedModel = (replies: Reply[]) =>
|
|
48
|
+
withGenerateStreaming({
|
|
49
|
+
doGenerate: mock(async () => {
|
|
50
|
+
const reply = replies.shift();
|
|
51
|
+
if (!reply) {
|
|
52
|
+
throw new Error("script exhausted");
|
|
53
|
+
}
|
|
54
|
+
reply.started?.release();
|
|
55
|
+
if (reply.hang) {
|
|
56
|
+
await reply.hang.promise;
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
content: reply.toolCall
|
|
60
|
+
? [
|
|
61
|
+
{
|
|
62
|
+
input: "{}",
|
|
63
|
+
toolCallId: reply.toolCall.id,
|
|
64
|
+
toolName: reply.toolCall.name,
|
|
65
|
+
type: "tool-call" as const,
|
|
66
|
+
},
|
|
67
|
+
]
|
|
68
|
+
: [{text: reply.text ?? "", type: "text" as const}],
|
|
69
|
+
finishReason: reply.toolCall ? ("tool-calls" as const) : ("stop" as const),
|
|
70
|
+
usage: {inputTokens: 1, outputTokens: 1, totalTokens: 2},
|
|
71
|
+
warnings: [],
|
|
72
|
+
};
|
|
73
|
+
}),
|
|
74
|
+
modelId: "mock-model",
|
|
75
|
+
provider: "mock",
|
|
76
|
+
specificationVersion: "v2" as const,
|
|
77
|
+
supportedUrls: {},
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
interface ProcessHandle {
|
|
81
|
+
die: () => void;
|
|
82
|
+
harness: Harness;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const liveHandles: ProcessHandle[] = [];
|
|
86
|
+
const hungGates: Gate[] = [];
|
|
87
|
+
|
|
88
|
+
/** One simulated process sharing the test database; `die` freezes every lease renewal. */
|
|
89
|
+
const openProcess = async ({
|
|
90
|
+
agent,
|
|
91
|
+
beforeCommitEnd,
|
|
92
|
+
model,
|
|
93
|
+
ownerId,
|
|
94
|
+
}: {
|
|
95
|
+
agent: HarnessAgentDefinition;
|
|
96
|
+
beforeCommitEnd?: (context: {die: () => void; phase: string}) => void;
|
|
97
|
+
model: ReturnType<typeof scriptedModel>;
|
|
98
|
+
ownerId: string;
|
|
99
|
+
}): Promise<ProcessHandle> => {
|
|
100
|
+
let isDead = false;
|
|
101
|
+
const die = (): void => {
|
|
102
|
+
isDead = true;
|
|
103
|
+
};
|
|
104
|
+
const harness = await Harness.open({
|
|
105
|
+
// A dead process must not consume the shared script: its model calls fail at once.
|
|
106
|
+
models: () =>
|
|
107
|
+
withGenerateStreaming({
|
|
108
|
+
...model,
|
|
109
|
+
doGenerate: async (options: unknown) => {
|
|
110
|
+
if (isDead) {
|
|
111
|
+
throw new Error(`${ownerId} is dead`);
|
|
112
|
+
}
|
|
113
|
+
return (model.doGenerate as (options: unknown) => ReturnType<typeof model.doGenerate>)(
|
|
114
|
+
options
|
|
115
|
+
);
|
|
116
|
+
},
|
|
117
|
+
}) as unknown as LanguageModel,
|
|
118
|
+
registry: [agent],
|
|
119
|
+
runner: new InProcessRunner({
|
|
120
|
+
heartbeatInterval: {milliseconds: HEARTBEAT_MS},
|
|
121
|
+
leaseDuration: {milliseconds: LEASE_MS},
|
|
122
|
+
ownerId,
|
|
123
|
+
pollInterval: {milliseconds: 20},
|
|
124
|
+
}),
|
|
125
|
+
testHooks: {
|
|
126
|
+
beforeCommitEnd: beforeCommitEnd ? ({phase}) => beforeCommitEnd({die, phase}) : undefined,
|
|
127
|
+
isHeartbeatSuspended: () => isDead,
|
|
128
|
+
},
|
|
129
|
+
});
|
|
130
|
+
const handle = {die, harness};
|
|
131
|
+
liveHandles.push(handle);
|
|
132
|
+
await harness.start();
|
|
133
|
+
return handle;
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
/** A `chart` tool whose first execution hangs until the test lets it go. */
|
|
137
|
+
const hangingTool = (replay: HarnessReplayPolicy) => {
|
|
138
|
+
const hung = gate();
|
|
139
|
+
hungGates.push(hung);
|
|
140
|
+
const started = gate();
|
|
141
|
+
let runs = 0;
|
|
142
|
+
const tool = defineTool({
|
|
143
|
+
description: "Write a note to the chart",
|
|
144
|
+
execute: async () => {
|
|
145
|
+
runs += 1;
|
|
146
|
+
if (runs === 1) {
|
|
147
|
+
started.release();
|
|
148
|
+
await hung.promise;
|
|
149
|
+
return "written by the dead process";
|
|
150
|
+
}
|
|
151
|
+
return `written on run ${runs}`;
|
|
152
|
+
},
|
|
153
|
+
name: "chart",
|
|
154
|
+
parameters: z.object({}),
|
|
155
|
+
replay,
|
|
156
|
+
});
|
|
157
|
+
return {hung, runs: () => runs, started, tool};
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
const toolMessages = async (conversationId: string) => {
|
|
161
|
+
const rows = await MessageModel.find({conversationId, role: "tool"}).sort({seq: 1});
|
|
162
|
+
return rows.map(({parts, status}) => ({
|
|
163
|
+
output: (parts[0] as {output?: unknown} | undefined)?.output,
|
|
164
|
+
status,
|
|
165
|
+
}));
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
describe("Agent turns across a crash", () => {
|
|
169
|
+
beforeAll(async () => {
|
|
170
|
+
// The shared preload's beforeEach imports this lazily under a 5 s hook timeout; a
|
|
171
|
+
// cold first import can exceed it, so load it here first.
|
|
172
|
+
await import("../langfuseClient");
|
|
173
|
+
createLocalObservabilityPlugin();
|
|
174
|
+
SpanModel = registerObsSpan();
|
|
175
|
+
TraceModel = registerObsTrace();
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
beforeEach(async () => {
|
|
179
|
+
await Promise.all([
|
|
180
|
+
TaskModel.deleteMany({}),
|
|
181
|
+
OwnerModel.deleteMany({}),
|
|
182
|
+
ConversationModel.deleteMany({}),
|
|
183
|
+
MessageModel.deleteMany({}),
|
|
184
|
+
SpanModel.deleteMany({}),
|
|
185
|
+
TraceModel.deleteMany({}),
|
|
186
|
+
]);
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
afterEach(async () => {
|
|
190
|
+
for (const hung of hungGates.splice(0)) {
|
|
191
|
+
hung.release();
|
|
192
|
+
}
|
|
193
|
+
await Promise.all(liveHandles.splice(0).map(({harness}) => harness.stop()));
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it("reports an interrupted replay-never tool as 'interrupted, not retried' and carries on (AC3)", async () => {
|
|
197
|
+
const chart = hangingTool("never");
|
|
198
|
+
const model = scriptedModel([
|
|
199
|
+
{toolCall: {id: "c1", name: "chart"}},
|
|
200
|
+
{text: "Please verify the chart."},
|
|
201
|
+
]);
|
|
202
|
+
const agent = defineAgent({
|
|
203
|
+
instructions: "Write notes.",
|
|
204
|
+
model: {modelId: "mock-model", provider: "mock"},
|
|
205
|
+
name: "test.writer",
|
|
206
|
+
tools: [chart.tool],
|
|
207
|
+
});
|
|
208
|
+
const first = await openProcess({agent, model, ownerId: "process-1"});
|
|
209
|
+
const conversation = await first.harness.createConversation({agent});
|
|
210
|
+
const turn = await conversation.submit({content: "Write it", requestId: "r1"});
|
|
211
|
+
await chart.started.promise;
|
|
212
|
+
first.die();
|
|
213
|
+
|
|
214
|
+
const second = await openProcess({agent, model, ownerId: "process-2"});
|
|
215
|
+
const done = await second.harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
216
|
+
|
|
217
|
+
expect(done.status).toBe("completed");
|
|
218
|
+
expect(done.outcome?.result).toMatchObject({text: "Please verify the chart."});
|
|
219
|
+
expect(chart.runs()).toBe(1);
|
|
220
|
+
const tool = await TaskModel.findExactlyOne({name: AGENT_TOOL_TASK_NAME});
|
|
221
|
+
expect(tool.status).toBe("failed");
|
|
222
|
+
expect(tool.outcome?.error).toBe(
|
|
223
|
+
"Interrupted, not retried: lease of process-1 expired mid-phase (replay: never)"
|
|
224
|
+
);
|
|
225
|
+
expect(await toolMessages(conversation.id)).toEqual([
|
|
226
|
+
{
|
|
227
|
+
output: "Interrupted, not retried: lease of process-1 expired mid-phase (replay: never)",
|
|
228
|
+
status: "error",
|
|
229
|
+
},
|
|
230
|
+
]);
|
|
231
|
+
const toolSpan = await SpanModel.findExactlyOne({kind: "TOOL", traceId: done.traceId});
|
|
232
|
+
expect(toolSpan.status).toBe("error");
|
|
233
|
+
expect(toolSpan.endedAt).toBeDefined();
|
|
234
|
+
expect(model.doGenerate).toHaveBeenCalledTimes(2);
|
|
235
|
+
const secondPrompt = (
|
|
236
|
+
model.doGenerate.mock.calls[1] as unknown as [{prompt: Array<{content: unknown}>}]
|
|
237
|
+
)[0].prompt;
|
|
238
|
+
expect(secondPrompt.at(-1)).toMatchObject({
|
|
239
|
+
content: [
|
|
240
|
+
{
|
|
241
|
+
output: {
|
|
242
|
+
type: "error-text",
|
|
243
|
+
value: "Interrupted, not retried: lease of process-1 expired mid-phase (replay: never)",
|
|
244
|
+
},
|
|
245
|
+
},
|
|
246
|
+
],
|
|
247
|
+
role: "tool",
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
// The dead process's tool finishes late; its result is fenced out.
|
|
251
|
+
chart.hung.release();
|
|
252
|
+
await first.harness.stop();
|
|
253
|
+
expect((await TaskModel.findExactlyOne({name: AGENT_TOOL_TASK_NAME})).status).toBe("failed");
|
|
254
|
+
expect(await toolMessages(conversation.id)).toHaveLength(1);
|
|
255
|
+
}, 20_000);
|
|
256
|
+
|
|
257
|
+
it("re-runs an interrupted replay-safe tool on the new process (AC3)", async () => {
|
|
258
|
+
const chart = hangingTool("safe");
|
|
259
|
+
const model = scriptedModel([{toolCall: {id: "c1", name: "chart"}}, {text: "Done."}]);
|
|
260
|
+
const agent = defineAgent({
|
|
261
|
+
instructions: "Write notes.",
|
|
262
|
+
model: {modelId: "mock-model", provider: "mock"},
|
|
263
|
+
name: "test.writer",
|
|
264
|
+
tools: [chart.tool],
|
|
265
|
+
});
|
|
266
|
+
const first = await openProcess({agent, model, ownerId: "process-1"});
|
|
267
|
+
const conversation = await first.harness.createConversation({agent});
|
|
268
|
+
const turn = await conversation.submit({content: "Write it", requestId: "r1"});
|
|
269
|
+
await chart.started.promise;
|
|
270
|
+
first.die();
|
|
271
|
+
|
|
272
|
+
const second = await openProcess({agent, model, ownerId: "process-2"});
|
|
273
|
+
const done = await second.harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
274
|
+
|
|
275
|
+
expect(done.status).toBe("completed");
|
|
276
|
+
expect(chart.runs()).toBe(2);
|
|
277
|
+
expect(await toolMessages(conversation.id)).toEqual([
|
|
278
|
+
{output: "written on run 2", status: "ok"},
|
|
279
|
+
]);
|
|
280
|
+
const tool = await TaskModel.findExactlyOne({name: AGENT_TOOL_TASK_NAME});
|
|
281
|
+
expect(tool.status).toBe("completed");
|
|
282
|
+
}, 20_000);
|
|
283
|
+
|
|
284
|
+
it("re-runs an interrupted tools phase without re-running its tools", async () => {
|
|
285
|
+
let toolsCommits = 0;
|
|
286
|
+
let runs = 0;
|
|
287
|
+
const chart = defineTool({
|
|
288
|
+
description: "Write a note to the chart",
|
|
289
|
+
execute: async () => {
|
|
290
|
+
runs += 1;
|
|
291
|
+
return "written";
|
|
292
|
+
},
|
|
293
|
+
name: "chart",
|
|
294
|
+
parameters: z.object({}),
|
|
295
|
+
});
|
|
296
|
+
const model = scriptedModel([{toolCall: {id: "c1", name: "chart"}}, {text: "Done."}]);
|
|
297
|
+
const agent = defineAgent({
|
|
298
|
+
instructions: "Write notes.",
|
|
299
|
+
model: {modelId: "mock-model", provider: "mock"},
|
|
300
|
+
name: "test.writer",
|
|
301
|
+
tools: [chart],
|
|
302
|
+
});
|
|
303
|
+
const first = await openProcess({
|
|
304
|
+
agent,
|
|
305
|
+
// The second `tools` commit stores the results: crash right there.
|
|
306
|
+
beforeCommitEnd: ({die, phase}) => {
|
|
307
|
+
if (phase !== "tools") {
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
toolsCommits += 1;
|
|
311
|
+
if (toolsCommits === 2) {
|
|
312
|
+
die();
|
|
313
|
+
throw new Error("process died mid-commit");
|
|
314
|
+
}
|
|
315
|
+
},
|
|
316
|
+
model,
|
|
317
|
+
ownerId: "process-1",
|
|
318
|
+
});
|
|
319
|
+
const conversation = await first.harness.createConversation({agent});
|
|
320
|
+
const turn = await conversation.submit({content: "Write it", requestId: "r1"});
|
|
321
|
+
for (let i = 0; i < 300 && toolsCommits < 2; i++) {
|
|
322
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
323
|
+
}
|
|
324
|
+
expect(toolsCommits).toBe(2);
|
|
325
|
+
|
|
326
|
+
const second = await openProcess({agent, model, ownerId: "process-2"});
|
|
327
|
+
const done = await second.harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
328
|
+
expect(done.status).toBe("completed");
|
|
329
|
+
expect(runs).toBe(1);
|
|
330
|
+
expect(await TaskModel.countDocuments({name: AGENT_TOOL_TASK_NAME})).toBe(1);
|
|
331
|
+
expect(await toolMessages(conversation.id)).toEqual([{output: "written", status: "ok"}]);
|
|
332
|
+
const seqs = (await MessageModel.find({conversationId: conversation.id}).sort({seq: 1})).map(
|
|
333
|
+
({seq}) => seq
|
|
334
|
+
);
|
|
335
|
+
expect(seqs).toEqual([1, 2, 3, 4]);
|
|
336
|
+
}, 20_000);
|
|
337
|
+
|
|
338
|
+
it("re-requests an interrupted model call and keeps exactly one answer", async () => {
|
|
339
|
+
const hung = gate();
|
|
340
|
+
hungGates.push(hung);
|
|
341
|
+
const started = gate();
|
|
342
|
+
const model = scriptedModel([
|
|
343
|
+
{hang: hung, started, text: "from the dead process"},
|
|
344
|
+
{text: "Fresh answer."},
|
|
345
|
+
]);
|
|
346
|
+
const agent = defineAgent({
|
|
347
|
+
instructions: "Answer.",
|
|
348
|
+
model: {modelId: "mock-model", provider: "mock"},
|
|
349
|
+
name: "test.answerer",
|
|
350
|
+
});
|
|
351
|
+
const first = await openProcess({agent, model, ownerId: "process-1"});
|
|
352
|
+
const conversation = await first.harness.createConversation({agent});
|
|
353
|
+
const turn = await conversation.submit({content: "Question", requestId: "r1"});
|
|
354
|
+
await started.promise;
|
|
355
|
+
first.die();
|
|
356
|
+
|
|
357
|
+
const second = await openProcess({agent, model, ownerId: "process-2"});
|
|
358
|
+
const done = await second.harness.waitForTask(turn._id, {timeout: {seconds: 10}});
|
|
359
|
+
expect(done.status).toBe("completed");
|
|
360
|
+
expect(done.outcome?.result).toMatchObject({text: "Fresh answer."});
|
|
361
|
+
expect(model.doGenerate).toHaveBeenCalledTimes(2);
|
|
362
|
+
|
|
363
|
+
// The dead process's late answer is fenced out.
|
|
364
|
+
hung.release();
|
|
365
|
+
await first.harness.stop();
|
|
366
|
+
const messages = await MessageModel.find({conversationId: conversation.id}).sort({seq: 1});
|
|
367
|
+
expect(messages.map(({parts, seq}) => ({parts, seq}))).toEqual([
|
|
368
|
+
{parts: [{text: "Question", type: "text"}], seq: 1},
|
|
369
|
+
{parts: [{text: "Fresh answer.", type: "text"}], seq: 2},
|
|
370
|
+
]);
|
|
371
|
+
expect((await ConversationModel.findExactlyOne({_id: conversation.id})).seq).toBe(2);
|
|
372
|
+
expect(await SpanModel.countDocuments({kind: "LLM", traceId: done.traceId})).toBe(1);
|
|
373
|
+
const interruption = await SpanModel.findExactlyOne({
|
|
374
|
+
name: "request",
|
|
375
|
+
status: "error",
|
|
376
|
+
traceId: done.traceId,
|
|
377
|
+
});
|
|
378
|
+
expect(interruption.error).toContain("re-running (replay: safe)");
|
|
379
|
+
}, 20_000);
|
|
380
|
+
});
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type {HarnessApprovalPolicy} from "../types/harness";
|
|
2
|
+
import {HarnessDefinitionError} from "./definitionError";
|
|
3
|
+
|
|
4
|
+
/** Validate an approval policy map (`defineTask` / `defineExtension` `approvals`). */
|
|
5
|
+
export const assertValidApprovalPolicies = (
|
|
6
|
+
label: string,
|
|
7
|
+
approvals: Readonly<Record<string, HarnessApprovalPolicy>> | undefined
|
|
8
|
+
): void => {
|
|
9
|
+
if (approvals === undefined) {
|
|
10
|
+
return;
|
|
11
|
+
}
|
|
12
|
+
if (typeof approvals !== "object" || approvals === null || Array.isArray(approvals)) {
|
|
13
|
+
throw new HarnessDefinitionError(`${label}: approvals must be an object keyed by approval key`);
|
|
14
|
+
}
|
|
15
|
+
for (const [key, policy] of Object.entries(approvals)) {
|
|
16
|
+
if (!key.trim()) {
|
|
17
|
+
throw new HarnessDefinitionError(`${label}: approval keys must be non-empty`);
|
|
18
|
+
}
|
|
19
|
+
if (
|
|
20
|
+
!Array.isArray(policy?.approvers) ||
|
|
21
|
+
policy.approvers.some((approver) => typeof approver !== "function")
|
|
22
|
+
) {
|
|
23
|
+
throw new HarnessDefinitionError(
|
|
24
|
+
`${label}: approvals.${key}.approvers must be an array of permission functions`
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
};
|