@terreno/ai 57.8.0 → 57.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursorrules +2 -2
- package/README.md +2 -1
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +16 -5
- package/dist/aiApp.js +17 -7
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +127 -0
- package/dist/harness/errors.js +101 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +520 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +186 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +57 -0
- package/dist/harness/runners/inProcessRunner.js +196 -0
- package/dist/harness/runtime.d.ts +34 -0
- package/dist/harness/runtime.js +551 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +88 -2
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +20 -7
- package/dist/routes/gpt.js +179 -488
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +431 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fileUploadsGate.d.ts +8 -0
- package/dist/service/fileUploadsGate.js +18 -0
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/gemini.d.ts +3 -3
- package/dist/service/gemini.js +3 -3
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/service/vertex.d.ts +6 -6
- package/dist/service/vertex.js +5 -5
- package/dist/types/harness.d.ts +934 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +219 -13
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +17 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +31 -8
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +119 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +757 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +767 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +201 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +912 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +402 -0
- package/src/harness/runners/inProcessRunner.ts +220 -0
- package/src/harness/runtime.ts +714 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.test.ts +2 -2
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +106 -4
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +84 -9
- package/src/routes/files.ts +23 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +214 -552
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +676 -5
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +614 -0
- package/src/service/aiService.ts +575 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fileUploadsGate.ts +23 -0
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/gemini.test.ts +16 -10
- package/src/service/gemini.ts +3 -3
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/service/vertex.test.ts +49 -49
- package/src/service/vertex.ts +7 -8
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1095 -0
- package/src/types/index.ts +237 -12
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import {afterEach, beforeEach, describe, expect, it, mock} from "bun:test";
|
|
2
|
+
import {APIError} from "@terreno/api";
|
|
2
3
|
import {jsonSchema, type LanguageModel} from "ai";
|
|
3
4
|
import {assert} from "chai";
|
|
4
5
|
import mongoose from "mongoose";
|
|
5
6
|
|
|
6
7
|
import {AIRequest} from "../models/aiRequest";
|
|
8
|
+
import {MemoryTraceSink} from "../observability/local/traceStore";
|
|
9
|
+
import {ObservabilityApp, resetObservabilityApp} from "../observability/observabilityApp";
|
|
10
|
+
import type {ObservabilityPlugin, PromptVersionRef} from "../observability/types";
|
|
7
11
|
import {AIService, TemperaturePresets} from "./aiService";
|
|
8
12
|
|
|
9
13
|
// Create a mock LanguageModelV2
|
|
@@ -44,6 +48,7 @@ describe("AIService", () => {
|
|
|
44
48
|
|
|
45
49
|
afterEach(async () => {
|
|
46
50
|
await AIRequest.deleteMany({});
|
|
51
|
+
resetObservabilityApp();
|
|
47
52
|
});
|
|
48
53
|
|
|
49
54
|
describe("constructor", () => {
|
|
@@ -542,6 +547,27 @@ describe("AIService", () => {
|
|
|
542
547
|
expect(messages[1]).toEqual({content: "[Generated image]", role: "assistant"});
|
|
543
548
|
});
|
|
544
549
|
|
|
550
|
+
it("skips streaming placeholders and empty failed replies", () => {
|
|
551
|
+
const model = createMockModel();
|
|
552
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
553
|
+
|
|
554
|
+
const messages = service.buildMessages([
|
|
555
|
+
{text: "First", type: "user"},
|
|
556
|
+
{status: "error", text: "", type: "assistant"},
|
|
557
|
+
{text: "Second", type: "user"},
|
|
558
|
+
{status: "error", text: "Partial answer", type: "assistant"},
|
|
559
|
+
{text: "Third", type: "user"},
|
|
560
|
+
{status: "streaming", text: "Half", type: "assistant"},
|
|
561
|
+
]);
|
|
562
|
+
|
|
563
|
+
expect(messages).toEqual([
|
|
564
|
+
{content: "First", role: "user"},
|
|
565
|
+
{content: "Second", role: "user"},
|
|
566
|
+
{content: "Partial answer", role: "assistant"},
|
|
567
|
+
{content: "Third", role: "user"},
|
|
568
|
+
]);
|
|
569
|
+
});
|
|
570
|
+
|
|
545
571
|
it("should convert multi-modal user prompts with image content", () => {
|
|
546
572
|
const model = createMockModel();
|
|
547
573
|
const service = new AIService({model: model as unknown as LanguageModel});
|
|
@@ -606,6 +632,262 @@ describe("AIService", () => {
|
|
|
606
632
|
expect(messages.length).toBe(2);
|
|
607
633
|
expect(messages[0]).toEqual({content: "You are helpful", role: "system"});
|
|
608
634
|
});
|
|
635
|
+
|
|
636
|
+
describe("asks", () => {
|
|
637
|
+
const planAsk = {
|
|
638
|
+
options: [
|
|
639
|
+
{id: "starter", label: "Starter"},
|
|
640
|
+
{id: "team", label: "Team"},
|
|
641
|
+
],
|
|
642
|
+
prompt: "Which plan should I set up?",
|
|
643
|
+
select: "one",
|
|
644
|
+
};
|
|
645
|
+
const regionAsk = {
|
|
646
|
+
options: [
|
|
647
|
+
{id: "us", label: "US"},
|
|
648
|
+
{id: "eu", label: "EU"},
|
|
649
|
+
],
|
|
650
|
+
prompt: "Where should your data live?",
|
|
651
|
+
select: "one",
|
|
652
|
+
};
|
|
653
|
+
const buildMessages = (prompts: Parameters<AIService["buildMessages"]>[0]) =>
|
|
654
|
+
new AIService({model: createMockModel() as unknown as LanguageModel}).buildMessages(
|
|
655
|
+
prompts
|
|
656
|
+
);
|
|
657
|
+
|
|
658
|
+
it("includes a completed ask as a tool call and its result, and still skips host tool rows", () => {
|
|
659
|
+
const messages = buildMessages([
|
|
660
|
+
{text: "Set up my workspace", type: "user"},
|
|
661
|
+
{
|
|
662
|
+
args: {},
|
|
663
|
+
text: "Tool call: lookupPlans",
|
|
664
|
+
toolCallId: "call_lookup",
|
|
665
|
+
toolName: "lookupPlans",
|
|
666
|
+
type: "tool-call",
|
|
667
|
+
},
|
|
668
|
+
{
|
|
669
|
+
result: {plans: 2},
|
|
670
|
+
text: "Tool result: lookupPlans",
|
|
671
|
+
toolCallId: "call_lookup",
|
|
672
|
+
toolName: "lookupPlans",
|
|
673
|
+
type: "tool-result",
|
|
674
|
+
},
|
|
675
|
+
{
|
|
676
|
+
args: planAsk,
|
|
677
|
+
ask: {kind: "choice", status: "answered"},
|
|
678
|
+
text: "Tool call: ask_choice",
|
|
679
|
+
toolCallId: "call_plan",
|
|
680
|
+
toolName: "ask_choice",
|
|
681
|
+
type: "tool-call",
|
|
682
|
+
},
|
|
683
|
+
{
|
|
684
|
+
result: {action: "accept", content: {selected: ["team"]}},
|
|
685
|
+
text: "Tool result: ask_choice",
|
|
686
|
+
toolCallId: "call_plan",
|
|
687
|
+
toolName: "ask_choice",
|
|
688
|
+
type: "tool-result",
|
|
689
|
+
},
|
|
690
|
+
{text: "Setting up the Team plan.", type: "assistant"},
|
|
691
|
+
{text: "Thanks", type: "user"},
|
|
692
|
+
]);
|
|
693
|
+
|
|
694
|
+
expect(messages).toEqual([
|
|
695
|
+
{content: "Set up my workspace", role: "user"},
|
|
696
|
+
{
|
|
697
|
+
content: [
|
|
698
|
+
{input: planAsk, toolCallId: "call_plan", toolName: "ask_choice", type: "tool-call"},
|
|
699
|
+
],
|
|
700
|
+
role: "assistant",
|
|
701
|
+
},
|
|
702
|
+
{
|
|
703
|
+
content: [
|
|
704
|
+
{
|
|
705
|
+
output: {type: "json", value: {action: "accept", content: {selected: ["team"]}}},
|
|
706
|
+
toolCallId: "call_plan",
|
|
707
|
+
toolName: "ask_choice",
|
|
708
|
+
type: "tool-result",
|
|
709
|
+
},
|
|
710
|
+
],
|
|
711
|
+
role: "tool",
|
|
712
|
+
},
|
|
713
|
+
{content: "Setting up the Team plan.", role: "assistant"},
|
|
714
|
+
{content: "Thanks", role: "user"},
|
|
715
|
+
]);
|
|
716
|
+
});
|
|
717
|
+
|
|
718
|
+
it("puts consecutive ask calls from one step in one assistant message, in call order", () => {
|
|
719
|
+
const messages = buildMessages([
|
|
720
|
+
{text: "Set up my workspace", type: "user"},
|
|
721
|
+
{
|
|
722
|
+
args: planAsk,
|
|
723
|
+
ask: {kind: "choice", status: "answered"},
|
|
724
|
+
text: "Tool call: ask_choice",
|
|
725
|
+
toolCallId: "call_plan",
|
|
726
|
+
toolName: "ask_choice",
|
|
727
|
+
type: "tool-call",
|
|
728
|
+
},
|
|
729
|
+
{
|
|
730
|
+
args: regionAsk,
|
|
731
|
+
ask: {kind: "choice", status: "cancelled"},
|
|
732
|
+
text: "Tool call: ask_choice",
|
|
733
|
+
toolCallId: "call_region",
|
|
734
|
+
toolName: "ask_choice",
|
|
735
|
+
type: "tool-call",
|
|
736
|
+
},
|
|
737
|
+
{
|
|
738
|
+
result: {action: "cancel", reason: "one_ask_at_a_time"},
|
|
739
|
+
text: "Tool result: ask_choice",
|
|
740
|
+
toolCallId: "call_region",
|
|
741
|
+
toolName: "ask_choice",
|
|
742
|
+
type: "tool-result",
|
|
743
|
+
},
|
|
744
|
+
{
|
|
745
|
+
result: {action: "decline"},
|
|
746
|
+
text: "Tool result: ask_choice",
|
|
747
|
+
toolCallId: "call_plan",
|
|
748
|
+
toolName: "ask_choice",
|
|
749
|
+
type: "tool-result",
|
|
750
|
+
},
|
|
751
|
+
{text: "No plan for now.", type: "assistant"},
|
|
752
|
+
]);
|
|
753
|
+
|
|
754
|
+
expect(messages).toEqual([
|
|
755
|
+
{content: "Set up my workspace", role: "user"},
|
|
756
|
+
{
|
|
757
|
+
content: [
|
|
758
|
+
{input: planAsk, toolCallId: "call_plan", toolName: "ask_choice", type: "tool-call"},
|
|
759
|
+
{
|
|
760
|
+
input: regionAsk,
|
|
761
|
+
toolCallId: "call_region",
|
|
762
|
+
toolName: "ask_choice",
|
|
763
|
+
type: "tool-call",
|
|
764
|
+
},
|
|
765
|
+
],
|
|
766
|
+
role: "assistant",
|
|
767
|
+
},
|
|
768
|
+
{
|
|
769
|
+
content: [
|
|
770
|
+
{
|
|
771
|
+
output: {type: "json", value: {action: "decline"}},
|
|
772
|
+
toolCallId: "call_plan",
|
|
773
|
+
toolName: "ask_choice",
|
|
774
|
+
type: "tool-result",
|
|
775
|
+
},
|
|
776
|
+
{
|
|
777
|
+
output: {type: "json", value: {action: "cancel", reason: "one_ask_at_a_time"}},
|
|
778
|
+
toolCallId: "call_region",
|
|
779
|
+
toolName: "ask_choice",
|
|
780
|
+
type: "tool-result",
|
|
781
|
+
},
|
|
782
|
+
],
|
|
783
|
+
role: "tool",
|
|
784
|
+
},
|
|
785
|
+
{content: "No plan for now.", role: "assistant"},
|
|
786
|
+
]);
|
|
787
|
+
});
|
|
788
|
+
|
|
789
|
+
it("skips an ask that is still waiting for an answer", () => {
|
|
790
|
+
const messages = buildMessages([
|
|
791
|
+
{text: "Set up my workspace", type: "user"},
|
|
792
|
+
{
|
|
793
|
+
args: planAsk,
|
|
794
|
+
ask: {kind: "choice", status: "pending"},
|
|
795
|
+
text: "Tool call: ask_choice",
|
|
796
|
+
toolCallId: "call_plan",
|
|
797
|
+
toolName: "ask_choice",
|
|
798
|
+
type: "tool-call",
|
|
799
|
+
},
|
|
800
|
+
]);
|
|
801
|
+
|
|
802
|
+
expect(messages).toEqual([{content: "Set up my workspace", role: "user"}]);
|
|
803
|
+
});
|
|
804
|
+
|
|
805
|
+
it("skips approval asks, which are display-only", () => {
|
|
806
|
+
const messages = buildMessages([
|
|
807
|
+
{text: "Delete my completed todos", type: "user"},
|
|
808
|
+
{
|
|
809
|
+
args: {prompt: "Allow deleteCompletedTodos?"},
|
|
810
|
+
ask: {kind: "confirm", origin: "approval", status: "answered"},
|
|
811
|
+
text: "Tool call: deleteCompletedTodos",
|
|
812
|
+
toolCallId: "approval_delete",
|
|
813
|
+
toolName: "deleteCompletedTodos",
|
|
814
|
+
type: "tool-call",
|
|
815
|
+
},
|
|
816
|
+
{
|
|
817
|
+
result: {action: "accept", content: {confirmed: true}},
|
|
818
|
+
text: "Tool result: deleteCompletedTodos",
|
|
819
|
+
toolCallId: "approval_delete",
|
|
820
|
+
toolName: "deleteCompletedTodos",
|
|
821
|
+
type: "tool-result",
|
|
822
|
+
},
|
|
823
|
+
{text: "Deleted 2 todos.", type: "assistant"},
|
|
824
|
+
]);
|
|
825
|
+
|
|
826
|
+
expect(messages).toEqual([
|
|
827
|
+
{content: "Delete my completed todos", role: "user"},
|
|
828
|
+
{content: "Deleted 2 todos.", role: "assistant"},
|
|
829
|
+
]);
|
|
830
|
+
});
|
|
831
|
+
|
|
832
|
+
it("skips ask rows without a tool call id or tool name, and sends null for a missing result", () => {
|
|
833
|
+
const messages = buildMessages([
|
|
834
|
+
{text: "Set up my workspace", type: "user"},
|
|
835
|
+
{
|
|
836
|
+
args: planAsk,
|
|
837
|
+
ask: {kind: "choice", status: "answered"},
|
|
838
|
+
text: "Tool call: ask_choice",
|
|
839
|
+
type: "tool-call",
|
|
840
|
+
},
|
|
841
|
+
{
|
|
842
|
+
args: planAsk,
|
|
843
|
+
ask: {kind: "choice", status: "answered"},
|
|
844
|
+
text: "Tool call: ask_choice",
|
|
845
|
+
toolCallId: "call_nameless",
|
|
846
|
+
type: "tool-call",
|
|
847
|
+
},
|
|
848
|
+
{
|
|
849
|
+
result: {action: "decline"},
|
|
850
|
+
text: "Tool result: ask_choice",
|
|
851
|
+
toolCallId: "call_nameless",
|
|
852
|
+
type: "tool-result",
|
|
853
|
+
},
|
|
854
|
+
{
|
|
855
|
+
ask: {kind: "choice", status: "answered"},
|
|
856
|
+
text: "Tool call: ask_choice",
|
|
857
|
+
toolCallId: "call_bare",
|
|
858
|
+
toolName: "ask_choice",
|
|
859
|
+
type: "tool-call",
|
|
860
|
+
},
|
|
861
|
+
{
|
|
862
|
+
text: "Tool result: ask_choice",
|
|
863
|
+
toolCallId: "call_bare",
|
|
864
|
+
toolName: "ask_choice",
|
|
865
|
+
type: "tool-result",
|
|
866
|
+
},
|
|
867
|
+
]);
|
|
868
|
+
|
|
869
|
+
expect(messages).toEqual([
|
|
870
|
+
{content: "Set up my workspace", role: "user"},
|
|
871
|
+
{
|
|
872
|
+
content: [
|
|
873
|
+
{input: {}, toolCallId: "call_bare", toolName: "ask_choice", type: "tool-call"},
|
|
874
|
+
],
|
|
875
|
+
role: "assistant",
|
|
876
|
+
},
|
|
877
|
+
{
|
|
878
|
+
content: [
|
|
879
|
+
{
|
|
880
|
+
output: {type: "json", value: null},
|
|
881
|
+
toolCallId: "call_bare",
|
|
882
|
+
toolName: "ask_choice",
|
|
883
|
+
type: "tool-result",
|
|
884
|
+
},
|
|
885
|
+
],
|
|
886
|
+
role: "tool",
|
|
887
|
+
},
|
|
888
|
+
]);
|
|
889
|
+
});
|
|
890
|
+
});
|
|
609
891
|
});
|
|
610
892
|
|
|
611
893
|
describe("generateChatStream", () => {
|
|
@@ -679,6 +961,338 @@ describe("AIService", () => {
|
|
|
679
961
|
});
|
|
680
962
|
});
|
|
681
963
|
|
|
964
|
+
describe("observability traces and prompt resolve", () => {
|
|
965
|
+
const PRODUCTION_BODY = "You are the production greeter.";
|
|
966
|
+
const productionVersion: PromptVersionRef = {
|
|
967
|
+
body: PRODUCTION_BODY,
|
|
968
|
+
label: "production",
|
|
969
|
+
name: "greeter",
|
|
970
|
+
sensitive: true,
|
|
971
|
+
version: 2,
|
|
972
|
+
};
|
|
973
|
+
|
|
974
|
+
const registerLocalApp = ({
|
|
975
|
+
priceMap,
|
|
976
|
+
promptVersion,
|
|
977
|
+
traceSink,
|
|
978
|
+
}: {
|
|
979
|
+
priceMap?: {inputPerMTok: number; outputPerMTok: number};
|
|
980
|
+
promptVersion?: PromptVersionRef;
|
|
981
|
+
traceSink: MemoryTraceSink | {export: (trace: unknown) => Promise<void>};
|
|
982
|
+
}): void => {
|
|
983
|
+
const plugin: ObservabilityPlugin = {
|
|
984
|
+
capabilities: new Set([
|
|
985
|
+
"datasets",
|
|
986
|
+
"experiments",
|
|
987
|
+
"prompts",
|
|
988
|
+
"reviewQueue",
|
|
989
|
+
"scores",
|
|
990
|
+
"traces",
|
|
991
|
+
]),
|
|
992
|
+
datasetStore: {},
|
|
993
|
+
experimentRunner: {},
|
|
994
|
+
id: "local",
|
|
995
|
+
promptRegistry: {
|
|
996
|
+
get: async ({label, name}) => {
|
|
997
|
+
if (!promptVersion) {
|
|
998
|
+
return undefined;
|
|
999
|
+
}
|
|
1000
|
+
if (name === promptVersion.name && (label ?? "production") === promptVersion.label) {
|
|
1001
|
+
return promptVersion;
|
|
1002
|
+
}
|
|
1003
|
+
return undefined;
|
|
1004
|
+
},
|
|
1005
|
+
},
|
|
1006
|
+
reviewQueue: {},
|
|
1007
|
+
traceSink: traceSink as MemoryTraceSink,
|
|
1008
|
+
};
|
|
1009
|
+
new ObservabilityApp({
|
|
1010
|
+
plugins: [plugin],
|
|
1011
|
+
priceMap: priceMap ? {"mock-model": priceMap} : undefined,
|
|
1012
|
+
});
|
|
1013
|
+
};
|
|
1014
|
+
|
|
1015
|
+
it("exports a trace to every TraceSink", async () => {
|
|
1016
|
+
const sink = new MemoryTraceSink();
|
|
1017
|
+
registerLocalApp({traceSink: sink});
|
|
1018
|
+
const model = createMockModel("Hello world");
|
|
1019
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1020
|
+
const userId = new mongoose.Types.ObjectId();
|
|
1021
|
+
|
|
1022
|
+
await service.generateText({
|
|
1023
|
+
prompt: "Say hello",
|
|
1024
|
+
sessionId: "sess-1",
|
|
1025
|
+
userId,
|
|
1026
|
+
});
|
|
1027
|
+
|
|
1028
|
+
expect(sink.traces.length).toBe(1);
|
|
1029
|
+
expect(sink.traces[0].userId).toBe(userId.toString());
|
|
1030
|
+
expect(sink.traces[0].sessionId).toBe("sess-1");
|
|
1031
|
+
expect(sink.traces[0].status).toBe("ok");
|
|
1032
|
+
expect(sink.traces[0].spans[0].kind).toBe("LLM");
|
|
1033
|
+
expect("costUsd" in (sink.traces[0].usage ?? {})).toBe(false);
|
|
1034
|
+
});
|
|
1035
|
+
|
|
1036
|
+
it("exports a CHAIN root with TOOL children when childSpans are provided", async () => {
|
|
1037
|
+
const sink = new MemoryTraceSink();
|
|
1038
|
+
registerLocalApp({traceSink: sink});
|
|
1039
|
+
const model = createMockModel("Hello world");
|
|
1040
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1041
|
+
const observability = await service.resolveGenerateObservability({});
|
|
1042
|
+
|
|
1043
|
+
await service.recordGenerate({
|
|
1044
|
+
childSpans: [
|
|
1045
|
+
{
|
|
1046
|
+
durationMs: 5,
|
|
1047
|
+
endedAt: "2026-01-01T00:00:00.005Z",
|
|
1048
|
+
id: "tool-span",
|
|
1049
|
+
input: {q: "hello"},
|
|
1050
|
+
kind: "TOOL",
|
|
1051
|
+
name: "search",
|
|
1052
|
+
output: {results: ["item1"]},
|
|
1053
|
+
startedAt: "2026-01-01T00:00:00.000Z",
|
|
1054
|
+
status: "ok",
|
|
1055
|
+
},
|
|
1056
|
+
],
|
|
1057
|
+
observability,
|
|
1058
|
+
prompt: "search",
|
|
1059
|
+
requestType: "general",
|
|
1060
|
+
response: "done",
|
|
1061
|
+
responseTime: 12,
|
|
1062
|
+
startTime: Date.parse("2026-01-01T00:00:00.000Z"),
|
|
1063
|
+
});
|
|
1064
|
+
|
|
1065
|
+
assert.equal(sink.traces.length, 1);
|
|
1066
|
+
assert.equal(sink.traces[0].spans[0]?.kind, "CHAIN");
|
|
1067
|
+
assert.equal(sink.traces[0].spans.length, 2);
|
|
1068
|
+
const toolSpan = sink.traces[0].spans.find((span) => span.kind === "TOOL");
|
|
1069
|
+
assert.isDefined(toolSpan);
|
|
1070
|
+
assert.equal(toolSpan?.parentSpanId, sink.traces[0].spans[0]?.id);
|
|
1071
|
+
assert.deepEqual(toolSpan?.input, {q: "hello"});
|
|
1072
|
+
assert.deepEqual(toolSpan?.output, {results: ["item1"]});
|
|
1073
|
+
});
|
|
1074
|
+
|
|
1075
|
+
it("does not export a trace when skipTrace is true", async () => {
|
|
1076
|
+
const sink = new MemoryTraceSink();
|
|
1077
|
+
registerLocalApp({traceSink: sink});
|
|
1078
|
+
const model = createMockModel("Hello world");
|
|
1079
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1080
|
+
|
|
1081
|
+
await service.generateText({prompt: "Say hello", skipTrace: true});
|
|
1082
|
+
|
|
1083
|
+
expect(sink.traces.length).toBe(0);
|
|
1084
|
+
const logs = await AIRequest.find({prompt: "Say hello"});
|
|
1085
|
+
expect(logs.length).toBe(1);
|
|
1086
|
+
});
|
|
1087
|
+
|
|
1088
|
+
it("logs a throwing sink and still writes AIRequest", async () => {
|
|
1089
|
+
registerLocalApp({
|
|
1090
|
+
traceSink: {
|
|
1091
|
+
export: async () => {
|
|
1092
|
+
throw new Error("sink down");
|
|
1093
|
+
},
|
|
1094
|
+
},
|
|
1095
|
+
});
|
|
1096
|
+
const model = createMockModel("Hello world");
|
|
1097
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1098
|
+
|
|
1099
|
+
const result = await service.generateText({prompt: "Say hello"});
|
|
1100
|
+
|
|
1101
|
+
expect(result).toBe("Hello world");
|
|
1102
|
+
const logs = await AIRequest.find({prompt: "Say hello"});
|
|
1103
|
+
expect(logs.length).toBe(1);
|
|
1104
|
+
});
|
|
1105
|
+
|
|
1106
|
+
it("sets costUsd from the price map", async () => {
|
|
1107
|
+
const sink = new MemoryTraceSink();
|
|
1108
|
+
registerLocalApp({
|
|
1109
|
+
priceMap: {inputPerMTok: 1000, outputPerMTok: 2000},
|
|
1110
|
+
traceSink: sink,
|
|
1111
|
+
});
|
|
1112
|
+
const model = createMockModel("Hello world");
|
|
1113
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1114
|
+
|
|
1115
|
+
await service.generateText({prompt: "Say hello"});
|
|
1116
|
+
|
|
1117
|
+
expect(sink.traces[0].usage?.costUsd).toBeCloseTo(0.025);
|
|
1118
|
+
});
|
|
1119
|
+
|
|
1120
|
+
it("omits costUsd when the model is unpriced", async () => {
|
|
1121
|
+
const sink = new MemoryTraceSink();
|
|
1122
|
+
registerLocalApp({traceSink: sink});
|
|
1123
|
+
const model = createMockModel("Hello world");
|
|
1124
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1125
|
+
|
|
1126
|
+
await service.generateText({prompt: "Say hello"});
|
|
1127
|
+
|
|
1128
|
+
expect(sink.traces[0].usage?.costUsd).toBeUndefined();
|
|
1129
|
+
});
|
|
1130
|
+
|
|
1131
|
+
it("resolves promptName before the model call even when skipTrace is true", async () => {
|
|
1132
|
+
const sink = new MemoryTraceSink();
|
|
1133
|
+
registerLocalApp({promptVersion: productionVersion, traceSink: sink});
|
|
1134
|
+
const model = createMockModel("Hello world");
|
|
1135
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1136
|
+
|
|
1137
|
+
await service.generateText({
|
|
1138
|
+
prompt: "User says hi",
|
|
1139
|
+
promptLabel: "production",
|
|
1140
|
+
promptName: "greeter",
|
|
1141
|
+
skipTrace: true,
|
|
1142
|
+
});
|
|
1143
|
+
|
|
1144
|
+
expect(sink.traces.length).toBe(0);
|
|
1145
|
+
expect(model.doGenerate.mock.calls.length).toBe(1);
|
|
1146
|
+
const payload = JSON.stringify(model.doGenerate.mock.calls[0]);
|
|
1147
|
+
expect(payload).toContain(PRODUCTION_BODY);
|
|
1148
|
+
});
|
|
1149
|
+
|
|
1150
|
+
it("returns 400 and does not call the model when the production label is missing", async () => {
|
|
1151
|
+
const sink = new MemoryTraceSink();
|
|
1152
|
+
registerLocalApp({promptVersion: productionVersion, traceSink: sink});
|
|
1153
|
+
const model = createMockModel("Hello world");
|
|
1154
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1155
|
+
|
|
1156
|
+
try {
|
|
1157
|
+
await service.generateText({
|
|
1158
|
+
prompt: "User says hi",
|
|
1159
|
+
promptLabel: "production",
|
|
1160
|
+
promptName: "missing-prompt",
|
|
1161
|
+
});
|
|
1162
|
+
throw new Error("expected APIError");
|
|
1163
|
+
} catch (error) {
|
|
1164
|
+
expect(error).toBeInstanceOf(APIError);
|
|
1165
|
+
expect((error as APIError).status).toBe(400);
|
|
1166
|
+
}
|
|
1167
|
+
expect(model.doGenerate.mock.calls.length).toBe(0);
|
|
1168
|
+
expect(sink.traces.length).toBe(0);
|
|
1169
|
+
});
|
|
1170
|
+
|
|
1171
|
+
it("returns 400 and does not call the model when no prompt registry is registered", async () => {
|
|
1172
|
+
resetObservabilityApp();
|
|
1173
|
+
const model = createMockModel("Hello world");
|
|
1174
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1175
|
+
|
|
1176
|
+
try {
|
|
1177
|
+
await service.generateText({
|
|
1178
|
+
prompt: "User says hi",
|
|
1179
|
+
promptName: "greeter",
|
|
1180
|
+
});
|
|
1181
|
+
throw new Error("expected APIError");
|
|
1182
|
+
} catch (error) {
|
|
1183
|
+
expect(error).toBeInstanceOf(APIError);
|
|
1184
|
+
expect((error as APIError).status).toBe(400);
|
|
1185
|
+
}
|
|
1186
|
+
expect(model.doGenerate.mock.calls.length).toBe(0);
|
|
1187
|
+
});
|
|
1188
|
+
|
|
1189
|
+
it("inherits sensitive from the resolved prompt version", async () => {
|
|
1190
|
+
const sink = new MemoryTraceSink();
|
|
1191
|
+
registerLocalApp({promptVersion: productionVersion, traceSink: sink});
|
|
1192
|
+
const model = createMockModel("Hello world");
|
|
1193
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1194
|
+
|
|
1195
|
+
await service.generateText({
|
|
1196
|
+
prompt: "User says hi",
|
|
1197
|
+
promptName: "greeter",
|
|
1198
|
+
});
|
|
1199
|
+
|
|
1200
|
+
expect(sink.traces[0].sensitive).toBe(true);
|
|
1201
|
+
expect(sink.traces[0].prompts).toEqual([{label: "production", name: "greeter", version: 2}]);
|
|
1202
|
+
});
|
|
1203
|
+
|
|
1204
|
+
it("still writes AIRequest for a chat stream when a sink is registered", async () => {
|
|
1205
|
+
const sink = new MemoryTraceSink();
|
|
1206
|
+
registerLocalApp({traceSink: sink});
|
|
1207
|
+
const model = createMockModel();
|
|
1208
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1209
|
+
|
|
1210
|
+
const chunks: string[] = [];
|
|
1211
|
+
for await (const chunk of service.generateChatStream({
|
|
1212
|
+
messages: [{content: "Hello", role: "user"}],
|
|
1213
|
+
})) {
|
|
1214
|
+
chunks.push(chunk);
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
expect(chunks.join("")).toBe("Mock response");
|
|
1218
|
+
const logs = await AIRequest.find({});
|
|
1219
|
+
expect(logs.length).toBe(1);
|
|
1220
|
+
expect(sink.traces.length).toBe(1);
|
|
1221
|
+
});
|
|
1222
|
+
});
|
|
1223
|
+
|
|
1224
|
+
describe("generateBlocks", () => {
|
|
1225
|
+
const validDocument = '{"v":1,"blocks":[{"type":"heading","text":"Hello"}]}';
|
|
1226
|
+
const overLimitDocument =
|
|
1227
|
+
'{"v":1,"datasets":{"signups":{"source":"ref","id":"ds1","limit":5000}},"blocks":[{"type":"heading","text":"Hello"}]}';
|
|
1228
|
+
|
|
1229
|
+
it("returns a validated document at deterministic temperature and logs ui_blocks", async () => {
|
|
1230
|
+
const model = createMockModel(validDocument);
|
|
1231
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1232
|
+
|
|
1233
|
+
const result = await service.generateBlocks({prompt: "Say hello"});
|
|
1234
|
+
|
|
1235
|
+
expect(result.blocks[0]).toMatchObject({text: "Hello", type: "heading"});
|
|
1236
|
+
expect(model.doGenerate).toHaveBeenCalledTimes(1);
|
|
1237
|
+
const call = model.doGenerate.mock.calls[0]?.[0] as {temperature?: number} | undefined;
|
|
1238
|
+
expect(call?.temperature).toBe(0);
|
|
1239
|
+
expect(JSON.stringify(call)).toContain("Your entire reply is one document");
|
|
1240
|
+
const logs = await AIRequest.find({requestType: "ui_blocks"});
|
|
1241
|
+
expect(logs).toHaveLength(1);
|
|
1242
|
+
expect(logs[0].error).toBeUndefined();
|
|
1243
|
+
});
|
|
1244
|
+
|
|
1245
|
+
it("repairs once and puts the error code in the second prompt", async () => {
|
|
1246
|
+
const model = createMockModel(overLimitDocument);
|
|
1247
|
+
let calls = 0;
|
|
1248
|
+
model.doGenerate = mock(async () => {
|
|
1249
|
+
calls += 1;
|
|
1250
|
+
const text = calls === 1 ? overLimitDocument : validDocument;
|
|
1251
|
+
return {
|
|
1252
|
+
content: [{text, type: "text" as const}],
|
|
1253
|
+
finishReason: "stop" as const,
|
|
1254
|
+
usage: {inputTokens: 5, outputTokens: 10},
|
|
1255
|
+
};
|
|
1256
|
+
});
|
|
1257
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1258
|
+
|
|
1259
|
+
const result = await service.generateBlocks({prompt: "Chart signups"});
|
|
1260
|
+
|
|
1261
|
+
expect(result.blocks[0]).toMatchObject({text: "Hello", type: "heading"});
|
|
1262
|
+
expect(model.doGenerate).toHaveBeenCalledTimes(2);
|
|
1263
|
+
const second = JSON.stringify(model.doGenerate.mock.calls[1]?.[0]);
|
|
1264
|
+
expect(second).toContain("TOO_MANY_POINTS");
|
|
1265
|
+
});
|
|
1266
|
+
|
|
1267
|
+
it("throws 502 when the model fails before a document exists", async () => {
|
|
1268
|
+
const model = createMockModel(validDocument);
|
|
1269
|
+
model.doGenerate = mock(async () => {
|
|
1270
|
+
throw new Error("model unavailable");
|
|
1271
|
+
});
|
|
1272
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1273
|
+
|
|
1274
|
+
await expect(service.generateBlocks({prompt: "Say hello"})).rejects.toMatchObject({
|
|
1275
|
+
status: 502,
|
|
1276
|
+
title: "Block generation failed",
|
|
1277
|
+
});
|
|
1278
|
+
expect(model.doGenerate).toHaveBeenCalledTimes(1);
|
|
1279
|
+
});
|
|
1280
|
+
|
|
1281
|
+
it("throws 422 and logs error codes when the repair still fails", async () => {
|
|
1282
|
+
const model = createMockModel(overLimitDocument);
|
|
1283
|
+
const service = new AIService({model: model as unknown as LanguageModel});
|
|
1284
|
+
|
|
1285
|
+
await expect(service.generateBlocks({prompt: "Still too big"})).rejects.toMatchObject({
|
|
1286
|
+
status: 422,
|
|
1287
|
+
});
|
|
1288
|
+
|
|
1289
|
+
expect(model.doGenerate).toHaveBeenCalledTimes(2);
|
|
1290
|
+
const logs = await AIRequest.find({requestType: "ui_blocks"});
|
|
1291
|
+
expect(logs).toHaveLength(1);
|
|
1292
|
+
expect(logs[0].metadata?.errorCodes).toEqual(["TOO_MANY_POINTS"]);
|
|
1293
|
+
});
|
|
1294
|
+
});
|
|
1295
|
+
|
|
682
1296
|
describe("TemperaturePresets", () => {
|
|
683
1297
|
it("should have correct values", () => {
|
|
684
1298
|
expect(TemperaturePresets.DETERMINISTIC).toBe(0);
|