@terreno/ai 57.9.0 → 57.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +8 -2
- package/dist/aiApp.js +12 -6
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +127 -0
- package/dist/harness/errors.js +101 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +520 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +186 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +57 -0
- package/dist/harness/runners/inProcessRunner.js +196 -0
- package/dist/harness/runtime.d.ts +34 -0
- package/dist/harness/runtime.js +551 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +73 -0
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +18 -7
- package/dist/routes/gpt.js +79 -689
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +426 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/types/harness.d.ts +934 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +192 -14
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +17 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +19 -6
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +119 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +757 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +767 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +201 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +912 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +402 -0
- package/src/harness/runners/inProcessRunner.ts +220 -0
- package/src/harness/runtime.ts +714 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +89 -1
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +63 -9
- package/src/routes/files.ts +21 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +91 -816
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +222 -2
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +593 -0
- package/src/service/aiService.ts +568 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1095 -0
- package/src/types/index.ts +209 -13
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.AIDataset = void 0;
|
|
7
|
+
const api_1 = require("@terreno/api");
|
|
8
|
+
const mongoose_1 = __importDefault(require("mongoose"));
|
|
9
|
+
const columnSchema = new mongoose_1.default.Schema({
|
|
10
|
+
name: {
|
|
11
|
+
description: "Column name used by chart and table blocks",
|
|
12
|
+
required: true,
|
|
13
|
+
type: String,
|
|
14
|
+
},
|
|
15
|
+
type: {
|
|
16
|
+
description: "Cell type: string, number, or date",
|
|
17
|
+
enum: ["string", "number", "date"],
|
|
18
|
+
required: true,
|
|
19
|
+
type: String,
|
|
20
|
+
},
|
|
21
|
+
}, { _id: false });
|
|
22
|
+
const aiDatasetSchema = new mongoose_1.default.Schema({
|
|
23
|
+
columns: {
|
|
24
|
+
description: "Named columns and their cell types, at most 12",
|
|
25
|
+
required: true,
|
|
26
|
+
type: [columnSchema],
|
|
27
|
+
},
|
|
28
|
+
expiresAt: {
|
|
29
|
+
description: "When this dataset expires. Absent datasets are kept.",
|
|
30
|
+
type: Date,
|
|
31
|
+
},
|
|
32
|
+
historyId: {
|
|
33
|
+
description: "The conversation whose tool stored these rows",
|
|
34
|
+
index: true,
|
|
35
|
+
ref: "GptHistory",
|
|
36
|
+
required: true,
|
|
37
|
+
type: mongoose_1.default.Schema.Types.ObjectId,
|
|
38
|
+
},
|
|
39
|
+
rowCount: {
|
|
40
|
+
description: "How many rows were stored",
|
|
41
|
+
required: true,
|
|
42
|
+
type: Number,
|
|
43
|
+
},
|
|
44
|
+
rows: {
|
|
45
|
+
description: "Table cells, one array per row, aligned with columns",
|
|
46
|
+
required: true,
|
|
47
|
+
type: mongoose_1.default.Schema.Types.Mixed,
|
|
48
|
+
},
|
|
49
|
+
userId: {
|
|
50
|
+
description: "The user who ran the tool that stored these rows",
|
|
51
|
+
index: true,
|
|
52
|
+
ref: "User",
|
|
53
|
+
required: true,
|
|
54
|
+
type: mongoose_1.default.Schema.Types.ObjectId,
|
|
55
|
+
},
|
|
56
|
+
}, { strict: "throw", toJSON: { virtuals: true }, toObject: { virtuals: true } });
|
|
57
|
+
aiDatasetSchema.plugin(api_1.createdUpdatedPlugin);
|
|
58
|
+
aiDatasetSchema.plugin(api_1.isDeletedPlugin);
|
|
59
|
+
aiDatasetSchema.plugin(api_1.findOneOrNone);
|
|
60
|
+
aiDatasetSchema.plugin(api_1.findExactlyOne);
|
|
61
|
+
// Documents with no expiresAt are kept. expireAfterSeconds 0 deletes at the stored instant.
|
|
62
|
+
aiDatasetSchema.index({ expiresAt: 1 }, { expireAfterSeconds: 0 });
|
|
63
|
+
// Virtual ownerId alias so Permissions.IsOwner works with userId field
|
|
64
|
+
aiDatasetSchema.virtual("ownerId").get(function () {
|
|
65
|
+
return this.userId;
|
|
66
|
+
});
|
|
67
|
+
exports.AIDataset = mongoose_1.default.model("AIDataset", aiDatasetSchema);
|
package/dist/models/aiRequest.js
CHANGED
|
@@ -24,7 +24,7 @@ const aiRequestSchema = new mongoose_1.default.Schema({
|
|
|
24
24
|
},
|
|
25
25
|
prompt: { description: "The input prompt sent to the AI model", required: true, type: String },
|
|
26
26
|
requestType: {
|
|
27
|
-
description: "The type of AI request (
|
|
27
|
+
description: "The type of AI request (general, translation, summarization, ui_action, and the other default types)",
|
|
28
28
|
required: true,
|
|
29
29
|
type: String,
|
|
30
30
|
},
|
|
@@ -5,7 +5,72 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.GptHistory = void 0;
|
|
7
7
|
const api_1 = require("@terreno/api");
|
|
8
|
+
const blocks_1 = require("@terreno/blocks");
|
|
8
9
|
const mongoose_1 = __importDefault(require("mongoose"));
|
|
10
|
+
const ASK_STATUSES = ["pending", "answered", "cancelled"];
|
|
11
|
+
const ASK_ORIGINS = ["approval"];
|
|
12
|
+
const askOriginField = {
|
|
13
|
+
description: "approval when the server asked before running a host tool that needs approval; unset when the model asked",
|
|
14
|
+
enum: ASK_ORIGINS,
|
|
15
|
+
type: String,
|
|
16
|
+
};
|
|
17
|
+
const promptAskSchema = new mongoose_1.default.Schema({
|
|
18
|
+
kind: {
|
|
19
|
+
description: "Ask kind; the model asked with the tool ask_<kind>",
|
|
20
|
+
enum: blocks_1.ASK_KINDS,
|
|
21
|
+
required: true,
|
|
22
|
+
type: String,
|
|
23
|
+
},
|
|
24
|
+
origin: askOriginField,
|
|
25
|
+
status: {
|
|
26
|
+
description: "pending while the user can answer; answered or cancelled once the ask is resolved",
|
|
27
|
+
enum: ASK_STATUSES,
|
|
28
|
+
required: true,
|
|
29
|
+
type: String,
|
|
30
|
+
},
|
|
31
|
+
}, { _id: false, strict: "throw" });
|
|
32
|
+
const pendingAskSchema = new mongoose_1.default.Schema({
|
|
33
|
+
approvalId: {
|
|
34
|
+
description: "AI SDK approval request an approval ask answers; the same as toolCallId",
|
|
35
|
+
type: String,
|
|
36
|
+
},
|
|
37
|
+
created: { description: "When the model asked", required: true, type: Date },
|
|
38
|
+
input: {
|
|
39
|
+
description: "The validated ask input the model sent",
|
|
40
|
+
required: true,
|
|
41
|
+
type: mongoose_1.default.Schema.Types.Mixed,
|
|
42
|
+
},
|
|
43
|
+
kind: {
|
|
44
|
+
description: "Ask kind; the model asked with the tool ask_<kind>",
|
|
45
|
+
enum: blocks_1.ASK_KINDS,
|
|
46
|
+
required: true,
|
|
47
|
+
type: String,
|
|
48
|
+
},
|
|
49
|
+
origin: askOriginField,
|
|
50
|
+
promptIndex: {
|
|
51
|
+
description: "Number of leading prompts that form the paused turn's history, replayed before responseMessages on resume",
|
|
52
|
+
required: true,
|
|
53
|
+
type: Number,
|
|
54
|
+
},
|
|
55
|
+
responseMessages: {
|
|
56
|
+
description: "AI SDK response messages of the paused turn, replayed verbatim with the answer on resume",
|
|
57
|
+
required: true,
|
|
58
|
+
type: mongoose_1.default.Schema.Types.Mixed,
|
|
59
|
+
},
|
|
60
|
+
simple: {
|
|
61
|
+
description: "Simple card (short text and up to three answer buttons) made when the ask was made",
|
|
62
|
+
required: true,
|
|
63
|
+
type: mongoose_1.default.Schema.Types.Mixed,
|
|
64
|
+
},
|
|
65
|
+
toolCallId: {
|
|
66
|
+
description: "Tool call id of the ask; an answer must name it",
|
|
67
|
+
required: true,
|
|
68
|
+
type: String,
|
|
69
|
+
},
|
|
70
|
+
toolName: { description: "Host tool an approval ask asks to run", type: String },
|
|
71
|
+
},
|
|
72
|
+
// `minimize` would drop empty objects the AI SDK requires on replay, such as a tool call's `input: {}`.
|
|
73
|
+
{ _id: false, minimize: false, strict: "throw" });
|
|
9
74
|
const contentPartSchema = new mongoose_1.default.Schema({
|
|
10
75
|
filename: { description: "Original filename of the attached file", type: String },
|
|
11
76
|
gcsKey: {
|
|
@@ -24,6 +89,10 @@ const contentPartSchema = new mongoose_1.default.Schema({
|
|
|
24
89
|
}, { _id: false, strict: "throw" });
|
|
25
90
|
const gptHistoryPromptSchema = new mongoose_1.default.Schema({
|
|
26
91
|
args: { description: "Arguments passed to a tool call", type: mongoose_1.default.Schema.Types.Mixed },
|
|
92
|
+
ask: {
|
|
93
|
+
description: "Set on tool-call rows where the model asked the user a question",
|
|
94
|
+
type: promptAskSchema,
|
|
95
|
+
},
|
|
27
96
|
content: { description: "Multipart content attached to this prompt", type: [contentPartSchema] },
|
|
28
97
|
model: { description: "AI model identifier used for this prompt", type: String },
|
|
29
98
|
rating: {
|
|
@@ -63,6 +132,10 @@ const gptHistoryPromptSchema = new mongoose_1.default.Schema({
|
|
|
63
132
|
},
|
|
64
133
|
}, { _id: false, strict: "throw" });
|
|
65
134
|
const gptHistorySchema = new mongoose_1.default.Schema({
|
|
135
|
+
pendingAsk: {
|
|
136
|
+
description: "The ask this conversation is waiting on; cleared when the user answers or the ask is cancelled",
|
|
137
|
+
type: pendingAskSchema,
|
|
138
|
+
},
|
|
66
139
|
projectId: {
|
|
67
140
|
description: "Project this conversation belongs to",
|
|
68
141
|
index: true,
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.observabilityAdminScreens = exports.AI_OBSERVABILITY_GROUP = void 0;
|
|
4
|
+
exports.AI_OBSERVABILITY_GROUP = "AI Observability";
|
|
5
|
+
const observabilityScreen = ({ displayName, name, resource, }) => ({
|
|
6
|
+
adminAccess: { action: "list", resource },
|
|
7
|
+
displayName,
|
|
8
|
+
group: exports.AI_OBSERVABILITY_GROUP,
|
|
9
|
+
name,
|
|
10
|
+
});
|
|
11
|
+
const observabilityAdminScreens = ({ localOn }) => {
|
|
12
|
+
const screens = [
|
|
13
|
+
observabilityScreen({ displayName: "Prompts", name: "ai-prompts", resource: "aiPrompt" }),
|
|
14
|
+
observabilityScreen({ displayName: "Traces", name: "ai-traces", resource: "aiTrace" }),
|
|
15
|
+
observabilityScreen({
|
|
16
|
+
displayName: "Evaluators",
|
|
17
|
+
name: "ai-evaluators",
|
|
18
|
+
resource: "aiEvaluator",
|
|
19
|
+
}),
|
|
20
|
+
observabilityScreen({ displayName: "Datasets", name: "ai-datasets", resource: "aiDataset" }),
|
|
21
|
+
observabilityScreen({
|
|
22
|
+
displayName: "Experiments",
|
|
23
|
+
name: "ai-experiments",
|
|
24
|
+
resource: "aiExperiment",
|
|
25
|
+
}),
|
|
26
|
+
];
|
|
27
|
+
if (localOn) {
|
|
28
|
+
screens.push(observabilityScreen({ displayName: "Review queue", name: "ai-review", resource: "aiReview" }));
|
|
29
|
+
}
|
|
30
|
+
return screens;
|
|
31
|
+
};
|
|
32
|
+
exports.observabilityAdminScreens = observabilityAdminScreens;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const compileTemplate: (template: string, variables: Record<string, string>) => string;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.compileTemplate = void 0;
|
|
4
|
+
const PLACEHOLDER = /\{\{([A-Za-z_][A-Za-z0-9_]*)\}\}/g;
|
|
5
|
+
const compileTemplate = (template, variables) => {
|
|
6
|
+
return template.replace(PLACEHOLDER, (_match, key) => {
|
|
7
|
+
if (Object.hasOwn(variables, key)) {
|
|
8
|
+
return variables[key];
|
|
9
|
+
}
|
|
10
|
+
return "";
|
|
11
|
+
});
|
|
12
|
+
};
|
|
13
|
+
exports.compileTemplate = compileTemplate;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface ParsedDatasetImportRow {
|
|
2
|
+
expectedOutput?: unknown;
|
|
3
|
+
input: unknown;
|
|
4
|
+
metadata?: Record<string, unknown>;
|
|
5
|
+
outcomeClass?: "fn" | "fp" | "tn" | "tp";
|
|
6
|
+
proofread?: boolean;
|
|
7
|
+
tags: string[];
|
|
8
|
+
}
|
|
9
|
+
export declare const parseDatasetJsonImport: (payload: unknown) => ParsedDatasetImportRow[];
|
|
10
|
+
export declare const parseDatasetCsvImport: (content: string) => ParsedDatasetImportRow[];
|
|
11
|
+
export declare const flattenValidationPath: (path: string) => string;
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.flattenValidationPath = exports.parseDatasetCsvImport = exports.parseDatasetJsonImport = void 0;
|
|
4
|
+
const api_1 = require("@terreno/api");
|
|
5
|
+
const sync_1 = require("csv-parse/sync");
|
|
6
|
+
const jsonPath_1 = require("./jsonPath");
|
|
7
|
+
const parseBooleanCell = (value) => {
|
|
8
|
+
if (value === undefined || value.trim() === "") {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
const normalized = value.trim().toLowerCase();
|
|
12
|
+
if (normalized === "true" || normalized === "1" || normalized === "yes") {
|
|
13
|
+
return true;
|
|
14
|
+
}
|
|
15
|
+
if (normalized === "false" || normalized === "0" || normalized === "no") {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
return undefined;
|
|
19
|
+
};
|
|
20
|
+
const parseTagsCell = (value) => {
|
|
21
|
+
if (!value || value.trim() === "") {
|
|
22
|
+
return [];
|
|
23
|
+
}
|
|
24
|
+
return value
|
|
25
|
+
.split("|")
|
|
26
|
+
.map((tag) => {
|
|
27
|
+
return tag.trim();
|
|
28
|
+
})
|
|
29
|
+
.filter((tag) => {
|
|
30
|
+
return tag.length > 0;
|
|
31
|
+
});
|
|
32
|
+
};
|
|
33
|
+
const coerceCellValue = (value) => {
|
|
34
|
+
const trimmed = value.trim();
|
|
35
|
+
if (trimmed === "") {
|
|
36
|
+
return "";
|
|
37
|
+
}
|
|
38
|
+
if (trimmed === "true") {
|
|
39
|
+
return true;
|
|
40
|
+
}
|
|
41
|
+
if (trimmed === "false") {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
if (/^-?\d+(\.\d+)?$/.test(trimmed)) {
|
|
45
|
+
return Number(trimmed);
|
|
46
|
+
}
|
|
47
|
+
if ((trimmed.startsWith("{") && trimmed.endsWith("}")) ||
|
|
48
|
+
(trimmed.startsWith("[") && trimmed.endsWith("]"))) {
|
|
49
|
+
try {
|
|
50
|
+
return JSON.parse(trimmed);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return trimmed;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return trimmed;
|
|
57
|
+
};
|
|
58
|
+
const rowFromStructured = (row) => {
|
|
59
|
+
if (row.input === undefined) {
|
|
60
|
+
throw new api_1.APIError({ status: 400, title: "structured import row requires input" });
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
expectedOutput: row.expectedOutput,
|
|
64
|
+
input: row.input,
|
|
65
|
+
metadata: row.metadata,
|
|
66
|
+
outcomeClass: row.outcomeClass,
|
|
67
|
+
proofread: row.proofread,
|
|
68
|
+
tags: row.tags ?? [],
|
|
69
|
+
};
|
|
70
|
+
};
|
|
71
|
+
const rowFromBareObject = (row) => {
|
|
72
|
+
const { expectedOutput, metadata, outcomeClass, proofread, tags, ...input } = row;
|
|
73
|
+
return {
|
|
74
|
+
expectedOutput,
|
|
75
|
+
input,
|
|
76
|
+
metadata: metadata && typeof metadata === "object" && !Array.isArray(metadata)
|
|
77
|
+
? metadata
|
|
78
|
+
: undefined,
|
|
79
|
+
outcomeClass: outcomeClass === "fn" ||
|
|
80
|
+
outcomeClass === "fp" ||
|
|
81
|
+
outcomeClass === "tn" ||
|
|
82
|
+
outcomeClass === "tp"
|
|
83
|
+
? outcomeClass
|
|
84
|
+
: undefined,
|
|
85
|
+
proofread: typeof proofread === "boolean" ? proofread : undefined,
|
|
86
|
+
tags: Array.isArray(tags)
|
|
87
|
+
? tags.filter((tag) => {
|
|
88
|
+
return typeof tag === "string";
|
|
89
|
+
})
|
|
90
|
+
: [],
|
|
91
|
+
};
|
|
92
|
+
};
|
|
93
|
+
const rowFromCsvRecord = (record) => {
|
|
94
|
+
const input = {};
|
|
95
|
+
const expectedOutput = {};
|
|
96
|
+
let explicitExpectedOutput;
|
|
97
|
+
let explicitInput;
|
|
98
|
+
let proofread;
|
|
99
|
+
let outcomeClass;
|
|
100
|
+
let tags = [];
|
|
101
|
+
for (const [column, rawValue] of Object.entries(record)) {
|
|
102
|
+
if (column === "proofread") {
|
|
103
|
+
proofread = parseBooleanCell(rawValue);
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
if (column === "tags") {
|
|
107
|
+
tags = parseTagsCell(rawValue);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (column === "outcomeClass") {
|
|
111
|
+
const normalized = rawValue?.trim();
|
|
112
|
+
if (normalized === "fn" ||
|
|
113
|
+
normalized === "fp" ||
|
|
114
|
+
normalized === "tn" ||
|
|
115
|
+
normalized === "tp") {
|
|
116
|
+
outcomeClass = normalized;
|
|
117
|
+
}
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
if (column === "input") {
|
|
121
|
+
explicitInput = coerceCellValue(rawValue);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (column === "expectedOutput") {
|
|
125
|
+
explicitExpectedOutput = coerceCellValue(rawValue);
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
if (column.startsWith("input.")) {
|
|
129
|
+
(0, jsonPath_1.setValueAtPath)(input, column.slice("input.".length), coerceCellValue(rawValue));
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
if (column.startsWith("expectedOutput.")) {
|
|
133
|
+
(0, jsonPath_1.setValueAtPath)(expectedOutput, column.slice("expectedOutput.".length), coerceCellValue(rawValue));
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
input[column] = coerceCellValue(rawValue);
|
|
137
|
+
}
|
|
138
|
+
const hasExpectedOutput = Object.keys(expectedOutput).length > 0;
|
|
139
|
+
return {
|
|
140
|
+
expectedOutput: explicitExpectedOutput !== undefined
|
|
141
|
+
? explicitExpectedOutput
|
|
142
|
+
: hasExpectedOutput
|
|
143
|
+
? expectedOutput
|
|
144
|
+
: undefined,
|
|
145
|
+
input: explicitInput !== undefined ? explicitInput : Object.keys(input).length > 0 ? input : record,
|
|
146
|
+
outcomeClass,
|
|
147
|
+
proofread,
|
|
148
|
+
tags,
|
|
149
|
+
};
|
|
150
|
+
};
|
|
151
|
+
const parseDatasetJsonImport = (payload) => {
|
|
152
|
+
if (!Array.isArray(payload)) {
|
|
153
|
+
throw new api_1.APIError({ status: 400, title: "JSON import must be an array" });
|
|
154
|
+
}
|
|
155
|
+
return payload.map((row, index) => {
|
|
156
|
+
if (!row || typeof row !== "object" || Array.isArray(row)) {
|
|
157
|
+
throw new api_1.APIError({ status: 400, title: `import row ${index + 1} must be an object` });
|
|
158
|
+
}
|
|
159
|
+
const record = row;
|
|
160
|
+
if ("input" in record) {
|
|
161
|
+
return rowFromStructured(record);
|
|
162
|
+
}
|
|
163
|
+
return rowFromBareObject(record);
|
|
164
|
+
});
|
|
165
|
+
};
|
|
166
|
+
exports.parseDatasetJsonImport = parseDatasetJsonImport;
|
|
167
|
+
const parseDatasetCsvImport = (content) => {
|
|
168
|
+
const records = (0, sync_1.parse)(content, {
|
|
169
|
+
columns: true,
|
|
170
|
+
relaxQuotes: true,
|
|
171
|
+
skip_empty_lines: true,
|
|
172
|
+
trim: true,
|
|
173
|
+
});
|
|
174
|
+
return records.map((record) => {
|
|
175
|
+
return rowFromCsvRecord(record);
|
|
176
|
+
});
|
|
177
|
+
};
|
|
178
|
+
exports.parseDatasetCsvImport = parseDatasetCsvImport;
|
|
179
|
+
const flattenValidationPath = (path) => {
|
|
180
|
+
if (!path || path === "/") {
|
|
181
|
+
return "/";
|
|
182
|
+
}
|
|
183
|
+
return path.startsWith("/") ? path : `/${path}`;
|
|
184
|
+
};
|
|
185
|
+
exports.flattenValidationPath = flattenValidationPath;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { EvaluatorDimension } from "../types/observability";
|
|
2
|
+
import type { ObservabilityGenerateClient } from "./types";
|
|
3
|
+
export type EvaluatorAiClient = Pick<ObservabilityGenerateClient, "generateJsonObject">;
|
|
4
|
+
export interface EvaluatorExecutionContext {
|
|
5
|
+
evaluator: {
|
|
6
|
+
assertion?: {
|
|
7
|
+
constraint: string;
|
|
8
|
+
path: string;
|
|
9
|
+
};
|
|
10
|
+
dimensions: EvaluatorDimension[];
|
|
11
|
+
judgePromptName?: string;
|
|
12
|
+
name: string;
|
|
13
|
+
type: "human" | "json-assert" | "llm-judge";
|
|
14
|
+
};
|
|
15
|
+
expectedOutput?: unknown;
|
|
16
|
+
input: unknown;
|
|
17
|
+
output: unknown;
|
|
18
|
+
outputSchema?: Record<string, unknown>;
|
|
19
|
+
}
|
|
20
|
+
export interface EvaluatorExecutionResult {
|
|
21
|
+
confidence?: number;
|
|
22
|
+
error?: string;
|
|
23
|
+
scores?: Record<string, boolean | number | string>;
|
|
24
|
+
}
|
|
25
|
+
export declare const runJsonAssertEvaluator: (context: EvaluatorExecutionContext) => EvaluatorExecutionResult;
|
|
26
|
+
export declare const runLlmJudgeEvaluator: (context: EvaluatorExecutionContext, ai: EvaluatorAiClient, judgeOutputSchema: Record<string, unknown>) => Promise<EvaluatorExecutionResult>;
|
|
27
|
+
export declare const runEvaluator: (context: EvaluatorExecutionContext, deps: {
|
|
28
|
+
ai?: EvaluatorAiClient;
|
|
29
|
+
judgeOutputSchema?: Record<string, unknown>;
|
|
30
|
+
}) => Promise<EvaluatorExecutionResult>;
|
|
31
|
+
export declare const judgeSchemaMissingDimensions: (dimensions: EvaluatorDimension[], outputSchema?: Record<string, unknown>) => string[];
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.judgeSchemaMissingDimensions = exports.runEvaluator = exports.runLlmJudgeEvaluator = exports.runJsonAssertEvaluator = void 0;
|
|
4
|
+
const ai_1 = require("ai");
|
|
5
|
+
const jsonPath_1 = require("./jsonPath");
|
|
6
|
+
const schemaValidation_1 = require("./schemaValidation");
|
|
7
|
+
const parseConstraintValue = (constraint) => {
|
|
8
|
+
const raw = constraint.slice("eq:".length).trim();
|
|
9
|
+
try {
|
|
10
|
+
return JSON.parse(raw);
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return raw;
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
const evaluatePathConstraint = (value, constraint) => {
|
|
17
|
+
if (constraint === "exists") {
|
|
18
|
+
return value !== undefined && value !== null;
|
|
19
|
+
}
|
|
20
|
+
if (constraint === "notEmpty") {
|
|
21
|
+
if (value === undefined || value === null) {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
if (typeof value === "string") {
|
|
25
|
+
return value.trim().length > 0;
|
|
26
|
+
}
|
|
27
|
+
if (Array.isArray(value)) {
|
|
28
|
+
return value.length > 0;
|
|
29
|
+
}
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
if (constraint.startsWith("type:")) {
|
|
33
|
+
const expected = constraint.slice("type:".length);
|
|
34
|
+
if (expected === "array") {
|
|
35
|
+
return Array.isArray(value);
|
|
36
|
+
}
|
|
37
|
+
return typeof value === expected;
|
|
38
|
+
}
|
|
39
|
+
if (constraint.startsWith("eq:")) {
|
|
40
|
+
return value === parseConstraintValue(constraint);
|
|
41
|
+
}
|
|
42
|
+
if (constraint.startsWith("gte:")) {
|
|
43
|
+
return Number(value) >= Number(constraint.slice("gte:".length));
|
|
44
|
+
}
|
|
45
|
+
if (constraint.startsWith("lte:")) {
|
|
46
|
+
return Number(value) <= Number(constraint.slice("lte:".length));
|
|
47
|
+
}
|
|
48
|
+
return false;
|
|
49
|
+
};
|
|
50
|
+
const coerceDimensionValue = (dimension, raw) => {
|
|
51
|
+
if (raw === undefined || raw === null) {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
if (dimension.dataType === "boolean") {
|
|
55
|
+
if (typeof raw === "boolean") {
|
|
56
|
+
return raw;
|
|
57
|
+
}
|
|
58
|
+
if (raw === "true") {
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
if (raw === "false") {
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
return Boolean(raw);
|
|
65
|
+
}
|
|
66
|
+
if (dimension.dataType === "numeric") {
|
|
67
|
+
const parsed = Number(raw);
|
|
68
|
+
if (Number.isNaN(parsed)) {
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
return parsed;
|
|
72
|
+
}
|
|
73
|
+
return String(raw);
|
|
74
|
+
};
|
|
75
|
+
const buildJudgePrompt = (context) => {
|
|
76
|
+
return [
|
|
77
|
+
"Evaluate the model output for the dataset item.",
|
|
78
|
+
"",
|
|
79
|
+
`Input JSON:\n${JSON.stringify(context.input, null, 2)}`,
|
|
80
|
+
"",
|
|
81
|
+
`Model output JSON:\n${JSON.stringify(context.output, null, 2)}`,
|
|
82
|
+
context.expectedOutput !== undefined
|
|
83
|
+
? `\nExpected output JSON:\n${JSON.stringify(context.expectedOutput, null, 2)}`
|
|
84
|
+
: "",
|
|
85
|
+
].join("\n");
|
|
86
|
+
};
|
|
87
|
+
const extractDeclaredScores = (dimensions, payload) => {
|
|
88
|
+
const scores = {};
|
|
89
|
+
for (const dimension of dimensions) {
|
|
90
|
+
const coerced = coerceDimensionValue(dimension, payload[dimension.key]);
|
|
91
|
+
if (coerced !== undefined) {
|
|
92
|
+
scores[dimension.key] = coerced;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return scores;
|
|
96
|
+
};
|
|
97
|
+
const runJsonAssertEvaluator = (context) => {
|
|
98
|
+
const dimension = context.evaluator.dimensions[0];
|
|
99
|
+
if (!dimension) {
|
|
100
|
+
return { error: "json-assert evaluator requires at least one dimension" };
|
|
101
|
+
}
|
|
102
|
+
const usesOutputSchema = !context.evaluator.assertion?.path ||
|
|
103
|
+
context.evaluator.assertion.path === "__outputSchema__" ||
|
|
104
|
+
context.evaluator.name === "schema-assert";
|
|
105
|
+
if (usesOutputSchema) {
|
|
106
|
+
const errors = (0, schemaValidation_1.validateAgainstSchema)(context.outputSchema, context.output);
|
|
107
|
+
const valid = errors.length === 0;
|
|
108
|
+
return {
|
|
109
|
+
scores: {
|
|
110
|
+
[dimension.key]: valid,
|
|
111
|
+
},
|
|
112
|
+
...(valid
|
|
113
|
+
? {}
|
|
114
|
+
: {
|
|
115
|
+
error: errors.map((row) => `${row.path}: ${row.message}`).join("; "),
|
|
116
|
+
}),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
const actual = (0, jsonPath_1.getValueAtPath)(context.output, context.evaluator.assertion?.path ?? "");
|
|
120
|
+
const passed = evaluatePathConstraint(actual, context.evaluator.assertion?.constraint ?? "");
|
|
121
|
+
return {
|
|
122
|
+
scores: {
|
|
123
|
+
[dimension.key]: passed,
|
|
124
|
+
},
|
|
125
|
+
...(passed
|
|
126
|
+
? {}
|
|
127
|
+
: {
|
|
128
|
+
error: `Assertion failed at ${context.evaluator.assertion?.path}`,
|
|
129
|
+
}),
|
|
130
|
+
};
|
|
131
|
+
};
|
|
132
|
+
exports.runJsonAssertEvaluator = runJsonAssertEvaluator;
|
|
133
|
+
const runLlmJudgeEvaluator = async (context, ai, judgeOutputSchema) => {
|
|
134
|
+
const judgePromptName = context.evaluator.judgePromptName;
|
|
135
|
+
if (!judgePromptName) {
|
|
136
|
+
return { error: "llm-judge evaluator is missing judgePromptName" };
|
|
137
|
+
}
|
|
138
|
+
try {
|
|
139
|
+
const payload = (await ai.generateJsonObject({
|
|
140
|
+
prompt: buildJudgePrompt(context),
|
|
141
|
+
promptName: judgePromptName,
|
|
142
|
+
schema: (0, ai_1.jsonSchema)(judgeOutputSchema),
|
|
143
|
+
schemaName: `${context.evaluator.name}-judge`,
|
|
144
|
+
skipTrace: true,
|
|
145
|
+
}));
|
|
146
|
+
const scores = extractDeclaredScores(context.evaluator.dimensions, payload);
|
|
147
|
+
const confidenceRaw = payload.confidence;
|
|
148
|
+
const confidence = typeof confidenceRaw === "number"
|
|
149
|
+
? confidenceRaw
|
|
150
|
+
: confidenceRaw !== undefined
|
|
151
|
+
? Number(confidenceRaw)
|
|
152
|
+
: undefined;
|
|
153
|
+
const missing = context.evaluator.dimensions
|
|
154
|
+
.filter((dimension) => {
|
|
155
|
+
return dimension.required && scores[dimension.key] === undefined;
|
|
156
|
+
})
|
|
157
|
+
.map((dimension) => dimension.key);
|
|
158
|
+
if (missing.length > 0) {
|
|
159
|
+
return {
|
|
160
|
+
confidence: Number.isNaN(confidence) ? undefined : confidence,
|
|
161
|
+
error: `Judge output missing required dimensions: ${missing.join(", ")}`,
|
|
162
|
+
scores,
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
return {
|
|
166
|
+
confidence: Number.isNaN(confidence) ? undefined : confidence,
|
|
167
|
+
scores,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
return {
|
|
172
|
+
error: error instanceof Error ? error.message : "llm-judge generation failed",
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
exports.runLlmJudgeEvaluator = runLlmJudgeEvaluator;
|
|
177
|
+
const runEvaluator = async (context, deps) => {
|
|
178
|
+
if (context.evaluator.type === "human") {
|
|
179
|
+
return { error: "human evaluators are not executed automatically" };
|
|
180
|
+
}
|
|
181
|
+
if (context.evaluator.type === "json-assert") {
|
|
182
|
+
return (0, exports.runJsonAssertEvaluator)(context);
|
|
183
|
+
}
|
|
184
|
+
if (!deps.ai || !deps.judgeOutputSchema) {
|
|
185
|
+
return { error: "llm-judge execution requires an AI client and judge output schema" };
|
|
186
|
+
}
|
|
187
|
+
return (0, exports.runLlmJudgeEvaluator)(context, deps.ai, deps.judgeOutputSchema);
|
|
188
|
+
};
|
|
189
|
+
exports.runEvaluator = runEvaluator;
|
|
190
|
+
const judgeSchemaMissingDimensions = (dimensions, outputSchema) => {
|
|
191
|
+
const properties = outputSchema && typeof outputSchema.properties === "object"
|
|
192
|
+
? outputSchema.properties
|
|
193
|
+
: {};
|
|
194
|
+
return dimensions
|
|
195
|
+
.filter((dimension) => {
|
|
196
|
+
return dimension.required && properties[dimension.key] === undefined;
|
|
197
|
+
})
|
|
198
|
+
.map((dimension) => dimension.key);
|
|
199
|
+
};
|
|
200
|
+
exports.judgeSchemaMissingDimensions = judgeSchemaMissingDimensions;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { EvaluatorDimension, EvaluatorRunModes } from "../types/observability";
|
|
2
|
+
export interface EvaluatorTemplate {
|
|
3
|
+
assertion?: {
|
|
4
|
+
constraint: string;
|
|
5
|
+
path: string;
|
|
6
|
+
};
|
|
7
|
+
description: string;
|
|
8
|
+
dimensions: EvaluatorDimension[];
|
|
9
|
+
instructions?: string;
|
|
10
|
+
judgePromptName?: string;
|
|
11
|
+
name: string;
|
|
12
|
+
runModes: EvaluatorRunModes;
|
|
13
|
+
target: "dataset item" | "full trace" | "generation span";
|
|
14
|
+
type: "human" | "json-assert" | "llm-judge";
|
|
15
|
+
}
|
|
16
|
+
export declare const EVALUATOR_TEMPLATES: EvaluatorTemplate[];
|
|
17
|
+
export declare const getEvaluatorTemplate: (name: string) => EvaluatorTemplate | undefined;
|