@terreno/ai 57.8.0 → 57.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursorrules +2 -2
- package/README.md +2 -1
- package/bunfig.toml +1 -1
- package/dist/aiApp.d.ts +16 -5
- package/dist/aiApp.js +17 -7
- package/dist/harness/adminScreens.d.ts +6 -0
- package/dist/harness/adminScreens.js +11 -0
- package/dist/harness/agentLoop.d.ts +61 -0
- package/dist/harness/agentLoop.js +798 -0
- package/dist/harness/agentTaskNames.d.ts +4 -0
- package/dist/harness/agentTaskNames.js +7 -0
- package/dist/harness/approvalPolicy.d.ts +3 -0
- package/dist/harness/approvalPolicy.js +23 -0
- package/dist/harness/approvals.d.ts +101 -0
- package/dist/harness/approvals.js +468 -0
- package/dist/harness/commit.d.ts +238 -0
- package/dist/harness/commit.js +678 -0
- package/dist/harness/conversation.d.ts +75 -0
- package/dist/harness/conversation.js +397 -0
- package/dist/harness/defineAgent.d.ts +9 -0
- package/dist/harness/defineAgent.js +65 -0
- package/dist/harness/defineTask.d.ts +11 -0
- package/dist/harness/defineTask.js +59 -0
- package/dist/harness/defineTool.d.ts +7 -0
- package/dist/harness/defineTool.js +36 -0
- package/dist/harness/definitionError.d.ts +8 -0
- package/dist/harness/definitionError.js +15 -0
- package/dist/harness/errors.d.ts +127 -0
- package/dist/harness/errors.js +101 -0
- package/dist/harness/eventHub.d.ts +45 -0
- package/dist/harness/eventHub.js +135 -0
- package/dist/harness/events.d.ts +95 -0
- package/dist/harness/events.js +267 -0
- package/dist/harness/executionEnv.d.ts +31 -0
- package/dist/harness/executionEnv.js +2 -0
- package/dist/harness/extensions.d.ts +82 -0
- package/dist/harness/extensions.js +274 -0
- package/dist/harness/harness.d.ts +163 -0
- package/dist/harness/harness.js +520 -0
- package/dist/harness/harnessApp.d.ts +44 -0
- package/dist/harness/harnessApp.js +62 -0
- package/dist/harness/internalRuntime.d.ts +16 -0
- package/dist/harness/internalRuntime.js +18 -0
- package/dist/harness/leases.d.ts +35 -0
- package/dist/harness/leases.js +86 -0
- package/dist/harness/memo.d.ts +24 -0
- package/dist/harness/memo.js +78 -0
- package/dist/harness/modelCall.d.ts +45 -0
- package/dist/harness/modelCall.js +147 -0
- package/dist/harness/models/harnessApproval.d.ts +2 -0
- package/dist/harness/models/harnessApproval.js +100 -0
- package/dist/harness/models/harnessConversation.d.ts +2 -0
- package/dist/harness/models/harnessConversation.js +117 -0
- package/dist/harness/models/harnessEvent.d.ts +3 -0
- package/dist/harness/models/harnessEvent.js +73 -0
- package/dist/harness/models/harnessInboxEvent.d.ts +2 -0
- package/dist/harness/models/harnessInboxEvent.js +58 -0
- package/dist/harness/models/harnessMemo.d.ts +2 -0
- package/dist/harness/models/harnessMemo.js +42 -0
- package/dist/harness/models/harnessMessage.d.ts +2 -0
- package/dist/harness/models/harnessMessage.js +73 -0
- package/dist/harness/models/harnessOwner.d.ts +2 -0
- package/dist/harness/models/harnessOwner.js +36 -0
- package/dist/harness/models/harnessTask.d.ts +2 -0
- package/dist/harness/models/harnessTask.js +186 -0
- package/dist/harness/ownership.d.ts +89 -0
- package/dist/harness/ownership.js +438 -0
- package/dist/harness/registry.d.ts +29 -0
- package/dist/harness/registry.js +93 -0
- package/dist/harness/retryBackoff.d.ts +31 -0
- package/dist/harness/retryBackoff.js +50 -0
- package/dist/harness/routes/approvals.d.ts +13 -0
- package/dist/harness/routes/approvals.js +93 -0
- package/dist/harness/routes/conversations.d.ts +11 -0
- package/dist/harness/routes/conversations.js +66 -0
- package/dist/harness/routes/events.d.ts +19 -0
- package/dist/harness/routes/events.js +215 -0
- package/dist/harness/routes/tasks.d.ts +11 -0
- package/dist/harness/routes/tasks.js +108 -0
- package/dist/harness/runners/inProcessRunner.d.ts +57 -0
- package/dist/harness/runners/inProcessRunner.js +196 -0
- package/dist/harness/runtime.d.ts +34 -0
- package/dist/harness/runtime.js +551 -0
- package/dist/harness/subagent.d.ts +53 -0
- package/dist/harness/subagent.js +134 -0
- package/dist/harness/suspend.d.ts +8 -0
- package/dist/harness/suspend.js +16 -0
- package/dist/harness/transaction.d.ts +3 -0
- package/dist/harness/transaction.js +22 -0
- package/dist/harness/waits.d.ts +100 -0
- package/dist/harness/waits.js +214 -0
- package/dist/index.d.ts +19 -1
- package/dist/index.js +45 -2
- package/dist/models/aiDataset.d.ts +2 -0
- package/dist/models/aiDataset.js +67 -0
- package/dist/models/aiRequest.js +1 -1
- package/dist/models/gptHistory.js +88 -2
- package/dist/observability/adminScreens.d.ts +5 -0
- package/dist/observability/adminScreens.js +32 -0
- package/dist/observability/compileTemplate.d.ts +1 -0
- package/dist/observability/compileTemplate.js +13 -0
- package/dist/observability/datasetImport.d.ts +11 -0
- package/dist/observability/datasetImport.js +185 -0
- package/dist/observability/evaluate.d.ts +31 -0
- package/dist/observability/evaluate.js +200 -0
- package/dist/observability/evaluatorTemplates.d.ts +17 -0
- package/dist/observability/evaluatorTemplates.js +111 -0
- package/dist/observability/jsonPath.d.ts +2 -0
- package/dist/observability/jsonPath.js +57 -0
- package/dist/observability/local/datasetStore.d.ts +87 -0
- package/dist/observability/local/datasetStore.js +326 -0
- package/dist/observability/local/evaluatorStore.d.ts +44 -0
- package/dist/observability/local/evaluatorStore.js +182 -0
- package/dist/observability/local/experimentRunner.d.ts +96 -0
- package/dist/observability/local/experimentRunner.js +419 -0
- package/dist/observability/local/localPlugin.d.ts +22 -0
- package/dist/observability/local/localPlugin.js +76 -0
- package/dist/observability/local/models/obsDataset.d.ts +2 -0
- package/dist/observability/local/models/obsDataset.js +37 -0
- package/dist/observability/local/models/obsDatasetItem.d.ts +2 -0
- package/dist/observability/local/models/obsDatasetItem.js +76 -0
- package/dist/observability/local/models/obsEvaluator.d.ts +2 -0
- package/dist/observability/local/models/obsEvaluator.js +98 -0
- package/dist/observability/local/models/obsExperiment.d.ts +2 -0
- package/dist/observability/local/models/obsExperiment.js +87 -0
- package/dist/observability/local/models/obsExperimentItem.d.ts +2 -0
- package/dist/observability/local/models/obsExperimentItem.js +45 -0
- package/dist/observability/local/models/obsPrompt.d.ts +2 -0
- package/dist/observability/local/models/obsPrompt.js +42 -0
- package/dist/observability/local/models/obsPromptLabel.d.ts +2 -0
- package/dist/observability/local/models/obsPromptLabel.js +39 -0
- package/dist/observability/local/models/obsPromptVersion.d.ts +2 -0
- package/dist/observability/local/models/obsPromptVersion.js +96 -0
- package/dist/observability/local/models/obsReviewItem.d.ts +2 -0
- package/dist/observability/local/models/obsReviewItem.js +75 -0
- package/dist/observability/local/models/obsScore.d.ts +2 -0
- package/dist/observability/local/models/obsScore.js +58 -0
- package/dist/observability/local/models/obsSpan.d.ts +2 -0
- package/dist/observability/local/models/obsSpan.js +64 -0
- package/dist/observability/local/models/obsTrace.d.ts +2 -0
- package/dist/observability/local/models/obsTrace.js +93 -0
- package/dist/observability/local/promptStore.d.ts +180 -0
- package/dist/observability/local/promptStore.js +444 -0
- package/dist/observability/local/reviewStore.d.ts +50 -0
- package/dist/observability/local/reviewStore.js +203 -0
- package/dist/observability/local/traceStore.d.ts +103 -0
- package/dist/observability/local/traceStore.js +325 -0
- package/dist/observability/observabilityApp.d.ts +24 -0
- package/dist/observability/observabilityApp.js +157 -0
- package/dist/observability/observabilityAppRegistry.d.ts +14 -0
- package/dist/observability/observabilityAppRegistry.js +16 -0
- package/dist/observability/permissions.d.ts +7 -0
- package/dist/observability/permissions.js +60 -0
- package/dist/observability/reviewPanels.d.ts +18 -0
- package/dist/observability/reviewPanels.js +46 -0
- package/dist/observability/routes/datasets.d.ts +7 -0
- package/dist/observability/routes/datasets.js +149 -0
- package/dist/observability/routes/evaluators.d.ts +7 -0
- package/dist/observability/routes/evaluators.js +123 -0
- package/dist/observability/routes/experiments.d.ts +7 -0
- package/dist/observability/routes/experiments.js +95 -0
- package/dist/observability/routes/observabilityRouteAccess.d.ts +13 -0
- package/dist/observability/routes/observabilityRouteAccess.js +67 -0
- package/dist/observability/routes/prompts.d.ts +12 -0
- package/dist/observability/routes/prompts.js +172 -0
- package/dist/observability/routes/requestAiService.d.ts +13 -0
- package/dist/observability/routes/requestAiService.js +21 -0
- package/dist/observability/routes/review.d.ts +7 -0
- package/dist/observability/routes/review.js +79 -0
- package/dist/observability/routes/status.d.ts +5 -0
- package/dist/observability/routes/status.js +31 -0
- package/dist/observability/routes/testMultiStage.d.ts +9 -0
- package/dist/observability/routes/testMultiStage.js +67 -0
- package/dist/observability/routes/traces.d.ts +7 -0
- package/dist/observability/routes/traces.js +129 -0
- package/dist/observability/schemaValidation.d.ts +5 -0
- package/dist/observability/schemaValidation.js +26 -0
- package/dist/observability/sopThresholds.d.ts +2 -0
- package/dist/observability/sopThresholds.js +21 -0
- package/dist/observability/status.d.ts +23 -0
- package/dist/observability/status.js +34 -0
- package/dist/observability/testMultiStageSchemas.d.ts +10 -0
- package/dist/observability/testMultiStageSchemas.js +79 -0
- package/dist/observability/testMultiStageWorkflow.d.ts +20 -0
- package/dist/observability/testMultiStageWorkflow.js +218 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.js +67 -0
- package/dist/routes/files.js +20 -7
- package/dist/routes/gpt.js +179 -488
- package/dist/routes/gptActions.d.ts +9 -0
- package/dist/routes/gptActions.js +212 -0
- package/dist/routes/gptDatasets.d.ts +5 -0
- package/dist/routes/gptDatasets.js +94 -0
- package/dist/routes/gptHistories.d.ts +3 -1
- package/dist/routes/gptHistories.js +188 -2
- package/dist/service/aiDatasets.d.ts +57 -0
- package/dist/service/aiDatasets.js +253 -0
- package/dist/service/aiService.d.ts +50 -1
- package/dist/service/aiService.js +431 -77
- package/dist/service/askFiles.d.ts +49 -0
- package/dist/service/askFiles.js +162 -0
- package/dist/service/asks.d.ts +76 -0
- package/dist/service/asks.js +284 -0
- package/dist/service/chatTurn.d.ts +128 -0
- package/dist/service/chatTurn.js +1674 -0
- package/dist/service/fileStorage.d.ts +4 -0
- package/dist/service/fileStorage.js +7 -2
- package/dist/service/fileUploadsGate.d.ts +8 -0
- package/dist/service/fileUploadsGate.js +18 -0
- package/dist/service/fillActionIds.d.ts +6 -0
- package/dist/service/fillActionIds.js +79 -0
- package/dist/service/gemini.d.ts +3 -3
- package/dist/service/gemini.js +3 -3
- package/dist/service/jsonFenceModel.d.ts +7 -0
- package/dist/service/jsonFenceModel.js +45 -0
- package/dist/service/prompts.d.ts +52 -0
- package/dist/service/prompts.js +96 -1
- package/dist/service/sanitizeHtml.d.ts +11 -0
- package/dist/service/sanitizeHtml.js +108 -0
- package/dist/service/vertex.d.ts +6 -6
- package/dist/service/vertex.js +5 -5
- package/dist/types/harness.d.ts +934 -0
- package/dist/types/harness.js +133 -0
- package/dist/types/index.d.ts +219 -13
- package/dist/types/index.js +2 -0
- package/dist/types/observability.d.ts +288 -0
- package/dist/types/observability.js +2 -0
- package/package.json +17 -5
- package/src/aiApp.test.ts +207 -1
- package/src/aiApp.ts +31 -8
- package/src/harness/adminScreens.test.ts +34 -0
- package/src/harness/adminScreens.ts +11 -0
- package/src/harness/agentLoop.test.ts +1034 -0
- package/src/harness/agentLoop.ts +1082 -0
- package/src/harness/agentRecovery.test.ts +380 -0
- package/src/harness/agentTaskNames.ts +4 -0
- package/src/harness/approvalPolicy.ts +28 -0
- package/src/harness/approvals.test.ts +1094 -0
- package/src/harness/approvals.ts +644 -0
- package/src/harness/commit.ts +1049 -0
- package/src/harness/conversation.ts +503 -0
- package/src/harness/defineAgent.test.ts +92 -0
- package/src/harness/defineAgent.ts +82 -0
- package/src/harness/defineTask.test.ts +59 -0
- package/src/harness/defineTask.ts +76 -0
- package/src/harness/defineTool.ts +45 -0
- package/src/harness/definitionError.ts +13 -0
- package/src/harness/errors.test.ts +90 -0
- package/src/harness/errors.ts +119 -0
- package/src/harness/eventHub.test.ts +124 -0
- package/src/harness/eventHub.ts +159 -0
- package/src/harness/events.test.ts +757 -0
- package/src/harness/events.ts +384 -0
- package/src/harness/executionEnv.ts +34 -0
- package/src/harness/extensions.test.ts +1218 -0
- package/src/harness/extensions.ts +382 -0
- package/src/harness/harness.test.ts +693 -0
- package/src/harness/harness.ts +767 -0
- package/src/harness/harnessApp.ts +76 -0
- package/src/harness/internalRuntime.ts +44 -0
- package/src/harness/leases.ts +132 -0
- package/src/harness/memo.ts +107 -0
- package/src/harness/modelCall.test.ts +134 -0
- package/src/harness/modelCall.ts +200 -0
- package/src/harness/models/harnessApproval.ts +109 -0
- package/src/harness/models/harnessConversation.ts +142 -0
- package/src/harness/models/harnessEvent.ts +92 -0
- package/src/harness/models/harnessInboxEvent.ts +73 -0
- package/src/harness/models/harnessMemo.ts +43 -0
- package/src/harness/models/harnessMessage.ts +85 -0
- package/src/harness/models/harnessModels.test.ts +60 -0
- package/src/harness/models/harnessOwner.ts +40 -0
- package/src/harness/models/harnessTask.ts +201 -0
- package/src/harness/ownership.test.ts +1087 -0
- package/src/harness/ownership.ts +618 -0
- package/src/harness/recovery.test.ts +912 -0
- package/src/harness/registry.ts +124 -0
- package/src/harness/retries.test.ts +292 -0
- package/src/harness/retryBackoff.test.ts +100 -0
- package/src/harness/retryBackoff.ts +77 -0
- package/src/harness/routes/approvals.ts +123 -0
- package/src/harness/routes/conversations.test.ts +518 -0
- package/src/harness/routes/conversations.ts +102 -0
- package/src/harness/routes/events.ts +280 -0
- package/src/harness/routes/tasks.test.ts +198 -0
- package/src/harness/routes/tasks.ts +146 -0
- package/src/harness/runners/inProcessRunner.test.ts +402 -0
- package/src/harness/runners/inProcessRunner.ts +220 -0
- package/src/harness/runtime.ts +714 -0
- package/src/harness/subagent.test.ts +696 -0
- package/src/harness/subagent.ts +192 -0
- package/src/harness/suspend.ts +13 -0
- package/src/harness/transaction.ts +17 -0
- package/src/harness/versioning.test.ts +288 -0
- package/src/harness/waits.test.ts +693 -0
- package/src/harness/waits.ts +356 -0
- package/src/index.test.ts +13 -0
- package/src/index.ts +70 -0
- package/src/isolated/adminScreensConfig.isolated.ts +143 -0
- package/src/models/aiDataset.test.ts +34 -0
- package/src/models/aiDataset.ts +78 -0
- package/src/models/aiRequest.test.ts +2 -2
- package/src/models/aiRequest.ts +2 -1
- package/src/models/gptHistory.test.ts +156 -0
- package/src/models/gptHistory.ts +106 -4
- package/src/observability/adminScreens.test.ts +71 -0
- package/src/observability/adminScreens.ts +42 -0
- package/src/observability/compileTemplate.test.ts +14 -0
- package/src/observability/compileTemplate.ts +10 -0
- package/src/observability/datasetImport.test.ts +95 -0
- package/src/observability/datasetImport.ts +220 -0
- package/src/observability/evaluate.test.ts +549 -0
- package/src/observability/evaluate.ts +254 -0
- package/src/observability/evaluatorTemplates.ts +124 -0
- package/src/observability/jsonPath.test.ts +52 -0
- package/src/observability/jsonPath.ts +60 -0
- package/src/observability/local/datasetStore.test.ts +230 -0
- package/src/observability/local/datasetStore.ts +452 -0
- package/src/observability/local/evaluatorStore.test.ts +138 -0
- package/src/observability/local/evaluatorStore.ts +224 -0
- package/src/observability/local/experimentRunner.test.ts +489 -0
- package/src/observability/local/experimentRunner.ts +548 -0
- package/src/observability/local/localPlugin.ts +88 -0
- package/src/observability/local/models/obsDataset.ts +38 -0
- package/src/observability/local/models/obsDatasetItem.ts +80 -0
- package/src/observability/local/models/obsEvaluator.ts +102 -0
- package/src/observability/local/models/obsExperiment.ts +91 -0
- package/src/observability/local/models/obsExperimentItem.ts +52 -0
- package/src/observability/local/models/obsPrompt.ts +43 -0
- package/src/observability/local/models/obsPromptLabel.ts +43 -0
- package/src/observability/local/models/obsPromptVersion.ts +107 -0
- package/src/observability/local/models/obsReviewItem.ts +79 -0
- package/src/observability/local/models/obsScore.ts +59 -0
- package/src/observability/local/models/obsSpan.ts +65 -0
- package/src/observability/local/models/obsTrace.ts +94 -0
- package/src/observability/local/obsModels.test.ts +119 -0
- package/src/observability/local/promptStore.test.ts +351 -0
- package/src/observability/local/promptStore.ts +665 -0
- package/src/observability/local/reviewStore.test.ts +164 -0
- package/src/observability/local/reviewStore.ts +264 -0
- package/src/observability/local/traceStore.test.ts +264 -0
- package/src/observability/local/traceStore.ts +440 -0
- package/src/observability/observabilityApp.test.ts +92 -0
- package/src/observability/observabilityApp.ts +185 -0
- package/src/observability/observabilityAppRegistry.test.ts +30 -0
- package/src/observability/observabilityAppRegistry.ts +35 -0
- package/src/observability/permissions.test.ts +96 -0
- package/src/observability/permissions.ts +78 -0
- package/src/observability/reviewPanels.test.ts +35 -0
- package/src/observability/reviewPanels.ts +64 -0
- package/src/observability/routes/datasets.test.ts +176 -0
- package/src/observability/routes/datasets.ts +252 -0
- package/src/observability/routes/evaluators.test.ts +119 -0
- package/src/observability/routes/evaluators.ts +207 -0
- package/src/observability/routes/experiments.test.ts +282 -0
- package/src/observability/routes/experiments.ts +152 -0
- package/src/observability/routes/observabilityRbac.test.ts +767 -0
- package/src/observability/routes/observabilityRouteAccess.ts +106 -0
- package/src/observability/routes/prompts.test.ts +320 -0
- package/src/observability/routes/prompts.ts +263 -0
- package/src/observability/routes/requestAiService.ts +36 -0
- package/src/observability/routes/review.test.ts +120 -0
- package/src/observability/routes/review.ts +125 -0
- package/src/observability/routes/status.test.ts +116 -0
- package/src/observability/routes/status.ts +47 -0
- package/src/observability/routes/testMultiStage.test.ts +214 -0
- package/src/observability/routes/testMultiStage.ts +96 -0
- package/src/observability/routes/traces.test.ts +167 -0
- package/src/observability/routes/traces.ts +172 -0
- package/src/observability/schemaValidation.ts +29 -0
- package/src/observability/sopThresholds.ts +20 -0
- package/src/observability/status.test.ts +38 -0
- package/src/observability/status.ts +56 -0
- package/src/observability/testMultiStageSchemas.ts +95 -0
- package/src/observability/testMultiStageWorkflow.test.ts +114 -0
- package/src/observability/testMultiStageWorkflow.ts +307 -0
- package/src/observability/types.ts +262 -0
- package/src/routes/files.test.ts +84 -9
- package/src/routes/files.ts +23 -7
- package/src/routes/gpt.test.ts +3279 -0
- package/src/routes/gpt.ts +214 -552
- package/src/routes/gptActions.test.ts +241 -0
- package/src/routes/gptActions.ts +255 -0
- package/src/routes/gptApprovals.test.ts +762 -0
- package/src/routes/gptDatasets.test.ts +86 -0
- package/src/routes/gptDatasets.ts +104 -0
- package/src/routes/gptFiles.test.ts +589 -0
- package/src/routes/gptHistories.test.ts +933 -0
- package/src/routes/gptHistories.ts +266 -5
- package/src/routes/routes.test.ts +676 -5
- package/src/service/aiDatasets.test.ts +123 -0
- package/src/service/aiDatasets.ts +316 -0
- package/src/service/aiService.test.ts +614 -0
- package/src/service/aiService.ts +575 -111
- package/src/service/askFiles.test.ts +96 -0
- package/src/service/askFiles.ts +222 -0
- package/src/service/asks.test.ts +946 -0
- package/src/service/asks.ts +423 -0
- package/src/service/chatTurn.ts +2287 -0
- package/src/service/fileStorage.test.ts +11 -0
- package/src/service/fileStorage.ts +10 -2
- package/src/service/fileUploadsGate.ts +23 -0
- package/src/service/fillActionIds.test.ts +65 -0
- package/src/service/fillActionIds.ts +98 -0
- package/src/service/gemini.test.ts +16 -10
- package/src/service/gemini.ts +3 -3
- package/src/service/jsonFenceModel.test.ts +47 -0
- package/src/service/jsonFenceModel.ts +53 -0
- package/src/service/prompts.test.ts +21 -0
- package/src/service/prompts.ts +147 -0
- package/src/service/sanitizeHtml.test.ts +61 -0
- package/src/service/sanitizeHtml.ts +109 -0
- package/src/service/vertex.test.ts +49 -49
- package/src/service/vertex.ts +7 -8
- package/src/tests/bunSetup.test.ts +7 -4
- package/src/tests/bunSetup.ts +7 -7
- package/src/tests/chatHarness.ts +478 -0
- package/src/tests/generateStream.ts +49 -0
- package/src/tests/harnessErrors.ts +22 -0
- package/src/tests/harnessStreaming.ts +227 -0
- package/src/tests/helpers.ts +9 -2
- package/src/types/harness.ts +1095 -0
- package/src/types/index.ts +237 -12
- package/src/types/observability.ts +344 -0
- package/src/types/sanitize-html.d.ts +25 -0
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.LocalExperimentRunner = exports.EXPERIMENT_LIST_DEFAULT_LIMIT = void 0;
|
|
7
|
+
const api_1 = require("@terreno/api");
|
|
8
|
+
const luxon_1 = require("luxon");
|
|
9
|
+
const mongoose_1 = __importDefault(require("mongoose"));
|
|
10
|
+
const compileTemplate_1 = require("../compileTemplate");
|
|
11
|
+
const evaluate_1 = require("../evaluate");
|
|
12
|
+
const sopThresholds_1 = require("../sopThresholds");
|
|
13
|
+
const obsExperiment_1 = require("./models/obsExperiment");
|
|
14
|
+
const obsExperimentItem_1 = require("./models/obsExperimentItem");
|
|
15
|
+
exports.EXPERIMENT_LIST_DEFAULT_LIMIT = 50;
|
|
16
|
+
const EXPERIMENT_LIST_MAX_LIMIT = 100;
|
|
17
|
+
const defaultBackgroundTaskRunner = {
|
|
18
|
+
enqueue: async ({ execute, taskType }) => {
|
|
19
|
+
const task = await api_1.BackgroundTask.create({
|
|
20
|
+
status: "pending",
|
|
21
|
+
taskType,
|
|
22
|
+
});
|
|
23
|
+
const taskId = String(task._id);
|
|
24
|
+
setImmediate(async () => {
|
|
25
|
+
task.status = "running";
|
|
26
|
+
task.startedAt = luxon_1.DateTime.now().toJSDate();
|
|
27
|
+
await task.save();
|
|
28
|
+
try {
|
|
29
|
+
await execute();
|
|
30
|
+
task.status = "completed";
|
|
31
|
+
task.completedAt = luxon_1.DateTime.now().toJSDate();
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
task.status = "failed";
|
|
35
|
+
task.error = error instanceof Error ? error.message : "experiment failed";
|
|
36
|
+
task.completedAt = luxon_1.DateTime.now().toJSDate();
|
|
37
|
+
}
|
|
38
|
+
await task.save();
|
|
39
|
+
});
|
|
40
|
+
return { taskId };
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
const compareGate = (actual, op, expected) => {
|
|
44
|
+
if (op === "eq") {
|
|
45
|
+
return actual === expected;
|
|
46
|
+
}
|
|
47
|
+
if (op === "gte") {
|
|
48
|
+
return actual >= expected;
|
|
49
|
+
}
|
|
50
|
+
return actual <= expected;
|
|
51
|
+
};
|
|
52
|
+
const computeGateValue = (values, aggregate) => {
|
|
53
|
+
const filtered = values.filter((value) => {
|
|
54
|
+
return value !== undefined;
|
|
55
|
+
});
|
|
56
|
+
if (filtered.length === 0) {
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
if (aggregate === "trueRate") {
|
|
60
|
+
const trueCount = filtered.filter((value) => {
|
|
61
|
+
return value === true || value === 1;
|
|
62
|
+
}).length;
|
|
63
|
+
return trueCount / filtered.length;
|
|
64
|
+
}
|
|
65
|
+
const numeric = filtered.map((value) => {
|
|
66
|
+
return Number(value);
|
|
67
|
+
});
|
|
68
|
+
return numeric.reduce((sum, value) => sum + value, 0) / numeric.length;
|
|
69
|
+
};
|
|
70
|
+
const parseGenerationOutput = (output) => {
|
|
71
|
+
try {
|
|
72
|
+
return JSON.parse(output);
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return output;
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
const gateBucketKey = (version, threshold) => {
|
|
79
|
+
return `${version}:${threshold.evaluatorName}:${threshold.dimension}:${threshold.aggregate}`;
|
|
80
|
+
};
|
|
81
|
+
class LocalExperimentRunner {
|
|
82
|
+
deps;
|
|
83
|
+
aiDeps;
|
|
84
|
+
constructor(deps) {
|
|
85
|
+
this.deps = deps;
|
|
86
|
+
}
|
|
87
|
+
configureAi(aiDeps) {
|
|
88
|
+
this.aiDeps = aiDeps;
|
|
89
|
+
}
|
|
90
|
+
requireAiDeps() {
|
|
91
|
+
if (!this.aiDeps?.aiService) {
|
|
92
|
+
throw new api_1.APIError({ status: 400, title: "experiments require an AI service" });
|
|
93
|
+
}
|
|
94
|
+
return this.aiDeps;
|
|
95
|
+
}
|
|
96
|
+
resolveAiClient(modelOverride) {
|
|
97
|
+
const aiDeps = this.requireAiDeps();
|
|
98
|
+
if (!modelOverride) {
|
|
99
|
+
return aiDeps.aiService;
|
|
100
|
+
}
|
|
101
|
+
if (!aiDeps.aiServiceFactory) {
|
|
102
|
+
throw new api_1.APIError({
|
|
103
|
+
status: 400,
|
|
104
|
+
title: `modelOverride "${modelOverride}" requires aiServiceFactory on ObservabilityApp`,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
const client = aiDeps.aiServiceFactory(modelOverride);
|
|
108
|
+
if (!client) {
|
|
109
|
+
throw new api_1.APIError({
|
|
110
|
+
status: 400,
|
|
111
|
+
title: `Unsupported modelOverride "${modelOverride}"`,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
return client;
|
|
115
|
+
}
|
|
116
|
+
async estimate(input) {
|
|
117
|
+
const items = await this.deps.datasetStore.listExperimentItems(input.datasetId, input.includeUnproofread ?? false);
|
|
118
|
+
const generations = items.length * input.versions.length;
|
|
119
|
+
const evaluatorCount = input.evaluatorIds.length;
|
|
120
|
+
const secondsPerGeneration = 2;
|
|
121
|
+
const judgeSeconds = generations * evaluatorCount * 0.5;
|
|
122
|
+
const modelId = (input.modelOverride
|
|
123
|
+
? this.resolveAiClient(input.modelOverride).modelId
|
|
124
|
+
: this.aiDeps?.aiService.modelId) ?? "unknown";
|
|
125
|
+
const price = this.aiDeps?.priceMap?.[modelId];
|
|
126
|
+
const tokensPerGeneration = 1000;
|
|
127
|
+
const costUsd = price && generations > 0
|
|
128
|
+
? (price.inputPerMTok * tokensPerGeneration * generations) / 1_000_000
|
|
129
|
+
: undefined;
|
|
130
|
+
return {
|
|
131
|
+
costUsd,
|
|
132
|
+
generations,
|
|
133
|
+
wallClockSeconds: generations * secondsPerGeneration + judgeSeconds,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
async assertAutomaticEvaluators(evaluatorIds) {
|
|
137
|
+
for (const id of evaluatorIds) {
|
|
138
|
+
const evaluator = await this.deps.evaluatorStore.get(id);
|
|
139
|
+
if (evaluator.type !== "llm-judge" && evaluator.type !== "json-assert") {
|
|
140
|
+
throw new api_1.APIError({
|
|
141
|
+
status: 400,
|
|
142
|
+
title: "Experiments only accept llm-judge and json-assert evaluators",
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
async create(input) {
|
|
148
|
+
if (input.versions.length < 2 || input.versions.length > 3) {
|
|
149
|
+
throw new api_1.APIError({ status: 400, title: "experiments must compare 2–3 prompt versions" });
|
|
150
|
+
}
|
|
151
|
+
await this.assertAutomaticEvaluators(input.evaluatorIds);
|
|
152
|
+
this.resolveAiClient(input.modelOverride);
|
|
153
|
+
await this.deps.datasetStore.get(input.datasetId);
|
|
154
|
+
const estimate = await this.estimate({
|
|
155
|
+
datasetId: input.datasetId,
|
|
156
|
+
evaluatorIds: input.evaluatorIds,
|
|
157
|
+
includeUnproofread: input.includeUnproofread,
|
|
158
|
+
modelOverride: input.modelOverride,
|
|
159
|
+
versions: input.versions,
|
|
160
|
+
});
|
|
161
|
+
const experiment = await (0, obsExperiment_1.registerObsExperiment)().create({
|
|
162
|
+
datasetId: new mongoose_1.default.Types.ObjectId(input.datasetId),
|
|
163
|
+
estimate,
|
|
164
|
+
evaluatorIds: input.evaluatorIds.map((id) => {
|
|
165
|
+
return new mongoose_1.default.Types.ObjectId(id);
|
|
166
|
+
}),
|
|
167
|
+
includeUnproofread: input.includeUnproofread ?? false,
|
|
168
|
+
modelOverride: input.modelOverride,
|
|
169
|
+
name: input.name,
|
|
170
|
+
promptName: input.promptName,
|
|
171
|
+
status: "pending",
|
|
172
|
+
thresholds: input.thresholds ?? sopThresholds_1.SOP_DEFAULT_THRESHOLDS,
|
|
173
|
+
versions: input.versions,
|
|
174
|
+
});
|
|
175
|
+
const runner = this.deps.backgroundTaskRunner ?? defaultBackgroundTaskRunner;
|
|
176
|
+
const { taskId } = await runner.enqueue({
|
|
177
|
+
execute: async () => {
|
|
178
|
+
await this.executeExperiment(String(experiment._id));
|
|
179
|
+
},
|
|
180
|
+
taskType: `obs-experiment-${String(experiment._id)}`,
|
|
181
|
+
});
|
|
182
|
+
experiment.backgroundTaskId = new mongoose_1.default.Types.ObjectId(taskId);
|
|
183
|
+
experiment.status = "running";
|
|
184
|
+
await experiment.save();
|
|
185
|
+
return this.get(String(experiment._id));
|
|
186
|
+
}
|
|
187
|
+
async list(params) {
|
|
188
|
+
const page = params?.page && params.page > 0 ? params.page : 1;
|
|
189
|
+
const limit = params?.limit && params.limit > 0
|
|
190
|
+
? Math.min(params.limit, EXPERIMENT_LIST_MAX_LIMIT)
|
|
191
|
+
: exports.EXPERIMENT_LIST_DEFAULT_LIMIT;
|
|
192
|
+
const filter = {};
|
|
193
|
+
if (params?.promptName) {
|
|
194
|
+
filter.promptName = params.promptName;
|
|
195
|
+
}
|
|
196
|
+
const ObsExperiment = (0, obsExperiment_1.registerObsExperiment)();
|
|
197
|
+
const total = await ObsExperiment.countDocuments(filter);
|
|
198
|
+
const rows = await ObsExperiment.find(filter)
|
|
199
|
+
.sort({ created: -1 })
|
|
200
|
+
.skip((page - 1) * limit)
|
|
201
|
+
.limit(limit);
|
|
202
|
+
return {
|
|
203
|
+
data: rows.map((row) => {
|
|
204
|
+
return this.toSummaryView(row);
|
|
205
|
+
}),
|
|
206
|
+
meta: { limit, page, total },
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
async get(id) {
|
|
210
|
+
const experiment = await (0, obsExperiment_1.registerObsExperiment)().findOneOrNone({ _id: id });
|
|
211
|
+
if (!experiment) {
|
|
212
|
+
throw new api_1.APIError({ status: 404, title: `Unknown experiment "${id}"` });
|
|
213
|
+
}
|
|
214
|
+
const items = await (0, obsExperimentItem_1.registerObsExperimentItem)().find({ experimentId: experiment._id });
|
|
215
|
+
const sortedItems = items.sort((left, right) => {
|
|
216
|
+
if (left.failed !== right.failed) {
|
|
217
|
+
return left.failed ? -1 : 1;
|
|
218
|
+
}
|
|
219
|
+
return left.created.getTime() - right.created.getTime();
|
|
220
|
+
});
|
|
221
|
+
return {
|
|
222
|
+
...this.toSummaryView(experiment),
|
|
223
|
+
items: sortedItems.map((row) => {
|
|
224
|
+
return {
|
|
225
|
+
datasetItemId: String(row.datasetItemId),
|
|
226
|
+
failed: row.failed,
|
|
227
|
+
id: String(row._id),
|
|
228
|
+
versionResults: row.versionResults,
|
|
229
|
+
};
|
|
230
|
+
}),
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
toSummaryView(experiment) {
|
|
234
|
+
return {
|
|
235
|
+
backgroundTaskId: experiment.backgroundTaskId
|
|
236
|
+
? String(experiment.backgroundTaskId)
|
|
237
|
+
: undefined,
|
|
238
|
+
created: luxon_1.DateTime.fromJSDate(experiment.created).toUTC().toISO() ?? "",
|
|
239
|
+
datasetId: String(experiment.datasetId),
|
|
240
|
+
estimate: experiment.estimate,
|
|
241
|
+
evaluatorIds: experiment.evaluatorIds.map((value) => String(value)),
|
|
242
|
+
id: String(experiment._id),
|
|
243
|
+
includeUnproofread: experiment.includeUnproofread,
|
|
244
|
+
items: [],
|
|
245
|
+
modelOverride: experiment.modelOverride,
|
|
246
|
+
name: experiment.name,
|
|
247
|
+
promptName: experiment.promptName,
|
|
248
|
+
results: experiment.results,
|
|
249
|
+
status: experiment.status,
|
|
250
|
+
thresholds: experiment.thresholds,
|
|
251
|
+
updated: luxon_1.DateTime.fromJSDate(experiment.updated).toUTC().toISO() ?? "",
|
|
252
|
+
versions: experiment.versions,
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
async promote(id, version) {
|
|
256
|
+
const experiment = await this.get(id);
|
|
257
|
+
if (!experiment.results) {
|
|
258
|
+
throw new api_1.APIError({ status: 409, title: "Experiment has not finished running" });
|
|
259
|
+
}
|
|
260
|
+
if (!experiment.versions.includes(version)) {
|
|
261
|
+
throw new api_1.APIError({
|
|
262
|
+
status: 400,
|
|
263
|
+
title: `Version ${version} was not part of experiment ${id}`,
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
const failingGate = experiment.results.gates.find((gate) => {
|
|
267
|
+
return gate.version === version && !gate.passed;
|
|
268
|
+
});
|
|
269
|
+
if (failingGate) {
|
|
270
|
+
throw new api_1.APIError({
|
|
271
|
+
status: 409,
|
|
272
|
+
title: `Promote blocked: gate failed for v${version} ${failingGate.evaluatorName}.${failingGate.dimension}`,
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
return this.deps.promptStore.moveLabel(experiment.promptName, {
|
|
276
|
+
label: "production",
|
|
277
|
+
version,
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
async executeExperiment(id) {
|
|
281
|
+
const experiment = await (0, obsExperiment_1.registerObsExperiment)().findOneOrNone({ _id: id });
|
|
282
|
+
if (!experiment) {
|
|
283
|
+
throw new api_1.APIError({ status: 404, title: `Unknown experiment "${id}"` });
|
|
284
|
+
}
|
|
285
|
+
const aiService = this.resolveAiClient(experiment.modelOverride);
|
|
286
|
+
const datasetItems = await this.deps.datasetStore.listExperimentItems(String(experiment.datasetId), experiment.includeUnproofread);
|
|
287
|
+
const evaluators = await Promise.all(experiment.evaluatorIds.map((evaluatorId) => {
|
|
288
|
+
return this.deps.evaluatorStore.get(String(evaluatorId));
|
|
289
|
+
}));
|
|
290
|
+
const evaluatorById = new Map(evaluators.map((evaluator) => {
|
|
291
|
+
return [evaluator.id, evaluator];
|
|
292
|
+
}));
|
|
293
|
+
const judgeSchemas = new Map();
|
|
294
|
+
for (const evaluator of evaluators) {
|
|
295
|
+
if (evaluator.type === "llm-judge" && evaluator.judgePromptName) {
|
|
296
|
+
const version = await this.deps.promptStore.getVersionByLabel(evaluator.judgePromptName);
|
|
297
|
+
judgeSchemas.set(evaluator.id, version?.outputSchema);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
let completed = 0;
|
|
301
|
+
const gateBuckets = new Map();
|
|
302
|
+
const lowConfidenceItemIds = [];
|
|
303
|
+
const outlierItemIds = [];
|
|
304
|
+
for (const datasetItem of datasetItems) {
|
|
305
|
+
const versionResults = {};
|
|
306
|
+
let rowFailed = false;
|
|
307
|
+
for (const versionNumber of experiment.versions) {
|
|
308
|
+
const promptVersion = await this.deps.promptStore.getVersionByNumber(experiment.promptName, versionNumber);
|
|
309
|
+
if (!promptVersion) {
|
|
310
|
+
throw new api_1.APIError({
|
|
311
|
+
status: 404,
|
|
312
|
+
title: `Unknown version ${versionNumber} for prompt "${experiment.promptName}"`,
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
const variables = promptVersion.variables.length > 0 &&
|
|
316
|
+
datasetItem.input &&
|
|
317
|
+
typeof datasetItem.input === "object" &&
|
|
318
|
+
!Array.isArray(datasetItem.input)
|
|
319
|
+
? datasetItem.input
|
|
320
|
+
: { input: JSON.stringify(datasetItem.input) };
|
|
321
|
+
const compiled = (0, compileTemplate_1.compileTemplate)(promptVersion.template ?? "", variables);
|
|
322
|
+
const outputText = await aiService.generateText({
|
|
323
|
+
prompt: compiled,
|
|
324
|
+
skipTrace: true,
|
|
325
|
+
systemPrompt: promptVersion.system,
|
|
326
|
+
});
|
|
327
|
+
const parsedOutput = parseGenerationOutput(outputText);
|
|
328
|
+
const evaluatorScores = {};
|
|
329
|
+
for (const evaluatorId of experiment.evaluatorIds) {
|
|
330
|
+
const evaluator = evaluatorById.get(String(evaluatorId));
|
|
331
|
+
if (!evaluator) {
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
const result = await (0, evaluate_1.runEvaluator)({
|
|
335
|
+
evaluator,
|
|
336
|
+
expectedOutput: datasetItem.expectedOutput,
|
|
337
|
+
input: datasetItem.input,
|
|
338
|
+
output: parsedOutput,
|
|
339
|
+
outputSchema: promptVersion.outputSchema,
|
|
340
|
+
}, {
|
|
341
|
+
ai: aiService,
|
|
342
|
+
judgeOutputSchema: judgeSchemas.get(evaluator.id),
|
|
343
|
+
});
|
|
344
|
+
evaluatorScores[evaluator.id] = {
|
|
345
|
+
confidence: result.confidence,
|
|
346
|
+
error: result.error,
|
|
347
|
+
scores: result.scores,
|
|
348
|
+
};
|
|
349
|
+
if (result.error) {
|
|
350
|
+
rowFailed = true;
|
|
351
|
+
outlierItemIds.push(datasetItem.id);
|
|
352
|
+
}
|
|
353
|
+
if (result.confidence !== undefined &&
|
|
354
|
+
result.confidence < evaluator.confidenceAlertBelow) {
|
|
355
|
+
lowConfidenceItemIds.push(datasetItem.id);
|
|
356
|
+
outlierItemIds.push(datasetItem.id);
|
|
357
|
+
}
|
|
358
|
+
for (const threshold of experiment.thresholds) {
|
|
359
|
+
if (threshold.evaluatorName !== evaluator.name || !result.scores) {
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
const bucketKey = gateBucketKey(versionNumber, threshold);
|
|
363
|
+
const raw = result.scores?.[threshold.dimension];
|
|
364
|
+
const bucket = gateBuckets.get(bucketKey) ?? [];
|
|
365
|
+
if (typeof raw === "boolean" || typeof raw === "number") {
|
|
366
|
+
bucket.push(raw);
|
|
367
|
+
}
|
|
368
|
+
gateBuckets.set(bucketKey, bucket);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
versionResults[String(versionNumber)] = {
|
|
372
|
+
evaluatorScores,
|
|
373
|
+
output: parsedOutput,
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
await (0, obsExperimentItem_1.registerObsExperimentItem)().create({
|
|
377
|
+
datasetItemId: new mongoose_1.default.Types.ObjectId(datasetItem.id),
|
|
378
|
+
experimentId: experiment._id,
|
|
379
|
+
failed: rowFailed,
|
|
380
|
+
versionResults,
|
|
381
|
+
});
|
|
382
|
+
completed += 1;
|
|
383
|
+
experiment.results = {
|
|
384
|
+
gates: this.computeGates(experiment.versions, experiment.thresholds, gateBuckets),
|
|
385
|
+
lowConfidenceItemIds: [...new Set(lowConfidenceItemIds)],
|
|
386
|
+
outlierItemIds: [...new Set(outlierItemIds)],
|
|
387
|
+
progress: { completed, total: datasetItems.length },
|
|
388
|
+
totalCostUsd: experiment.estimate?.costUsd,
|
|
389
|
+
};
|
|
390
|
+
await experiment.save();
|
|
391
|
+
}
|
|
392
|
+
experiment.status = "completed";
|
|
393
|
+
experiment.results = {
|
|
394
|
+
...experiment.results,
|
|
395
|
+
gates: this.computeGates(experiment.versions, experiment.thresholds, gateBuckets),
|
|
396
|
+
lowConfidenceItemIds: [...new Set(lowConfidenceItemIds)],
|
|
397
|
+
outlierItemIds: [...new Set(outlierItemIds)],
|
|
398
|
+
progress: { completed: datasetItems.length, total: datasetItems.length },
|
|
399
|
+
totalCostUsd: experiment.estimate?.costUsd,
|
|
400
|
+
};
|
|
401
|
+
await experiment.save();
|
|
402
|
+
}
|
|
403
|
+
computeGates(versions, thresholds, gateBuckets) {
|
|
404
|
+
return versions.flatMap((version) => {
|
|
405
|
+
return thresholds.map((threshold) => {
|
|
406
|
+
const bucketKey = gateBucketKey(version, threshold);
|
|
407
|
+
const actual = computeGateValue(gateBuckets.get(bucketKey) ?? [], threshold.aggregate);
|
|
408
|
+
const passed = actual === undefined ? false : compareGate(actual, threshold.op, threshold.value);
|
|
409
|
+
return {
|
|
410
|
+
...threshold,
|
|
411
|
+
actual,
|
|
412
|
+
passed,
|
|
413
|
+
version,
|
|
414
|
+
};
|
|
415
|
+
});
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
exports.LocalExperimentRunner = LocalExperimentRunner;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { ModelPrice, ObservabilityPlugin } from "../types";
|
|
2
|
+
import { LocalDatasetStore } from "./datasetStore";
|
|
3
|
+
import { LocalEvaluatorStore } from "./evaluatorStore";
|
|
4
|
+
import { type BackgroundTaskRunner, LocalExperimentRunner } from "./experimentRunner";
|
|
5
|
+
import { LocalPromptStore } from "./promptStore";
|
|
6
|
+
import { LocalTraceStore } from "./traceStore";
|
|
7
|
+
export interface LocalObservabilityPluginBundle {
|
|
8
|
+
datasetStore: LocalDatasetStore;
|
|
9
|
+
evaluatorStore: LocalEvaluatorStore;
|
|
10
|
+
experimentRunner: LocalExperimentRunner;
|
|
11
|
+
plugin: ObservabilityPlugin;
|
|
12
|
+
promptStore: LocalPromptStore;
|
|
13
|
+
traceStore: LocalTraceStore;
|
|
14
|
+
}
|
|
15
|
+
export declare const createLocalObservabilityBundle: (options?: {
|
|
16
|
+
backgroundTaskRunner?: BackgroundTaskRunner;
|
|
17
|
+
priceMap?: Record<string, ModelPrice>;
|
|
18
|
+
}) => LocalObservabilityPluginBundle;
|
|
19
|
+
export declare const createLocalObservabilityPlugin: (options?: {
|
|
20
|
+
backgroundTaskRunner?: BackgroundTaskRunner;
|
|
21
|
+
priceMap?: Record<string, ModelPrice>;
|
|
22
|
+
}) => ObservabilityPlugin;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createLocalObservabilityPlugin = exports.createLocalObservabilityBundle = void 0;
|
|
4
|
+
const datasetStore_1 = require("./datasetStore");
|
|
5
|
+
const evaluatorStore_1 = require("./evaluatorStore");
|
|
6
|
+
const experimentRunner_1 = require("./experimentRunner");
|
|
7
|
+
const obsDataset_1 = require("./models/obsDataset");
|
|
8
|
+
const obsDatasetItem_1 = require("./models/obsDatasetItem");
|
|
9
|
+
const obsEvaluator_1 = require("./models/obsEvaluator");
|
|
10
|
+
const obsExperiment_1 = require("./models/obsExperiment");
|
|
11
|
+
const obsExperimentItem_1 = require("./models/obsExperimentItem");
|
|
12
|
+
const obsPrompt_1 = require("./models/obsPrompt");
|
|
13
|
+
const obsPromptLabel_1 = require("./models/obsPromptLabel");
|
|
14
|
+
const obsPromptVersion_1 = require("./models/obsPromptVersion");
|
|
15
|
+
const obsReviewItem_1 = require("./models/obsReviewItem");
|
|
16
|
+
const obsScore_1 = require("./models/obsScore");
|
|
17
|
+
const obsSpan_1 = require("./models/obsSpan");
|
|
18
|
+
const obsTrace_1 = require("./models/obsTrace");
|
|
19
|
+
const promptStore_1 = require("./promptStore");
|
|
20
|
+
const traceStore_1 = require("./traceStore");
|
|
21
|
+
const registerLocalObservabilityModels = () => {
|
|
22
|
+
(0, obsPrompt_1.registerObsPrompt)();
|
|
23
|
+
(0, obsPromptVersion_1.registerObsPromptVersion)();
|
|
24
|
+
(0, obsPromptLabel_1.registerObsPromptLabel)();
|
|
25
|
+
(0, obsTrace_1.registerObsTrace)();
|
|
26
|
+
(0, obsSpan_1.registerObsSpan)();
|
|
27
|
+
(0, obsScore_1.registerObsScore)();
|
|
28
|
+
(0, obsEvaluator_1.registerObsEvaluator)();
|
|
29
|
+
(0, obsReviewItem_1.registerObsReviewItem)();
|
|
30
|
+
(0, obsDataset_1.registerObsDataset)();
|
|
31
|
+
(0, obsDatasetItem_1.registerObsDatasetItem)();
|
|
32
|
+
(0, obsExperiment_1.registerObsExperiment)();
|
|
33
|
+
(0, obsExperimentItem_1.registerObsExperimentItem)();
|
|
34
|
+
};
|
|
35
|
+
const createLocalObservabilityBundle = (options = {}) => {
|
|
36
|
+
registerLocalObservabilityModels();
|
|
37
|
+
const promptStore = new promptStore_1.LocalPromptStore();
|
|
38
|
+
const traces = new traceStore_1.LocalTraceStore();
|
|
39
|
+
const datasetStore = new datasetStore_1.LocalDatasetStore(promptStore);
|
|
40
|
+
const evaluatorStore = new evaluatorStore_1.LocalEvaluatorStore(promptStore);
|
|
41
|
+
const experimentRunner = new experimentRunner_1.LocalExperimentRunner({
|
|
42
|
+
backgroundTaskRunner: options.backgroundTaskRunner,
|
|
43
|
+
datasetStore,
|
|
44
|
+
evaluatorStore,
|
|
45
|
+
promptStore,
|
|
46
|
+
});
|
|
47
|
+
return {
|
|
48
|
+
datasetStore,
|
|
49
|
+
evaluatorStore,
|
|
50
|
+
experimentRunner,
|
|
51
|
+
plugin: {
|
|
52
|
+
capabilities: new Set([
|
|
53
|
+
"datasets",
|
|
54
|
+
"experiments",
|
|
55
|
+
"prompts",
|
|
56
|
+
"reviewQueue",
|
|
57
|
+
"scores",
|
|
58
|
+
"traces",
|
|
59
|
+
]),
|
|
60
|
+
datasetStore,
|
|
61
|
+
experimentRunner,
|
|
62
|
+
id: "local",
|
|
63
|
+
promptRegistry: promptStore,
|
|
64
|
+
reviewQueue: {},
|
|
65
|
+
scoreSink: new traceStore_1.LocalScoreSink(traces),
|
|
66
|
+
traceSink: new traceStore_1.LocalTraceSink(traces),
|
|
67
|
+
},
|
|
68
|
+
promptStore,
|
|
69
|
+
traceStore: traces,
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
exports.createLocalObservabilityBundle = createLocalObservabilityBundle;
|
|
73
|
+
const createLocalObservabilityPlugin = (options = {}) => {
|
|
74
|
+
return (0, exports.createLocalObservabilityBundle)(options).plugin;
|
|
75
|
+
};
|
|
76
|
+
exports.createLocalObservabilityPlugin = createLocalObservabilityPlugin;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.registerObsDataset = void 0;
|
|
7
|
+
const api_1 = require("@terreno/api");
|
|
8
|
+
const mongoose_1 = __importDefault(require("mongoose"));
|
|
9
|
+
const obsDatasetSchema = new mongoose_1.default.Schema({
|
|
10
|
+
description: { description: "Operator-facing dataset description", type: String },
|
|
11
|
+
expectedOutputSchema: {
|
|
12
|
+
description: "Optional JSON Schema for expected output labels",
|
|
13
|
+
type: mongoose_1.default.Schema.Types.Mixed,
|
|
14
|
+
},
|
|
15
|
+
inputSchemaPromptName: {
|
|
16
|
+
description: "Optional prompt whose production inputSchema validates imported items",
|
|
17
|
+
type: String,
|
|
18
|
+
},
|
|
19
|
+
name: { description: "Unique dataset name", required: true, type: String },
|
|
20
|
+
tags: {
|
|
21
|
+
default: [],
|
|
22
|
+
description: "Free-form dataset tags for filtering",
|
|
23
|
+
type: [String],
|
|
24
|
+
},
|
|
25
|
+
}, { strict: "throw", toJSON: { virtuals: true }, toObject: { virtuals: true } });
|
|
26
|
+
obsDatasetSchema.plugin(api_1.createdUpdatedPlugin);
|
|
27
|
+
obsDatasetSchema.plugin(api_1.isDeletedPlugin);
|
|
28
|
+
obsDatasetSchema.plugin(api_1.findOneOrNone);
|
|
29
|
+
obsDatasetSchema.plugin(api_1.findExactlyOne);
|
|
30
|
+
obsDatasetSchema.index({ name: 1 }, { unique: true });
|
|
31
|
+
const registerObsDataset = () => {
|
|
32
|
+
if (mongoose_1.default.models.ObsDataset) {
|
|
33
|
+
return mongoose_1.default.models.ObsDataset;
|
|
34
|
+
}
|
|
35
|
+
return mongoose_1.default.model("ObsDataset", obsDatasetSchema);
|
|
36
|
+
};
|
|
37
|
+
exports.registerObsDataset = registerObsDataset;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.registerObsDatasetItem = void 0;
|
|
7
|
+
const api_1 = require("@terreno/api");
|
|
8
|
+
const mongoose_1 = __importDefault(require("mongoose"));
|
|
9
|
+
const obsDatasetItemSchema = new mongoose_1.default.Schema({
|
|
10
|
+
annotatedBy: {
|
|
11
|
+
description: "Who approved the expected output for provenance display",
|
|
12
|
+
type: {
|
|
13
|
+
label: { description: "Human-readable attribution label", required: true, type: String },
|
|
14
|
+
reviewItemId: { description: "Optional review queue item id", type: String },
|
|
15
|
+
userId: { description: "Optional user id for the annotator", type: String },
|
|
16
|
+
},
|
|
17
|
+
},
|
|
18
|
+
datasetId: {
|
|
19
|
+
description: "Parent dataset id",
|
|
20
|
+
ref: "ObsDataset",
|
|
21
|
+
required: true,
|
|
22
|
+
type: mongoose_1.default.Schema.Types.ObjectId,
|
|
23
|
+
},
|
|
24
|
+
expectedOutput: {
|
|
25
|
+
description: "Human- or machine-authored expected output label",
|
|
26
|
+
type: mongoose_1.default.Schema.Types.Mixed,
|
|
27
|
+
},
|
|
28
|
+
input: {
|
|
29
|
+
description: "Dataset item input payload",
|
|
30
|
+
required: true,
|
|
31
|
+
type: mongoose_1.default.Schema.Types.Mixed,
|
|
32
|
+
},
|
|
33
|
+
metadata: {
|
|
34
|
+
description: "Additional import metadata",
|
|
35
|
+
type: mongoose_1.default.Schema.Types.Mixed,
|
|
36
|
+
},
|
|
37
|
+
origin: {
|
|
38
|
+
description: "How this item entered the dataset",
|
|
39
|
+
enum: ["manual", "synthetic", "trace"],
|
|
40
|
+
required: true,
|
|
41
|
+
type: String,
|
|
42
|
+
},
|
|
43
|
+
outcomeClass: {
|
|
44
|
+
description: "Optional confusion-matrix class for classification features",
|
|
45
|
+
enum: ["fn", "fp", "tn", "tp"],
|
|
46
|
+
type: String,
|
|
47
|
+
},
|
|
48
|
+
proofread: {
|
|
49
|
+
default: false,
|
|
50
|
+
description: "When true, the item is treated as human-annotated",
|
|
51
|
+
type: Boolean,
|
|
52
|
+
},
|
|
53
|
+
sourceTraceId: {
|
|
54
|
+
description: "Trace copied into this item, when origin is trace",
|
|
55
|
+
ref: "ObsTrace",
|
|
56
|
+
type: mongoose_1.default.Schema.Types.ObjectId,
|
|
57
|
+
},
|
|
58
|
+
tags: {
|
|
59
|
+
default: [],
|
|
60
|
+
description: "Item-level tags for filtering experiments",
|
|
61
|
+
type: [String],
|
|
62
|
+
},
|
|
63
|
+
}, { strict: "throw", toJSON: { virtuals: true }, toObject: { virtuals: true } });
|
|
64
|
+
obsDatasetItemSchema.plugin(api_1.createdUpdatedPlugin);
|
|
65
|
+
obsDatasetItemSchema.plugin(api_1.isDeletedPlugin);
|
|
66
|
+
obsDatasetItemSchema.plugin(api_1.findOneOrNone);
|
|
67
|
+
obsDatasetItemSchema.plugin(api_1.findExactlyOne);
|
|
68
|
+
obsDatasetItemSchema.index({ datasetId: 1, origin: 1, proofread: 1 });
|
|
69
|
+
obsDatasetItemSchema.index({ created: -1, datasetId: 1 });
|
|
70
|
+
const registerObsDatasetItem = () => {
|
|
71
|
+
if (mongoose_1.default.models.ObsDatasetItem) {
|
|
72
|
+
return mongoose_1.default.models.ObsDatasetItem;
|
|
73
|
+
}
|
|
74
|
+
return mongoose_1.default.model("ObsDatasetItem", obsDatasetItemSchema);
|
|
75
|
+
};
|
|
76
|
+
exports.registerObsDatasetItem = registerObsDatasetItem;
|