@opensearch-project/agent-health 0.3.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -6
- package/cli/dist/index.js +10072 -4502
- package/deployment/cloudformation/agent-health-observability.yaml +762 -0
- package/dist/assets/index-CCQRDlO0.js +243 -0
- package/dist/assets/index-CNHQVbcj.css +1 -0
- package/dist/index.html +2 -2
- package/docs/ARCHITECTURE.md +450 -0
- package/docs/BACKEND_JOB_QUEUE.md +405 -0
- package/docs/CLAUDE_CODE_TELEMETRY.md +283 -0
- package/docs/CLI.md +431 -0
- package/docs/CODING_AGENT_ANALYTICS.md +298 -0
- package/docs/CONFIGURATION.md +388 -0
- package/docs/CONNECTORS.md +536 -0
- package/docs/INSTRUMENT_WITH_OTEL.md +390 -0
- package/docs/ML-COMMONS-SETUP.md +289 -0
- package/docs/NPX_PACKAGING.md +195 -0
- package/docs/PERFORMANCE-MONITORING.md +200 -0
- package/docs/PERFORMANCE.md +390 -0
- package/docs/PI_PROFILING.md +169 -0
- package/docs/PLAN-non-agui-agent-support.md +525 -0
- package/docs/SDK.md +577 -0
- package/docs/SKILLS.md +264 -0
- package/docs/blogs/2026-02-28-opensearch-agent-health.md +200 -0
- package/docs/blogs/getting-started-blog.md +608 -0
- package/docs/diagrams/Agent-health.excalidraw +5656 -0
- package/docs/diagrams/architecture.png +0 -0
- package/docs/plans/field-redesign.md +468 -0
- package/docs/rfcs/001-coding-agent-analytics.md +374 -0
- package/docs/rfcs/002-enterprise-leaderboard.md +267 -0
- package/docs/rfcs/003-remote-aggregation.md +146 -0
- package/docs/rfcs/004-test-sdk-v2.md +599 -0
- package/docs/skills/AGENT_HEALTH.md +598 -0
- package/docs/skills/AGENT_PROFILE.md +191 -0
- package/docs/skills/add-connector/SKILL.md +68 -0
- package/docs/skills/agent-health-profile/SKILL.md +40 -0
- package/docs/skills/config-auth/SKILL.md +194 -0
- package/docs/skills/config-auth/evals/evals.json +35 -0
- package/docs/skills/create-pr/SKILL.md +73 -0
- package/docs/skills/instrument-otel/SKILL.md +84 -0
- package/docs/skills/write-test/SKILL.md +124 -0
- package/docs/ui prd.md +376 -0
- package/examples/README.md +53 -0
- package/examples/config/agent-health.config.example.ts +155 -0
- package/examples/connectors/echo-connector.ts +131 -0
- package/examples/eval-files/demo.eval.js +128 -0
- package/examples/eval-files/sdk-hooks-demo.eval.js +99 -0
- package/examples/pi-profiling/README.md +77 -0
- package/examples/pi-profiling/agent-health-profile.ts +417 -0
- package/lib/dist/lib/agentUtils.d.ts +29 -0
- package/lib/dist/lib/agentUtils.d.ts.map +1 -0
- package/lib/dist/lib/agentUtils.js +43 -0
- package/lib/dist/lib/agentUtils.js.map +1 -0
- package/lib/dist/lib/benchmarkExport.d.ts +14 -0
- package/lib/dist/lib/benchmarkExport.d.ts.map +1 -0
- package/lib/dist/lib/benchmarkExport.js +41 -0
- package/lib/dist/lib/benchmarkExport.js.map +1 -0
- package/lib/dist/lib/benchmarkVersionUtils.d.ts +37 -0
- package/lib/dist/lib/benchmarkVersionUtils.d.ts.map +1 -0
- package/lib/dist/lib/benchmarkVersionUtils.js +68 -0
- package/lib/dist/lib/benchmarkVersionUtils.js.map +1 -0
- package/lib/dist/lib/config/defineConfig.d.ts +27 -0
- package/lib/dist/lib/config/defineConfig.d.ts.map +1 -0
- package/lib/dist/lib/config/defineConfig.js +28 -0
- package/lib/dist/lib/config/defineConfig.js.map +1 -0
- package/lib/dist/lib/config/index.d.ts +9 -0
- package/lib/dist/lib/config/index.d.ts.map +1 -0
- package/lib/dist/lib/config/index.js +8 -0
- package/lib/dist/lib/config/index.js.map +1 -0
- package/lib/dist/lib/config/loader.d.ts +39 -0
- package/lib/dist/lib/config/loader.d.ts.map +1 -0
- package/lib/dist/lib/config/loader.js +258 -0
- package/lib/dist/lib/config/loader.js.map +1 -0
- package/lib/dist/lib/config/statePaths.d.ts +61 -0
- package/lib/dist/lib/config/statePaths.d.ts.map +1 -0
- package/lib/dist/lib/config/statePaths.js +188 -0
- package/lib/dist/lib/config/statePaths.js.map +1 -0
- package/lib/dist/lib/config/types.d.ts +231 -0
- package/lib/dist/lib/config/types.d.ts.map +1 -0
- package/lib/dist/lib/config/types.js +6 -0
- package/lib/dist/lib/config/types.js.map +1 -0
- package/lib/dist/lib/config.d.ts +39 -0
- package/lib/dist/lib/config.d.ts.map +1 -0
- package/lib/dist/lib/config.js +118 -0
- package/lib/dist/lib/config.js.map +1 -0
- package/lib/dist/lib/constants.d.ts +70 -0
- package/lib/dist/lib/constants.d.ts.map +1 -0
- package/lib/dist/lib/constants.js +365 -0
- package/lib/dist/lib/constants.js.map +1 -0
- package/lib/dist/lib/contextUtilization.d.ts +23 -0
- package/lib/dist/lib/contextUtilization.d.ts.map +1 -0
- package/lib/dist/lib/contextUtilization.js +72 -0
- package/lib/dist/lib/contextUtilization.js.map +1 -0
- package/lib/dist/lib/dashboardMetrics.d.ts +87 -0
- package/lib/dist/lib/dashboardMetrics.d.ts.map +1 -0
- package/lib/dist/lib/dashboardMetrics.js +242 -0
- package/lib/dist/lib/dashboardMetrics.js.map +1 -0
- package/lib/dist/lib/dataSourceConfig.d.ts +108 -0
- package/lib/dist/lib/dataSourceConfig.d.ts.map +1 -0
- package/lib/dist/lib/dataSourceConfig.js +166 -0
- package/lib/dist/lib/dataSourceConfig.js.map +1 -0
- package/lib/dist/lib/debug.d.ts +26 -0
- package/lib/dist/lib/debug.d.ts.map +1 -0
- package/lib/dist/lib/debug.js +132 -0
- package/lib/dist/lib/debug.js.map +1 -0
- package/lib/dist/lib/diagnostics.d.ts +28 -0
- package/lib/dist/lib/diagnostics.d.ts.map +1 -0
- package/lib/dist/lib/diagnostics.js +65 -0
- package/lib/dist/lib/diagnostics.js.map +1 -0
- package/lib/dist/lib/envCompat.d.ts +27 -0
- package/lib/dist/lib/envCompat.d.ts.map +1 -0
- package/lib/dist/lib/envCompat.js +73 -0
- package/lib/dist/lib/envCompat.js.map +1 -0
- package/lib/dist/lib/findPackageRoot.d.ts +7 -0
- package/lib/dist/lib/findPackageRoot.d.ts.map +1 -0
- package/lib/dist/lib/findPackageRoot.js +57 -0
- package/lib/dist/lib/findPackageRoot.js.map +1 -0
- package/lib/dist/lib/hooks.d.ts +36 -0
- package/lib/dist/lib/hooks.d.ts.map +1 -0
- package/lib/dist/lib/hooks.js +112 -0
- package/lib/dist/lib/hooks.js.map +1 -0
- package/lib/dist/lib/index.d.ts +47 -0
- package/lib/dist/lib/index.d.ts.map +1 -0
- package/lib/dist/lib/index.js +62 -0
- package/lib/dist/lib/index.js.map +1 -0
- package/lib/dist/lib/labels.d.ts +90 -0
- package/lib/dist/lib/labels.d.ts.map +1 -0
- package/lib/dist/lib/labels.js +158 -0
- package/lib/dist/lib/labels.js.map +1 -0
- package/lib/dist/lib/markdown.d.ts +16 -0
- package/lib/dist/lib/markdown.d.ts.map +1 -0
- package/lib/dist/lib/markdown.js +42 -0
- package/lib/dist/lib/markdown.js.map +1 -0
- package/lib/dist/lib/matchers/expect.d.ts +3 -0
- package/lib/dist/lib/matchers/expect.d.ts.map +1 -0
- package/lib/dist/lib/matchers/expect.js +225 -0
- package/lib/dist/lib/matchers/expect.js.map +1 -0
- package/lib/dist/lib/matchers/index.d.ts +8 -0
- package/lib/dist/lib/matchers/index.d.ts.map +1 -0
- package/lib/dist/lib/matchers/index.js +9 -0
- package/lib/dist/lib/matchers/index.js.map +1 -0
- package/lib/dist/lib/matchers/judgeAccessor.d.ts +113 -0
- package/lib/dist/lib/matchers/judgeAccessor.d.ts.map +1 -0
- package/lib/dist/lib/matchers/judgeAccessor.js +183 -0
- package/lib/dist/lib/matchers/judgeAccessor.js.map +1 -0
- package/lib/dist/lib/matchers/session.d.ts +39 -0
- package/lib/dist/lib/matchers/session.d.ts.map +1 -0
- package/lib/dist/lib/matchers/session.js +116 -0
- package/lib/dist/lib/matchers/session.js.map +1 -0
- package/lib/dist/lib/matchers/traces.d.ts +55 -0
- package/lib/dist/lib/matchers/traces.d.ts.map +1 -0
- package/lib/dist/lib/matchers/traces.js +116 -0
- package/lib/dist/lib/matchers/traces.js.map +1 -0
- package/lib/dist/lib/matchers/types.d.ts +75 -0
- package/lib/dist/lib/matchers/types.d.ts.map +1 -0
- package/lib/dist/lib/matchers/types.js +6 -0
- package/lib/dist/lib/matchers/types.js.map +1 -0
- package/lib/dist/lib/packagePaths.d.ts +29 -0
- package/lib/dist/lib/packagePaths.d.ts.map +1 -0
- package/lib/dist/lib/packagePaths.js +63 -0
- package/lib/dist/lib/packagePaths.js.map +1 -0
- package/lib/dist/lib/performance.d.ts +51 -0
- package/lib/dist/lib/performance.d.ts.map +1 -0
- package/lib/dist/lib/performance.js +159 -0
- package/lib/dist/lib/performance.js.map +1 -0
- package/lib/dist/lib/portConfig.d.ts +29 -0
- package/lib/dist/lib/portConfig.d.ts.map +1 -0
- package/lib/dist/lib/portConfig.js +64 -0
- package/lib/dist/lib/portConfig.js.map +1 -0
- package/lib/dist/lib/preferences.d.ts +63 -0
- package/lib/dist/lib/preferences.d.ts.map +1 -0
- package/lib/dist/lib/preferences.js +117 -0
- package/lib/dist/lib/preferences.js.map +1 -0
- package/lib/dist/lib/resolveAgentModel.d.ts +22 -0
- package/lib/dist/lib/resolveAgentModel.d.ts.map +1 -0
- package/lib/dist/lib/resolveAgentModel.js +37 -0
- package/lib/dist/lib/resolveAgentModel.js.map +1 -0
- package/lib/dist/lib/runStats.d.ts +92 -0
- package/lib/dist/lib/runStats.d.ts.map +1 -0
- package/lib/dist/lib/runStats.js +160 -0
- package/lib/dist/lib/runStats.js.map +1 -0
- package/lib/dist/lib/telemetry/constants.d.ts +60 -0
- package/lib/dist/lib/telemetry/constants.d.ts.map +1 -0
- package/lib/dist/lib/telemetry/constants.js +87 -0
- package/lib/dist/lib/telemetry/constants.js.map +1 -0
- package/lib/dist/lib/telemetry/evalSpans.d.ts +61 -0
- package/lib/dist/lib/telemetry/evalSpans.d.ts.map +1 -0
- package/lib/dist/lib/telemetry/evalSpans.js +254 -0
- package/lib/dist/lib/telemetry/evalSpans.js.map +1 -0
- package/lib/dist/lib/telemetry/index.d.ts +11 -0
- package/lib/dist/lib/telemetry/index.d.ts.map +1 -0
- package/lib/dist/lib/telemetry/index.js +15 -0
- package/lib/dist/lib/telemetry/index.js.map +1 -0
- package/lib/dist/lib/telemetry/opensearchExporter.d.ts +43 -0
- package/lib/dist/lib/telemetry/opensearchExporter.d.ts.map +1 -0
- package/lib/dist/lib/telemetry/opensearchExporter.js +217 -0
- package/lib/dist/lib/telemetry/opensearchExporter.js.map +1 -0
- package/lib/dist/lib/telemetry/provider.d.ts +55 -0
- package/lib/dist/lib/telemetry/provider.d.ts.map +1 -0
- package/lib/dist/lib/telemetry/provider.js +140 -0
- package/lib/dist/lib/telemetry/provider.js.map +1 -0
- package/lib/dist/lib/testCaseLabels.d.ts +34 -0
- package/lib/dist/lib/testCaseLabels.d.ts.map +1 -0
- package/lib/dist/lib/testCaseLabels.js +88 -0
- package/lib/dist/lib/testCaseLabels.js.map +1 -0
- package/lib/dist/lib/testCaseValidation.d.ts +140 -0
- package/lib/dist/lib/testCaseValidation.d.ts.map +1 -0
- package/lib/dist/lib/testCaseValidation.js +162 -0
- package/lib/dist/lib/testCaseValidation.js.map +1 -0
- package/lib/dist/lib/testCases/agentFixture.d.ts +80 -0
- package/lib/dist/lib/testCases/agentFixture.d.ts.map +1 -0
- package/lib/dist/lib/testCases/agentFixture.js +43 -0
- package/lib/dist/lib/testCases/agentFixture.js.map +1 -0
- package/lib/dist/lib/testCases/authoringSurface.d.ts +10 -0
- package/lib/dist/lib/testCases/authoringSurface.d.ts.map +1 -0
- package/lib/dist/lib/testCases/authoringSurface.js +54 -0
- package/lib/dist/lib/testCases/authoringSurface.js.map +1 -0
- package/lib/dist/lib/testCases/codemod.d.ts +13 -0
- package/lib/dist/lib/testCases/codemod.d.ts.map +1 -0
- package/lib/dist/lib/testCases/codemod.js +169 -0
- package/lib/dist/lib/testCases/codemod.js.map +1 -0
- package/lib/dist/lib/testCases/define.d.ts +114 -0
- package/lib/dist/lib/testCases/define.d.ts.map +1 -0
- package/lib/dist/lib/testCases/define.js +253 -0
- package/lib/dist/lib/testCases/define.js.map +1 -0
- package/lib/dist/lib/testCases/evaluators.d.ts +80 -0
- package/lib/dist/lib/testCases/evaluators.d.ts.map +1 -0
- package/lib/dist/lib/testCases/evaluators.js +105 -0
- package/lib/dist/lib/testCases/evaluators.js.map +1 -0
- package/lib/dist/lib/testCases/index.d.ts +14 -0
- package/lib/dist/lib/testCases/index.d.ts.map +1 -0
- package/lib/dist/lib/testCases/index.js +12 -0
- package/lib/dist/lib/testCases/index.js.map +1 -0
- package/lib/dist/lib/testCases/judge.d.ts +165 -0
- package/lib/dist/lib/testCases/judge.d.ts.map +1 -0
- package/lib/dist/lib/testCases/judge.js +359 -0
- package/lib/dist/lib/testCases/judge.js.map +1 -0
- package/lib/dist/lib/testCases/loader.d.ts +26 -0
- package/lib/dist/lib/testCases/loader.d.ts.map +1 -0
- package/lib/dist/lib/testCases/loader.js +149 -0
- package/lib/dist/lib/testCases/loader.js.map +1 -0
- package/lib/dist/lib/testCases/types.d.ts +242 -0
- package/lib/dist/lib/testCases/types.d.ts.map +1 -0
- package/lib/dist/lib/testCases/types.js +6 -0
- package/lib/dist/lib/testCases/types.js.map +1 -0
- package/lib/dist/lib/theme.d.ts +6 -0
- package/lib/dist/lib/theme.d.ts.map +1 -0
- package/lib/dist/lib/theme.js +36 -0
- package/lib/dist/lib/theme.js.map +1 -0
- package/lib/dist/lib/uiTelemetry.d.ts +7 -0
- package/lib/dist/lib/uiTelemetry.d.ts.map +1 -0
- package/lib/dist/lib/uiTelemetry.js +25 -0
- package/lib/dist/lib/uiTelemetry.js.map +1 -0
- package/lib/dist/lib/utils.d.ts +96 -0
- package/lib/dist/lib/utils.d.ts.map +1 -0
- package/lib/dist/lib/utils.js +232 -0
- package/lib/dist/lib/utils.js.map +1 -0
- package/lib/dist/lib/workflow/consolidate.d.ts +12 -0
- package/lib/dist/lib/workflow/consolidate.d.ts.map +1 -0
- package/lib/dist/lib/workflow/consolidate.js +33 -0
- package/lib/dist/lib/workflow/consolidate.js.map +1 -0
- package/lib/dist/lib/workflow/index.d.ts +13 -0
- package/lib/dist/lib/workflow/index.d.ts.map +1 -0
- package/lib/dist/lib/workflow/index.js +12 -0
- package/lib/dist/lib/workflow/index.js.map +1 -0
- package/lib/dist/lib/workflow/ledger.d.ts +30 -0
- package/lib/dist/lib/workflow/ledger.d.ts.map +1 -0
- package/lib/dist/lib/workflow/ledger.js +41 -0
- package/lib/dist/lib/workflow/ledger.js.map +1 -0
- package/lib/dist/lib/workflow/pool.d.ts +13 -0
- package/lib/dist/lib/workflow/pool.d.ts.map +1 -0
- package/lib/dist/lib/workflow/pool.js +44 -0
- package/lib/dist/lib/workflow/pool.js.map +1 -0
- package/lib/dist/lib/workflow/source.d.ts +22 -0
- package/lib/dist/lib/workflow/source.d.ts.map +1 -0
- package/lib/dist/lib/workflow/source.js +29 -0
- package/lib/dist/lib/workflow/source.js.map +1 -0
- package/lib/dist/lib/workflow/stepB.d.ts +71 -0
- package/lib/dist/lib/workflow/stepB.d.ts.map +1 -0
- package/lib/dist/lib/workflow/stepB.js +99 -0
- package/lib/dist/lib/workflow/stepB.js.map +1 -0
- package/lib/dist/lib/workflow/types.d.ts +86 -0
- package/lib/dist/lib/workflow/types.d.ts.map +1 -0
- package/lib/dist/lib/workflow/types.js +6 -0
- package/lib/dist/lib/workflow/types.js.map +1 -0
- package/lib/dist/lib/workflow/workflow.d.ts +119 -0
- package/lib/dist/lib/workflow/workflow.d.ts.map +1 -0
- package/lib/dist/lib/workflow/workflow.js +195 -0
- package/lib/dist/lib/workflow/workflow.js.map +1 -0
- package/lib/dist/services/agent/aguiConverter.d.ts +50 -0
- package/lib/dist/services/agent/aguiConverter.d.ts.map +1 -0
- package/lib/dist/services/agent/aguiConverter.js +449 -0
- package/lib/dist/services/agent/aguiConverter.js.map +1 -0
- package/lib/dist/services/agent/index.d.ts +10 -0
- package/lib/dist/services/agent/index.d.ts.map +1 -0
- package/lib/dist/services/agent/index.js +12 -0
- package/lib/dist/services/agent/index.js.map +1 -0
- package/lib/dist/services/agent/payloadBuilder.d.ts +33 -0
- package/lib/dist/services/agent/payloadBuilder.d.ts.map +1 -0
- package/lib/dist/services/agent/payloadBuilder.js +75 -0
- package/lib/dist/services/agent/payloadBuilder.js.map +1 -0
- package/lib/dist/services/agent/sseStream.d.ts +43 -0
- package/lib/dist/services/agent/sseStream.d.ts.map +1 -0
- package/lib/dist/services/agent/sseStream.js +223 -0
- package/lib/dist/services/agent/sseStream.js.map +1 -0
- package/lib/dist/services/connectors/agui/AGUIStreamingConnector.d.ts +44 -0
- package/lib/dist/services/connectors/agui/AGUIStreamingConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/agui/AGUIStreamingConnector.js +95 -0
- package/lib/dist/services/connectors/agui/AGUIStreamingConnector.js.map +1 -0
- package/lib/dist/services/connectors/base/BaseConnector.d.ts +81 -0
- package/lib/dist/services/connectors/base/BaseConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/base/BaseConnector.js +170 -0
- package/lib/dist/services/connectors/base/BaseConnector.js.map +1 -0
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +116 -0
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +403 -0
- package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -0
- package/lib/dist/services/connectors/index.d.ts +13 -0
- package/lib/dist/services/connectors/index.d.ts.map +1 -0
- package/lib/dist/services/connectors/index.js +32 -0
- package/lib/dist/services/connectors/index.js.map +1 -0
- package/lib/dist/services/connectors/kiro/KiroConnector.d.ts +48 -0
- package/lib/dist/services/connectors/kiro/KiroConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/kiro/KiroConnector.js +158 -0
- package/lib/dist/services/connectors/kiro/KiroConnector.js.map +1 -0
- package/lib/dist/services/connectors/langgraph/LangGraphConnector.d.ts +36 -0
- package/lib/dist/services/connectors/langgraph/LangGraphConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/langgraph/LangGraphConnector.js +175 -0
- package/lib/dist/services/connectors/langgraph/LangGraphConnector.js.map +1 -0
- package/lib/dist/services/connectors/mock/MockConnector.d.ts +37 -0
- package/lib/dist/services/connectors/mock/MockConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/mock/MockConnector.js +120 -0
- package/lib/dist/services/connectors/mock/MockConnector.js.map +1 -0
- package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.d.ts +42 -0
- package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.js +133 -0
- package/lib/dist/services/connectors/openai-compatible/OpenAICompatibleConnector.js.map +1 -0
- package/lib/dist/services/connectors/pi/PiConnector.d.ts +87 -0
- package/lib/dist/services/connectors/pi/PiConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/pi/PiConnector.js +274 -0
- package/lib/dist/services/connectors/pi/PiConnector.js.map +1 -0
- package/lib/dist/services/connectors/registry.d.ts +57 -0
- package/lib/dist/services/connectors/registry.d.ts.map +1 -0
- package/lib/dist/services/connectors/registry.js +106 -0
- package/lib/dist/services/connectors/registry.js.map +1 -0
- package/lib/dist/services/connectors/rest/RESTConnector.d.ts +38 -0
- package/lib/dist/services/connectors/rest/RESTConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/rest/RESTConnector.js +117 -0
- package/lib/dist/services/connectors/rest/RESTConnector.js.map +1 -0
- package/lib/dist/services/connectors/server.d.ts +13 -0
- package/lib/dist/services/connectors/server.d.ts.map +1 -0
- package/lib/dist/services/connectors/server.js +34 -0
- package/lib/dist/services/connectors/server.js.map +1 -0
- package/lib/dist/services/connectors/strands/StrandsConnector.d.ts +48 -0
- package/lib/dist/services/connectors/strands/StrandsConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/strands/StrandsConnector.js +221 -0
- package/lib/dist/services/connectors/strands/StrandsConnector.js.map +1 -0
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts +88 -0
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -0
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +418 -0
- package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -0
- package/lib/dist/services/connectors/types.d.ts +213 -0
- package/lib/dist/services/connectors/types.d.ts.map +1 -0
- package/lib/dist/services/connectors/types.js +6 -0
- package/lib/dist/services/connectors/types.js.map +1 -0
- package/lib/dist/services/evaluation/bedrockJudge.d.ts +64 -0
- package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -0
- package/lib/dist/services/evaluation/bedrockJudge.js +167 -0
- package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -0
- package/lib/dist/services/evaluation/evaluatorError.d.ts +56 -0
- package/lib/dist/services/evaluation/evaluatorError.d.ts.map +1 -0
- package/lib/dist/services/evaluation/evaluatorError.js +56 -0
- package/lib/dist/services/evaluation/evaluatorError.js.map +1 -0
- package/lib/dist/services/evaluation/index.d.ts +106 -0
- package/lib/dist/services/evaluation/index.d.ts.map +1 -0
- package/lib/dist/services/evaluation/index.js +684 -0
- package/lib/dist/services/evaluation/index.js.map +1 -0
- package/lib/dist/services/evaluation/mockTrajectory.d.ts +3 -0
- package/lib/dist/services/evaluation/mockTrajectory.d.ts.map +1 -0
- package/lib/dist/services/evaluation/mockTrajectory.js +72 -0
- package/lib/dist/services/evaluation/mockTrajectory.js.map +1 -0
- package/lib/dist/services/opensearch/client.d.ts +26 -0
- package/lib/dist/services/opensearch/client.d.ts.map +1 -0
- package/lib/dist/services/opensearch/client.js +131 -0
- package/lib/dist/services/opensearch/client.js.map +1 -0
- package/lib/dist/services/opensearch/index.d.ts +16 -0
- package/lib/dist/services/opensearch/index.d.ts.map +1 -0
- package/lib/dist/services/opensearch/index.js +25 -0
- package/lib/dist/services/opensearch/index.js.map +1 -0
- package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts +123 -0
- package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -0
- package/lib/dist/services/storage/asyncBenchmarkStorage.js +429 -0
- package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -0
- package/lib/dist/services/storage/asyncRunStorage.d.ts +127 -0
- package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -0
- package/lib/dist/services/storage/asyncRunStorage.js +448 -0
- package/lib/dist/services/storage/asyncRunStorage.js.map +1 -0
- package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +156 -0
- package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -0
- package/lib/dist/services/storage/asyncTestCaseStorage.js +285 -0
- package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -0
- package/lib/dist/services/storage/index.d.ts +17 -0
- package/lib/dist/services/storage/index.d.ts.map +1 -0
- package/lib/dist/services/storage/index.js +20 -0
- package/lib/dist/services/storage/index.js.map +1 -0
- package/lib/dist/services/storage/migration.d.ts +54 -0
- package/lib/dist/services/storage/migration.d.ts.map +1 -0
- package/lib/dist/services/storage/migration.js +296 -0
- package/lib/dist/services/storage/migration.js.map +1 -0
- package/lib/dist/services/storage/opensearchClient.d.ts +924 -0
- package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -0
- package/lib/dist/services/storage/opensearchClient.js +435 -0
- package/lib/dist/services/storage/opensearchClient.js.map +1 -0
- package/lib/dist/services/traces/browserRecovery.d.ts +26 -0
- package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -0
- package/lib/dist/services/traces/browserRecovery.js +81 -0
- package/lib/dist/services/traces/browserRecovery.js.map +1 -0
- package/lib/dist/services/traces/categoryStyles.d.ts +21 -0
- package/lib/dist/services/traces/categoryStyles.d.ts.map +1 -0
- package/lib/dist/services/traces/categoryStyles.js +56 -0
- package/lib/dist/services/traces/categoryStyles.js.map +1 -0
- package/lib/dist/services/traces/executionOrderTransform.d.ts +35 -0
- package/lib/dist/services/traces/executionOrderTransform.d.ts.map +1 -0
- package/lib/dist/services/traces/executionOrderTransform.js +313 -0
- package/lib/dist/services/traces/executionOrderTransform.js.map +1 -0
- package/lib/dist/services/traces/fetchSpansForRun.d.ts +86 -0
- package/lib/dist/services/traces/fetchSpansForRun.d.ts.map +1 -0
- package/lib/dist/services/traces/fetchSpansForRun.js +69 -0
- package/lib/dist/services/traces/fetchSpansForRun.js.map +1 -0
- package/lib/dist/services/traces/flowTransform.d.ts +24 -0
- package/lib/dist/services/traces/flowTransform.d.ts.map +1 -0
- package/lib/dist/services/traces/flowTransform.js +228 -0
- package/lib/dist/services/traces/flowTransform.js.map +1 -0
- package/lib/dist/services/traces/index.d.ts +121 -0
- package/lib/dist/services/traces/index.d.ts.map +1 -0
- package/lib/dist/services/traces/index.js +255 -0
- package/lib/dist/services/traces/index.js.map +1 -0
- package/lib/dist/services/traces/intentTransform.d.ts +20 -0
- package/lib/dist/services/traces/intentTransform.d.ts.map +1 -0
- package/lib/dist/services/traces/intentTransform.js +131 -0
- package/lib/dist/services/traces/intentTransform.js.map +1 -0
- package/lib/dist/services/traces/judgeAgentsHints.d.ts +63 -0
- package/lib/dist/services/traces/judgeAgentsHints.d.ts.map +1 -0
- package/lib/dist/services/traces/judgeAgentsHints.js +89 -0
- package/lib/dist/services/traces/judgeAgentsHints.js.map +1 -0
- package/lib/dist/services/traces/messageExtraction.d.ts +15 -0
- package/lib/dist/services/traces/messageExtraction.d.ts.map +1 -0
- package/lib/dist/services/traces/messageExtraction.js +251 -0
- package/lib/dist/services/traces/messageExtraction.js.map +1 -0
- package/lib/dist/services/traces/spanCategorization.d.ts +63 -0
- package/lib/dist/services/traces/spanCategorization.d.ts.map +1 -0
- package/lib/dist/services/traces/spanCategorization.js +276 -0
- package/lib/dist/services/traces/spanCategorization.js.map +1 -0
- package/lib/dist/services/traces/spanPreprocessing.d.ts +37 -0
- package/lib/dist/services/traces/spanPreprocessing.d.ts.map +1 -0
- package/lib/dist/services/traces/spanPreprocessing.js +102 -0
- package/lib/dist/services/traces/spanPreprocessing.js.map +1 -0
- package/lib/dist/services/traces/spansToTrajectory.d.ts +36 -0
- package/lib/dist/services/traces/spansToTrajectory.d.ts.map +1 -0
- package/lib/dist/services/traces/spansToTrajectory.js +387 -0
- package/lib/dist/services/traces/spansToTrajectory.js.map +1 -0
- package/lib/dist/services/traces/toolSimilarity.d.ts +35 -0
- package/lib/dist/services/traces/toolSimilarity.d.ts.map +1 -0
- package/lib/dist/services/traces/toolSimilarity.js +203 -0
- package/lib/dist/services/traces/toolSimilarity.js.map +1 -0
- package/lib/dist/services/traces/traceComparison.d.ts +31 -0
- package/lib/dist/services/traces/traceComparison.d.ts.map +1 -0
- package/lib/dist/services/traces/traceComparison.js +318 -0
- package/lib/dist/services/traces/traceComparison.js.map +1 -0
- package/lib/dist/services/traces/traceGrouping.d.ts +19 -0
- package/lib/dist/services/traces/traceGrouping.d.ts.map +1 -0
- package/lib/dist/services/traces/traceGrouping.js +107 -0
- package/lib/dist/services/traces/traceGrouping.js.map +1 -0
- package/lib/dist/services/traces/tracePoller.d.ts +84 -0
- package/lib/dist/services/traces/tracePoller.d.ts.map +1 -0
- package/lib/dist/services/traces/tracePoller.js +309 -0
- package/lib/dist/services/traces/tracePoller.js.map +1 -0
- package/lib/dist/services/traces/traceStats.d.ts +45 -0
- package/lib/dist/services/traces/traceStats.d.ts.map +1 -0
- package/lib/dist/services/traces/traceStats.js +114 -0
- package/lib/dist/services/traces/traceStats.js.map +1 -0
- package/lib/dist/services/traces/traceSummary.d.ts +47 -0
- package/lib/dist/services/traces/traceSummary.d.ts.map +1 -0
- package/lib/dist/services/traces/traceSummary.js +68 -0
- package/lib/dist/services/traces/traceSummary.js.map +1 -0
- package/lib/dist/services/traces/utils.d.ts +33 -0
- package/lib/dist/services/traces/utils.d.ts.map +1 -0
- package/lib/dist/services/traces/utils.js +114 -0
- package/lib/dist/services/traces/utils.js.map +1 -0
- package/lib/dist/types/agui.d.ts +13 -0
- package/lib/dist/types/agui.d.ts.map +1 -0
- package/lib/dist/types/agui.js +16 -0
- package/lib/dist/types/agui.js.map +1 -0
- package/lib/dist/types/index.d.ts +1175 -0
- package/lib/dist/types/index.d.ts.map +1 -0
- package/lib/dist/types/index.js +12 -0
- package/lib/dist/types/index.js.map +1 -0
- package/lib/dist/types/skills.d.ts +146 -0
- package/lib/dist/types/skills.d.ts.map +1 -0
- package/lib/dist/types/skills.js +6 -0
- package/lib/dist/types/skills.js.map +1 -0
- package/observio-sample-agent/pi-package/README.md +112 -0
- package/observio-sample-agent/pi-package/extensions/agent-health.ts +373 -0
- package/observio-sample-agent/pi-package/package.json +17 -0
- package/observio-sample-agent/pi-package/prompts/agent-health.md +37 -0
- package/observio-sample-agent/pi-package/skills/create-pr/SKILL.md +88 -0
- package/observio-sample-agent/pi-package/skills/fix-bug/SKILL.md +71 -0
- package/observio-sample-agent/pi-package/skills/implement-feature/SKILL.md +156 -0
- package/observio-sample-agent/pi-package/skills/instrument-otel/SKILL.md +208 -0
- package/observio-sample-agent/pi-package/skills/setup-collector/SKILL.md +146 -0
- package/observio-sample-agent/pi-package/skills/write-test/SKILL.md +115 -0
- package/package.json +64 -13
- package/server/dist/app.js +32651 -17637
- package/server/dist/index.js +29875 -14638
- package/tsconfig.lib.json +71 -0
- package/dist/assets/index-EvPLSTAS.js +0 -267
- package/dist/assets/index-RXasQKUs.css +0 -1
- package/lib/dist/config/index.js +0 -404
- package/lib/dist/index.js +0 -1665
|
@@ -0,0 +1,924 @@
|
|
|
1
|
+
export interface StorageTestCase {
|
|
2
|
+
id: string;
|
|
3
|
+
name: string;
|
|
4
|
+
description?: string;
|
|
5
|
+
version: number;
|
|
6
|
+
initialPrompt: string;
|
|
7
|
+
tools?: unknown[];
|
|
8
|
+
messages?: unknown[];
|
|
9
|
+
context?: unknown[];
|
|
10
|
+
forwardedProps?: Record<string, unknown>;
|
|
11
|
+
expectedPPL?: string;
|
|
12
|
+
expectedOutcomes?: string[];
|
|
13
|
+
expectedTrajectory?: unknown[];
|
|
14
|
+
labels?: string[];
|
|
15
|
+
category?: string;
|
|
16
|
+
subcategory?: string;
|
|
17
|
+
difficulty?: 'Easy' | 'Medium' | 'Hard';
|
|
18
|
+
tags?: string[];
|
|
19
|
+
author?: string;
|
|
20
|
+
createdAt: string;
|
|
21
|
+
updatedAt: string;
|
|
22
|
+
lastRunAt?: string;
|
|
23
|
+
/** Source provenance for SDK / code-imported test cases. Used by the
|
|
24
|
+
* CollapsibleTestCaseDefinition UI to distinguish SDK tests (path is
|
|
25
|
+
* the source of truth) from JSON tests (the row itself is the source
|
|
26
|
+
* of truth). Already round-trips through OpenSearch — the storage
|
|
27
|
+
* layer was just dropping it on read. */
|
|
28
|
+
sourceFile?: string;
|
|
29
|
+
sourceHash?: string;
|
|
30
|
+
}
|
|
31
|
+
export interface StorageBenchmarkRunConfig {
|
|
32
|
+
id: string;
|
|
33
|
+
name: string;
|
|
34
|
+
description?: string;
|
|
35
|
+
agentKey: string;
|
|
36
|
+
agentId?: string;
|
|
37
|
+
modelId: string;
|
|
38
|
+
headers?: Record<string, string>;
|
|
39
|
+
iterationCount?: number;
|
|
40
|
+
createdAt: string;
|
|
41
|
+
results?: Record<string, {
|
|
42
|
+
reportId: string;
|
|
43
|
+
status: string;
|
|
44
|
+
error?: string;
|
|
45
|
+
}>;
|
|
46
|
+
status?: string;
|
|
47
|
+
error?: string;
|
|
48
|
+
stats?: {
|
|
49
|
+
passed: number;
|
|
50
|
+
failed: number;
|
|
51
|
+
pending: number;
|
|
52
|
+
total: number;
|
|
53
|
+
errored?: number;
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
export interface StorageBenchmark {
|
|
57
|
+
id: string;
|
|
58
|
+
name: string;
|
|
59
|
+
description?: string;
|
|
60
|
+
author?: string;
|
|
61
|
+
createdAt: string;
|
|
62
|
+
llmJudgePrompt?: string;
|
|
63
|
+
testCaseIds: string[];
|
|
64
|
+
runs: StorageBenchmarkRunConfig[];
|
|
65
|
+
}
|
|
66
|
+
/** @deprecated Use StorageBenchmarkRunConfig instead */
|
|
67
|
+
export type StorageExperimentRunConfig = StorageBenchmarkRunConfig;
|
|
68
|
+
/** @deprecated Use StorageBenchmark instead */
|
|
69
|
+
export type StorageExperiment = StorageBenchmark;
|
|
70
|
+
export interface StorageRunAnnotation {
|
|
71
|
+
id: string;
|
|
72
|
+
text: string;
|
|
73
|
+
tags?: string[];
|
|
74
|
+
author?: string;
|
|
75
|
+
createdAt: string;
|
|
76
|
+
updatedAt: string;
|
|
77
|
+
}
|
|
78
|
+
export interface StorageRun {
|
|
79
|
+
id: string;
|
|
80
|
+
name?: string;
|
|
81
|
+
description?: string;
|
|
82
|
+
experimentId: string;
|
|
83
|
+
experimentRunId: string;
|
|
84
|
+
testCaseId: string;
|
|
85
|
+
testCaseVersionId: string;
|
|
86
|
+
agentId: string;
|
|
87
|
+
modelId: string;
|
|
88
|
+
iteration: number;
|
|
89
|
+
author?: string;
|
|
90
|
+
createdAt: string;
|
|
91
|
+
status: 'running' | 'completed' | 'failed';
|
|
92
|
+
passFailStatus?: 'passed' | 'failed';
|
|
93
|
+
traceId?: string;
|
|
94
|
+
tags?: string[];
|
|
95
|
+
actualOutcomes?: unknown[];
|
|
96
|
+
llmJudgeReasoning?: string;
|
|
97
|
+
metrics?: {
|
|
98
|
+
accuracy?: number;
|
|
99
|
+
faithfulness?: number;
|
|
100
|
+
latency_score?: number;
|
|
101
|
+
trajectory_alignment_score?: number;
|
|
102
|
+
};
|
|
103
|
+
annotations?: StorageRunAnnotation[];
|
|
104
|
+
trajectory?: unknown[];
|
|
105
|
+
rawEvents?: unknown[];
|
|
106
|
+
logs?: unknown[];
|
|
107
|
+
improvementStrategies?: {
|
|
108
|
+
category: string;
|
|
109
|
+
issue: string;
|
|
110
|
+
recommendation: string;
|
|
111
|
+
priority: 'high' | 'medium' | 'low';
|
|
112
|
+
}[];
|
|
113
|
+
connectorProtocol?: string;
|
|
114
|
+
}
|
|
115
|
+
export interface StorageAnalyticsRecord {
|
|
116
|
+
analyticsId: string;
|
|
117
|
+
runId: string;
|
|
118
|
+
experimentId: string;
|
|
119
|
+
experimentRunId: string;
|
|
120
|
+
testCaseId: string;
|
|
121
|
+
testCaseVersionId?: string;
|
|
122
|
+
traceId?: string;
|
|
123
|
+
experimentName?: string;
|
|
124
|
+
testCaseName?: string;
|
|
125
|
+
testCaseCategory?: string;
|
|
126
|
+
testCaseDifficulty?: string;
|
|
127
|
+
agentId: string;
|
|
128
|
+
modelId: string;
|
|
129
|
+
iteration: number;
|
|
130
|
+
tags?: string[];
|
|
131
|
+
passFailStatus?: string;
|
|
132
|
+
status?: string;
|
|
133
|
+
createdAt: string;
|
|
134
|
+
author?: string;
|
|
135
|
+
[key: string]: unknown;
|
|
136
|
+
}
|
|
137
|
+
export declare const storageAdmin: {
|
|
138
|
+
/**
|
|
139
|
+
* Check storage health/connectivity
|
|
140
|
+
*/
|
|
141
|
+
health(): Promise<{
|
|
142
|
+
status: string;
|
|
143
|
+
cluster?: unknown;
|
|
144
|
+
error?: string;
|
|
145
|
+
/** Active storage backend. 'file' means OpenSearch is unavailable / not configured; 'error' means configured-but-unreachable. */
|
|
146
|
+
backend?: "file" | "opensearch" | "error";
|
|
147
|
+
/**
|
|
148
|
+
* Real OpenSearch connectivity, present when OpenSearch is configured but
|
|
149
|
+
* the active backend fell back to file storage. Top-level `status` reflects
|
|
150
|
+
* the active (possibly file) backend, NOT OpenSearch — use this to report
|
|
151
|
+
* true OpenSearch connectivity. Shape mirrors testStorageConnection().
|
|
152
|
+
*/
|
|
153
|
+
opensearch?: {
|
|
154
|
+
status: string;
|
|
155
|
+
message?: string;
|
|
156
|
+
latencyMs?: number;
|
|
157
|
+
clusterName?: string;
|
|
158
|
+
clusterStatus?: string;
|
|
159
|
+
};
|
|
160
|
+
}>;
|
|
161
|
+
/**
|
|
162
|
+
* Initialize all indexes with mappings
|
|
163
|
+
*/
|
|
164
|
+
initIndexes(): Promise<{
|
|
165
|
+
success: boolean;
|
|
166
|
+
results: Record<string, {
|
|
167
|
+
status: string;
|
|
168
|
+
error?: string;
|
|
169
|
+
}>;
|
|
170
|
+
}>;
|
|
171
|
+
/**
|
|
172
|
+
* Get storage statistics
|
|
173
|
+
*/
|
|
174
|
+
stats(): Promise<{
|
|
175
|
+
stats: Record<string, {
|
|
176
|
+
count: number;
|
|
177
|
+
error?: string;
|
|
178
|
+
}>;
|
|
179
|
+
}>;
|
|
180
|
+
};
|
|
181
|
+
export declare const testCaseStorage: {
|
|
182
|
+
/**
|
|
183
|
+
* Get all test cases (latest versions only)
|
|
184
|
+
* @param options.fields - 'summary' for lightweight list-view payload
|
|
185
|
+
* @param options.size - page size for pagination
|
|
186
|
+
* @param options.after - cursor token for next page
|
|
187
|
+
* @param options.includeSample - whether to include sample/demo data
|
|
188
|
+
*/
|
|
189
|
+
getAll(options?: {
|
|
190
|
+
fields?: "summary";
|
|
191
|
+
size?: number;
|
|
192
|
+
after?: string;
|
|
193
|
+
includeSample?: boolean;
|
|
194
|
+
}): Promise<{
|
|
195
|
+
testCases: StorageTestCase[];
|
|
196
|
+
total: number;
|
|
197
|
+
after?: string | null;
|
|
198
|
+
hasMore?: boolean;
|
|
199
|
+
}>;
|
|
200
|
+
/**
|
|
201
|
+
* Get test cases by specific IDs (latest versions only)
|
|
202
|
+
* Used for efficient filtered fetching (e.g., only test cases in a benchmark)
|
|
203
|
+
*/
|
|
204
|
+
getByIds(ids: string[]): Promise<StorageTestCase[]>;
|
|
205
|
+
/**
|
|
206
|
+
* Get test case by ID (latest version)
|
|
207
|
+
*/
|
|
208
|
+
getById(id: string): Promise<StorageTestCase | null>;
|
|
209
|
+
/**
|
|
210
|
+
* Get all versions of a test case
|
|
211
|
+
*/
|
|
212
|
+
getVersions(id: string): Promise<StorageTestCase[]>;
|
|
213
|
+
/**
|
|
214
|
+
* Get specific version of a test case
|
|
215
|
+
*/
|
|
216
|
+
getVersion(id: string, version: number): Promise<StorageTestCase | null>;
|
|
217
|
+
/**
|
|
218
|
+
* Create new test case (starts at version 1)
|
|
219
|
+
*/
|
|
220
|
+
create(testCase: Omit<StorageTestCase, "id" | "version" | "createdAt" | "updatedAt">): Promise<StorageTestCase>;
|
|
221
|
+
/**
|
|
222
|
+
* Update test case (creates new version)
|
|
223
|
+
*/
|
|
224
|
+
update(id: string, testCase: Partial<StorageTestCase>): Promise<StorageTestCase>;
|
|
225
|
+
/**
|
|
226
|
+
* Delete test case (all versions)
|
|
227
|
+
*/
|
|
228
|
+
delete(id: string): Promise<{
|
|
229
|
+
deleted: number;
|
|
230
|
+
}>;
|
|
231
|
+
/**
|
|
232
|
+
* Bulk create test cases
|
|
233
|
+
*/
|
|
234
|
+
bulkCreate(testCases: Partial<StorageTestCase>[]): Promise<{
|
|
235
|
+
created: number;
|
|
236
|
+
errors: boolean;
|
|
237
|
+
}>;
|
|
238
|
+
};
|
|
239
|
+
export declare const benchmarkStorage: {
|
|
240
|
+
/**
|
|
241
|
+
* Get all benchmarks
|
|
242
|
+
* @param options.includeSample - whether to include sample/demo data
|
|
243
|
+
*/
|
|
244
|
+
getAll(options?: {
|
|
245
|
+
includeSample?: boolean;
|
|
246
|
+
}): Promise<StorageBenchmark[]>;
|
|
247
|
+
/**
|
|
248
|
+
* Get benchmark by ID
|
|
249
|
+
* @param options.fields - 'polling' for lightweight payload (excludes versions, testCaseSnapshots, headers)
|
|
250
|
+
* @param options.runsSize - max number of runs to return
|
|
251
|
+
* @param options.runsOffset - offset into runs array for pagination
|
|
252
|
+
*/
|
|
253
|
+
getById(id: string, options?: {
|
|
254
|
+
fields?: "polling";
|
|
255
|
+
runsSize?: number;
|
|
256
|
+
runsOffset?: number;
|
|
257
|
+
}): Promise<StorageBenchmark | null>;
|
|
258
|
+
/**
|
|
259
|
+
* Create benchmark
|
|
260
|
+
*/
|
|
261
|
+
create(benchmark: Omit<StorageBenchmark, "id" | "createdAt">): Promise<StorageBenchmark>;
|
|
262
|
+
/**
|
|
263
|
+
* Update benchmark (for run management)
|
|
264
|
+
*/
|
|
265
|
+
update(id: string, benchmark: Partial<StorageBenchmark>): Promise<StorageBenchmark>;
|
|
266
|
+
/**
|
|
267
|
+
* Delete benchmark
|
|
268
|
+
*/
|
|
269
|
+
delete(id: string): Promise<{
|
|
270
|
+
deleted: boolean;
|
|
271
|
+
}>;
|
|
272
|
+
/**
|
|
273
|
+
* Bulk create benchmarks
|
|
274
|
+
*/
|
|
275
|
+
bulkCreate(benchmarks: Partial<StorageBenchmark>[]): Promise<{
|
|
276
|
+
created: number;
|
|
277
|
+
errors: boolean;
|
|
278
|
+
}>;
|
|
279
|
+
/**
|
|
280
|
+
* Update benchmark metadata only (name, description)
|
|
281
|
+
*/
|
|
282
|
+
updateMetadata(id: string, updates: {
|
|
283
|
+
name?: string;
|
|
284
|
+
description?: string;
|
|
285
|
+
}): Promise<StorageBenchmark>;
|
|
286
|
+
/**
|
|
287
|
+
* Get all versions of a benchmark
|
|
288
|
+
*/
|
|
289
|
+
getVersions(id: string): Promise<{
|
|
290
|
+
versions: any[];
|
|
291
|
+
total: number;
|
|
292
|
+
}>;
|
|
293
|
+
/**
|
|
294
|
+
* Get specific version of a benchmark
|
|
295
|
+
*/
|
|
296
|
+
getVersion(id: string, version: number): Promise<any>;
|
|
297
|
+
};
|
|
298
|
+
/** @deprecated Use benchmarkStorage instead */
|
|
299
|
+
export declare const experimentStorage: {
|
|
300
|
+
/**
|
|
301
|
+
* Get all benchmarks
|
|
302
|
+
* @param options.includeSample - whether to include sample/demo data
|
|
303
|
+
*/
|
|
304
|
+
getAll(options?: {
|
|
305
|
+
includeSample?: boolean;
|
|
306
|
+
}): Promise<StorageBenchmark[]>;
|
|
307
|
+
/**
|
|
308
|
+
* Get benchmark by ID
|
|
309
|
+
* @param options.fields - 'polling' for lightweight payload (excludes versions, testCaseSnapshots, headers)
|
|
310
|
+
* @param options.runsSize - max number of runs to return
|
|
311
|
+
* @param options.runsOffset - offset into runs array for pagination
|
|
312
|
+
*/
|
|
313
|
+
getById(id: string, options?: {
|
|
314
|
+
fields?: "polling";
|
|
315
|
+
runsSize?: number;
|
|
316
|
+
runsOffset?: number;
|
|
317
|
+
}): Promise<StorageBenchmark | null>;
|
|
318
|
+
/**
|
|
319
|
+
* Create benchmark
|
|
320
|
+
*/
|
|
321
|
+
create(benchmark: Omit<StorageBenchmark, "id" | "createdAt">): Promise<StorageBenchmark>;
|
|
322
|
+
/**
|
|
323
|
+
* Update benchmark (for run management)
|
|
324
|
+
*/
|
|
325
|
+
update(id: string, benchmark: Partial<StorageBenchmark>): Promise<StorageBenchmark>;
|
|
326
|
+
/**
|
|
327
|
+
* Delete benchmark
|
|
328
|
+
*/
|
|
329
|
+
delete(id: string): Promise<{
|
|
330
|
+
deleted: boolean;
|
|
331
|
+
}>;
|
|
332
|
+
/**
|
|
333
|
+
* Bulk create benchmarks
|
|
334
|
+
*/
|
|
335
|
+
bulkCreate(benchmarks: Partial<StorageBenchmark>[]): Promise<{
|
|
336
|
+
created: number;
|
|
337
|
+
errors: boolean;
|
|
338
|
+
}>;
|
|
339
|
+
/**
|
|
340
|
+
* Update benchmark metadata only (name, description)
|
|
341
|
+
*/
|
|
342
|
+
updateMetadata(id: string, updates: {
|
|
343
|
+
name?: string;
|
|
344
|
+
description?: string;
|
|
345
|
+
}): Promise<StorageBenchmark>;
|
|
346
|
+
/**
|
|
347
|
+
* Get all versions of a benchmark
|
|
348
|
+
*/
|
|
349
|
+
getVersions(id: string): Promise<{
|
|
350
|
+
versions: any[];
|
|
351
|
+
total: number;
|
|
352
|
+
}>;
|
|
353
|
+
/**
|
|
354
|
+
* Get specific version of a benchmark
|
|
355
|
+
*/
|
|
356
|
+
getVersion(id: string, version: number): Promise<any>;
|
|
357
|
+
};
|
|
358
|
+
export declare const runStorage: {
|
|
359
|
+
/**
|
|
360
|
+
* Get all runs with pagination
|
|
361
|
+
*/
|
|
362
|
+
getAll(options?: {
|
|
363
|
+
size?: number;
|
|
364
|
+
from?: number;
|
|
365
|
+
_source?: string[];
|
|
366
|
+
}): Promise<{
|
|
367
|
+
runs: StorageRun[];
|
|
368
|
+
total: number;
|
|
369
|
+
size: number;
|
|
370
|
+
from: number;
|
|
371
|
+
}>;
|
|
372
|
+
/**
|
|
373
|
+
* Batch-get runs by ID in one request (server fans out in parallel).
|
|
374
|
+
* Used by the comparison page to load every cell's report at once.
|
|
375
|
+
*/
|
|
376
|
+
getByIds(ids: string[]): Promise<StorageRun[]>;
|
|
377
|
+
/**
|
|
378
|
+
* Get run by ID
|
|
379
|
+
*/
|
|
380
|
+
getById(id: string): Promise<StorageRun | null>;
|
|
381
|
+
/**
|
|
382
|
+
* Create run
|
|
383
|
+
*/
|
|
384
|
+
create(run: Omit<StorageRun, "id" | "createdAt" | "annotations">, options?: {
|
|
385
|
+
analytics?: boolean;
|
|
386
|
+
}): Promise<StorageRun>;
|
|
387
|
+
/**
|
|
388
|
+
* Delete run
|
|
389
|
+
*/
|
|
390
|
+
delete(id: string): Promise<{
|
|
391
|
+
deleted: boolean;
|
|
392
|
+
}>;
|
|
393
|
+
/**
|
|
394
|
+
* Partial update of a run
|
|
395
|
+
* Used for updating trace-mode runs after traces become available
|
|
396
|
+
*/
|
|
397
|
+
partialUpdate(id: string, updates: Partial<StorageRun>): Promise<StorageRun>;
|
|
398
|
+
/**
|
|
399
|
+
* Get run counts grouped by test case ID (single aggregation query)
|
|
400
|
+
*/
|
|
401
|
+
getCountsByTestCase(): Promise<Record<string, number>>;
|
|
402
|
+
/**
|
|
403
|
+
* Get runs by test case ID
|
|
404
|
+
*/
|
|
405
|
+
getByTestCase(testCaseId: string, size?: number, from?: number): Promise<{
|
|
406
|
+
runs: StorageRun[];
|
|
407
|
+
total: number;
|
|
408
|
+
}>;
|
|
409
|
+
/**
|
|
410
|
+
* Get runs by benchmark ID
|
|
411
|
+
*/
|
|
412
|
+
getByBenchmark(benchmarkId: string, size?: number): Promise<StorageRun[]>;
|
|
413
|
+
/**
|
|
414
|
+
* Get runs by benchmark run config ID
|
|
415
|
+
*/
|
|
416
|
+
getByBenchmarkRun(benchmarkId: string, runId: string, size?: number): Promise<StorageRun[]>;
|
|
417
|
+
/**
|
|
418
|
+
* Get all iterations for a test case in a benchmark
|
|
419
|
+
*/
|
|
420
|
+
getIterations(benchmarkId: string, testCaseId: string, benchmarkRunId?: string): Promise<{
|
|
421
|
+
runs: StorageRun[];
|
|
422
|
+
total: number;
|
|
423
|
+
maxIteration: number;
|
|
424
|
+
}>;
|
|
425
|
+
/** @deprecated Use getByBenchmark instead */
|
|
426
|
+
getByExperiment(experimentId: string, size?: number): Promise<StorageRun[]>;
|
|
427
|
+
/** @deprecated Use getByBenchmarkRun instead */
|
|
428
|
+
getByExperimentRun(experimentId: string, runId: string, size?: number): Promise<StorageRun[]>;
|
|
429
|
+
/**
|
|
430
|
+
* Search runs with filters
|
|
431
|
+
*/
|
|
432
|
+
search(filters: {
|
|
433
|
+
experimentId?: string;
|
|
434
|
+
testCaseId?: string;
|
|
435
|
+
experimentRunId?: string;
|
|
436
|
+
agentId?: string;
|
|
437
|
+
modelId?: string;
|
|
438
|
+
status?: string;
|
|
439
|
+
passFailStatus?: string;
|
|
440
|
+
tags?: string[];
|
|
441
|
+
dateRange?: {
|
|
442
|
+
start: string;
|
|
443
|
+
end: string;
|
|
444
|
+
};
|
|
445
|
+
size?: number;
|
|
446
|
+
from?: number;
|
|
447
|
+
}): Promise<{
|
|
448
|
+
runs: StorageRun[];
|
|
449
|
+
total: number;
|
|
450
|
+
}>;
|
|
451
|
+
/**
|
|
452
|
+
* Add annotation to run
|
|
453
|
+
*/
|
|
454
|
+
addAnnotation(runId: string, annotation: Omit<StorageRunAnnotation, "id" | "createdAt" | "updatedAt">): Promise<StorageRunAnnotation>;
|
|
455
|
+
/**
|
|
456
|
+
* Update annotation
|
|
457
|
+
*/
|
|
458
|
+
updateAnnotation(runId: string, annotationId: string, updates: Partial<StorageRunAnnotation>): Promise<StorageRunAnnotation>;
|
|
459
|
+
/**
|
|
460
|
+
* Delete annotation
|
|
461
|
+
*/
|
|
462
|
+
deleteAnnotation(runId: string, annotationId: string): Promise<{
|
|
463
|
+
deleted: boolean;
|
|
464
|
+
}>;
|
|
465
|
+
/**
|
|
466
|
+
* Bulk create runs
|
|
467
|
+
*/
|
|
468
|
+
bulkCreate(runs: Partial<StorageRun>[]): Promise<{
|
|
469
|
+
created: number;
|
|
470
|
+
errors: boolean;
|
|
471
|
+
}>;
|
|
472
|
+
};
|
|
473
|
+
export declare const analyticsStorage: {
|
|
474
|
+
/**
|
|
475
|
+
* Query analytics records
|
|
476
|
+
*/
|
|
477
|
+
query(filters: {
|
|
478
|
+
experimentId?: string;
|
|
479
|
+
testCaseId?: string;
|
|
480
|
+
agentId?: string;
|
|
481
|
+
modelId?: string;
|
|
482
|
+
passFailStatus?: string;
|
|
483
|
+
size?: number;
|
|
484
|
+
from?: number;
|
|
485
|
+
}): Promise<{
|
|
486
|
+
records: StorageAnalyticsRecord[];
|
|
487
|
+
total: number;
|
|
488
|
+
}>;
|
|
489
|
+
/**
|
|
490
|
+
* Get aggregated metrics
|
|
491
|
+
*/
|
|
492
|
+
aggregations(experimentId?: string, groupBy?: string): Promise<{
|
|
493
|
+
aggregations: Array<{
|
|
494
|
+
key: string;
|
|
495
|
+
metrics: {
|
|
496
|
+
avgAccuracy?: number;
|
|
497
|
+
avgFaithfulness?: number;
|
|
498
|
+
avgLatency?: number;
|
|
499
|
+
avgTrajectory?: number;
|
|
500
|
+
};
|
|
501
|
+
passCount: number;
|
|
502
|
+
failCount: number;
|
|
503
|
+
totalRuns: number;
|
|
504
|
+
}>;
|
|
505
|
+
groupBy: string;
|
|
506
|
+
}>;
|
|
507
|
+
/**
|
|
508
|
+
* Complex search with custom filters and aggregations
|
|
509
|
+
*/
|
|
510
|
+
search(options: {
|
|
511
|
+
filters?: Record<string, unknown>;
|
|
512
|
+
aggs?: Record<string, unknown>;
|
|
513
|
+
size?: number;
|
|
514
|
+
from?: number;
|
|
515
|
+
}): Promise<{
|
|
516
|
+
records: StorageAnalyticsRecord[];
|
|
517
|
+
total: number;
|
|
518
|
+
aggregations: Record<string, unknown>;
|
|
519
|
+
}>;
|
|
520
|
+
/**
|
|
521
|
+
* Backfill analytics from existing runs
|
|
522
|
+
*/
|
|
523
|
+
backfill(): Promise<{
|
|
524
|
+
backfilled: number;
|
|
525
|
+
errors: number;
|
|
526
|
+
total: number;
|
|
527
|
+
}>;
|
|
528
|
+
};
|
|
529
|
+
export declare const opensearchStorage: {
|
|
530
|
+
admin: {
|
|
531
|
+
/**
|
|
532
|
+
* Check storage health/connectivity
|
|
533
|
+
*/
|
|
534
|
+
health(): Promise<{
|
|
535
|
+
status: string;
|
|
536
|
+
cluster?: unknown;
|
|
537
|
+
error?: string;
|
|
538
|
+
/** Active storage backend. 'file' means OpenSearch is unavailable / not configured; 'error' means configured-but-unreachable. */
|
|
539
|
+
backend?: "file" | "opensearch" | "error";
|
|
540
|
+
/**
|
|
541
|
+
* Real OpenSearch connectivity, present when OpenSearch is configured but
|
|
542
|
+
* the active backend fell back to file storage. Top-level `status` reflects
|
|
543
|
+
* the active (possibly file) backend, NOT OpenSearch — use this to report
|
|
544
|
+
* true OpenSearch connectivity. Shape mirrors testStorageConnection().
|
|
545
|
+
*/
|
|
546
|
+
opensearch?: {
|
|
547
|
+
status: string;
|
|
548
|
+
message?: string;
|
|
549
|
+
latencyMs?: number;
|
|
550
|
+
clusterName?: string;
|
|
551
|
+
clusterStatus?: string;
|
|
552
|
+
};
|
|
553
|
+
}>;
|
|
554
|
+
/**
|
|
555
|
+
* Initialize all indexes with mappings
|
|
556
|
+
*/
|
|
557
|
+
initIndexes(): Promise<{
|
|
558
|
+
success: boolean;
|
|
559
|
+
results: Record<string, {
|
|
560
|
+
status: string;
|
|
561
|
+
error?: string;
|
|
562
|
+
}>;
|
|
563
|
+
}>;
|
|
564
|
+
/**
|
|
565
|
+
* Get storage statistics
|
|
566
|
+
*/
|
|
567
|
+
stats(): Promise<{
|
|
568
|
+
stats: Record<string, {
|
|
569
|
+
count: number;
|
|
570
|
+
error?: string;
|
|
571
|
+
}>;
|
|
572
|
+
}>;
|
|
573
|
+
};
|
|
574
|
+
testCases: {
|
|
575
|
+
/**
|
|
576
|
+
* Get all test cases (latest versions only)
|
|
577
|
+
* @param options.fields - 'summary' for lightweight list-view payload
|
|
578
|
+
* @param options.size - page size for pagination
|
|
579
|
+
* @param options.after - cursor token for next page
|
|
580
|
+
* @param options.includeSample - whether to include sample/demo data
|
|
581
|
+
*/
|
|
582
|
+
getAll(options?: {
|
|
583
|
+
fields?: "summary";
|
|
584
|
+
size?: number;
|
|
585
|
+
after?: string;
|
|
586
|
+
includeSample?: boolean;
|
|
587
|
+
}): Promise<{
|
|
588
|
+
testCases: StorageTestCase[];
|
|
589
|
+
total: number;
|
|
590
|
+
after?: string | null;
|
|
591
|
+
hasMore?: boolean;
|
|
592
|
+
}>;
|
|
593
|
+
/**
|
|
594
|
+
* Get test cases by specific IDs (latest versions only)
|
|
595
|
+
* Used for efficient filtered fetching (e.g., only test cases in a benchmark)
|
|
596
|
+
*/
|
|
597
|
+
getByIds(ids: string[]): Promise<StorageTestCase[]>;
|
|
598
|
+
/**
|
|
599
|
+
* Get test case by ID (latest version)
|
|
600
|
+
*/
|
|
601
|
+
getById(id: string): Promise<StorageTestCase | null>;
|
|
602
|
+
/**
|
|
603
|
+
* Get all versions of a test case
|
|
604
|
+
*/
|
|
605
|
+
getVersions(id: string): Promise<StorageTestCase[]>;
|
|
606
|
+
/**
|
|
607
|
+
* Get specific version of a test case
|
|
608
|
+
*/
|
|
609
|
+
getVersion(id: string, version: number): Promise<StorageTestCase | null>;
|
|
610
|
+
/**
|
|
611
|
+
* Create new test case (starts at version 1)
|
|
612
|
+
*/
|
|
613
|
+
create(testCase: Omit<StorageTestCase, "id" | "version" | "createdAt" | "updatedAt">): Promise<StorageTestCase>;
|
|
614
|
+
/**
|
|
615
|
+
* Update test case (creates new version)
|
|
616
|
+
*/
|
|
617
|
+
update(id: string, testCase: Partial<StorageTestCase>): Promise<StorageTestCase>;
|
|
618
|
+
/**
|
|
619
|
+
* Delete test case (all versions)
|
|
620
|
+
*/
|
|
621
|
+
delete(id: string): Promise<{
|
|
622
|
+
deleted: number;
|
|
623
|
+
}>;
|
|
624
|
+
/**
|
|
625
|
+
* Bulk create test cases
|
|
626
|
+
*/
|
|
627
|
+
bulkCreate(testCases: Partial<StorageTestCase>[]): Promise<{
|
|
628
|
+
created: number;
|
|
629
|
+
errors: boolean;
|
|
630
|
+
}>;
|
|
631
|
+
};
|
|
632
|
+
benchmarks: {
|
|
633
|
+
/**
|
|
634
|
+
* Get all benchmarks
|
|
635
|
+
* @param options.includeSample - whether to include sample/demo data
|
|
636
|
+
*/
|
|
637
|
+
getAll(options?: {
|
|
638
|
+
includeSample?: boolean;
|
|
639
|
+
}): Promise<StorageBenchmark[]>;
|
|
640
|
+
/**
|
|
641
|
+
* Get benchmark by ID
|
|
642
|
+
* @param options.fields - 'polling' for lightweight payload (excludes versions, testCaseSnapshots, headers)
|
|
643
|
+
* @param options.runsSize - max number of runs to return
|
|
644
|
+
* @param options.runsOffset - offset into runs array for pagination
|
|
645
|
+
*/
|
|
646
|
+
getById(id: string, options?: {
|
|
647
|
+
fields?: "polling";
|
|
648
|
+
runsSize?: number;
|
|
649
|
+
runsOffset?: number;
|
|
650
|
+
}): Promise<StorageBenchmark | null>;
|
|
651
|
+
/**
|
|
652
|
+
* Create benchmark
|
|
653
|
+
*/
|
|
654
|
+
create(benchmark: Omit<StorageBenchmark, "id" | "createdAt">): Promise<StorageBenchmark>;
|
|
655
|
+
/**
|
|
656
|
+
* Update benchmark (for run management)
|
|
657
|
+
*/
|
|
658
|
+
update(id: string, benchmark: Partial<StorageBenchmark>): Promise<StorageBenchmark>;
|
|
659
|
+
/**
|
|
660
|
+
* Delete benchmark
|
|
661
|
+
*/
|
|
662
|
+
delete(id: string): Promise<{
|
|
663
|
+
deleted: boolean;
|
|
664
|
+
}>;
|
|
665
|
+
/**
|
|
666
|
+
* Bulk create benchmarks
|
|
667
|
+
*/
|
|
668
|
+
bulkCreate(benchmarks: Partial<StorageBenchmark>[]): Promise<{
|
|
669
|
+
created: number;
|
|
670
|
+
errors: boolean;
|
|
671
|
+
}>;
|
|
672
|
+
/**
|
|
673
|
+
* Update benchmark metadata only (name, description)
|
|
674
|
+
*/
|
|
675
|
+
updateMetadata(id: string, updates: {
|
|
676
|
+
name?: string;
|
|
677
|
+
description?: string;
|
|
678
|
+
}): Promise<StorageBenchmark>;
|
|
679
|
+
/**
|
|
680
|
+
* Get all versions of a benchmark
|
|
681
|
+
*/
|
|
682
|
+
getVersions(id: string): Promise<{
|
|
683
|
+
versions: any[];
|
|
684
|
+
total: number;
|
|
685
|
+
}>;
|
|
686
|
+
/**
|
|
687
|
+
* Get specific version of a benchmark
|
|
688
|
+
*/
|
|
689
|
+
getVersion(id: string, version: number): Promise<any>;
|
|
690
|
+
};
|
|
691
|
+
runs: {
|
|
692
|
+
/**
|
|
693
|
+
* Get all runs with pagination
|
|
694
|
+
*/
|
|
695
|
+
getAll(options?: {
|
|
696
|
+
size?: number;
|
|
697
|
+
from?: number;
|
|
698
|
+
_source?: string[];
|
|
699
|
+
}): Promise<{
|
|
700
|
+
runs: StorageRun[];
|
|
701
|
+
total: number;
|
|
702
|
+
size: number;
|
|
703
|
+
from: number;
|
|
704
|
+
}>;
|
|
705
|
+
/**
|
|
706
|
+
* Batch-get runs by ID in one request (server fans out in parallel).
|
|
707
|
+
* Used by the comparison page to load every cell's report at once.
|
|
708
|
+
*/
|
|
709
|
+
getByIds(ids: string[]): Promise<StorageRun[]>;
|
|
710
|
+
/**
|
|
711
|
+
* Get run by ID
|
|
712
|
+
*/
|
|
713
|
+
getById(id: string): Promise<StorageRun | null>;
|
|
714
|
+
/**
|
|
715
|
+
* Create run
|
|
716
|
+
*/
|
|
717
|
+
create(run: Omit<StorageRun, "id" | "createdAt" | "annotations">, options?: {
|
|
718
|
+
analytics?: boolean;
|
|
719
|
+
}): Promise<StorageRun>;
|
|
720
|
+
/**
|
|
721
|
+
* Delete run
|
|
722
|
+
*/
|
|
723
|
+
delete(id: string): Promise<{
|
|
724
|
+
deleted: boolean;
|
|
725
|
+
}>;
|
|
726
|
+
/**
|
|
727
|
+
* Partial update of a run
|
|
728
|
+
* Used for updating trace-mode runs after traces become available
|
|
729
|
+
*/
|
|
730
|
+
partialUpdate(id: string, updates: Partial<StorageRun>): Promise<StorageRun>;
|
|
731
|
+
/**
|
|
732
|
+
* Get run counts grouped by test case ID (single aggregation query)
|
|
733
|
+
*/
|
|
734
|
+
getCountsByTestCase(): Promise<Record<string, number>>;
|
|
735
|
+
/**
|
|
736
|
+
* Get runs by test case ID
|
|
737
|
+
*/
|
|
738
|
+
getByTestCase(testCaseId: string, size?: number, from?: number): Promise<{
|
|
739
|
+
runs: StorageRun[];
|
|
740
|
+
total: number;
|
|
741
|
+
}>;
|
|
742
|
+
/**
|
|
743
|
+
* Get runs by benchmark ID
|
|
744
|
+
*/
|
|
745
|
+
getByBenchmark(benchmarkId: string, size?: number): Promise<StorageRun[]>;
|
|
746
|
+
/**
|
|
747
|
+
* Get runs by benchmark run config ID
|
|
748
|
+
*/
|
|
749
|
+
getByBenchmarkRun(benchmarkId: string, runId: string, size?: number): Promise<StorageRun[]>;
|
|
750
|
+
/**
|
|
751
|
+
* Get all iterations for a test case in a benchmark
|
|
752
|
+
*/
|
|
753
|
+
getIterations(benchmarkId: string, testCaseId: string, benchmarkRunId?: string): Promise<{
|
|
754
|
+
runs: StorageRun[];
|
|
755
|
+
total: number;
|
|
756
|
+
maxIteration: number;
|
|
757
|
+
}>;
|
|
758
|
+
/** @deprecated Use getByBenchmark instead */
|
|
759
|
+
getByExperiment(experimentId: string, size?: number): Promise<StorageRun[]>;
|
|
760
|
+
/** @deprecated Use getByBenchmarkRun instead */
|
|
761
|
+
getByExperimentRun(experimentId: string, runId: string, size?: number): Promise<StorageRun[]>;
|
|
762
|
+
/**
|
|
763
|
+
* Search runs with filters
|
|
764
|
+
*/
|
|
765
|
+
search(filters: {
|
|
766
|
+
experimentId?: string;
|
|
767
|
+
testCaseId?: string;
|
|
768
|
+
experimentRunId?: string;
|
|
769
|
+
agentId?: string;
|
|
770
|
+
modelId?: string;
|
|
771
|
+
status?: string;
|
|
772
|
+
passFailStatus?: string;
|
|
773
|
+
tags?: string[];
|
|
774
|
+
dateRange?: {
|
|
775
|
+
start: string;
|
|
776
|
+
end: string;
|
|
777
|
+
};
|
|
778
|
+
size?: number;
|
|
779
|
+
from?: number;
|
|
780
|
+
}): Promise<{
|
|
781
|
+
runs: StorageRun[];
|
|
782
|
+
total: number;
|
|
783
|
+
}>;
|
|
784
|
+
/**
|
|
785
|
+
* Add annotation to run
|
|
786
|
+
*/
|
|
787
|
+
addAnnotation(runId: string, annotation: Omit<StorageRunAnnotation, "id" | "createdAt" | "updatedAt">): Promise<StorageRunAnnotation>;
|
|
788
|
+
/**
|
|
789
|
+
* Update annotation
|
|
790
|
+
*/
|
|
791
|
+
updateAnnotation(runId: string, annotationId: string, updates: Partial<StorageRunAnnotation>): Promise<StorageRunAnnotation>;
|
|
792
|
+
/**
|
|
793
|
+
* Delete annotation
|
|
794
|
+
*/
|
|
795
|
+
deleteAnnotation(runId: string, annotationId: string): Promise<{
|
|
796
|
+
deleted: boolean;
|
|
797
|
+
}>;
|
|
798
|
+
/**
|
|
799
|
+
* Bulk create runs
|
|
800
|
+
*/
|
|
801
|
+
bulkCreate(runs: Partial<StorageRun>[]): Promise<{
|
|
802
|
+
created: number;
|
|
803
|
+
errors: boolean;
|
|
804
|
+
}>;
|
|
805
|
+
};
|
|
806
|
+
analytics: {
|
|
807
|
+
/**
|
|
808
|
+
* Query analytics records
|
|
809
|
+
*/
|
|
810
|
+
query(filters: {
|
|
811
|
+
experimentId?: string;
|
|
812
|
+
testCaseId?: string;
|
|
813
|
+
agentId?: string;
|
|
814
|
+
modelId?: string;
|
|
815
|
+
passFailStatus?: string;
|
|
816
|
+
size?: number;
|
|
817
|
+
from?: number;
|
|
818
|
+
}): Promise<{
|
|
819
|
+
records: StorageAnalyticsRecord[];
|
|
820
|
+
total: number;
|
|
821
|
+
}>;
|
|
822
|
+
/**
|
|
823
|
+
* Get aggregated metrics
|
|
824
|
+
*/
|
|
825
|
+
aggregations(experimentId?: string, groupBy?: string): Promise<{
|
|
826
|
+
aggregations: Array<{
|
|
827
|
+
key: string;
|
|
828
|
+
metrics: {
|
|
829
|
+
avgAccuracy?: number;
|
|
830
|
+
avgFaithfulness?: number;
|
|
831
|
+
avgLatency?: number;
|
|
832
|
+
avgTrajectory?: number;
|
|
833
|
+
};
|
|
834
|
+
passCount: number;
|
|
835
|
+
failCount: number;
|
|
836
|
+
totalRuns: number;
|
|
837
|
+
}>;
|
|
838
|
+
groupBy: string;
|
|
839
|
+
}>;
|
|
840
|
+
/**
|
|
841
|
+
* Complex search with custom filters and aggregations
|
|
842
|
+
*/
|
|
843
|
+
search(options: {
|
|
844
|
+
filters?: Record<string, unknown>;
|
|
845
|
+
aggs?: Record<string, unknown>;
|
|
846
|
+
size?: number;
|
|
847
|
+
from?: number;
|
|
848
|
+
}): Promise<{
|
|
849
|
+
records: StorageAnalyticsRecord[];
|
|
850
|
+
total: number;
|
|
851
|
+
aggregations: Record<string, unknown>;
|
|
852
|
+
}>;
|
|
853
|
+
/**
|
|
854
|
+
* Backfill analytics from existing runs
|
|
855
|
+
*/
|
|
856
|
+
backfill(): Promise<{
|
|
857
|
+
backfilled: number;
|
|
858
|
+
errors: number;
|
|
859
|
+
total: number;
|
|
860
|
+
}>;
|
|
861
|
+
};
|
|
862
|
+
/** @deprecated Use benchmarks instead */
|
|
863
|
+
experiments: {
|
|
864
|
+
/**
|
|
865
|
+
* Get all benchmarks
|
|
866
|
+
* @param options.includeSample - whether to include sample/demo data
|
|
867
|
+
*/
|
|
868
|
+
getAll(options?: {
|
|
869
|
+
includeSample?: boolean;
|
|
870
|
+
}): Promise<StorageBenchmark[]>;
|
|
871
|
+
/**
|
|
872
|
+
* Get benchmark by ID
|
|
873
|
+
* @param options.fields - 'polling' for lightweight payload (excludes versions, testCaseSnapshots, headers)
|
|
874
|
+
* @param options.runsSize - max number of runs to return
|
|
875
|
+
* @param options.runsOffset - offset into runs array for pagination
|
|
876
|
+
*/
|
|
877
|
+
getById(id: string, options?: {
|
|
878
|
+
fields?: "polling";
|
|
879
|
+
runsSize?: number;
|
|
880
|
+
runsOffset?: number;
|
|
881
|
+
}): Promise<StorageBenchmark | null>;
|
|
882
|
+
/**
|
|
883
|
+
* Create benchmark
|
|
884
|
+
*/
|
|
885
|
+
create(benchmark: Omit<StorageBenchmark, "id" | "createdAt">): Promise<StorageBenchmark>;
|
|
886
|
+
/**
|
|
887
|
+
* Update benchmark (for run management)
|
|
888
|
+
*/
|
|
889
|
+
update(id: string, benchmark: Partial<StorageBenchmark>): Promise<StorageBenchmark>;
|
|
890
|
+
/**
|
|
891
|
+
* Delete benchmark
|
|
892
|
+
*/
|
|
893
|
+
delete(id: string): Promise<{
|
|
894
|
+
deleted: boolean;
|
|
895
|
+
}>;
|
|
896
|
+
/**
|
|
897
|
+
* Bulk create benchmarks
|
|
898
|
+
*/
|
|
899
|
+
bulkCreate(benchmarks: Partial<StorageBenchmark>[]): Promise<{
|
|
900
|
+
created: number;
|
|
901
|
+
errors: boolean;
|
|
902
|
+
}>;
|
|
903
|
+
/**
|
|
904
|
+
* Update benchmark metadata only (name, description)
|
|
905
|
+
*/
|
|
906
|
+
updateMetadata(id: string, updates: {
|
|
907
|
+
name?: string;
|
|
908
|
+
description?: string;
|
|
909
|
+
}): Promise<StorageBenchmark>;
|
|
910
|
+
/**
|
|
911
|
+
* Get all versions of a benchmark
|
|
912
|
+
*/
|
|
913
|
+
getVersions(id: string): Promise<{
|
|
914
|
+
versions: any[];
|
|
915
|
+
total: number;
|
|
916
|
+
}>;
|
|
917
|
+
/**
|
|
918
|
+
* Get specific version of a benchmark
|
|
919
|
+
*/
|
|
920
|
+
getVersion(id: string, version: number): Promise<any>;
|
|
921
|
+
};
|
|
922
|
+
};
|
|
923
|
+
export default opensearchStorage;
|
|
924
|
+
//# sourceMappingURL=opensearchClient.d.ts.map
|