@agentlensai/server 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/cloud/auth/rbac.d.ts +1 -1
- package/dist/cloud/auth/rbac.d.ts.map +1 -1
- package/dist/cloud/auth/rbac.js +2 -2
- package/dist/cloud/auth/rbac.js.map +1 -1
- package/dist/cloud/billing/stripe-client.d.ts.map +1 -1
- package/dist/cloud/billing/stripe-client.js +6 -1
- package/dist/cloud/billing/stripe-client.js.map +1 -1
- package/dist/cloud/ingestion/gateway.d.ts.map +1 -1
- package/dist/cloud/ingestion/gateway.js +0 -1
- package/dist/cloud/ingestion/gateway.js.map +1 -1
- package/dist/cloud/middleware/validate-org-access.d.ts +14 -0
- package/dist/cloud/middleware/validate-org-access.d.ts.map +1 -0
- package/dist/cloud/middleware/validate-org-access.js +38 -0
- package/dist/cloud/middleware/validate-org-access.js.map +1 -0
- package/dist/cloud/routes/index.d.ts +13 -0
- package/dist/cloud/routes/index.d.ts.map +1 -0
- package/dist/cloud/routes/index.js +98 -0
- package/dist/cloud/routes/index.js.map +1 -0
- package/dist/config.d.ts +33 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +71 -1
- package/dist/config.js.map +1 -1
- package/dist/db/api-key-lookup.d.ts +25 -0
- package/dist/db/api-key-lookup.d.ts.map +1 -0
- package/dist/db/api-key-lookup.js +38 -0
- package/dist/db/api-key-lookup.js.map +1 -0
- package/dist/db/connection.postgres.d.ts +44 -0
- package/dist/db/connection.postgres.d.ts.map +1 -0
- package/dist/db/connection.postgres.js +79 -0
- package/dist/db/connection.postgres.js.map +1 -0
- package/dist/db/cost-budget-store.d.ts +30 -0
- package/dist/db/cost-budget-store.d.ts.map +1 -0
- package/dist/db/cost-budget-store.js +201 -0
- package/dist/db/cost-budget-store.js.map +1 -0
- package/dist/db/drizzle/0000_initial.sql +336 -0
- package/dist/db/drizzle/0001_indexes.sql +20 -0
- package/dist/db/drizzle/0002_pgvector.sql +19 -0
- package/dist/db/drizzle/drizzle/0000_initial.sql +336 -0
- package/dist/db/drizzle/drizzle/0001_indexes.sql +20 -0
- package/dist/db/drizzle/drizzle/0002_pgvector.sql +19 -0
- package/dist/db/drizzle/drizzle/meta/0000_snapshot.json +2593 -0
- package/dist/db/drizzle/drizzle/meta/_journal.json +27 -0
- package/dist/db/drizzle/meta/0000_snapshot.json +2593 -0
- package/dist/db/drizzle/meta/_journal.json +27 -0
- package/dist/db/embedding-store.d.ts +2 -1
- package/dist/db/embedding-store.d.ts.map +1 -1
- package/dist/db/embedding-store.interface.d.ts +19 -0
- package/dist/db/embedding-store.interface.d.ts.map +1 -0
- package/dist/db/embedding-store.interface.js +7 -0
- package/dist/db/embedding-store.interface.js.map +1 -0
- package/dist/db/embedding-store.js +3 -1
- package/dist/db/embedding-store.js.map +1 -1
- package/dist/db/eval-store.d.ts +88 -0
- package/dist/db/eval-store.d.ts.map +1 -0
- package/dist/db/eval-store.js +408 -0
- package/dist/db/eval-store.js.map +1 -0
- package/dist/db/guardrail-store.d.ts +9 -0
- package/dist/db/guardrail-store.d.ts.map +1 -1
- package/dist/db/guardrail-store.js +57 -3
- package/dist/db/guardrail-store.js.map +1 -1
- package/dist/db/index.d.ts +7 -0
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +4 -12
- package/dist/db/index.js.map +1 -1
- package/dist/db/migrate.d.ts +5 -22
- package/dist/db/migrate.d.ts.map +1 -1
- package/dist/db/migrate.js +7 -637
- package/dist/db/migrate.js.map +1 -1
- package/dist/db/migrate.postgres.d.ts +16 -0
- package/dist/db/migrate.postgres.d.ts.map +1 -0
- package/dist/db/migrate.postgres.js +23 -0
- package/dist/db/migrate.postgres.js.map +1 -0
- package/dist/db/migrate.sqlite.d.ts +26 -0
- package/dist/db/migrate.sqlite.d.ts.map +1 -0
- package/dist/db/migrate.sqlite.js +920 -0
- package/dist/db/migrate.sqlite.js.map +1 -0
- package/dist/db/postgres-embedding-store.d.ts +23 -0
- package/dist/db/postgres-embedding-store.d.ts.map +1 -0
- package/dist/db/postgres-embedding-store.js +218 -0
- package/dist/db/postgres-embedding-store.js.map +1 -0
- package/dist/db/postgres-store.d.ts +80 -0
- package/dist/db/postgres-store.d.ts.map +1 -0
- package/dist/db/postgres-store.js +910 -0
- package/dist/db/postgres-store.js.map +1 -0
- package/dist/db/prompt-store.d.ts +57 -0
- package/dist/db/prompt-store.d.ts.map +1 -0
- package/dist/db/prompt-store.js +300 -0
- package/dist/db/prompt-store.js.map +1 -0
- package/dist/db/repositories/agent-repository.d.ts +21 -0
- package/dist/db/repositories/agent-repository.d.ts.map +1 -0
- package/dist/db/repositories/agent-repository.js +142 -0
- package/dist/db/repositories/agent-repository.js.map +1 -0
- package/dist/db/repositories/alert-repository.d.ts +27 -0
- package/dist/db/repositories/alert-repository.d.ts.map +1 -0
- package/dist/db/repositories/alert-repository.js +164 -0
- package/dist/db/repositories/alert-repository.js.map +1 -0
- package/dist/db/repositories/analytics-repository.d.ts +24 -0
- package/dist/db/repositories/analytics-repository.d.ts.map +1 -0
- package/dist/db/repositories/analytics-repository.js +147 -0
- package/dist/db/repositories/analytics-repository.js.map +1 -0
- package/dist/db/repositories/event-repository.d.ts +81 -0
- package/dist/db/repositories/event-repository.d.ts.map +1 -0
- package/dist/db/repositories/event-repository.js +331 -0
- package/dist/db/repositories/event-repository.js.map +1 -0
- package/dist/db/repositories/notification-channel-repository.d.ts +28 -0
- package/dist/db/repositories/notification-channel-repository.d.ts.map +1 -0
- package/dist/db/repositories/notification-channel-repository.js +151 -0
- package/dist/db/repositories/notification-channel-repository.js.map +1 -0
- package/dist/db/repositories/session-repository.d.ts +26 -0
- package/dist/db/repositories/session-repository.d.ts.map +1 -0
- package/dist/db/repositories/session-repository.js +240 -0
- package/dist/db/repositories/session-repository.js.map +1 -0
- package/dist/db/schema.postgres.d.ts +4681 -0
- package/dist/db/schema.postgres.d.ts.map +1 -0
- package/dist/db/schema.postgres.js +458 -0
- package/dist/db/schema.postgres.js.map +1 -0
- package/dist/db/schema.sqlite.d.ts +2221 -671
- package/dist/db/schema.sqlite.d.ts.map +1 -1
- package/dist/db/schema.sqlite.js +137 -2
- package/dist/db/schema.sqlite.js.map +1 -1
- package/dist/db/services/retention-service.d.ts +13 -0
- package/dist/db/services/retention-service.d.ts.map +1 -0
- package/dist/db/services/retention-service.js +48 -0
- package/dist/db/services/retention-service.js.map +1 -0
- package/dist/db/shared/query-helpers.d.ts +32 -0
- package/dist/db/shared/query-helpers.d.ts.map +1 -0
- package/dist/db/shared/query-helpers.js +180 -0
- package/dist/db/shared/query-helpers.js.map +1 -0
- package/dist/db/sqlite-store.d.ts +48 -55
- package/dist/db/sqlite-store.d.ts.map +1 -1
- package/dist/db/sqlite-store.js +78 -945
- package/dist/db/sqlite-store.js.map +1 -1
- package/dist/db/tenant-scoped-store.d.ts +18 -1
- package/dist/db/tenant-scoped-store.d.ts.map +1 -1
- package/dist/db/tenant-scoped-store.js +6 -0
- package/dist/db/tenant-scoped-store.js.map +1 -1
- package/dist/index.d.ts +28 -14
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +432 -97
- package/dist/index.js.map +1 -1
- package/dist/lib/alert-engine.d.ts +10 -0
- package/dist/lib/alert-engine.d.ts.map +1 -1
- package/dist/lib/alert-engine.js +73 -20
- package/dist/lib/alert-engine.js.map +1 -1
- package/dist/lib/audit-verify.d.ts +40 -0
- package/dist/lib/audit-verify.d.ts.map +1 -0
- package/dist/lib/audit-verify.js +128 -0
- package/dist/lib/audit-verify.js.map +1 -0
- package/dist/lib/audit.d.ts +37 -0
- package/dist/lib/audit.d.ts.map +1 -0
- package/dist/lib/audit.js +59 -0
- package/dist/lib/audit.js.map +1 -0
- package/dist/lib/budget-engine.d.ts +26 -0
- package/dist/lib/budget-engine.d.ts.map +1 -0
- package/dist/lib/budget-engine.js +201 -0
- package/dist/lib/budget-engine.js.map +1 -0
- package/dist/lib/compliance-export.d.ts +41 -0
- package/dist/lib/compliance-export.d.ts.map +1 -0
- package/dist/lib/compliance-export.js +124 -0
- package/dist/lib/compliance-export.js.map +1 -0
- package/dist/lib/compliance-report.d.ts +87 -0
- package/dist/lib/compliance-report.d.ts.map +1 -0
- package/dist/lib/compliance-report.js +148 -0
- package/dist/lib/compliance-report.js.map +1 -0
- package/dist/lib/context/retrieval.d.ts +5 -3
- package/dist/lib/context/retrieval.d.ts.map +1 -1
- package/dist/lib/context/retrieval.js +5 -2
- package/dist/lib/context/retrieval.js.map +1 -1
- package/dist/lib/cost-anomaly-detector.d.ts +23 -0
- package/dist/lib/cost-anomaly-detector.d.ts.map +1 -0
- package/dist/lib/cost-anomaly-detector.js +108 -0
- package/dist/lib/cost-anomaly-detector.js.map +1 -0
- package/dist/lib/db-resilience.d.ts +15 -0
- package/dist/lib/db-resilience.d.ts.map +1 -0
- package/dist/lib/db-resilience.js +49 -0
- package/dist/lib/db-resilience.js.map +1 -0
- package/dist/lib/diagnostics/cache.d.ts +29 -0
- package/dist/lib/diagnostics/cache.d.ts.map +1 -0
- package/dist/lib/diagnostics/cache.js +88 -0
- package/dist/lib/diagnostics/cache.js.map +1 -0
- package/dist/lib/diagnostics/context-builder.d.ts +41 -0
- package/dist/lib/diagnostics/context-builder.d.ts.map +1 -0
- package/dist/lib/diagnostics/context-builder.js +135 -0
- package/dist/lib/diagnostics/context-builder.js.map +1 -0
- package/dist/lib/diagnostics/index.d.ts +34 -0
- package/dist/lib/diagnostics/index.d.ts.map +1 -0
- package/dist/lib/diagnostics/index.js +223 -0
- package/dist/lib/diagnostics/index.js.map +1 -0
- package/dist/lib/diagnostics/llm-client.d.ts +24 -0
- package/dist/lib/diagnostics/llm-client.d.ts.map +1 -0
- package/dist/lib/diagnostics/llm-client.js +42 -0
- package/dist/lib/diagnostics/llm-client.js.map +1 -0
- package/dist/lib/diagnostics/prompt-templates.d.ts +18 -0
- package/dist/lib/diagnostics/prompt-templates.d.ts.map +1 -0
- package/dist/lib/diagnostics/prompt-templates.js +144 -0
- package/dist/lib/diagnostics/prompt-templates.js.map +1 -0
- package/dist/lib/diagnostics/providers/anthropic.d.ts +8 -0
- package/dist/lib/diagnostics/providers/anthropic.d.ts.map +1 -0
- package/dist/lib/diagnostics/providers/anthropic.js +79 -0
- package/dist/lib/diagnostics/providers/anthropic.js.map +1 -0
- package/dist/lib/diagnostics/providers/openai.d.ts +8 -0
- package/dist/lib/diagnostics/providers/openai.d.ts.map +1 -0
- package/dist/lib/diagnostics/providers/openai.js +70 -0
- package/dist/lib/diagnostics/providers/openai.js.map +1 -0
- package/dist/lib/diagnostics/providers/types.d.ts +23 -0
- package/dist/lib/diagnostics/providers/types.d.ts.map +1 -0
- package/dist/lib/diagnostics/providers/types.js +5 -0
- package/dist/lib/diagnostics/providers/types.js.map +1 -0
- package/dist/lib/diagnostics/response-parser.d.ts +60 -0
- package/dist/lib/diagnostics/response-parser.d.ts.map +1 -0
- package/dist/lib/diagnostics/response-parser.js +55 -0
- package/dist/lib/diagnostics/response-parser.js.map +1 -0
- package/dist/lib/diagnostics/types.d.ts +60 -0
- package/dist/lib/diagnostics/types.d.ts.map +1 -0
- package/dist/lib/diagnostics/types.js +7 -0
- package/dist/lib/diagnostics/types.js.map +1 -0
- package/dist/lib/embeddings/index.d.ts +6 -3
- package/dist/lib/embeddings/index.d.ts.map +1 -1
- package/dist/lib/embeddings/index.js +7 -15
- package/dist/lib/embeddings/index.js.map +1 -1
- package/dist/lib/embeddings/worker.d.ts +2 -2
- package/dist/lib/embeddings/worker.d.ts.map +1 -1
- package/dist/lib/embeddings/worker.js +3 -1
- package/dist/lib/embeddings/worker.js.map +1 -1
- package/dist/lib/error-sanitizer.d.ts +28 -0
- package/dist/lib/error-sanitizer.d.ts.map +1 -0
- package/dist/lib/error-sanitizer.js +106 -0
- package/dist/lib/error-sanitizer.js.map +1 -0
- package/dist/lib/eval/index.d.ts +15 -0
- package/dist/lib/eval/index.d.ts.map +1 -0
- package/dist/lib/eval/index.js +24 -0
- package/dist/lib/eval/index.js.map +1 -0
- package/dist/lib/eval/runner.d.ts +28 -0
- package/dist/lib/eval/runner.d.ts.map +1 -0
- package/dist/lib/eval/runner.js +260 -0
- package/dist/lib/eval/runner.js.map +1 -0
- package/dist/lib/eval/scorers/contains.d.ts +10 -0
- package/dist/lib/eval/scorers/contains.d.ts.map +1 -0
- package/dist/lib/eval/scorers/contains.js +33 -0
- package/dist/lib/eval/scorers/contains.js.map +1 -0
- package/dist/lib/eval/scorers/exact-match.d.ts +10 -0
- package/dist/lib/eval/scorers/exact-match.d.ts.map +1 -0
- package/dist/lib/eval/scorers/exact-match.js +33 -0
- package/dist/lib/eval/scorers/exact-match.js.map +1 -0
- package/dist/lib/eval/scorers/index.d.ts +20 -0
- package/dist/lib/eval/scorers/index.d.ts.map +1 -0
- package/dist/lib/eval/scorers/index.js +19 -0
- package/dist/lib/eval/scorers/index.js.map +1 -0
- package/dist/lib/eval/scorers/llm-judge.d.ts +22 -0
- package/dist/lib/eval/scorers/llm-judge.d.ts.map +1 -0
- package/dist/lib/eval/scorers/llm-judge.js +79 -0
- package/dist/lib/eval/scorers/llm-judge.js.map +1 -0
- package/dist/lib/eval/scorers/regex.d.ts +10 -0
- package/dist/lib/eval/scorers/regex.d.ts.map +1 -0
- package/dist/lib/eval/scorers/regex.js +36 -0
- package/dist/lib/eval/scorers/regex.js.map +1 -0
- package/dist/lib/guardrails/actions.d.ts +6 -0
- package/dist/lib/guardrails/actions.d.ts.map +1 -1
- package/dist/lib/guardrails/actions.js +82 -0
- package/dist/lib/guardrails/actions.js.map +1 -1
- package/dist/lib/guardrails/conditions.d.ts +47 -0
- package/dist/lib/guardrails/conditions.d.ts.map +1 -1
- package/dist/lib/guardrails/conditions.js +55 -10
- package/dist/lib/guardrails/conditions.js.map +1 -1
- package/dist/lib/guardrails/content-engine.d.ts +19 -0
- package/dist/lib/guardrails/content-engine.d.ts.map +1 -0
- package/dist/lib/guardrails/content-engine.js +154 -0
- package/dist/lib/guardrails/content-engine.js.map +1 -0
- package/dist/lib/guardrails/engine.d.ts +33 -0
- package/dist/lib/guardrails/engine.d.ts.map +1 -1
- package/dist/lib/guardrails/engine.js +37 -2
- package/dist/lib/guardrails/engine.js.map +1 -1
- package/dist/lib/guardrails/scanners/base-scanner.d.ts +23 -0
- package/dist/lib/guardrails/scanners/base-scanner.d.ts.map +1 -0
- package/dist/lib/guardrails/scanners/base-scanner.js +7 -0
- package/dist/lib/guardrails/scanners/base-scanner.js.map +1 -0
- package/dist/lib/guardrails/scanners/patterns/pii-patterns.d.ts +13 -0
- package/dist/lib/guardrails/scanners/patterns/pii-patterns.d.ts.map +1 -0
- package/dist/lib/guardrails/scanners/patterns/pii-patterns.js +49 -0
- package/dist/lib/guardrails/scanners/patterns/pii-patterns.js.map +1 -0
- package/dist/lib/guardrails/scanners/patterns/secret-patterns.d.ts +6 -0
- package/dist/lib/guardrails/scanners/patterns/secret-patterns.d.ts.map +1 -0
- package/dist/lib/guardrails/scanners/patterns/secret-patterns.js +69 -0
- package/dist/lib/guardrails/scanners/patterns/secret-patterns.js.map +1 -0
- package/dist/lib/guardrails/scanners/pii-scanner.d.ts +10 -0
- package/dist/lib/guardrails/scanners/pii-scanner.d.ts.map +1 -0
- package/dist/lib/guardrails/scanners/pii-scanner.js +57 -0
- package/dist/lib/guardrails/scanners/pii-scanner.js.map +1 -0
- package/dist/lib/guardrails/scanners/scanner-registry.d.ts +14 -0
- package/dist/lib/guardrails/scanners/scanner-registry.d.ts.map +1 -0
- package/dist/lib/guardrails/scanners/scanner-registry.js +51 -0
- package/dist/lib/guardrails/scanners/scanner-registry.js.map +1 -0
- package/dist/lib/guardrails/scanners/secrets-scanner.d.ts +9 -0
- package/dist/lib/guardrails/scanners/secrets-scanner.d.ts.map +1 -0
- package/dist/lib/guardrails/scanners/secrets-scanner.js +47 -0
- package/dist/lib/guardrails/scanners/secrets-scanner.js.map +1 -0
- package/dist/lib/logger.d.ts +8 -0
- package/dist/lib/logger.d.ts.map +1 -0
- package/dist/lib/logger.js +31 -0
- package/dist/lib/logger.js.map +1 -0
- package/dist/lib/lore-client.d.ts +128 -0
- package/dist/lib/lore-client.d.ts.map +1 -0
- package/dist/lib/lore-client.js +188 -0
- package/dist/lib/lore-client.js.map +1 -0
- package/dist/lib/mesh-client.d.ts +31 -0
- package/dist/lib/mesh-client.d.ts.map +1 -0
- package/dist/lib/mesh-client.js +72 -0
- package/dist/lib/mesh-client.js.map +1 -0
- package/dist/lib/notifications/grouping-buffer.d.ts +25 -0
- package/dist/lib/notifications/grouping-buffer.d.ts.map +1 -0
- package/dist/lib/notifications/grouping-buffer.js +73 -0
- package/dist/lib/notifications/grouping-buffer.js.map +1 -0
- package/dist/lib/notifications/provider.d.ts +10 -0
- package/dist/lib/notifications/provider.d.ts.map +1 -0
- package/dist/lib/notifications/provider.js +5 -0
- package/dist/lib/notifications/provider.js.map +1 -0
- package/dist/lib/notifications/providers/email.d.ts +14 -0
- package/dist/lib/notifications/providers/email.d.ts.map +1 -0
- package/dist/lib/notifications/providers/email.js +88 -0
- package/dist/lib/notifications/providers/email.js.map +1 -0
- package/dist/lib/notifications/providers/pagerduty.d.ts +16 -0
- package/dist/lib/notifications/providers/pagerduty.d.ts.map +1 -0
- package/dist/lib/notifications/providers/pagerduty.js +94 -0
- package/dist/lib/notifications/providers/pagerduty.js.map +1 -0
- package/dist/lib/notifications/providers/slack.d.ts +14 -0
- package/dist/lib/notifications/providers/slack.d.ts.map +1 -0
- package/dist/lib/notifications/providers/slack.js +106 -0
- package/dist/lib/notifications/providers/slack.js.map +1 -0
- package/dist/lib/notifications/providers/webhook.d.ts +16 -0
- package/dist/lib/notifications/providers/webhook.d.ts.map +1 -0
- package/dist/lib/notifications/providers/webhook.js +78 -0
- package/dist/lib/notifications/providers/webhook.js.map +1 -0
- package/dist/lib/notifications/router.d.ts +30 -0
- package/dist/lib/notifications/router.d.ts.map +1 -0
- package/dist/lib/notifications/router.js +137 -0
- package/dist/lib/notifications/router.js.map +1 -0
- package/dist/lib/notifications/ssrf.d.ts +13 -0
- package/dist/lib/notifications/ssrf.d.ts.map +1 -0
- package/dist/lib/notifications/ssrf.js +37 -0
- package/dist/lib/notifications/ssrf.js.map +1 -0
- package/dist/lib/optimization/analyzers/model-downgrade.d.ts +15 -0
- package/dist/lib/optimization/analyzers/model-downgrade.d.ts.map +1 -0
- package/dist/lib/optimization/analyzers/model-downgrade.js +58 -0
- package/dist/lib/optimization/analyzers/model-downgrade.js.map +1 -0
- package/dist/lib/optimization/analyzers/prompt-optimization.d.ts +17 -0
- package/dist/lib/optimization/analyzers/prompt-optimization.d.ts.map +1 -0
- package/dist/lib/optimization/analyzers/prompt-optimization.js +160 -0
- package/dist/lib/optimization/analyzers/prompt-optimization.js.map +1 -0
- package/dist/lib/optimization/analyzers/types.d.ts +23 -0
- package/dist/lib/optimization/analyzers/types.d.ts.map +1 -0
- package/dist/lib/optimization/analyzers/types.js +5 -0
- package/dist/lib/optimization/analyzers/types.js.map +1 -0
- package/dist/lib/optimization/classifier.d.ts +4 -3
- package/dist/lib/optimization/classifier.d.ts.map +1 -1
- package/dist/lib/optimization/classifier.js +15 -9
- package/dist/lib/optimization/classifier.js.map +1 -1
- package/dist/lib/optimization/cost-optimizer.d.ts +21 -0
- package/dist/lib/optimization/cost-optimizer.d.ts.map +1 -0
- package/dist/lib/optimization/cost-optimizer.js +114 -0
- package/dist/lib/optimization/cost-optimizer.js.map +1 -0
- package/dist/lib/optimization/engine.d.ts.map +1 -1
- package/dist/lib/optimization/engine.js +45 -6
- package/dist/lib/optimization/engine.js.map +1 -1
- package/dist/lib/optimization/forecast.d.ts +39 -0
- package/dist/lib/optimization/forecast.d.ts.map +1 -0
- package/dist/lib/optimization/forecast.js +128 -0
- package/dist/lib/optimization/forecast.js.map +1 -0
- package/dist/lib/secrets.d.ts +30 -0
- package/dist/lib/secrets.d.ts.map +1 -0
- package/dist/lib/secrets.js +103 -0
- package/dist/lib/secrets.js.map +1 -0
- package/dist/lib/threshold-monitor.d.ts +53 -0
- package/dist/lib/threshold-monitor.d.ts.map +1 -0
- package/dist/lib/threshold-monitor.js +112 -0
- package/dist/lib/threshold-monitor.js.map +1 -0
- package/dist/middleware/audit.d.ts +16 -0
- package/dist/middleware/audit.d.ts.map +1 -0
- package/dist/middleware/audit.js +16 -0
- package/dist/middleware/audit.js.map +1 -0
- package/dist/middleware/auth-errors.d.ts +67 -0
- package/dist/middleware/auth-errors.d.ts.map +1 -0
- package/dist/middleware/auth-errors.js +84 -0
- package/dist/middleware/auth-errors.js.map +1 -0
- package/dist/middleware/auth.d.ts +5 -2
- package/dist/middleware/auth.d.ts.map +1 -1
- package/dist/middleware/auth.js +44 -17
- package/dist/middleware/auth.js.map +1 -1
- package/dist/middleware/body-limit.d.ts +9 -0
- package/dist/middleware/body-limit.d.ts.map +1 -0
- package/dist/middleware/body-limit.js +15 -0
- package/dist/middleware/body-limit.js.map +1 -0
- package/dist/middleware/cors-config.d.ts +30 -0
- package/dist/middleware/cors-config.d.ts.map +1 -0
- package/dist/middleware/cors-config.js +55 -0
- package/dist/middleware/cors-config.js.map +1 -0
- package/dist/middleware/rate-limit.d.ts +9 -0
- package/dist/middleware/rate-limit.d.ts.map +1 -0
- package/dist/middleware/rate-limit.js +56 -0
- package/dist/middleware/rate-limit.js.map +1 -0
- package/dist/middleware/rbac.d.ts +30 -0
- package/dist/middleware/rbac.d.ts.map +1 -0
- package/dist/middleware/rbac.js +87 -0
- package/dist/middleware/rbac.js.map +1 -0
- package/dist/middleware/security-headers.d.ts +12 -0
- package/dist/middleware/security-headers.d.ts.map +1 -0
- package/dist/middleware/security-headers.js +57 -0
- package/dist/middleware/security-headers.js.map +1 -0
- package/dist/middleware/unified-auth.d.ts +49 -0
- package/dist/middleware/unified-auth.d.ts.map +1 -0
- package/dist/middleware/unified-auth.js +246 -0
- package/dist/middleware/unified-auth.js.map +1 -0
- package/dist/middleware/validation.d.ts +31 -0
- package/dist/middleware/validation.d.ts.map +1 -0
- package/dist/middleware/validation.js +45 -0
- package/dist/middleware/validation.js.map +1 -0
- package/dist/routes/alerts.d.ts.map +1 -1
- package/dist/routes/alerts.js +4 -3
- package/dist/routes/alerts.js.map +1 -1
- package/dist/routes/analytics.d.ts +2 -1
- package/dist/routes/analytics.d.ts.map +1 -1
- package/dist/routes/analytics.js +175 -95
- package/dist/routes/analytics.js.map +1 -1
- package/dist/routes/api-keys.d.ts +5 -0
- package/dist/routes/api-keys.d.ts.map +1 -1
- package/dist/routes/api-keys.js +89 -8
- package/dist/routes/api-keys.js.map +1 -1
- package/dist/routes/audit-verify.d.ts +12 -0
- package/dist/routes/audit-verify.d.ts.map +1 -0
- package/dist/routes/audit-verify.js +73 -0
- package/dist/routes/audit-verify.js.map +1 -0
- package/dist/routes/audit.d.ts +4 -6
- package/dist/routes/audit.d.ts.map +1 -1
- package/dist/routes/audit.js +54 -157
- package/dist/routes/audit.js.map +1 -1
- package/dist/routes/auth.d.ts +21 -0
- package/dist/routes/auth.d.ts.map +1 -0
- package/dist/routes/auth.js +235 -0
- package/dist/routes/auth.js.map +1 -0
- package/dist/routes/benchmarks.d.ts.map +1 -1
- package/dist/routes/benchmarks.js +63 -11
- package/dist/routes/benchmarks.js.map +1 -1
- package/dist/routes/capabilities-top.d.ts.map +1 -1
- package/dist/routes/capabilities-top.js +1 -4
- package/dist/routes/capabilities-top.js.map +1 -1
- package/dist/routes/capabilities.d.ts.map +1 -1
- package/dist/routes/capabilities.js +1 -7
- package/dist/routes/capabilities.js.map +1 -1
- package/dist/routes/compliance.d.ts +17 -0
- package/dist/routes/compliance.d.ts.map +1 -0
- package/dist/routes/compliance.js +151 -0
- package/dist/routes/compliance.js.map +1 -0
- package/dist/routes/config.d.ts +1 -13
- package/dist/routes/config.d.ts.map +1 -1
- package/dist/routes/context.d.ts.map +1 -1
- package/dist/routes/context.js +6 -5
- package/dist/routes/context.js.map +1 -1
- package/dist/routes/cost-budgets.d.ts +20 -0
- package/dist/routes/cost-budgets.d.ts.map +1 -0
- package/dist/routes/cost-budgets.js +194 -0
- package/dist/routes/cost-budgets.js.map +1 -0
- package/dist/routes/delegation.d.ts.map +1 -1
- package/dist/routes/delegation.js +67 -41
- package/dist/routes/delegation.js.map +1 -1
- package/dist/routes/delegations-top.d.ts.map +1 -1
- package/dist/routes/delegations-top.js +1 -3
- package/dist/routes/delegations-top.js.map +1 -1
- package/dist/routes/diagnose.d.ts +16 -0
- package/dist/routes/diagnose.d.ts.map +1 -0
- package/dist/routes/diagnose.js +82 -0
- package/dist/routes/diagnose.js.map +1 -0
- package/dist/routes/discovery.d.ts.map +1 -1
- package/dist/routes/discovery.js +50 -38
- package/dist/routes/discovery.js.map +1 -1
- package/dist/routes/eval.d.ts +24 -0
- package/dist/routes/eval.d.ts.map +1 -0
- package/dist/routes/eval.js +281 -0
- package/dist/routes/eval.js.map +1 -0
- package/dist/routes/events.d.ts.map +1 -1
- package/dist/routes/events.js +11 -6
- package/dist/routes/events.js.map +1 -1
- package/dist/routes/guardrails.d.ts +2 -1
- package/dist/routes/guardrails.d.ts.map +1 -1
- package/dist/routes/guardrails.js +85 -14
- package/dist/routes/guardrails.js.map +1 -1
- package/dist/routes/health.d.ts +14 -11
- package/dist/routes/health.d.ts.map +1 -1
- package/dist/routes/health.js +181 -61
- package/dist/routes/health.js.map +1 -1
- package/dist/routes/lore-proxy.d.ts +13 -0
- package/dist/routes/lore-proxy.d.ts.map +1 -0
- package/dist/routes/lore-proxy.js +229 -0
- package/dist/routes/lore-proxy.js.map +1 -0
- package/dist/routes/mesh-proxy.d.ts +7 -0
- package/dist/routes/mesh-proxy.d.ts.map +1 -0
- package/dist/routes/mesh-proxy.js +94 -0
- package/dist/routes/mesh-proxy.js.map +1 -0
- package/dist/routes/notifications.d.ts +19 -0
- package/dist/routes/notifications.d.ts.map +1 -0
- package/dist/routes/notifications.js +129 -0
- package/dist/routes/notifications.js.map +1 -0
- package/dist/routes/optimize.d.ts.map +1 -1
- package/dist/routes/optimize.js +44 -0
- package/dist/routes/optimize.js.map +1 -1
- package/dist/routes/otlp.d.ts +17 -0
- package/dist/routes/otlp.d.ts.map +1 -0
- package/dist/routes/otlp.js +544 -0
- package/dist/routes/otlp.js.map +1 -0
- package/dist/routes/prompts.d.ts +21 -0
- package/dist/routes/prompts.d.ts.map +1 -0
- package/dist/routes/prompts.js +173 -0
- package/dist/routes/prompts.js.map +1 -0
- package/dist/routes/recall.d.ts.map +1 -1
- package/dist/routes/recall.js +6 -4
- package/dist/routes/recall.js.map +1 -1
- package/dist/routes/replay.d.ts.map +1 -1
- package/dist/routes/replay.js +2 -1
- package/dist/routes/replay.js.map +1 -1
- package/dist/routes/server-info.d.ts +9 -0
- package/dist/routes/server-info.d.ts.map +1 -0
- package/dist/routes/server-info.js +18 -0
- package/dist/routes/server-info.js.map +1 -0
- package/dist/routes/sessions.d.ts +7 -7
- package/dist/routes/sessions.d.ts.map +1 -1
- package/dist/routes/sessions.js +112 -35
- package/dist/routes/sessions.js.map +1 -1
- package/dist/routes/stats.d.ts.map +1 -1
- package/dist/routes/stats.js +40 -0
- package/dist/routes/stats.js.map +1 -1
- package/dist/routes/stream.d.ts +2 -2
- package/dist/routes/stream.d.ts.map +1 -1
- package/dist/routes/stream.js +7 -11
- package/dist/routes/stream.js.map +1 -1
- package/dist/routes/tenant-helper.d.ts +15 -10
- package/dist/routes/tenant-helper.d.ts.map +1 -1
- package/dist/routes/tenant-helper.js +36 -22
- package/dist/routes/tenant-helper.js.map +1 -1
- package/dist/routes/trust.d.ts.map +1 -1
- package/dist/routes/trust.js +1 -3
- package/dist/routes/trust.js.map +1 -1
- package/dist/schemas/api-keys.d.ts +11 -0
- package/dist/schemas/api-keys.d.ts.map +1 -0
- package/dist/schemas/api-keys.js +10 -0
- package/dist/schemas/api-keys.js.map +1 -0
- package/dist/schemas/common.d.ts +34 -0
- package/dist/schemas/common.d.ts.map +1 -0
- package/dist/schemas/common.js +43 -0
- package/dist/schemas/common.js.map +1 -0
- package/dist/schemas/delegation.d.ts +23 -0
- package/dist/schemas/delegation.d.ts.map +1 -0
- package/dist/schemas/delegation.js +22 -0
- package/dist/schemas/delegation.js.map +1 -0
- package/dist/schemas/discovery.d.ts +17 -0
- package/dist/schemas/discovery.d.ts.map +1 -0
- package/dist/schemas/discovery.js +15 -0
- package/dist/schemas/discovery.js.map +1 -0
- package/dist/schemas/health.d.ts +75 -0
- package/dist/schemas/health.d.ts.map +1 -0
- package/dist/schemas/health.js +55 -0
- package/dist/schemas/health.js.map +1 -0
- package/dist/schemas/index.d.ts +6 -0
- package/dist/schemas/index.d.ts.map +1 -0
- package/dist/schemas/index.js +6 -0
- package/dist/schemas/index.js.map +1 -0
- package/dist/schemas/sessions.d.ts +67 -0
- package/dist/schemas/sessions.d.ts.map +1 -0
- package/dist/schemas/sessions.js +58 -0
- package/dist/schemas/sessions.js.map +1 -0
- package/dist/services/delegation-service.d.ts +1 -4
- package/dist/services/delegation-service.d.ts.map +1 -1
- package/dist/services/delegation-service.js +5 -31
- package/dist/services/delegation-service.js.map +1 -1
- package/package.json +29 -19
- package/dist/db/lesson-store.d.ts +0 -57
- package/dist/db/lesson-store.d.ts.map +0 -1
- package/dist/db/lesson-store.js +0 -217
- package/dist/db/lesson-store.js.map +0 -1
- package/dist/lib/embeddings/local.d.ts +0 -15
- package/dist/lib/embeddings/local.d.ts.map +0 -1
- package/dist/lib/embeddings/local.js +0 -65
- package/dist/lib/embeddings/local.js.map +0 -1
- package/dist/lib/redaction/human-review-layer.d.ts +0 -37
- package/dist/lib/redaction/human-review-layer.d.ts.map +0 -1
- package/dist/lib/redaction/human-review-layer.js +0 -62
- package/dist/lib/redaction/human-review-layer.js.map +0 -1
- package/dist/lib/redaction/index.d.ts +0 -12
- package/dist/lib/redaction/index.d.ts.map +0 -1
- package/dist/lib/redaction/index.js +0 -12
- package/dist/lib/redaction/index.js.map +0 -1
- package/dist/lib/redaction/pii-detection-layer.d.ts +0 -30
- package/dist/lib/redaction/pii-detection-layer.d.ts.map +0 -1
- package/dist/lib/redaction/pii-detection-layer.js +0 -183
- package/dist/lib/redaction/pii-detection-layer.js.map +0 -1
- package/dist/lib/redaction/pipeline.d.ts +0 -26
- package/dist/lib/redaction/pipeline.d.ts.map +0 -1
- package/dist/lib/redaction/pipeline.js +0 -91
- package/dist/lib/redaction/pipeline.js.map +0 -1
- package/dist/lib/redaction/secret-detection-layer.d.ts +0 -10
- package/dist/lib/redaction/secret-detection-layer.d.ts.map +0 -1
- package/dist/lib/redaction/secret-detection-layer.js +0 -79
- package/dist/lib/redaction/secret-detection-layer.js.map +0 -1
- package/dist/lib/redaction/secret-patterns.d.ts +0 -29
- package/dist/lib/redaction/secret-patterns.d.ts.map +0 -1
- package/dist/lib/redaction/secret-patterns.js +0 -133
- package/dist/lib/redaction/secret-patterns.js.map +0 -1
- package/dist/lib/redaction/semantic-denylist-layer.d.ts +0 -10
- package/dist/lib/redaction/semantic-denylist-layer.d.ts.map +0 -1
- package/dist/lib/redaction/semantic-denylist-layer.js +0 -64
- package/dist/lib/redaction/semantic-denylist-layer.js.map +0 -1
- package/dist/lib/redaction/tenant-deidentification-layer.d.ts +0 -10
- package/dist/lib/redaction/tenant-deidentification-layer.d.ts.map +0 -1
- package/dist/lib/redaction/tenant-deidentification-layer.js +0 -64
- package/dist/lib/redaction/tenant-deidentification-layer.js.map +0 -1
- package/dist/lib/redaction/url-path-scrubbing-layer.d.ts +0 -14
- package/dist/lib/redaction/url-path-scrubbing-layer.d.ts.map +0 -1
- package/dist/lib/redaction/url-path-scrubbing-layer.js +0 -156
- package/dist/lib/redaction/url-path-scrubbing-layer.js.map +0 -1
- package/dist/routes/community.d.ts +0 -24
- package/dist/routes/community.d.ts.map +0 -1
- package/dist/routes/community.js +0 -272
- package/dist/routes/community.js.map +0 -1
- package/dist/routes/lessons.d.ts +0 -19
- package/dist/routes/lessons.d.ts.map +0 -1
- package/dist/routes/lessons.js +0 -164
- package/dist/routes/lessons.js.map +0 -1
- package/dist/routes/redaction-test.d.ts +0 -14
- package/dist/routes/redaction-test.d.ts.map +0 -1
- package/dist/routes/redaction-test.js +0 -33
- package/dist/routes/redaction-test.js.map +0 -1
- package/dist/services/community-service.d.ts +0 -283
- package/dist/services/community-service.d.ts.map +0 -1
- package/dist/services/community-service.js +0 -816
- package/dist/services/community-service.js.map +0 -1
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "7",
|
|
3
|
+
"dialect": "postgresql",
|
|
4
|
+
"entries": [
|
|
5
|
+
{
|
|
6
|
+
"idx": 0,
|
|
7
|
+
"version": "7",
|
|
8
|
+
"when": 1771186404108,
|
|
9
|
+
"tag": "0000_initial",
|
|
10
|
+
"breakpoints": true
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"idx": 1,
|
|
14
|
+
"version": "7",
|
|
15
|
+
"when": 1739654400000,
|
|
16
|
+
"tag": "0001_indexes",
|
|
17
|
+
"breakpoints": false
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"idx": 2,
|
|
21
|
+
"version": "7",
|
|
22
|
+
"when": 1740048000000,
|
|
23
|
+
"tag": "0002_pgvector",
|
|
24
|
+
"breakpoints": false
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* - Tenant isolation
|
|
8
8
|
*/
|
|
9
9
|
import type { SqliteDb } from './index.js';
|
|
10
|
+
import type { IEmbeddingStore } from './embedding-store.interface.js';
|
|
10
11
|
/** Options for similarity search */
|
|
11
12
|
export interface SimilaritySearchOptions {
|
|
12
13
|
/** Filter by source type ('event', 'session', 'lesson') */
|
|
@@ -43,7 +44,7 @@ export interface StoredEmbedding {
|
|
|
43
44
|
dimensions: number;
|
|
44
45
|
createdAt: string;
|
|
45
46
|
}
|
|
46
|
-
export declare class EmbeddingStore {
|
|
47
|
+
export declare class EmbeddingStore implements IEmbeddingStore {
|
|
47
48
|
private readonly db;
|
|
48
49
|
constructor(db: SqliteDb);
|
|
49
50
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-store.d.ts","sourceRoot":"","sources":["../../src/db/embedding-store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"embedding-store.d.ts","sourceRoot":"","sources":["../../src/db/embedding-store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAG3C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AA0BtE,oCAAoC;AACpC,MAAM,WAAW,uBAAuB;IACtC,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8CAA8C;IAC9C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,8CAA8C;IAC9C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wCAAwC;AACxC,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,cAAe,YAAW,eAAe;IACxC,OAAO,CAAC,QAAQ,CAAC,EAAE;gBAAF,EAAE,EAAE,QAAQ;IAEzC;;;;;OAKG;IACH,KAAK,CACH,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,YAAY,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,GACjB,MAAM;IAqDT;;OAEG;IACH,WAAW,CACT,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,eAAe,GAAG,IAAI;IA6BzB;;;OAGG;IACH,gBAAgB,CACd,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,YAAY,EACzB,IAAI,GAAE,uBAA4B,GACjC,gBAAgB,EAAE;IAoDrB;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM;IAetE;;OAEG;IACH,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;CAShC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* IEmbeddingStore — shared interface for SQLite and PostgreSQL embedding stores.
|
|
3
|
+
*
|
|
4
|
+
* All methods return Promise-compatible values (sync returns are valid Promises).
|
|
5
|
+
*/
|
|
6
|
+
import type { SimilaritySearchOptions, SimilarityResult, StoredEmbedding } from './embedding-store.js';
|
|
7
|
+
export interface IEmbeddingStore {
|
|
8
|
+
/** Store an embedding with source-level deduplication. Returns the embedding ID. */
|
|
9
|
+
store(tenantId: string, sourceType: string, sourceId: string, textContent: string, embedding: Float32Array, model: string, dimensions: number): string | Promise<string>;
|
|
10
|
+
/** Get an embedding by source identifiers. */
|
|
11
|
+
getBySource(tenantId: string, sourceType: string, sourceId: string): StoredEmbedding | null | Promise<StoredEmbedding | null>;
|
|
12
|
+
/** Similarity search over embeddings. */
|
|
13
|
+
similaritySearch(tenantId: string, queryVector: Float32Array, opts?: SimilaritySearchOptions): SimilarityResult[] | Promise<SimilarityResult[]>;
|
|
14
|
+
/** Delete embedding(s) for a specific source. Returns count of deleted rows. */
|
|
15
|
+
delete(tenantId: string, sourceType: string, sourceId: string): number | Promise<number>;
|
|
16
|
+
/** Count embeddings for a tenant. */
|
|
17
|
+
count(tenantId: string): number | Promise<number>;
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=embedding-store.interface.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-store.interface.d.ts","sourceRoot":"","sources":["../../src/db/embedding-store.interface.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvG,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,KAAK,CACH,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,YAAY,EACvB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,GACjB,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE5B,8CAA8C;IAC9C,WAAW,CACT,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,eAAe,GAAG,IAAI,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IAE5D,yCAAyC;IACzC,gBAAgB,CACd,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,YAAY,EACzB,IAAI,CAAC,EAAE,uBAAuB,GAC7B,gBAAgB,EAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAEpD,gFAAgF;IAChF,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEzF,qCAAqC;IACrC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACnD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-store.interface.js","sourceRoot":"","sources":["../../src/db/embedding-store.interface.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
@@ -10,6 +10,8 @@ import { createHash, randomUUID } from 'node:crypto';
|
|
|
10
10
|
import { eq, and, sql } from 'drizzle-orm';
|
|
11
11
|
import { embeddings } from './schema.sqlite.js';
|
|
12
12
|
import { cosineSimilarity } from '../lib/embeddings/math.js';
|
|
13
|
+
import { createLogger } from '../lib/logger.js';
|
|
14
|
+
const log = createLogger('EmbeddingStore');
|
|
13
15
|
/** Maximum candidates loaded into memory for similarity search */
|
|
14
16
|
const MAX_CANDIDATES = 10_000;
|
|
15
17
|
/** Compute SHA-256 content hash of text */
|
|
@@ -128,7 +130,7 @@ export class EmbeddingStore {
|
|
|
128
130
|
.limit(MAX_CANDIDATES)
|
|
129
131
|
.all();
|
|
130
132
|
if (rows.length >= MAX_CANDIDATES) {
|
|
131
|
-
|
|
133
|
+
log.warn(`similaritySearch hit MAX_CANDIDATES limit (${MAX_CANDIDATES}) ` +
|
|
132
134
|
`for tenant=${tenantId}. Results may be incomplete.`);
|
|
133
135
|
}
|
|
134
136
|
// Compute cosine similarity for each candidate
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedding-store.js","sourceRoot":"","sources":["../../src/db/embedding-store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAE7D,kEAAkE;AAClE,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,2CAA2C;AAC3C,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC;AAED,+DAA+D;AAC/D,SAAS,kBAAkB,CAAC,SAAuB;IACjD,kEAAkE;IAClE,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC;AAChH,CAAC;AAED,0DAA0D;AAC1D,SAAS,oBAAoB,CAAC,IAAY;IACxC,uFAAuF;IACvF,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACzC,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAyCD,MAAM,OAAO,cAAc;IACI;IAA7B,YAA6B,EAAY;QAAZ,OAAE,GAAF,EAAE,CAAU;IAAG,CAAC;IAE7C;;;;;OAKG;IACH,KAAK,CACH,QAAgB,EAChB,UAAkB,EAClB,QAAgB,EAChB,WAAmB,EACnB,SAAuB,EACvB,KAAa,EACb,UAAkB;QAElB,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAErC,oDAAoD;QACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE;aACrB,MAAM,CAAC,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC7B,IAAI,CAAC,UAAU,CAAC;aAChB,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EACjC,EAAE,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,EACrC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAClC,CACF;aACA,GAAG,EAAE,CAAC;QAET,IAAI,QAAQ,EAAE,CAAC;YACb,6CAA6C;YAC7C,IAAI,CAAC,EAAE;iBACJ,MAAM,CAAC,UAAU,CAAC;iBAClB,GAAG,CAAC;gBACH,WAAW,EAAE,IAAI;gBACjB,WAAW;gBACX,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC;gBACxC,cAAc,EAAE,KAAK;gBACrB,UAAU;aACX,CAAC;iBACD,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;iBACrC,GAAG,EAAE,CAAC;YACT,OAAO,QAAQ,CAAC,EAAE,CAAC;QACrB,CAAC;QAED,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;QACxB,IAAI,CAAC,EAAE;aACJ,MAAM,CAAC,UAAU,CAAC;aAClB,MAAM,CAAC;YACN,EAAE;YACF,QAAQ;YACR,UAAU;YACV,QAAQ;YACR,WAAW,EAAE,IAAI;YACjB,WAAW;YACX,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC;YACxC,cAAc,EAAE,KAAK;YACrB,UAAU;YACV,SAAS,EAAE,GAAG;SACf,CAAC;aACD,GAAG,EAAE,CAAC;QAET,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACH,WAAW,CACT,QAAgB,EAChB,UAAkB,EAClB,QAAgB;QAEhB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,MAAM,EAAE;aACR,IAAI,CAAC,UAAU,CAAC;aAChB,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EACjC,EAAE,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,EACrC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAClC,CACF;aACA,GAAG,EAAE,CAAC;QAET,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,SAAS,EAAE,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC;YAC9C,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,gBAAgB,CACd,QAAgB,EAChB,WAAyB,EACzB,OAAgC,EAAE;QAElC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,GAAG,EAAE,EAAE,QAAQ,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC;QAElE,+BAA+B;QAC/B,MAAM,UAAU,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QACvD,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACT,UAAU,CAAC,IAAI,CAAC,GAAG,CAAA,GAAG,UAAU,CAAC,SAAS,OAAO,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,EAAE,EAAE,CAAC;YACP,UAAU,CAAC,IAAI,CAAC,GAAG,CAAA,GAAG,UAAU,CAAC,SAAS,OAAO,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,MAAM,EAAE;aACR,IAAI,CAAC,UAAU,CAAC;aAChB,KAAK,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;aACzB,KAAK,CAAC,cAAc,CAAC;aACrB,GAAG,EAAE,CAAC;QAET,IAAI,IAAI,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;YAClC,
|
|
1
|
+
{"version":3,"file":"embedding-store.js","sourceRoot":"","sources":["../../src/db/embedding-store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhD,MAAM,GAAG,GAAG,YAAY,CAAC,gBAAgB,CAAC,CAAC;AAE3C,kEAAkE;AAClE,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,2CAA2C;AAC3C,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC;AAED,+DAA+D;AAC/D,SAAS,kBAAkB,CAAC,SAAuB;IACjD,kEAAkE;IAClE,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC;AAChH,CAAC;AAED,0DAA0D;AAC1D,SAAS,oBAAoB,CAAC,IAAY;IACxC,uFAAuF;IACvF,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACzC,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAyCD,MAAM,OAAO,cAAc;IACI;IAA7B,YAA6B,EAAY;QAAZ,OAAE,GAAF,EAAE,CAAU;IAAG,CAAC;IAE7C;;;;;OAKG;IACH,KAAK,CACH,QAAgB,EAChB,UAAkB,EAClB,QAAgB,EAChB,WAAmB,EACnB,SAAuB,EACvB,KAAa,EACb,UAAkB;QAElB,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACtC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAErC,oDAAoD;QACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE;aACrB,MAAM,CAAC,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC;aAC7B,IAAI,CAAC,UAAU,CAAC;aAChB,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EACjC,EAAE,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,EACrC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAClC,CACF;aACA,GAAG,EAAE,CAAC;QAET,IAAI,QAAQ,EAAE,CAAC;YACb,6CAA6C;YAC7C,IAAI,CAAC,EAAE;iBACJ,MAAM,CAAC,UAAU,CAAC;iBAClB,GAAG,CAAC;gBACH,WAAW,EAAE,IAAI;gBACjB,WAAW;gBACX,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC;gBACxC,cAAc,EAAE,KAAK;gBACrB,UAAU;aACX,CAAC;iBACD,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;iBACrC,GAAG,EAAE,CAAC;YACT,OAAO,QAAQ,CAAC,EAAE,CAAC;QACrB,CAAC;QAED,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;QACxB,IAAI,CAAC,EAAE;aACJ,MAAM,CAAC,UAAU,CAAC;aAClB,MAAM,CAAC;YACN,EAAE;YACF,QAAQ;YACR,UAAU;YACV,QAAQ;YACR,WAAW,EAAE,IAAI;YACjB,WAAW;YACX,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC;YACxC,cAAc,EAAE,KAAK;YACrB,UAAU;YACV,SAAS,EAAE,GAAG;SACf,CAAC;aACD,GAAG,EAAE,CAAC;QAET,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACH,WAAW,CACT,QAAgB,EAChB,UAAkB,EAClB,QAAgB;QAEhB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,MAAM,EAAE;aACR,IAAI,CAAC,UAAU,CAAC;aAChB,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EACjC,EAAE,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,EACrC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAClC,CACF;aACA,GAAG,EAAE,CAAC;QAET,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,SAAS,EAAE,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC;YAC9C,cAAc,EAAE,GAAG,CAAC,cAAc;YAClC,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,gBAAgB,CACd,QAAgB,EAChB,WAAyB,EACzB,OAAgC,EAAE;QAElC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,GAAG,EAAE,EAAE,QAAQ,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC;QAElE,+BAA+B;QAC/B,MAAM,UAAU,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QACvD,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,IAAI,EAAE,CAAC;YACT,UAAU,CAAC,IAAI,CAAC,GAAG,CAAA,GAAG,UAAU,CAAC,SAAS,OAAO,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,EAAE,EAAE,CAAC;YACP,UAAU,CAAC,IAAI,CAAC,GAAG,CAAA,GAAG,UAAU,CAAC,SAAS,OAAO,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,MAAM,EAAE;aACR,IAAI,CAAC,UAAU,CAAC;aAChB,KAAK,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;aACzB,KAAK,CAAC,cAAc,CAAC;aACrB,GAAG,EAAE,CAAC;QAET,IAAI,IAAI,CAAC,MAAM,IAAI,cAAc,EAAE,CAAC;YAClC,GAAG,CAAC,IAAI,CACN,8CAA8C,cAAc,IAAI;gBAChE,cAAc,QAAQ,8BAA8B,CACrD,CAAC;QACJ,CAAC;QAED,+CAA+C;QAC/C,MAAM,OAAO,GAAuB,EAAE,CAAC;QACvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACtD,MAAM,KAAK,GAAG,gBAAgB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YACvD,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,OAAO,CAAC,IAAI,CAAC;oBACX,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,KAAK;oBACL,IAAI,EAAE,GAAG,CAAC,WAAW;oBACrB,cAAc,EAAE,GAAG,CAAC,cAAc;oBAClC,SAAS,EAAE,GAAG,CAAC,SAAS;iBACzB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,0CAA0C;QAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC1C,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,QAAgB,EAAE,UAAkB,EAAE,QAAgB;QAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE;aACnB,MAAM,CAAC,UAAU,CAAC;aAClB,KAAK,CACJ,GAAG,CACD,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EACjC,EAAE,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,EACrC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAClC,CACF;aACA,GAAG,EAAE,CAAC;QAET,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAgB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE;aACnB,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,CAAQ,UAAU,EAAE,CAAC;aACxC,IAAI,CAAC,UAAU,CAAC;aAChB,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;aACxC,GAAG,EAAE,CAAC;QAET,OAAO,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC;IAC5B,CAAC;CACF"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EvalStore (Feature 15 — Stories 2 & 3)
|
|
3
|
+
*
|
|
4
|
+
* CRUD operations for eval datasets, test cases, runs, and results.
|
|
5
|
+
* All operations are tenant-isolated. Follows BenchmarkStore patterns.
|
|
6
|
+
*/
|
|
7
|
+
import type { SqliteDb } from './index.js';
|
|
8
|
+
import type { EvalDataset, EvalTestCase, EvalInput, EvalRun, EvalRunStatus, EvalRunConfig, EvalResult, ScoreResult, ScorerType } from '@agentlensai/core';
|
|
9
|
+
export interface CreateDatasetInput {
|
|
10
|
+
name: string;
|
|
11
|
+
description?: string;
|
|
12
|
+
agentId?: string;
|
|
13
|
+
testCases?: CreateTestCaseInput[];
|
|
14
|
+
}
|
|
15
|
+
export interface CreateTestCaseInput {
|
|
16
|
+
input: EvalInput;
|
|
17
|
+
expectedOutput?: unknown;
|
|
18
|
+
tags?: string[];
|
|
19
|
+
metadata?: Record<string, unknown>;
|
|
20
|
+
scoringCriteria?: string;
|
|
21
|
+
sortOrder?: number;
|
|
22
|
+
}
|
|
23
|
+
export interface ListDatasetFilters {
|
|
24
|
+
agentId?: string;
|
|
25
|
+
limit?: number;
|
|
26
|
+
offset?: number;
|
|
27
|
+
}
|
|
28
|
+
export interface CreateRunInput {
|
|
29
|
+
datasetId: string;
|
|
30
|
+
agentId: string;
|
|
31
|
+
webhookUrl: string;
|
|
32
|
+
config: EvalRunConfig;
|
|
33
|
+
baselineRunId?: string;
|
|
34
|
+
}
|
|
35
|
+
export interface ListRunFilters {
|
|
36
|
+
datasetId?: string;
|
|
37
|
+
agentId?: string;
|
|
38
|
+
status?: EvalRunStatus;
|
|
39
|
+
limit?: number;
|
|
40
|
+
offset?: number;
|
|
41
|
+
}
|
|
42
|
+
export interface CreateResultInput {
|
|
43
|
+
runId: string;
|
|
44
|
+
testCaseId: string;
|
|
45
|
+
tenantId: string;
|
|
46
|
+
sessionId?: string;
|
|
47
|
+
actualOutput?: unknown;
|
|
48
|
+
score: number;
|
|
49
|
+
passed: boolean;
|
|
50
|
+
scorerType: ScorerType;
|
|
51
|
+
scorerDetails: ScoreResult;
|
|
52
|
+
latencyMs?: number;
|
|
53
|
+
costUsd?: number;
|
|
54
|
+
tokenCount?: number;
|
|
55
|
+
error?: string;
|
|
56
|
+
}
|
|
57
|
+
export declare class EvalStore {
|
|
58
|
+
private readonly db;
|
|
59
|
+
constructor(db: SqliteDb);
|
|
60
|
+
createDataset(tenantId: string, input: CreateDatasetInput): EvalDataset;
|
|
61
|
+
getDataset(tenantId: string, id: string): EvalDataset | undefined;
|
|
62
|
+
listDatasets(tenantId: string, filters?: ListDatasetFilters): {
|
|
63
|
+
datasets: EvalDataset[];
|
|
64
|
+
total: number;
|
|
65
|
+
};
|
|
66
|
+
updateDataset(tenantId: string, id: string, updates: {
|
|
67
|
+
name?: string;
|
|
68
|
+
description?: string;
|
|
69
|
+
agentId?: string;
|
|
70
|
+
}): EvalDataset | undefined;
|
|
71
|
+
getTestCases(datasetId: string): EvalTestCase[];
|
|
72
|
+
addTestCases(datasetId: string, tenantId: string, cases: CreateTestCaseInput[]): EvalTestCase[];
|
|
73
|
+
updateTestCase(tenantId: string, caseId: string, changes: Partial<CreateTestCaseInput>): EvalTestCase | undefined;
|
|
74
|
+
deleteTestCase(tenantId: string, caseId: string): boolean;
|
|
75
|
+
createVersion(tenantId: string, datasetId: string): EvalDataset;
|
|
76
|
+
createRun(tenantId: string, input: CreateRunInput): EvalRun;
|
|
77
|
+
getRun(tenantId: string, id: string): EvalRun | undefined;
|
|
78
|
+
listRuns(tenantId: string, filters?: ListRunFilters): {
|
|
79
|
+
runs: EvalRun[];
|
|
80
|
+
total: number;
|
|
81
|
+
};
|
|
82
|
+
updateRunStatus(id: string, status: EvalRunStatus, aggregates?: Partial<Pick<EvalRun, 'totalCases' | 'passedCases' | 'failedCases' | 'avgScore' | 'totalCostUsd' | 'totalDurationMs' | 'startedAt' | 'completedAt'>>, error?: string): void;
|
|
83
|
+
cancelRun(id: string): void;
|
|
84
|
+
saveResult(input: CreateResultInput): EvalResult;
|
|
85
|
+
getResults(runId: string): EvalResult[];
|
|
86
|
+
getResultForCase(runId: string, testCaseId: string): EvalResult | undefined;
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=eval-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-store.d.ts","sourceRoot":"","sources":["../../src/db/eval-store.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACZ,SAAS,EACT,OAAO,EACP,aAAa,EACb,aAAa,EACb,UAAU,EACV,WAAW,EACX,UAAU,EACX,MAAM,mBAAmB,CAAC;AAwE3B,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,SAAS,CAAC;IACjB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,UAAU,CAAC;IACvB,aAAa,EAAE,WAAW,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAiFD,qBAAa,SAAS;IACR,OAAO,CAAC,QAAQ,CAAC,EAAE;gBAAF,EAAE,EAAE,QAAQ;IAIzC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,WAAW;IAoCvE,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAejE,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,kBAAuB,GAAG;QAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE;IA+B5G,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,WAAW,GAAG,SAAS;IAmBxI,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,YAAY,EAAE;IAO/C,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,GAAG,YAAY,EAAE;IAuC/F,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,YAAY,GAAG,SAAS;IA+BjH,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO;IAkBzD,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,WAAW;IAgD/D,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO;IAmB3D,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS;IAQzD,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG;QAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE;IAsB5F,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,aAAa,GAAG,aAAa,GAAG,UAAU,GAAG,cAAc,GAAG,iBAAiB,GAAG,WAAW,GAAG,aAAa,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI;IAkB3O,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAQ3B,UAAU,CAAC,KAAK,EAAE,iBAAiB,GAAG,UAAU;IAqBhD,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,EAAE;IAOvC,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;CAM5E"}
|
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EvalStore (Feature 15 — Stories 2 & 3)
|
|
3
|
+
*
|
|
4
|
+
* CRUD operations for eval datasets, test cases, runs, and results.
|
|
5
|
+
* All operations are tenant-isolated. Follows BenchmarkStore patterns.
|
|
6
|
+
*/
|
|
7
|
+
import { randomUUID } from 'node:crypto';
|
|
8
|
+
import { sql } from 'drizzle-orm';
|
|
9
|
+
// ─── Row Converters ────────────────────────────────────────
|
|
10
|
+
function rowToDataset(row) {
|
|
11
|
+
const ds = {
|
|
12
|
+
id: row.id,
|
|
13
|
+
tenantId: row.tenant_id,
|
|
14
|
+
name: row.name,
|
|
15
|
+
version: row.version,
|
|
16
|
+
immutable: row.immutable === 1,
|
|
17
|
+
createdAt: row.created_at,
|
|
18
|
+
updatedAt: row.updated_at,
|
|
19
|
+
};
|
|
20
|
+
if (row.agent_id)
|
|
21
|
+
ds.agentId = row.agent_id;
|
|
22
|
+
if (row.description)
|
|
23
|
+
ds.description = row.description;
|
|
24
|
+
if (row.parent_id)
|
|
25
|
+
ds.parentId = row.parent_id;
|
|
26
|
+
return ds;
|
|
27
|
+
}
|
|
28
|
+
function rowToTestCase(row) {
|
|
29
|
+
return {
|
|
30
|
+
id: row.id,
|
|
31
|
+
datasetId: row.dataset_id,
|
|
32
|
+
tenantId: row.tenant_id,
|
|
33
|
+
input: JSON.parse(row.input),
|
|
34
|
+
expectedOutput: row.expected_output ? JSON.parse(row.expected_output) : undefined,
|
|
35
|
+
tags: JSON.parse(row.tags),
|
|
36
|
+
metadata: JSON.parse(row.metadata),
|
|
37
|
+
scoringCriteria: row.scoring_criteria ?? undefined,
|
|
38
|
+
sortOrder: row.sort_order,
|
|
39
|
+
createdAt: row.created_at,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function rowToRun(row) {
|
|
43
|
+
const run = {
|
|
44
|
+
id: row.id,
|
|
45
|
+
tenantId: row.tenant_id,
|
|
46
|
+
datasetId: row.dataset_id,
|
|
47
|
+
datasetVersion: row.dataset_version,
|
|
48
|
+
agentId: row.agent_id,
|
|
49
|
+
webhookUrl: row.webhook_url,
|
|
50
|
+
status: row.status,
|
|
51
|
+
config: JSON.parse(row.config),
|
|
52
|
+
totalCases: row.total_cases,
|
|
53
|
+
passedCases: row.passed_cases,
|
|
54
|
+
failedCases: row.failed_cases,
|
|
55
|
+
createdAt: row.created_at,
|
|
56
|
+
};
|
|
57
|
+
if (row.baseline_run_id)
|
|
58
|
+
run.baselineRunId = row.baseline_run_id;
|
|
59
|
+
if (row.avg_score !== null)
|
|
60
|
+
run.avgScore = row.avg_score;
|
|
61
|
+
if (row.total_cost_usd !== null)
|
|
62
|
+
run.totalCostUsd = row.total_cost_usd;
|
|
63
|
+
if (row.total_duration_ms !== null)
|
|
64
|
+
run.totalDurationMs = row.total_duration_ms;
|
|
65
|
+
if (row.started_at)
|
|
66
|
+
run.startedAt = row.started_at;
|
|
67
|
+
if (row.completed_at)
|
|
68
|
+
run.completedAt = row.completed_at;
|
|
69
|
+
return run;
|
|
70
|
+
}
|
|
71
|
+
function rowToResult(row) {
|
|
72
|
+
return {
|
|
73
|
+
id: row.id,
|
|
74
|
+
runId: row.run_id,
|
|
75
|
+
testCaseId: row.test_case_id,
|
|
76
|
+
tenantId: row.tenant_id,
|
|
77
|
+
sessionId: row.session_id ?? undefined,
|
|
78
|
+
actualOutput: row.actual_output ? JSON.parse(row.actual_output) : undefined,
|
|
79
|
+
score: row.score,
|
|
80
|
+
passed: row.passed === 1,
|
|
81
|
+
scorerType: row.scorer_type,
|
|
82
|
+
scorerDetails: JSON.parse(row.scorer_details),
|
|
83
|
+
latencyMs: row.latency_ms ?? undefined,
|
|
84
|
+
costUsd: row.cost_usd ?? undefined,
|
|
85
|
+
tokenCount: row.token_count ?? undefined,
|
|
86
|
+
error: row.error ?? undefined,
|
|
87
|
+
createdAt: row.created_at,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
// ─── Store Class ───────────────────────────────────────────
|
|
91
|
+
export class EvalStore {
|
|
92
|
+
db;
|
|
93
|
+
constructor(db) {
|
|
94
|
+
this.db = db;
|
|
95
|
+
}
|
|
96
|
+
// ─── Dataset CRUD ──────────────────────────────────────
|
|
97
|
+
createDataset(tenantId, input) {
|
|
98
|
+
const id = randomUUID();
|
|
99
|
+
const now = new Date().toISOString();
|
|
100
|
+
const client = this.db.$client;
|
|
101
|
+
const txn = client.transaction(() => {
|
|
102
|
+
this.db.run(sql `
|
|
103
|
+
INSERT INTO eval_datasets (id, tenant_id, agent_id, name, description, version, created_at, updated_at)
|
|
104
|
+
VALUES (${id}, ${tenantId}, ${input.agentId ?? null}, ${input.name}, ${input.description ?? null}, 1, ${now}, ${now})
|
|
105
|
+
`);
|
|
106
|
+
if (input.testCases && input.testCases.length > 0) {
|
|
107
|
+
for (let i = 0; i < input.testCases.length; i++) {
|
|
108
|
+
const tc = input.testCases[i];
|
|
109
|
+
const caseId = randomUUID();
|
|
110
|
+
this.db.run(sql `
|
|
111
|
+
INSERT INTO eval_test_cases (id, dataset_id, tenant_id, input, expected_output, tags, metadata, scoring_criteria, sort_order, created_at)
|
|
112
|
+
VALUES (
|
|
113
|
+
${caseId}, ${id}, ${tenantId},
|
|
114
|
+
${JSON.stringify(tc.input)},
|
|
115
|
+
${tc.expectedOutput !== undefined ? JSON.stringify(tc.expectedOutput) : null},
|
|
116
|
+
${JSON.stringify(tc.tags ?? [])},
|
|
117
|
+
${JSON.stringify(tc.metadata ?? {})},
|
|
118
|
+
${tc.scoringCriteria ?? null},
|
|
119
|
+
${tc.sortOrder ?? i},
|
|
120
|
+
${now}
|
|
121
|
+
)
|
|
122
|
+
`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
txn();
|
|
127
|
+
return this.getDataset(tenantId, id);
|
|
128
|
+
}
|
|
129
|
+
getDataset(tenantId, id) {
|
|
130
|
+
const row = this.db.get(sql `
|
|
131
|
+
SELECT * FROM eval_datasets WHERE id = ${id} AND tenant_id = ${tenantId}
|
|
132
|
+
`);
|
|
133
|
+
if (!row)
|
|
134
|
+
return undefined;
|
|
135
|
+
const countRow = this.db.get(sql `
|
|
136
|
+
SELECT COUNT(*) as cnt FROM eval_test_cases WHERE dataset_id = ${id}
|
|
137
|
+
`);
|
|
138
|
+
const ds = rowToDataset(row);
|
|
139
|
+
ds.testCaseCount = countRow?.cnt ?? 0;
|
|
140
|
+
return ds;
|
|
141
|
+
}
|
|
142
|
+
listDatasets(tenantId, filters = {}) {
|
|
143
|
+
const { agentId, limit = 20, offset = 0 } = filters;
|
|
144
|
+
let whereClause = sql `WHERE tenant_id = ${tenantId}`;
|
|
145
|
+
if (agentId) {
|
|
146
|
+
whereClause = sql `${whereClause} AND agent_id = ${agentId}`;
|
|
147
|
+
}
|
|
148
|
+
const countRow = this.db.get(sql `
|
|
149
|
+
SELECT COUNT(*) as cnt FROM eval_datasets ${whereClause}
|
|
150
|
+
`);
|
|
151
|
+
const total = countRow?.cnt ?? 0;
|
|
152
|
+
const rows = this.db.all(sql `
|
|
153
|
+
SELECT * FROM eval_datasets ${whereClause}
|
|
154
|
+
ORDER BY created_at DESC
|
|
155
|
+
LIMIT ${limit} OFFSET ${offset}
|
|
156
|
+
`);
|
|
157
|
+
const datasets = rows.map((row) => {
|
|
158
|
+
const ds = rowToDataset(row);
|
|
159
|
+
const countRow2 = this.db.get(sql `
|
|
160
|
+
SELECT COUNT(*) as cnt FROM eval_test_cases WHERE dataset_id = ${row.id}
|
|
161
|
+
`);
|
|
162
|
+
ds.testCaseCount = countRow2?.cnt ?? 0;
|
|
163
|
+
return ds;
|
|
164
|
+
});
|
|
165
|
+
return { datasets, total };
|
|
166
|
+
}
|
|
167
|
+
updateDataset(tenantId, id, updates) {
|
|
168
|
+
const existing = this.getDataset(tenantId, id);
|
|
169
|
+
if (!existing)
|
|
170
|
+
return undefined;
|
|
171
|
+
const now = new Date().toISOString();
|
|
172
|
+
this.db.run(sql `
|
|
173
|
+
UPDATE eval_datasets SET
|
|
174
|
+
name = ${updates.name ?? existing.name},
|
|
175
|
+
description = ${updates.description !== undefined ? updates.description : existing.description ?? null},
|
|
176
|
+
agent_id = ${updates.agentId !== undefined ? updates.agentId : existing.agentId ?? null},
|
|
177
|
+
updated_at = ${now}
|
|
178
|
+
WHERE id = ${id} AND tenant_id = ${tenantId}
|
|
179
|
+
`);
|
|
180
|
+
return this.getDataset(tenantId, id);
|
|
181
|
+
}
|
|
182
|
+
// ─── Test Case CRUD ────────────────────────────────────
|
|
183
|
+
getTestCases(datasetId) {
|
|
184
|
+
const rows = this.db.all(sql `
|
|
185
|
+
SELECT * FROM eval_test_cases WHERE dataset_id = ${datasetId} ORDER BY sort_order ASC
|
|
186
|
+
`);
|
|
187
|
+
return rows.map(rowToTestCase);
|
|
188
|
+
}
|
|
189
|
+
addTestCases(datasetId, tenantId, cases) {
|
|
190
|
+
// Check immutability
|
|
191
|
+
const ds = this.db.get(sql `
|
|
192
|
+
SELECT * FROM eval_datasets WHERE id = ${datasetId} AND tenant_id = ${tenantId}
|
|
193
|
+
`);
|
|
194
|
+
if (!ds)
|
|
195
|
+
throw new Error(`Dataset ${datasetId} not found`);
|
|
196
|
+
if (ds.immutable === 1)
|
|
197
|
+
throw new Error('Cannot modify an immutable (versioned) dataset');
|
|
198
|
+
const now = new Date().toISOString();
|
|
199
|
+
const ids = [];
|
|
200
|
+
for (let i = 0; i < cases.length; i++) {
|
|
201
|
+
const tc = cases[i];
|
|
202
|
+
const caseId = randomUUID();
|
|
203
|
+
ids.push(caseId);
|
|
204
|
+
this.db.run(sql `
|
|
205
|
+
INSERT INTO eval_test_cases (id, dataset_id, tenant_id, input, expected_output, tags, metadata, scoring_criteria, sort_order, created_at)
|
|
206
|
+
VALUES (
|
|
207
|
+
${caseId}, ${datasetId}, ${tenantId},
|
|
208
|
+
${JSON.stringify(tc.input)},
|
|
209
|
+
${tc.expectedOutput !== undefined ? JSON.stringify(tc.expectedOutput) : null},
|
|
210
|
+
${JSON.stringify(tc.tags ?? [])},
|
|
211
|
+
${JSON.stringify(tc.metadata ?? {})},
|
|
212
|
+
${tc.scoringCriteria ?? null},
|
|
213
|
+
${tc.sortOrder ?? i},
|
|
214
|
+
${now}
|
|
215
|
+
)
|
|
216
|
+
`);
|
|
217
|
+
}
|
|
218
|
+
// Update dataset's updated_at
|
|
219
|
+
this.db.run(sql `UPDATE eval_datasets SET updated_at = ${now} WHERE id = ${datasetId}`);
|
|
220
|
+
return ids.map(id => {
|
|
221
|
+
const row = this.db.get(sql `SELECT * FROM eval_test_cases WHERE id = ${id}`);
|
|
222
|
+
return rowToTestCase(row);
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
updateTestCase(tenantId, caseId, changes) {
|
|
226
|
+
const row = this.db.get(sql `
|
|
227
|
+
SELECT * FROM eval_test_cases WHERE id = ${caseId} AND tenant_id = ${tenantId}
|
|
228
|
+
`);
|
|
229
|
+
if (!row)
|
|
230
|
+
return undefined;
|
|
231
|
+
// Check immutability
|
|
232
|
+
const ds = this.db.get(sql `SELECT * FROM eval_datasets WHERE id = ${row.dataset_id}`);
|
|
233
|
+
if (ds && ds.immutable === 1)
|
|
234
|
+
throw new Error('Cannot modify an immutable (versioned) dataset');
|
|
235
|
+
const now = new Date().toISOString();
|
|
236
|
+
const existing = rowToTestCase(row);
|
|
237
|
+
this.db.run(sql `
|
|
238
|
+
UPDATE eval_test_cases SET
|
|
239
|
+
input = ${changes.input ? JSON.stringify(changes.input) : row.input},
|
|
240
|
+
expected_output = ${changes.expectedOutput !== undefined ? JSON.stringify(changes.expectedOutput) : row.expected_output},
|
|
241
|
+
tags = ${changes.tags ? JSON.stringify(changes.tags) : row.tags},
|
|
242
|
+
metadata = ${changes.metadata ? JSON.stringify(changes.metadata) : row.metadata},
|
|
243
|
+
scoring_criteria = ${changes.scoringCriteria !== undefined ? changes.scoringCriteria : row.scoring_criteria},
|
|
244
|
+
sort_order = ${changes.sortOrder !== undefined ? changes.sortOrder : row.sort_order}
|
|
245
|
+
WHERE id = ${caseId}
|
|
246
|
+
`);
|
|
247
|
+
// Update dataset's updated_at
|
|
248
|
+
this.db.run(sql `UPDATE eval_datasets SET updated_at = ${now} WHERE id = ${row.dataset_id}`);
|
|
249
|
+
const updated = this.db.get(sql `SELECT * FROM eval_test_cases WHERE id = ${caseId}`);
|
|
250
|
+
return updated ? rowToTestCase(updated) : undefined;
|
|
251
|
+
}
|
|
252
|
+
deleteTestCase(tenantId, caseId) {
|
|
253
|
+
const row = this.db.get(sql `
|
|
254
|
+
SELECT * FROM eval_test_cases WHERE id = ${caseId} AND tenant_id = ${tenantId}
|
|
255
|
+
`);
|
|
256
|
+
if (!row)
|
|
257
|
+
return false;
|
|
258
|
+
// Check immutability
|
|
259
|
+
const ds = this.db.get(sql `SELECT * FROM eval_datasets WHERE id = ${row.dataset_id}`);
|
|
260
|
+
if (ds && ds.immutable === 1)
|
|
261
|
+
throw new Error('Cannot modify an immutable (versioned) dataset');
|
|
262
|
+
this.db.run(sql `DELETE FROM eval_test_cases WHERE id = ${caseId}`);
|
|
263
|
+
const now = new Date().toISOString();
|
|
264
|
+
this.db.run(sql `UPDATE eval_datasets SET updated_at = ${now} WHERE id = ${row.dataset_id}`);
|
|
265
|
+
return true;
|
|
266
|
+
}
|
|
267
|
+
// ─── Versioning ────────────────────────────────────────
|
|
268
|
+
createVersion(tenantId, datasetId) {
|
|
269
|
+
const existing = this.getDataset(tenantId, datasetId);
|
|
270
|
+
if (!existing)
|
|
271
|
+
throw new Error(`Dataset ${datasetId} not found`);
|
|
272
|
+
const newId = randomUUID();
|
|
273
|
+
const now = new Date().toISOString();
|
|
274
|
+
const newVersion = existing.version + 1;
|
|
275
|
+
const client = this.db.$client;
|
|
276
|
+
const txn = client.transaction(() => {
|
|
277
|
+
// Mark original as immutable
|
|
278
|
+
this.db.run(sql `
|
|
279
|
+
UPDATE eval_datasets SET immutable = 1, updated_at = ${now}
|
|
280
|
+
WHERE id = ${datasetId} AND tenant_id = ${tenantId}
|
|
281
|
+
`);
|
|
282
|
+
// Create new dataset row
|
|
283
|
+
this.db.run(sql `
|
|
284
|
+
INSERT INTO eval_datasets (id, tenant_id, agent_id, name, description, version, parent_id, created_at, updated_at)
|
|
285
|
+
VALUES (${newId}, ${tenantId}, ${existing.agentId ?? null}, ${existing.name}, ${existing.description ?? null}, ${newVersion}, ${datasetId}, ${now}, ${now})
|
|
286
|
+
`);
|
|
287
|
+
// Copy test cases
|
|
288
|
+
const cases = this.getTestCases(datasetId);
|
|
289
|
+
for (const tc of cases) {
|
|
290
|
+
const caseId = randomUUID();
|
|
291
|
+
this.db.run(sql `
|
|
292
|
+
INSERT INTO eval_test_cases (id, dataset_id, tenant_id, input, expected_output, tags, metadata, scoring_criteria, sort_order, created_at)
|
|
293
|
+
VALUES (
|
|
294
|
+
${caseId}, ${newId}, ${tenantId},
|
|
295
|
+
${JSON.stringify(tc.input)},
|
|
296
|
+
${tc.expectedOutput !== undefined ? JSON.stringify(tc.expectedOutput) : null},
|
|
297
|
+
${JSON.stringify(tc.tags)},
|
|
298
|
+
${JSON.stringify(tc.metadata)},
|
|
299
|
+
${tc.scoringCriteria ?? null},
|
|
300
|
+
${tc.sortOrder},
|
|
301
|
+
${now}
|
|
302
|
+
)
|
|
303
|
+
`);
|
|
304
|
+
}
|
|
305
|
+
});
|
|
306
|
+
txn();
|
|
307
|
+
return this.getDataset(tenantId, newId);
|
|
308
|
+
}
|
|
309
|
+
// ─── Run CRUD ──────────────────────────────────────────
|
|
310
|
+
createRun(tenantId, input) {
|
|
311
|
+
const id = randomUUID();
|
|
312
|
+
const now = new Date().toISOString();
|
|
313
|
+
// Get dataset version
|
|
314
|
+
const ds = this.getDataset(tenantId, input.datasetId);
|
|
315
|
+
if (!ds)
|
|
316
|
+
throw new Error(`Dataset ${input.datasetId} not found`);
|
|
317
|
+
this.db.run(sql `
|
|
318
|
+
INSERT INTO eval_runs (id, tenant_id, dataset_id, dataset_version, agent_id, webhook_url, status, config, baseline_run_id, created_at)
|
|
319
|
+
VALUES (
|
|
320
|
+
${id}, ${tenantId}, ${input.datasetId}, ${ds.version}, ${input.agentId}, ${input.webhookUrl},
|
|
321
|
+
'pending', ${JSON.stringify(input.config)}, ${input.baselineRunId ?? null}, ${now}
|
|
322
|
+
)
|
|
323
|
+
`);
|
|
324
|
+
return this.getRun(tenantId, id);
|
|
325
|
+
}
|
|
326
|
+
getRun(tenantId, id) {
|
|
327
|
+
const row = this.db.get(sql `
|
|
328
|
+
SELECT * FROM eval_runs WHERE id = ${id} AND tenant_id = ${tenantId}
|
|
329
|
+
`);
|
|
330
|
+
if (!row)
|
|
331
|
+
return undefined;
|
|
332
|
+
return rowToRun(row);
|
|
333
|
+
}
|
|
334
|
+
listRuns(tenantId, filters = {}) {
|
|
335
|
+
const { datasetId, agentId, status, limit = 20, offset = 0 } = filters;
|
|
336
|
+
let whereClause = sql `WHERE tenant_id = ${tenantId}`;
|
|
337
|
+
if (datasetId)
|
|
338
|
+
whereClause = sql `${whereClause} AND dataset_id = ${datasetId}`;
|
|
339
|
+
if (agentId)
|
|
340
|
+
whereClause = sql `${whereClause} AND agent_id = ${agentId}`;
|
|
341
|
+
if (status)
|
|
342
|
+
whereClause = sql `${whereClause} AND status = ${status}`;
|
|
343
|
+
const countRow = this.db.get(sql `
|
|
344
|
+
SELECT COUNT(*) as cnt FROM eval_runs ${whereClause}
|
|
345
|
+
`);
|
|
346
|
+
const total = countRow?.cnt ?? 0;
|
|
347
|
+
const rows = this.db.all(sql `
|
|
348
|
+
SELECT * FROM eval_runs ${whereClause}
|
|
349
|
+
ORDER BY created_at DESC
|
|
350
|
+
LIMIT ${limit} OFFSET ${offset}
|
|
351
|
+
`);
|
|
352
|
+
return { runs: rows.map(rowToRun), total };
|
|
353
|
+
}
|
|
354
|
+
updateRunStatus(id, status, aggregates, error) {
|
|
355
|
+
const now = new Date().toISOString();
|
|
356
|
+
this.db.run(sql `
|
|
357
|
+
UPDATE eval_runs SET
|
|
358
|
+
status = ${status},
|
|
359
|
+
total_cases = COALESCE(${aggregates?.totalCases ?? null}, total_cases),
|
|
360
|
+
passed_cases = COALESCE(${aggregates?.passedCases ?? null}, passed_cases),
|
|
361
|
+
failed_cases = COALESCE(${aggregates?.failedCases ?? null}, failed_cases),
|
|
362
|
+
avg_score = COALESCE(${aggregates?.avgScore ?? null}, avg_score),
|
|
363
|
+
total_cost_usd = COALESCE(${aggregates?.totalCostUsd ?? null}, total_cost_usd),
|
|
364
|
+
total_duration_ms = COALESCE(${aggregates?.totalDurationMs ?? null}, total_duration_ms),
|
|
365
|
+
started_at = COALESCE(${aggregates?.startedAt ?? null}, started_at),
|
|
366
|
+
completed_at = COALESCE(${aggregates?.completedAt ?? null}, completed_at),
|
|
367
|
+
error = COALESCE(${error ?? null}, error)
|
|
368
|
+
WHERE id = ${id}
|
|
369
|
+
`);
|
|
370
|
+
}
|
|
371
|
+
cancelRun(id) {
|
|
372
|
+
this.db.run(sql `
|
|
373
|
+
UPDATE eval_runs SET status = 'cancelled' WHERE id = ${id} AND status IN ('pending', 'running')
|
|
374
|
+
`);
|
|
375
|
+
}
|
|
376
|
+
// ─── Result CRUD ───────────────────────────────────────
|
|
377
|
+
saveResult(input) {
|
|
378
|
+
const id = randomUUID();
|
|
379
|
+
const now = new Date().toISOString();
|
|
380
|
+
this.db.run(sql `
|
|
381
|
+
INSERT INTO eval_results (id, run_id, test_case_id, tenant_id, session_id, actual_output, score, passed, scorer_type, scorer_details, latency_ms, cost_usd, token_count, error, created_at)
|
|
382
|
+
VALUES (
|
|
383
|
+
${id}, ${input.runId}, ${input.testCaseId}, ${input.tenantId},
|
|
384
|
+
${input.sessionId ?? null},
|
|
385
|
+
${input.actualOutput !== undefined ? JSON.stringify(input.actualOutput) : null},
|
|
386
|
+
${input.score}, ${input.passed ? 1 : 0}, ${input.scorerType},
|
|
387
|
+
${JSON.stringify(input.scorerDetails)},
|
|
388
|
+
${input.latencyMs ?? null}, ${input.costUsd ?? null}, ${input.tokenCount ?? null},
|
|
389
|
+
${input.error ?? null}, ${now}
|
|
390
|
+
)
|
|
391
|
+
`);
|
|
392
|
+
const row = this.db.get(sql `SELECT * FROM eval_results WHERE id = ${id}`);
|
|
393
|
+
return rowToResult(row);
|
|
394
|
+
}
|
|
395
|
+
getResults(runId) {
|
|
396
|
+
const rows = this.db.all(sql `
|
|
397
|
+
SELECT * FROM eval_results WHERE run_id = ${runId} ORDER BY created_at ASC
|
|
398
|
+
`);
|
|
399
|
+
return rows.map(rowToResult);
|
|
400
|
+
}
|
|
401
|
+
getResultForCase(runId, testCaseId) {
|
|
402
|
+
const row = this.db.get(sql `
|
|
403
|
+
SELECT * FROM eval_results WHERE run_id = ${runId} AND test_case_id = ${testCaseId}
|
|
404
|
+
`);
|
|
405
|
+
return row ? rowToResult(row) : undefined;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
//# sourceMappingURL=eval-store.js.map
|