@agent-native/core 0.7.12 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/agent/engine/ai-sdk-engine.d.ts.map +1 -1
- package/dist/agent/engine/ai-sdk-engine.js +26 -8
- package/dist/agent/engine/ai-sdk-engine.js.map +1 -1
- package/dist/agent/engine/builder-engine.d.ts +19 -0
- package/dist/agent/engine/builder-engine.d.ts.map +1 -0
- package/dist/agent/engine/builder-engine.js +430 -0
- package/dist/agent/engine/builder-engine.js.map +1 -0
- package/dist/agent/engine/builtin.d.ts.map +1 -1
- package/dist/agent/engine/builtin.js +26 -10
- package/dist/agent/engine/builtin.js.map +1 -1
- package/dist/agent/engine/index.d.ts +1 -1
- package/dist/agent/engine/index.d.ts.map +1 -1
- package/dist/agent/engine/index.js +1 -1
- package/dist/agent/engine/index.js.map +1 -1
- package/dist/agent/engine/registry.d.ts +20 -1
- package/dist/agent/engine/registry.d.ts.map +1 -1
- package/dist/agent/engine/registry.js +49 -1
- package/dist/agent/engine/registry.js.map +1 -1
- package/dist/agent/engine/types.d.ts +30 -0
- package/dist/agent/engine/types.d.ts.map +1 -1
- package/dist/agent/engine/types.js +19 -1
- package/dist/agent/engine/types.js.map +1 -1
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +65 -7
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-manager.d.ts.map +1 -1
- package/dist/agent/run-manager.js +11 -1
- package/dist/agent/run-manager.js.map +1 -1
- package/dist/agent/thread-data-builder.d.ts +4 -0
- package/dist/agent/thread-data-builder.d.ts.map +1 -1
- package/dist/agent/thread-data-builder.js +1 -0
- package/dist/agent/thread-data-builder.js.map +1 -1
- package/dist/agent/types.d.ts +8 -0
- package/dist/agent/types.d.ts.map +1 -1
- package/dist/checkpoints/service.d.ts +1 -0
- package/dist/checkpoints/service.d.ts.map +1 -1
- package/dist/checkpoints/service.js +26 -2
- package/dist/checkpoints/service.js.map +1 -1
- package/dist/cli/create.d.ts +30 -0
- package/dist/cli/create.d.ts.map +1 -1
- package/dist/cli/create.js +25 -13
- package/dist/cli/create.js.map +1 -1
- package/dist/client/AgentPanel.js +1 -1
- package/dist/client/AgentPanel.js.map +1 -1
- package/dist/client/AssistantChat.d.ts.map +1 -1
- package/dist/client/AssistantChat.js +49 -10
- package/dist/client/AssistantChat.js.map +1 -1
- package/dist/client/ConnectBuilderCard.d.ts +1 -7
- package/dist/client/ConnectBuilderCard.d.ts.map +1 -1
- package/dist/client/ConnectBuilderCard.js +30 -132
- package/dist/client/ConnectBuilderCard.js.map +1 -1
- package/dist/client/MultiTabAssistantChat.d.ts.map +1 -1
- package/dist/client/MultiTabAssistantChat.js +30 -9
- package/dist/client/MultiTabAssistantChat.js.map +1 -1
- package/dist/client/analytics.d.ts +5 -8
- package/dist/client/analytics.d.ts.map +1 -1
- package/dist/client/analytics.js +53 -11
- package/dist/client/analytics.js.map +1 -1
- package/dist/client/builder-mark.d.ts +9 -0
- package/dist/client/builder-mark.d.ts.map +1 -0
- package/dist/client/builder-mark.js +10 -0
- package/dist/client/builder-mark.js.map +1 -0
- package/dist/client/components/ui/popover.d.ts +8 -0
- package/dist/client/components/ui/popover.d.ts.map +1 -0
- package/dist/client/components/ui/popover.js +11 -0
- package/dist/client/components/ui/popover.js.map +1 -0
- package/dist/client/composer/ComposerPlusMenu.d.ts +2 -0
- package/dist/client/composer/ComposerPlusMenu.d.ts.map +1 -0
- package/dist/client/composer/ComposerPlusMenu.js +244 -0
- package/dist/client/composer/ComposerPlusMenu.js.map +1 -0
- package/dist/client/composer/TiptapComposer.d.ts.map +1 -1
- package/dist/client/composer/TiptapComposer.js +9 -5
- package/dist/client/composer/TiptapComposer.js.map +1 -1
- package/dist/client/composer/useVoiceDictation.d.ts.map +1 -1
- package/dist/client/composer/useVoiceDictation.js +4 -2
- package/dist/client/composer/useVoiceDictation.js.map +1 -1
- package/dist/client/error-format.d.ts +2 -0
- package/dist/client/error-format.d.ts.map +1 -0
- package/dist/client/error-format.js +31 -0
- package/dist/client/error-format.js.map +1 -0
- package/dist/client/index.d.ts +3 -1
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +3 -1
- package/dist/client/index.js.map +1 -1
- package/dist/client/observability/ObservabilityDashboard.d.ts +5 -0
- package/dist/client/observability/ObservabilityDashboard.d.ts.map +1 -0
- package/dist/client/observability/ObservabilityDashboard.js +169 -0
- package/dist/client/observability/ObservabilityDashboard.js.map +1 -0
- package/dist/client/observability/ThumbsFeedback.d.ts +8 -0
- package/dist/client/observability/ThumbsFeedback.d.ts.map +1 -0
- package/dist/client/observability/ThumbsFeedback.js +64 -0
- package/dist/client/observability/ThumbsFeedback.js.map +1 -0
- package/dist/client/observability/index.d.ts +4 -0
- package/dist/client/observability/index.d.ts.map +1 -0
- package/dist/client/observability/index.js +4 -0
- package/dist/client/observability/index.js.map +1 -0
- package/dist/client/observability/useObservability.d.ts +128 -0
- package/dist/client/observability/useObservability.d.ts.map +1 -0
- package/dist/client/observability/useObservability.js +109 -0
- package/dist/client/observability/useObservability.js.map +1 -0
- package/dist/client/onboarding/OnboardingPanel.d.ts.map +1 -1
- package/dist/client/onboarding/OnboardingPanel.js +34 -92
- package/dist/client/onboarding/OnboardingPanel.js.map +1 -1
- package/dist/client/org/RequireActiveOrg.d.ts +33 -0
- package/dist/client/org/RequireActiveOrg.d.ts.map +1 -0
- package/dist/client/org/RequireActiveOrg.js +63 -0
- package/dist/client/org/RequireActiveOrg.js.map +1 -0
- package/dist/client/org/hooks.d.ts.map +1 -1
- package/dist/client/org/hooks.js +50 -15
- package/dist/client/org/hooks.js.map +1 -1
- package/dist/client/org/index.d.ts +1 -0
- package/dist/client/org/index.d.ts.map +1 -1
- package/dist/client/org/index.js +1 -0
- package/dist/client/org/index.js.map +1 -1
- package/dist/client/resources/ResourcesPanel.js +3 -3
- package/dist/client/resources/ResourcesPanel.js.map +1 -1
- package/dist/client/settings/AutomationsSection.js +1 -1
- package/dist/client/settings/AutomationsSection.js.map +1 -1
- package/dist/client/settings/BrowserSection.js +1 -1
- package/dist/client/settings/BrowserSection.js.map +1 -1
- package/dist/client/settings/SettingsPanel.d.ts.map +1 -1
- package/dist/client/settings/SettingsPanel.js +112 -12
- package/dist/client/settings/SettingsPanel.js.map +1 -1
- package/dist/client/settings/VoiceTranscriptionSection.d.ts.map +1 -1
- package/dist/client/settings/VoiceTranscriptionSection.js +10 -4
- package/dist/client/settings/VoiceTranscriptionSection.js.map +1 -1
- package/dist/client/settings/useBuilderStatus.d.ts +26 -0
- package/dist/client/settings/useBuilderStatus.d.ts.map +1 -1
- package/dist/client/settings/useBuilderStatus.js +128 -4
- package/dist/client/settings/useBuilderStatus.js.map +1 -1
- package/dist/client/sse-event-processor.d.ts +2 -0
- package/dist/client/sse-event-processor.d.ts.map +1 -1
- package/dist/client/sse-event-processor.js +6 -2
- package/dist/client/sse-event-processor.js.map +1 -1
- package/dist/client/transcription/BuilderTranscriptionCta.d.ts +9 -0
- package/dist/client/transcription/BuilderTranscriptionCta.d.ts.map +1 -0
- package/dist/client/transcription/BuilderTranscriptionCta.js +18 -0
- package/dist/client/transcription/BuilderTranscriptionCta.js.map +1 -0
- package/dist/client/transcription/use-live-transcription.d.ts +29 -0
- package/dist/client/transcription/use-live-transcription.d.ts.map +1 -0
- package/dist/client/transcription/use-live-transcription.js +156 -0
- package/dist/client/transcription/use-live-transcription.js.map +1 -0
- package/dist/client/use-builder-enabled.d.ts +17 -0
- package/dist/client/use-builder-enabled.d.ts.map +1 -0
- package/dist/client/use-builder-enabled.js +36 -0
- package/dist/client/use-builder-enabled.js.map +1 -0
- package/dist/client/use-db-sync.d.ts.map +1 -1
- package/dist/client/use-db-sync.js +4 -2
- package/dist/client/use-db-sync.js.map +1 -1
- package/dist/client/useProductionAgent.d.ts.map +1 -1
- package/dist/client/useProductionAgent.js +3 -1
- package/dist/client/useProductionAgent.js.map +1 -1
- package/dist/db/migrations.d.ts +9 -0
- package/dist/db/migrations.d.ts.map +1 -1
- package/dist/db/migrations.js +75 -10
- package/dist/db/migrations.js.map +1 -1
- package/dist/file-upload/builder.d.ts.map +1 -1
- package/dist/file-upload/builder.js +11 -4
- package/dist/file-upload/builder.js.map +1 -1
- package/dist/jobs/tools.d.ts.map +1 -1
- package/dist/jobs/tools.js +137 -161
- package/dist/jobs/tools.js.map +1 -1
- package/dist/notifications/actions.d.ts +2 -2
- package/dist/notifications/actions.d.ts.map +1 -1
- package/dist/notifications/actions.js +77 -69
- package/dist/notifications/actions.js.map +1 -1
- package/dist/observability/evals.d.ts +22 -0
- package/dist/observability/evals.d.ts.map +1 -0
- package/dist/observability/evals.js +371 -0
- package/dist/observability/evals.js.map +1 -0
- package/dist/observability/experiments.d.ts +24 -0
- package/dist/observability/experiments.d.ts.map +1 -0
- package/dist/observability/experiments.js +274 -0
- package/dist/observability/experiments.js.map +1 -0
- package/dist/observability/feedback.d.ts +14 -0
- package/dist/observability/feedback.d.ts.map +1 -0
- package/dist/observability/feedback.js +256 -0
- package/dist/observability/feedback.js.map +1 -0
- package/dist/observability/index.d.ts +6 -0
- package/dist/observability/index.d.ts.map +1 -0
- package/dist/observability/index.js +5 -0
- package/dist/observability/index.js.map +1 -0
- package/dist/observability/plugin.d.ts +2 -0
- package/dist/observability/plugin.d.ts.map +1 -0
- package/dist/observability/plugin.js +12 -0
- package/dist/observability/plugin.js.map +1 -0
- package/dist/observability/routes.d.ts +68 -0
- package/dist/observability/routes.d.ts.map +1 -0
- package/dist/observability/routes.js +301 -0
- package/dist/observability/routes.js.map +1 -0
- package/dist/observability/store.d.ts +77 -0
- package/dist/observability/store.d.ts.map +1 -0
- package/dist/observability/store.js +976 -0
- package/dist/observability/store.js.map +1 -0
- package/dist/observability/traces.d.ts +37 -0
- package/dist/observability/traces.d.ts.map +1 -0
- package/dist/observability/traces.js +182 -0
- package/dist/observability/traces.js.map +1 -0
- package/dist/observability/types.d.ts +159 -0
- package/dist/observability/types.d.ts.map +1 -0
- package/dist/observability/types.js +16 -0
- package/dist/observability/types.js.map +1 -0
- package/dist/onboarding/default-steps.d.ts.map +1 -1
- package/dist/onboarding/default-steps.js +6 -5
- package/dist/onboarding/default-steps.js.map +1 -1
- package/dist/onboarding/types.d.ts +10 -1
- package/dist/onboarding/types.d.ts.map +1 -1
- package/dist/org/context.d.ts +8 -1
- package/dist/org/context.d.ts.map +1 -1
- package/dist/org/context.js +163 -6
- package/dist/org/context.js.map +1 -1
- package/dist/org/handlers.d.ts.map +1 -1
- package/dist/org/handlers.js +49 -30
- package/dist/org/handlers.js.map +1 -1
- package/dist/progress/actions.d.ts +3 -0
- package/dist/progress/actions.d.ts.map +1 -1
- package/dist/progress/actions.js +86 -110
- package/dist/progress/actions.js.map +1 -1
- package/dist/progress/routes.d.ts +1 -1
- package/dist/progress/routes.js +1 -1
- package/dist/scripts/agent-engines/list-agent-engines.js +1 -1
- package/dist/scripts/agent-engines/list-agent-engines.js.map +1 -1
- package/dist/scripts/agent-engines/manage-agent-engine.d.ts +10 -0
- package/dist/scripts/agent-engines/manage-agent-engine.d.ts.map +1 -0
- package/dist/scripts/agent-engines/manage-agent-engine.js +47 -0
- package/dist/scripts/agent-engines/manage-agent-engine.js.map +1 -0
- package/dist/scripts/agent-engines/set-agent-engine.js +2 -2
- package/dist/scripts/agent-engines/set-agent-engine.js.map +1 -1
- package/dist/server/agent-chat-plugin.d.ts +39 -0
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +707 -443
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/dist/server/agent-teams.js +1 -1
- package/dist/server/agent-teams.js.map +1 -1
- package/dist/server/analytics.d.ts +5 -6
- package/dist/server/analytics.d.ts.map +1 -1
- package/dist/server/analytics.js +6 -14
- package/dist/server/analytics.js.map +1 -1
- package/dist/server/app-name.d.ts +5 -2
- package/dist/server/app-name.d.ts.map +1 -1
- package/dist/server/app-name.js +14 -3
- package/dist/server/app-name.js.map +1 -1
- package/dist/server/app-url.d.ts.map +1 -1
- package/dist/server/app-url.js +10 -1
- package/dist/server/app-url.js.map +1 -1
- package/dist/server/auth.d.ts +2 -0
- package/dist/server/auth.d.ts.map +1 -1
- package/dist/server/auth.js +153 -2
- package/dist/server/auth.js.map +1 -1
- package/dist/server/better-auth-instance.d.ts +2 -0
- package/dist/server/better-auth-instance.d.ts.map +1 -1
- package/dist/server/better-auth-instance.js +4 -0
- package/dist/server/better-auth-instance.js.map +1 -1
- package/dist/server/builder-browser.d.ts +59 -1
- package/dist/server/builder-browser.d.ts.map +1 -1
- package/dist/server/builder-browser.js +127 -11
- package/dist/server/builder-browser.js.map +1 -1
- package/dist/server/core-routes-plugin.d.ts.map +1 -1
- package/dist/server/core-routes-plugin.js +208 -6
- package/dist/server/core-routes-plugin.js.map +1 -1
- package/dist/server/credential-provider.d.ts +7 -0
- package/dist/server/credential-provider.d.ts.map +1 -1
- package/dist/server/credential-provider.js +10 -0
- package/dist/server/credential-provider.js.map +1 -1
- package/dist/server/onboarding-html.d.ts.map +1 -1
- package/dist/server/onboarding-html.js +29 -4
- package/dist/server/onboarding-html.js.map +1 -1
- package/dist/server/poll.d.ts.map +1 -1
- package/dist/server/poll.js +46 -5
- package/dist/server/poll.js.map +1 -1
- package/dist/server/ssr-handler.d.ts.map +1 -1
- package/dist/server/ssr-handler.js +2 -1
- package/dist/server/ssr-handler.js.map +1 -1
- package/dist/server/transcribe-voice.d.ts.map +1 -1
- package/dist/server/transcribe-voice.js +44 -5
- package/dist/server/transcribe-voice.js.map +1 -1
- package/dist/styles/agent-native.css +11 -2
- package/dist/templates/default/.agents/skills/progress/SKILL.md +14 -12
- package/dist/templates/default/app/root.tsx +7 -0
- package/dist/transcription/builder-transcription.d.ts +27 -0
- package/dist/transcription/builder-transcription.d.ts.map +1 -0
- package/dist/transcription/builder-transcription.js +41 -0
- package/dist/transcription/builder-transcription.js.map +1 -0
- package/dist/triggers/actions.d.ts +3 -0
- package/dist/triggers/actions.d.ts.map +1 -1
- package/dist/triggers/actions.js +189 -213
- package/dist/triggers/actions.js.map +1 -1
- package/docs/content/agent-mentions.md +1 -1
- package/docs/content/automations.md +22 -19
- package/docs/content/cloneable-saas.md +2 -2
- package/docs/content/deployment.md +21 -61
- package/docs/content/getting-started.md +1 -1
- package/docs/content/key-concepts.md +1 -1
- package/docs/content/{enterprise-workspace.md → multi-app-workspace.md} +3 -3
- package/docs/content/multi-tenancy.md +1 -1
- package/docs/content/progress.md +11 -11
- package/docs/content/template-dispatch.md +3 -3
- package/docs/content/workspace-management.md +1 -1
- package/package.json +9 -2
- package/src/templates/default/.agents/skills/progress/SKILL.md +14 -12
- package/src/templates/default/app/root.tsx +7 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Experiment, ExperimentVariant, ExperimentMetricResult } from "./types.js";
|
|
2
|
+
export declare function createExperiment(opts: {
|
|
3
|
+
name: string;
|
|
4
|
+
variants: Array<{
|
|
5
|
+
id: string;
|
|
6
|
+
weight: number;
|
|
7
|
+
config: Record<string, unknown>;
|
|
8
|
+
}>;
|
|
9
|
+
metrics: string[];
|
|
10
|
+
assignmentLevel?: "user" | "session";
|
|
11
|
+
}): Promise<Experiment>;
|
|
12
|
+
export declare function startExperiment(id: string): Promise<void>;
|
|
13
|
+
export declare function pauseExperiment(id: string): Promise<void>;
|
|
14
|
+
export declare function completeExperiment(id: string): Promise<void>;
|
|
15
|
+
export declare function resolveVariant(experimentId: string, userId: string): Promise<ExperimentVariant>;
|
|
16
|
+
export declare function resolveActiveExperimentConfig(userId: string): Promise<{
|
|
17
|
+
configs: Record<string, unknown>;
|
|
18
|
+
assignments: Array<{
|
|
19
|
+
experimentId: string;
|
|
20
|
+
variantId: string;
|
|
21
|
+
}>;
|
|
22
|
+
} | null>;
|
|
23
|
+
export declare function computeExperimentResults(experimentId: string): Promise<ExperimentMetricResult[]>;
|
|
24
|
+
//# sourceMappingURL=experiments.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../src/observability/experiments.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,UAAU,EACV,iBAAiB,EACjB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAkDpB,wBAAsB,gBAAgB,CAAC,IAAI,EAAE;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,KAAK,CAAC;QACd,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACjC,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACtC,GAAG,OAAO,CAAC,UAAU,CAAC,CActB;AAED,wBAAsB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG/D;AAED,wBAAsB,eAAe,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAG/D;AAED,wBAAsB,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAGlE;AAID,wBAAsB,cAAc,CAClC,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,iBAAiB,CAAC,CA8C5B;AAED,wBAAsB,6BAA6B,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,WAAW,EAAE,KAAK,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACjE,GAAG,IAAI,CAAC,CAmBR;AAID,wBAAsB,wBAAwB,CAC5C,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,sBAAsB,EAAE,CAAC,CA2JnC"}
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import { getDbExec } from "../db/client.js";
|
|
2
|
+
import { insertExperiment, updateExperiment, listExperiments, getExperiment, upsertAssignment, getAssignment, insertExperimentResult, ensureObservabilityTables, } from "./store.js";
|
|
3
|
+
// ─── Hashing ────────────────────────────────────────────────────────
|
|
4
|
+
function simpleHash(str) {
|
|
5
|
+
let hash = 0;
|
|
6
|
+
for (let i = 0; i < str.length; i++) {
|
|
7
|
+
hash = ((hash << 5) - hash + str.charCodeAt(i)) | 0;
|
|
8
|
+
}
|
|
9
|
+
return Math.abs(hash);
|
|
10
|
+
}
|
|
11
|
+
function generateId(prefix) {
|
|
12
|
+
return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
13
|
+
}
|
|
14
|
+
// ─── Active experiments cache (short TTL for hot path) ──────────────
|
|
15
|
+
let _cachedActive = null;
|
|
16
|
+
let _cachedActiveAt = 0;
|
|
17
|
+
const CACHE_TTL_MS = 5_000;
|
|
18
|
+
async function getActiveExperiments() {
|
|
19
|
+
const now = Date.now();
|
|
20
|
+
if (_cachedActive && now - _cachedActiveAt < CACHE_TTL_MS) {
|
|
21
|
+
return _cachedActive;
|
|
22
|
+
}
|
|
23
|
+
const all = await listExperiments();
|
|
24
|
+
_cachedActive = all.filter((e) => e.status === "running");
|
|
25
|
+
_cachedActiveAt = now;
|
|
26
|
+
return _cachedActive;
|
|
27
|
+
}
|
|
28
|
+
function invalidateCache() {
|
|
29
|
+
_cachedActive = null;
|
|
30
|
+
_cachedActiveAt = 0;
|
|
31
|
+
}
|
|
32
|
+
// ─── Experiment lifecycle ───────────────────────────────────────────
|
|
33
|
+
export async function createExperiment(opts) {
|
|
34
|
+
const experiment = {
|
|
35
|
+
id: generateId("exp"),
|
|
36
|
+
name: opts.name,
|
|
37
|
+
status: "draft",
|
|
38
|
+
variants: opts.variants,
|
|
39
|
+
metrics: opts.metrics,
|
|
40
|
+
assignmentLevel: opts.assignmentLevel ?? "user",
|
|
41
|
+
startedAt: null,
|
|
42
|
+
endedAt: null,
|
|
43
|
+
createdAt: Date.now(),
|
|
44
|
+
};
|
|
45
|
+
await insertExperiment(experiment);
|
|
46
|
+
return experiment;
|
|
47
|
+
}
|
|
48
|
+
export async function startExperiment(id) {
|
|
49
|
+
await updateExperiment(id, { status: "running" });
|
|
50
|
+
invalidateCache();
|
|
51
|
+
}
|
|
52
|
+
export async function pauseExperiment(id) {
|
|
53
|
+
await updateExperiment(id, { status: "paused" });
|
|
54
|
+
invalidateCache();
|
|
55
|
+
}
|
|
56
|
+
export async function completeExperiment(id) {
|
|
57
|
+
await updateExperiment(id, { status: "completed", endedAt: Date.now() });
|
|
58
|
+
invalidateCache();
|
|
59
|
+
}
|
|
60
|
+
// ─── Variant assignment ─────────────────────────────────────────────
|
|
61
|
+
export async function resolveVariant(experimentId, userId) {
|
|
62
|
+
const existing = await getAssignment(experimentId, userId);
|
|
63
|
+
if (existing) {
|
|
64
|
+
const experiment = await getExperiment(experimentId);
|
|
65
|
+
if (!experiment)
|
|
66
|
+
throw new Error(`Experiment ${experimentId} not found`);
|
|
67
|
+
const variant = experiment.variants.find((v) => v.id === existing.variantId);
|
|
68
|
+
if (!variant)
|
|
69
|
+
throw new Error(`Variant ${existing.variantId} not found in experiment ${experimentId}`);
|
|
70
|
+
return variant;
|
|
71
|
+
}
|
|
72
|
+
const experiment = await getExperiment(experimentId);
|
|
73
|
+
if (!experiment)
|
|
74
|
+
throw new Error(`Experiment ${experimentId} not found`);
|
|
75
|
+
if (experiment.variants.length === 0)
|
|
76
|
+
throw new Error("Experiment has no variants");
|
|
77
|
+
const totalWeight = experiment.variants.reduce((sum, v) => sum + v.weight, 0);
|
|
78
|
+
if (totalWeight <= 0)
|
|
79
|
+
throw new Error("Experiment has no valid variant weights");
|
|
80
|
+
const hashValue = simpleHash(experimentId + userId) % totalWeight;
|
|
81
|
+
let cumulative = 0;
|
|
82
|
+
let chosen;
|
|
83
|
+
for (const variant of experiment.variants) {
|
|
84
|
+
cumulative += variant.weight;
|
|
85
|
+
if (hashValue < cumulative) {
|
|
86
|
+
chosen = variant;
|
|
87
|
+
break;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// Fallback to last variant if rounding causes no match
|
|
91
|
+
if (!chosen)
|
|
92
|
+
chosen = experiment.variants[experiment.variants.length - 1];
|
|
93
|
+
// Fire-and-forget persistence
|
|
94
|
+
upsertAssignment({
|
|
95
|
+
experimentId,
|
|
96
|
+
userId,
|
|
97
|
+
variantId: chosen.id,
|
|
98
|
+
assignedAt: Date.now(),
|
|
99
|
+
}).catch(() => { });
|
|
100
|
+
return chosen;
|
|
101
|
+
}
|
|
102
|
+
export async function resolveActiveExperimentConfig(userId) {
|
|
103
|
+
const active = await getActiveExperiments();
|
|
104
|
+
if (active.length === 0)
|
|
105
|
+
return null;
|
|
106
|
+
const assignments = [];
|
|
107
|
+
const merged = {};
|
|
108
|
+
const variants = await Promise.all(active.map((exp) => resolveVariant(exp.id, userId)));
|
|
109
|
+
for (let i = 0; i < active.length; i++) {
|
|
110
|
+
const exp = active[i];
|
|
111
|
+
const variant = variants[i];
|
|
112
|
+
assignments.push({ experimentId: exp.id, variantId: variant.id });
|
|
113
|
+
Object.assign(merged, variant.config);
|
|
114
|
+
}
|
|
115
|
+
return { configs: merged, assignments };
|
|
116
|
+
}
|
|
117
|
+
// ─── Results computation ────────────────────────────────────────────
|
|
118
|
+
export async function computeExperimentResults(experimentId) {
|
|
119
|
+
const experiment = await getExperiment(experimentId);
|
|
120
|
+
if (!experiment)
|
|
121
|
+
throw new Error(`Experiment ${experimentId} not found`);
|
|
122
|
+
await ensureObservabilityTables();
|
|
123
|
+
const client = getDbExec();
|
|
124
|
+
const results = [];
|
|
125
|
+
const now = Date.now();
|
|
126
|
+
for (const variant of experiment.variants) {
|
|
127
|
+
const { rows: assignmentRows } = await client.execute({
|
|
128
|
+
sql: `SELECT user_id FROM agent_experiment_assignments WHERE experiment_id = ? AND variant_id = ?`,
|
|
129
|
+
args: [experimentId, variant.id],
|
|
130
|
+
});
|
|
131
|
+
if (assignmentRows.length === 0) {
|
|
132
|
+
const emptyMetrics = [
|
|
133
|
+
"avg_cost",
|
|
134
|
+
"avg_latency",
|
|
135
|
+
"avg_eval_score",
|
|
136
|
+
"tool_success_rate",
|
|
137
|
+
"satisfaction",
|
|
138
|
+
"sample_size",
|
|
139
|
+
];
|
|
140
|
+
for (const metric of emptyMetrics) {
|
|
141
|
+
const result = {
|
|
142
|
+
id: generateId("expres"),
|
|
143
|
+
experimentId,
|
|
144
|
+
variantId: variant.id,
|
|
145
|
+
metric,
|
|
146
|
+
value: 0,
|
|
147
|
+
sampleSize: 0,
|
|
148
|
+
confidenceLow: 0,
|
|
149
|
+
confidenceHigh: 0,
|
|
150
|
+
computedAt: now,
|
|
151
|
+
};
|
|
152
|
+
results.push(result);
|
|
153
|
+
insertExperimentResult(result).catch(() => { });
|
|
154
|
+
}
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
const userIds = assignmentRows.map((r) => String(r.user_id));
|
|
158
|
+
const placeholders = userIds.map(() => "?").join(", ");
|
|
159
|
+
// Join trace summaries with feedback to scope runs to this variant's assigned users
|
|
160
|
+
const { rows: userTraceRows } = await client.execute({
|
|
161
|
+
sql: `SELECT s.total_cost_cents_x100, s.total_duration_ms, s.successful_tools, s.tool_calls, s.run_id
|
|
162
|
+
FROM agent_trace_summaries s
|
|
163
|
+
INNER JOIN agent_feedback f ON f.run_id = s.run_id
|
|
164
|
+
WHERE f.user_id IN (${placeholders})
|
|
165
|
+
${experiment.startedAt ? "AND s.created_at >= ?" : ""}
|
|
166
|
+
GROUP BY s.run_id`,
|
|
167
|
+
args: experiment.startedAt ? [...userIds, experiment.startedAt] : userIds,
|
|
168
|
+
});
|
|
169
|
+
const costs = [];
|
|
170
|
+
const latencies = [];
|
|
171
|
+
const toolRates = [];
|
|
172
|
+
for (const row of userTraceRows) {
|
|
173
|
+
costs.push(Number(row.total_cost_cents_x100) / 100);
|
|
174
|
+
latencies.push(Number(row.total_duration_ms));
|
|
175
|
+
const totalTools = Number(row.tool_calls);
|
|
176
|
+
const successTools = Number(row.successful_tools);
|
|
177
|
+
toolRates.push(totalTools > 0 ? successTools / totalTools : 1);
|
|
178
|
+
}
|
|
179
|
+
// Eval scores for these runs
|
|
180
|
+
const runIds = userTraceRows.map((r) => String(r.run_id));
|
|
181
|
+
let evalScores = [];
|
|
182
|
+
if (runIds.length > 0) {
|
|
183
|
+
const runPlaceholders = runIds.map(() => "?").join(", ");
|
|
184
|
+
const { rows: evalRows } = await client.execute({
|
|
185
|
+
sql: `SELECT score FROM agent_evals WHERE run_id IN (${runPlaceholders})`,
|
|
186
|
+
args: runIds,
|
|
187
|
+
});
|
|
188
|
+
evalScores = evalRows.map((r) => Number(r.score));
|
|
189
|
+
}
|
|
190
|
+
// Satisfaction scores (inverse of frustration) for these users' threads
|
|
191
|
+
const { rows: satRows } = await client.execute({
|
|
192
|
+
sql: `SELECT frustration_score FROM agent_satisfaction_scores
|
|
193
|
+
WHERE thread_id IN (
|
|
194
|
+
SELECT DISTINCT f.thread_id FROM agent_feedback f
|
|
195
|
+
WHERE f.user_id IN (${placeholders}) AND f.thread_id IS NOT NULL
|
|
196
|
+
)
|
|
197
|
+
${experiment.startedAt ? "AND computed_at >= ?" : ""}`,
|
|
198
|
+
args: experiment.startedAt ? [...userIds, experiment.startedAt] : userIds,
|
|
199
|
+
});
|
|
200
|
+
const satisfactionScores = satRows.map((r) => 1 - Number(r.frustration_score) / 100);
|
|
201
|
+
const sampleSize = userTraceRows.length;
|
|
202
|
+
const metricEntries = [
|
|
203
|
+
{
|
|
204
|
+
metric: "avg_cost",
|
|
205
|
+
value: mean(costs),
|
|
206
|
+
std: stddev(costs),
|
|
207
|
+
n: costs.length,
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
metric: "avg_latency",
|
|
211
|
+
value: mean(latencies),
|
|
212
|
+
std: stddev(latencies),
|
|
213
|
+
n: latencies.length,
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
metric: "avg_eval_score",
|
|
217
|
+
value: mean(evalScores),
|
|
218
|
+
std: stddev(evalScores),
|
|
219
|
+
n: evalScores.length,
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
metric: "tool_success_rate",
|
|
223
|
+
value: mean(toolRates),
|
|
224
|
+
std: stddev(toolRates),
|
|
225
|
+
n: toolRates.length,
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
metric: "satisfaction",
|
|
229
|
+
value: mean(satisfactionScores),
|
|
230
|
+
std: stddev(satisfactionScores),
|
|
231
|
+
n: satisfactionScores.length,
|
|
232
|
+
},
|
|
233
|
+
{ metric: "sample_size", value: sampleSize, std: 0, n: sampleSize },
|
|
234
|
+
];
|
|
235
|
+
for (const entry of metricEntries) {
|
|
236
|
+
const [low, high] = confidenceInterval(entry.value, entry.std, entry.n);
|
|
237
|
+
const result = {
|
|
238
|
+
id: generateId("expres"),
|
|
239
|
+
experimentId,
|
|
240
|
+
variantId: variant.id,
|
|
241
|
+
metric: entry.metric,
|
|
242
|
+
value: entry.value,
|
|
243
|
+
sampleSize,
|
|
244
|
+
confidenceLow: low,
|
|
245
|
+
confidenceHigh: high,
|
|
246
|
+
computedAt: now,
|
|
247
|
+
};
|
|
248
|
+
results.push(result);
|
|
249
|
+
insertExperimentResult(result).catch(() => { });
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return results;
|
|
253
|
+
}
|
|
254
|
+
// ─── Stats helpers ──────────────────────────────────────────────────
|
|
255
|
+
function mean(values) {
|
|
256
|
+
if (values.length === 0)
|
|
257
|
+
return 0;
|
|
258
|
+
return values.reduce((a, b) => a + b, 0) / values.length;
|
|
259
|
+
}
|
|
260
|
+
function stddev(values) {
|
|
261
|
+
if (values.length < 2)
|
|
262
|
+
return 0;
|
|
263
|
+
const avg = mean(values);
|
|
264
|
+
const squaredDiffs = values.map((v) => (v - avg) ** 2);
|
|
265
|
+
return Math.sqrt(squaredDiffs.reduce((a, b) => a + b, 0) / (values.length - 1));
|
|
266
|
+
}
|
|
267
|
+
function confidenceInterval(avg, std, n) {
|
|
268
|
+
if (n < 2)
|
|
269
|
+
return [avg, avg];
|
|
270
|
+
const se = std / Math.sqrt(n);
|
|
271
|
+
const margin = 1.96 * se;
|
|
272
|
+
return [avg - margin, avg + margin];
|
|
273
|
+
}
|
|
274
|
+
//# sourceMappingURL=experiments.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"experiments.js","sourceRoot":"","sources":["../../src/observability/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAM5C,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,sBAAsB,EACtB,yBAAyB,GAC1B,MAAM,YAAY,CAAC;AAEpB,uEAAuE;AAEvE,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,UAAU,CAAC,MAAc;IAChC,OAAO,GAAG,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AAC7E,CAAC;AAED,uEAAuE;AAEvE,IAAI,aAAa,GAAwB,IAAI,CAAC;AAC9C,IAAI,eAAe,GAAG,CAAC,CAAC;AACxB,MAAM,YAAY,GAAG,KAAK,CAAC;AAE3B,KAAK,UAAU,oBAAoB;IACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,aAAa,IAAI,GAAG,GAAG,eAAe,GAAG,YAAY,EAAE,CAAC;QAC1D,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,eAAe,EAAE,CAAC;IACpC,aAAa,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;IAC1D,eAAe,GAAG,GAAG,CAAC;IACtB,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,SAAS,eAAe;IACtB,aAAa,GAAG,IAAI,CAAC;IACrB,eAAe,GAAG,CAAC,CAAC;AACtB,CAAC;AAED,uEAAuE;AAEvE,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAStC;IACC,MAAM,UAAU,GAAe;QAC7B,EAAE,EAAE,UAAU,CAAC,KAAK,CAAC;QACrB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM;QAC/C,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,IAAI;QACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IACF,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC;IACnC,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAAU;IAC9C,MAAM,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAClD,eAAe,EAAE,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAAU;IAC9C,MAAM,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IACjD,eAAe,EAAE,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EAAU;IACjD,MAAM,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACzE,eAAe,EAAE,CAAC;AACpB,CAAC;AAED,uEAAuE;AAEvE,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,YAAoB,EACpB,MAAc;IAEd,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC3D,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,CAAC;QACrD,IAAI,CAAC,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,cAAc,YAAY,YAAY,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,SAAS,CACnC,CAAC;QACF,IAAI,CAAC,OAAO;YACV,MAAM,IAAI,KAAK,CACb,WAAW,QAAQ,CAAC,SAAS,4BAA4B,YAAY,EAAE,CACxE,CAAC;QACJ,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,CAAC;IACrD,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,YAAY,YAAY,CAAC,CAAC;IAEzE,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9E,IAAI,WAAW,IAAI,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,UAAU,CAAC,YAAY,GAAG,MAAM,CAAC,GAAG,WAAW,CAAC;IAElE,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,MAAqC,CAAC;IAC1C,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;QAC1C,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,SAAS,GAAG,UAAU,EAAE,CAAC;YAC3B,MAAM,GAAG,OAAO,CAAC;YACjB,MAAM;QACR,CAAC;IACH,CAAC;IACD,uDAAuD;IACvD,IAAI,CAAC,MAAM;QAAE,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE1E,8BAA8B;IAC9B,gBAAgB,CAAC;QACf,YAAY;QACZ,MAAM;QACN,SAAS,EAAE,MAAM,CAAC,EAAE;QACpB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAEnB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,MAAc;IAIhE,MAAM,MAAM,GAAG,MAAM,oBAAoB,EAAE,CAAC;IAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAErC,MAAM,WAAW,GAAuD,EAAE,CAAC;IAC3E,MAAM,MAAM,GAA4B,EAAE,CAAC;IAE3C,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CACpD,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,WAAW,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AAC1C,CAAC;AAED,uEAAuE;AAEvE,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,YAAoB;IAEpB,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,CAAC;IACrD,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,YAAY,YAAY,CAAC,CAAC;IAEzE,MAAM,yBAAyB,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;QAC1C,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;YACpD,GAAG,EAAE,6FAA6F;YAClG,IAAI,EAAE,CAAC,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC;SACjC,CAAC,CAAC;QAEH,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG;gBACnB,UAAU;gBACV,aAAa;gBACb,gBAAgB;gBAChB,mBAAmB;gBACnB,cAAc;gBACd,aAAa;aACd,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;gBAClC,MAAM,MAAM,GAA2B;oBACrC,EAAE,EAAE,UAAU,CAAC,QAAQ,CAAC;oBACxB,YAAY;oBACZ,SAAS,EAAE,OAAO,CAAC,EAAE;oBACrB,MAAM;oBACN,KAAK,EAAE,CAAC;oBACR,UAAU,EAAE,CAAC;oBACb,aAAa,EAAE,CAAC;oBAChB,cAAc,EAAE,CAAC;oBACjB,UAAU,EAAE,GAAG;iBAChB,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,sBAAsB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACjD,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvD,oFAAoF;QACpF,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;YACnD,GAAG,EAAE;;;kCAGuB,YAAY;cAChC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,EAAE;8BACnC;YACxB,IAAI,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO;SAC1E,CAAC,CAAC;QAEH,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KAAK,MAAM,GAAG,IAAI,aAAsB,EAAE,CAAC;YACzC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,qBAAqB,CAAC,GAAG,GAAG,CAAC,CAAC;YACpD,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAC9C,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC1C,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAClD,SAAS,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,6BAA6B;QAC7B,MAAM,MAAM,GAAI,aAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QACrE,IAAI,UAAU,GAAa,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;gBAC9C,GAAG,EAAE,kDAAkD,eAAe,GAAG;gBACzE,IAAI,EAAE,MAAM;aACb,CAAC,CAAC;YACH,UAAU,GAAI,QAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,wEAAwE;QACxE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;YAC7C,GAAG,EAAE;;;oCAGyB,YAAY;;cAElC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,EAAE;YAC5D,IAAI,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO;SAC1E,CAAC,CAAC;QACH,MAAM,kBAAkB,GAAI,OAAiB,CAAC,GAAG,CAC/C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,GAAG,GAAG,CAC7C,CAAC;QAEF,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC;QAExC,MAAM,aAAa,GAKd;YACH;gBACE,MAAM,EAAE,UAAU;gBAClB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC;gBAClB,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC;gBAClB,CAAC,EAAE,KAAK,CAAC,MAAM;aAChB;YACD;gBACE,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;gBACtB,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC;gBACtB,CAAC,EAAE,SAAS,CAAC,MAAM;aACpB;YACD;gBACE,MAAM,EAAE,gBAAgB;gBACxB,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC;gBACvB,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;gBACvB,CAAC,EAAE,UAAU,CAAC,MAAM;aACrB;YACD;gBACE,MAAM,EAAE,mBAAmB;gBAC3B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;gBACtB,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC;gBACtB,CAAC,EAAE,SAAS,CAAC,MAAM;aACpB;YACD;gBACE,MAAM,EAAE,cAAc;gBACtB,KAAK,EAAE,IAAI,CAAC,kBAAkB,CAAC;gBAC/B,GAAG,EAAE,MAAM,CAAC,kBAAkB,CAAC;gBAC/B,CAAC,EAAE,kBAAkB,CAAC,MAAM;aAC7B;YACD,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE;SACpE,CAAC;QAEF,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,kBAAkB,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;YACxE,MAAM,MAAM,GAA2B;gBACrC,EAAE,EAAE,UAAU,CAAC,QAAQ,CAAC;gBACxB,YAAY;gBACZ,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,UAAU;gBACV,aAAa,EAAE,GAAG;gBAClB,cAAc,EAAE,IAAI;gBACpB,UAAU,EAAE,GAAG;aAChB,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,sBAAsB,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,uEAAuE;AAEvE,SAAS,IAAI,CAAC,MAAgB;IAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAC3D,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IACzB,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,IAAI,CACd,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CACzB,GAAW,EACX,GAAW,EACX,CAAS;IAET,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC7B,MAAM,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;IACzB,OAAO,CAAC,GAAG,GAAG,MAAM,EAAE,GAAG,GAAG,MAAM,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { FeedbackEntry, FeedbackType, SatisfactionScore } from "./types.js";
|
|
2
|
+
export interface SubmitFeedbackOpts {
|
|
3
|
+
threadId: string;
|
|
4
|
+
runId?: string;
|
|
5
|
+
messageSeq?: number;
|
|
6
|
+
feedbackType: FeedbackType;
|
|
7
|
+
value?: string;
|
|
8
|
+
userId?: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function submitFeedback(opts: SubmitFeedbackOpts): Promise<FeedbackEntry>;
|
|
11
|
+
export declare function computeSatisfactionScore(threadId: string, opts?: {
|
|
12
|
+
userId?: string | null;
|
|
13
|
+
}): Promise<SatisfactionScore>;
|
|
14
|
+
//# sourceMappingURL=feedback.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"feedback.d.ts","sourceRoot":"","sources":["../../src/observability/feedback.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EACZ,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAapB,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,YAAY,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,kBAAkB,GACvB,OAAO,CAAC,aAAa,CAAC,CA2BxB;AAyOD,wBAAsB,wBAAwB,CAC5C,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAO,GACpC,OAAO,CAAC,iBAAiB,CAAC,CAoC5B"}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
import { getDbExec } from "../db/client.js";
|
|
2
|
+
import { insertFeedback, upsertSatisfactionScore, ensureObservabilityTables, } from "./store.js";
|
|
3
|
+
function generateId() {
|
|
4
|
+
return `fb-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
5
|
+
}
|
|
6
|
+
export async function submitFeedback(opts) {
|
|
7
|
+
if (!opts.threadId)
|
|
8
|
+
throw new Error("threadId is required");
|
|
9
|
+
if (!opts.feedbackType)
|
|
10
|
+
throw new Error("feedbackType is required");
|
|
11
|
+
const validTypes = [
|
|
12
|
+
"thumbs_up",
|
|
13
|
+
"thumbs_down",
|
|
14
|
+
"category",
|
|
15
|
+
"text",
|
|
16
|
+
];
|
|
17
|
+
if (!validTypes.includes(opts.feedbackType)) {
|
|
18
|
+
throw new Error(`Invalid feedbackType: ${opts.feedbackType}`);
|
|
19
|
+
}
|
|
20
|
+
const entry = {
|
|
21
|
+
id: generateId(),
|
|
22
|
+
runId: opts.runId ?? null,
|
|
23
|
+
threadId: opts.threadId,
|
|
24
|
+
messageSeq: opts.messageSeq ?? null,
|
|
25
|
+
feedbackType: opts.feedbackType,
|
|
26
|
+
value: opts.value ?? "",
|
|
27
|
+
userId: opts.userId ?? null,
|
|
28
|
+
createdAt: Date.now(),
|
|
29
|
+
};
|
|
30
|
+
await insertFeedback(entry);
|
|
31
|
+
return entry;
|
|
32
|
+
}
|
|
33
|
+
async function getThreadMessages(threadId) {
|
|
34
|
+
await ensureObservabilityTables();
|
|
35
|
+
const client = getDbExec();
|
|
36
|
+
const { rows } = await client.execute({
|
|
37
|
+
sql: `SELECT thread_data FROM chat_threads WHERE id = ?`,
|
|
38
|
+
args: [threadId],
|
|
39
|
+
});
|
|
40
|
+
if (rows.length === 0)
|
|
41
|
+
return [];
|
|
42
|
+
const raw = rows[0].thread_data;
|
|
43
|
+
if (!raw)
|
|
44
|
+
return [];
|
|
45
|
+
try {
|
|
46
|
+
const data = JSON.parse(String(raw));
|
|
47
|
+
const messages = data.messages ?? data;
|
|
48
|
+
if (!Array.isArray(messages))
|
|
49
|
+
return [];
|
|
50
|
+
return messages
|
|
51
|
+
.filter((m) => m &&
|
|
52
|
+
typeof m.role === "string" &&
|
|
53
|
+
(typeof m.content === "string" ||
|
|
54
|
+
(Array.isArray(m.content) &&
|
|
55
|
+
m.content.some((p) => p.type === "text"))))
|
|
56
|
+
.map((m) => ({
|
|
57
|
+
role: m.role,
|
|
58
|
+
content: typeof m.content === "string"
|
|
59
|
+
? m.content
|
|
60
|
+
: m.content
|
|
61
|
+
.filter((p) => p.type === "text")
|
|
62
|
+
.map((p) => p.text ?? "")
|
|
63
|
+
.join(""),
|
|
64
|
+
createdAt: m.createdAt ? Number(m.createdAt) : undefined,
|
|
65
|
+
}));
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return [];
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function tokenize(text) {
|
|
72
|
+
return new Set(text
|
|
73
|
+
.toLowerCase()
|
|
74
|
+
.replace(/[^a-z0-9\s]/g, "")
|
|
75
|
+
.split(/\s+/)
|
|
76
|
+
.filter((w) => w.length > 1));
|
|
77
|
+
}
|
|
78
|
+
function jaccardSimilarity(a, b) {
|
|
79
|
+
if (a.size === 0 && b.size === 0)
|
|
80
|
+
return 0;
|
|
81
|
+
let intersection = 0;
|
|
82
|
+
for (const word of a) {
|
|
83
|
+
if (b.has(word))
|
|
84
|
+
intersection++;
|
|
85
|
+
}
|
|
86
|
+
const union = a.size + b.size - intersection;
|
|
87
|
+
return union === 0 ? 0 : intersection / union;
|
|
88
|
+
}
|
|
89
|
+
function computeRephrasingScore(userMessages) {
|
|
90
|
+
if (userMessages.length < 2)
|
|
91
|
+
return 0;
|
|
92
|
+
const tokenSets = userMessages.map(tokenize);
|
|
93
|
+
let maxConsecutiveSimilarity = 0;
|
|
94
|
+
let highSimilarityCount = 0;
|
|
95
|
+
for (let i = 1; i < tokenSets.length; i++) {
|
|
96
|
+
const sim = jaccardSimilarity(tokenSets[i - 1], tokenSets[i]);
|
|
97
|
+
if (sim > maxConsecutiveSimilarity)
|
|
98
|
+
maxConsecutiveSimilarity = sim;
|
|
99
|
+
if (sim >= 0.4)
|
|
100
|
+
highSimilarityCount++;
|
|
101
|
+
}
|
|
102
|
+
const pairCount = tokenSets.length - 1;
|
|
103
|
+
const rephrasingRatio = pairCount > 0 ? highSimilarityCount / pairCount : 0;
|
|
104
|
+
// Blend peak similarity with overall rephrasing frequency
|
|
105
|
+
return Math.min(100, ((maxConsecutiveSimilarity * 60 + rephrasingRatio * 40) * 100) / 100);
|
|
106
|
+
}
|
|
107
|
+
function computeAbandonmentScore(messages) {
|
|
108
|
+
if (messages.length === 0)
|
|
109
|
+
return 0;
|
|
110
|
+
const last = messages[messages.length - 1];
|
|
111
|
+
// Thread ends with a user message and no agent response
|
|
112
|
+
if (last.role === "user")
|
|
113
|
+
return 80;
|
|
114
|
+
// Thread ends with agent response, but check if last user message
|
|
115
|
+
// was very close to it (agent responded but user never replied back)
|
|
116
|
+
if (messages.length >= 3) {
|
|
117
|
+
const secondToLast = messages[messages.length - 2];
|
|
118
|
+
if (secondToLast.role === "user") {
|
|
119
|
+
const userMsg = secondToLast.content.trim();
|
|
120
|
+
// Short user messages right before end suggest giving up
|
|
121
|
+
if (userMsg.length < 15)
|
|
122
|
+
return 40;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return 0;
|
|
126
|
+
}
|
|
127
|
+
const NEGATIVE_PATTERNS = [
|
|
128
|
+
/\bno\b/i,
|
|
129
|
+
/\bwrong\b/i,
|
|
130
|
+
/\bnot what i/i,
|
|
131
|
+
/\btry again\b/i,
|
|
132
|
+
/\bnever mind\b/i,
|
|
133
|
+
/\bnevermind\b/i,
|
|
134
|
+
/\bthat's not\b/i,
|
|
135
|
+
/\bthats not\b/i,
|
|
136
|
+
/\bincorrect\b/i,
|
|
137
|
+
/\bdoesn't work\b/i,
|
|
138
|
+
/\bdoesnt work\b/i,
|
|
139
|
+
/\bstill wrong\b/i,
|
|
140
|
+
/\bnope\b/i,
|
|
141
|
+
/\bstop\b/i,
|
|
142
|
+
/\bforget it\b/i,
|
|
143
|
+
/\buseless\b/i,
|
|
144
|
+
/\bbroken\b/i,
|
|
145
|
+
];
|
|
146
|
+
function computeSentimentScore(userMessages) {
|
|
147
|
+
if (userMessages.length === 0)
|
|
148
|
+
return 0;
|
|
149
|
+
let negativeCount = 0;
|
|
150
|
+
let terseCount = 0;
|
|
151
|
+
for (const msg of userMessages) {
|
|
152
|
+
const trimmed = msg.trim();
|
|
153
|
+
// Terse single-word or very short responses
|
|
154
|
+
if (trimmed.split(/\s+/).length <= 2 && trimmed.length < 20) {
|
|
155
|
+
terseCount++;
|
|
156
|
+
}
|
|
157
|
+
for (const pattern of NEGATIVE_PATTERNS) {
|
|
158
|
+
if (pattern.test(trimmed)) {
|
|
159
|
+
negativeCount++;
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
const negativeRatio = negativeCount / userMessages.length;
|
|
165
|
+
const terseRatio = terseCount / userMessages.length;
|
|
166
|
+
return Math.min(100, (negativeRatio * 70 + terseRatio * 30) * 100);
|
|
167
|
+
}
|
|
168
|
+
function computeLengthTrendScore(userMessages) {
|
|
169
|
+
if (userMessages.length < 3)
|
|
170
|
+
return 0;
|
|
171
|
+
const lengths = userMessages.map((m) => m.trim().length);
|
|
172
|
+
const n = lengths.length;
|
|
173
|
+
// Simple linear regression: y = mx + b, we care about slope m
|
|
174
|
+
const xMean = (n - 1) / 2;
|
|
175
|
+
const yMean = lengths.reduce((a, b) => a + b, 0) / n;
|
|
176
|
+
let numerator = 0;
|
|
177
|
+
let denominator = 0;
|
|
178
|
+
for (let i = 0; i < n; i++) {
|
|
179
|
+
const xDiff = i - xMean;
|
|
180
|
+
numerator += xDiff * (lengths[i] - yMean);
|
|
181
|
+
denominator += xDiff * xDiff;
|
|
182
|
+
}
|
|
183
|
+
if (denominator === 0)
|
|
184
|
+
return 0;
|
|
185
|
+
const slope = numerator / denominator;
|
|
186
|
+
// Normalize: negative slope = messages getting shorter = frustration
|
|
187
|
+
// Scale by average length to get a relative measure
|
|
188
|
+
if (yMean === 0)
|
|
189
|
+
return 0;
|
|
190
|
+
const normalizedSlope = slope / yMean;
|
|
191
|
+
// Only negative slopes (shrinking messages) contribute to frustration
|
|
192
|
+
if (normalizedSlope >= 0)
|
|
193
|
+
return 0;
|
|
194
|
+
// Map normalized slope to 0-100; -1 (halving each message) = 100
|
|
195
|
+
return Math.min(100, Math.abs(normalizedSlope) * 100);
|
|
196
|
+
}
|
|
197
|
+
const RETRY_PATTERNS = [
|
|
198
|
+
/\btry again\b/i,
|
|
199
|
+
/\bthat's wrong\b/i,
|
|
200
|
+
/\bthats wrong\b/i,
|
|
201
|
+
/\bno,?\s*(that's|thats|it's|its)\b/i,
|
|
202
|
+
/\bredo\b/i,
|
|
203
|
+
/\bdo it again\b/i,
|
|
204
|
+
/\bone more time\b/i,
|
|
205
|
+
/\bregenerate\b/i,
|
|
206
|
+
/\bfix (it|this|that)\b/i,
|
|
207
|
+
/\btry (this|that) instead\b/i,
|
|
208
|
+
/\bi (said|meant|asked)\b/i,
|
|
209
|
+
/\bstill (not|wrong|broken|doesn't|doesnt)\b/i,
|
|
210
|
+
];
|
|
211
|
+
function computeRetryScore(userMessages) {
|
|
212
|
+
if (userMessages.length === 0)
|
|
213
|
+
return 0;
|
|
214
|
+
let retryCount = 0;
|
|
215
|
+
for (const msg of userMessages) {
|
|
216
|
+
for (const pattern of RETRY_PATTERNS) {
|
|
217
|
+
if (pattern.test(msg)) {
|
|
218
|
+
retryCount++;
|
|
219
|
+
break;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
const retryRatio = retryCount / userMessages.length;
|
|
224
|
+
return Math.min(100, retryRatio * 150);
|
|
225
|
+
}
|
|
226
|
+
export async function computeSatisfactionScore(threadId, opts = {}) {
|
|
227
|
+
const messages = await getThreadMessages(threadId);
|
|
228
|
+
const userMessages = messages
|
|
229
|
+
.filter((m) => m.role === "user")
|
|
230
|
+
.map((m) => m.content);
|
|
231
|
+
const rephrasingScore = computeRephrasingScore(userMessages);
|
|
232
|
+
const abandonmentScore = computeAbandonmentScore(messages);
|
|
233
|
+
const sentimentScore = computeSentimentScore(userMessages);
|
|
234
|
+
const lengthTrendScore = computeLengthTrendScore(userMessages);
|
|
235
|
+
const retryScore = computeRetryScore(userMessages);
|
|
236
|
+
// Weighted composite: rephrasing 30, abandonment 20, sentiment 15, length trend 15, retry 20
|
|
237
|
+
const frustrationScore = Math.min(100, rephrasingScore * 0.3 +
|
|
238
|
+
abandonmentScore * 0.2 +
|
|
239
|
+
sentimentScore * 0.15 +
|
|
240
|
+
lengthTrendScore * 0.15 +
|
|
241
|
+
retryScore * 0.2);
|
|
242
|
+
const score = {
|
|
243
|
+
id: `sat-${threadId}`,
|
|
244
|
+
threadId,
|
|
245
|
+
userId: opts.userId ?? null,
|
|
246
|
+
frustrationScore: Math.round(frustrationScore * 100) / 100,
|
|
247
|
+
rephrasingScore: Math.round(rephrasingScore * 100) / 100,
|
|
248
|
+
abandonmentScore: Math.round(abandonmentScore * 100) / 100,
|
|
249
|
+
sentimentScore: Math.round(sentimentScore * 100) / 100,
|
|
250
|
+
lengthTrendScore: Math.round(lengthTrendScore * 100) / 100,
|
|
251
|
+
computedAt: Date.now(),
|
|
252
|
+
};
|
|
253
|
+
await upsertSatisfactionScore(score);
|
|
254
|
+
return score;
|
|
255
|
+
}
|
|
256
|
+
//# sourceMappingURL=feedback.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"feedback.js","sourceRoot":"","sources":["../../src/observability/feedback.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAM5C,OAAO,EACL,cAAc,EACd,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,YAAY,CAAC;AAEpB,SAAS,UAAU;IACjB,OAAO,MAAM,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACtE,CAAC;AAaD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAwB;IAExB,IAAI,CAAC,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IAC5D,IAAI,CAAC,IAAI,CAAC,YAAY;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAEpE,MAAM,UAAU,GAAmB;QACjC,WAAW;QACX,aAAa;QACb,UAAU;QACV,MAAM;KACP,CAAC;IACF,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,KAAK,GAAkB;QAC3B,EAAE,EAAE,UAAU,EAAE;QAChB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;QACnC,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;QAC3B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IAEF,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;IAC5B,OAAO,KAAK,CAAC;AACf,CAAC;AAUD,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,MAAM,yBAAyB,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE,mDAAmD;QACxD,IAAI,EAAE,CAAC,QAAQ,CAAC;KACjB,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,MAAM,GAAG,GAAI,IAAI,CAAC,CAAC,CAA6B,CAAC,WAAW,CAAC;IAC7D,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IAEpB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAc,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QAClD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,CAAC;QAExC,OAAO,QAAQ;aACZ,MAAM,CACL,CAAC,CAAM,EAAE,EAAE,CACT,CAAC;YACD,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;YAC1B,CAAC,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ;gBAC5B,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;oBACvB,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CACtD;aACA,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YAChB,IAAI,EAAE,CAAC,CAAC,IAA4B;YACpC,OAAO,EACL,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ;gBAC3B,CAAC,CAAC,CAAC,CAAC,OAAO;gBACX,CAAC,CAAE,CAAC,CAAC,OAAiB;qBACjB,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;qBACrC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;qBAC7B,IAAI,CAAC,EAAE,CAAC;YACjB,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;SACzD,CAAC,CAAC,CAAC;IACR,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,GAAG,CACZ,IAAI;SACD,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,CAAc,EAAE,CAAc;IACvD,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,YAAY,EAAE,CAAC;IAClC,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,YAAY,CAAC;IAC7C,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC;AAChD,CAAC;AAED,SAAS,sBAAsB,CAAC,YAAsB;IACpD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtC,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,wBAAwB,GAAG,CAAC,CAAC;IACjC,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9D,IAAI,GAAG,GAAG,wBAAwB;YAAE,wBAAwB,GAAG,GAAG,CAAC;QACnE,IAAI,GAAG,IAAI,GAAG;YAAE,mBAAmB,EAAE,CAAC;IACxC,CAAC;IAED,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;IACvC,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,mBAAmB,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,0DAA0D;IAC1D,OAAO,IAAI,CAAC,GAAG,CACb,GAAG,EACH,CAAC,CAAC,wBAAwB,GAAG,EAAE,GAAG,eAAe,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CACrE,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAyB;IACxD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEpC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE3C,wDAAwD;IACxD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,EAAE,CAAC;IAEpC,kEAAkE;IAClE,qEAAqE;IACrE,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnD,IAAI,YAAY,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC5C,yDAAyD;YACzD,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;gBAAE,OAAO,EAAE,CAAC;QACrC,CAAC;IACH,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,iBAAiB,GAAG;IACxB,SAAS;IACT,YAAY;IACZ,eAAe;IACf,gBAAgB;IAChB,iBAAiB;IACjB,gBAAgB;IAChB,iBAAiB;IACjB,gBAAgB;IAChB,gBAAgB;IAChB,mBAAmB;IACnB,kBAAkB;IAClB,kBAAkB;IAClB,WAAW;IACX,WAAW;IACX,gBAAgB;IAChB,cAAc;IACd,aAAa;CACd,CAAC;AAEF,SAAS,qBAAqB,CAAC,YAAsB;IACnD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAExC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAE3B,4CAA4C;QAC5C,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YAC5D,UAAU,EAAE,CAAC;QACf,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;YACxC,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,aAAa,EAAE,CAAC;gBAChB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1D,MAAM,UAAU,GAAG,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IAEpD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,aAAa,GAAG,EAAE,GAAG,UAAU,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,uBAAuB,CAAC,YAAsB;IACrD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAEzB,8DAA8D;IAC9D,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAErD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC;QACxB,SAAS,IAAI,KAAK,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;QAC1C,WAAW,IAAI,KAAK,GAAG,KAAK,CAAC;IAC/B,CAAC;IAED,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,MAAM,KAAK,GAAG,SAAS,GAAG,WAAW,CAAC;IAEtC,qEAAqE;IACrE,oDAAoD;IACpD,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1B,MAAM,eAAe,GAAG,KAAK,GAAG,KAAK,CAAC;IAEtC,sEAAsE;IACtE,IAAI,eAAe,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAEnC,iEAAiE;IACjE,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,GAAG,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,cAAc,GAAG;IACrB,gBAAgB;IAChB,mBAAmB;IACnB,kBAAkB;IAClB,qCAAqC;IACrC,WAAW;IACX,kBAAkB;IAClB,oBAAoB;IACpB,iBAAiB;IACjB,yBAAyB;IACzB,8BAA8B;IAC9B,2BAA2B;IAC3B,8CAA8C;CAC/C,CAAC;AAEF,SAAS,iBAAiB,CAAC,YAAsB;IAC/C,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAExC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;YACrC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,UAAU,EAAE,CAAC;gBACb,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,GAAG,GAAG,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,QAAgB,EAChB,OAAmC,EAAE;IAErC,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,YAAY,GAAG,QAAQ;SAC1B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;SAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAEzB,MAAM,eAAe,GAAG,sBAAsB,CAAC,YAAY,CAAC,CAAC;IAC7D,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;IAC3D,MAAM,cAAc,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAC3D,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,YAAY,CAAC,CAAC;IAC/D,MAAM,UAAU,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAEnD,6FAA6F;IAC7F,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAC/B,GAAG,EACH,eAAe,GAAG,GAAG;QACnB,gBAAgB,GAAG,GAAG;QACtB,cAAc,GAAG,IAAI;QACrB,gBAAgB,GAAG,IAAI;QACvB,UAAU,GAAG,GAAG,CACnB,CAAC;IAEF,MAAM,KAAK,GAAsB;QAC/B,EAAE,EAAE,OAAO,QAAQ,EAAE;QACrB,QAAQ;QACR,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;QAC3B,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,GAAG,CAAC,GAAG,GAAG;QAC1D,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,GAAG,CAAC,GAAG,GAAG;QACxD,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,GAAG,CAAC,GAAG,GAAG;QAC1D,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,GAAG,CAAC,GAAG,GAAG;QACtD,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,GAAG,CAAC,GAAG,GAAG;QAC1D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;KACvB,CAAC;IAEF,MAAM,uBAAuB,CAAC,KAAK,CAAC,CAAC;IACrC,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export type { SpanType, SpanStatus, TraceSpan, TraceSummary, FeedbackType, FeedbackEntry, SatisfactionScore, EvalType, EvalResult, EvalDataset, EvalTestCase, EvalCriteria, ExperimentStatus, ExperimentVariant, Experiment, ExperimentAssignment, ExperimentMetricResult, ObservabilityConfig, ObservabilityExporterConfig, } from "./types.js";
|
|
2
|
+
export { DEFAULT_OBSERVABILITY_CONFIG } from "./types.js";
|
|
3
|
+
export { ensureObservabilityTables, insertTraceSpan, upsertTraceSummary, getTraceSpansForRun, getTraceSummaries, getTraceSummary, insertFeedback, getFeedback, getFeedbackStats, upsertSatisfactionScore, getSatisfactionScores, insertEvalResult, getEvalsForRun, getEvalStats, insertEvalDataset, listEvalDatasets, getEvalDataset, updateEvalDataset, insertExperiment, updateExperiment, listExperiments, getExperiment, upsertAssignment, getAssignment, insertExperimentResult, getExperimentResults, getObservabilityOverview, } from "./store.js";
|
|
4
|
+
export { createObservabilityPlugin } from "./plugin.js";
|
|
5
|
+
export { createObservabilityHandler } from "./routes.js";
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/observability/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,QAAQ,EACR,UAAU,EACV,SAAS,EACT,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,EACjB,UAAU,EACV,oBAAoB,EACpB,sBAAsB,EACtB,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,OAAO,EACL,yBAAyB,EACzB,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,sBAAsB,EACtB,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { DEFAULT_OBSERVABILITY_CONFIG } from "./types.js";
|
|
2
|
+
export { ensureObservabilityTables, insertTraceSpan, upsertTraceSummary, getTraceSpansForRun, getTraceSummaries, getTraceSummary, insertFeedback, getFeedback, getFeedbackStats, upsertSatisfactionScore, getSatisfactionScores, insertEvalResult, getEvalsForRun, getEvalStats, insertEvalDataset, listEvalDatasets, getEvalDataset, updateEvalDataset, insertExperiment, updateExperiment, listExperiments, getExperiment, upsertAssignment, getAssignment, insertExperimentResult, getExperimentResults, getObservabilityOverview, } from "./store.js";
|
|
3
|
+
export { createObservabilityPlugin } from "./plugin.js";
|
|
4
|
+
export { createObservabilityHandler } from "./routes.js";
|
|
5
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/observability/index.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,OAAO,EACL,yBAAyB,EACzB,eAAe,EACf,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,sBAAsB,EACtB,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../../src/observability/plugin.ts"],"names":[],"mappings":"AAQA,wBAAgB,yBAAyB,KACzB,UAAU,GAAG,mBAQ5B"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { getH3App, awaitBootstrap, } from "../server/framework-request-handler.js";
|
|
2
|
+
import { FRAMEWORK_ROUTE_PREFIX } from "../server/core-routes-plugin.js";
|
|
3
|
+
import { createObservabilityHandler } from "./routes.js";
|
|
4
|
+
import { ensureObservabilityTables } from "./store.js";
|
|
5
|
+
export function createObservabilityPlugin() {
|
|
6
|
+
return async (nitroApp) => {
|
|
7
|
+
await awaitBootstrap(nitroApp);
|
|
8
|
+
await ensureObservabilityTables().catch(() => { });
|
|
9
|
+
getH3App(nitroApp).use(`${FRAMEWORK_ROUTE_PREFIX}/observability`, createObservabilityHandler());
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=plugin.js.map
|