@jami-studio/core 0.92.25 → 0.92.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1321 +1,1313 @@
1
- /**
2
- * SQL persistence for the agent observability system.
3
- *
4
- * Creates and manages tables for traces, feedback, evals, experiments,
5
- * and satisfaction scores. Follows the same raw-SQL pattern as
6
- * run-store.ts and usage/store.ts — framework tables use getDbExec()
7
- * rather than Drizzle ORM (which is for template-level schemas).
8
- */
9
- import {
10
- getDbExec,
11
- intType,
12
- isPostgres,
13
- retryOnDdlRace,
14
- } from "../db/client.js";
15
- import {
16
- ensureTableExists,
17
- ensureColumnExists,
18
- ensureIndexExists,
19
- } from "../db/ddl-guard.js";
20
- import { isDuplicateColumnError } from "../db/migrations.js";
21
- import type {
22
- TraceSpan,
23
- TraceSummary,
24
- FeedbackEntry,
25
- SatisfactionScore,
26
- EvalResult,
27
- EvalDataset,
28
- Experiment,
29
- ExperimentAssignment,
30
- ExperimentMetricResult,
31
- } from "./types.js";
32
-
33
- function safeJsonParse<T>(value: unknown, fallback: T): T {
34
- if (!value) return fallback;
35
- try {
36
- return JSON.parse(String(value));
37
- } catch {
38
- return fallback;
39
- }
40
- }
41
-
42
- // Tables whose rows are owned by an end user — drives the boot-time
43
- // user_id ALTER loop and the per-user composite indexes below. Every
44
- // new user-owned observability table must be added here so the upgrade
45
- // path and the per-user query plan stay in sync.
46
- const USER_SCOPED_TABLES = [
47
- "agent_trace_spans",
48
- "agent_trace_summaries",
49
- "agent_satisfaction_scores",
50
- "agent_evals",
51
- "agent_feedback",
52
- ] as const;
53
-
54
- /**
55
- * Append an `AND user_id = ?` clause when a userId filter is requested.
56
- * Returns the fully-bound WHERE clause + args ready to splice into the
57
- * caller's SQL. Centralizes the pattern so tests can assert one shape.
58
- */
59
- function withUserFilter(
60
- baseWhere: string,
61
- baseArgs: any[],
62
- userId: string | undefined,
63
- ): { where: string; args: any[] } {
64
- if (userId == null) return { where: baseWhere, args: baseArgs };
65
- return {
66
- where: `${baseWhere} AND user_id = ?`,
67
- args: [...baseArgs, userId],
68
- };
69
- }
70
-
71
- let _initPromise: Promise<void> | undefined;
72
-
73
- export async function ensureObservabilityTables(): Promise<void> {
74
- if (!_initPromise) {
75
- _initPromise = (async () => {
76
- const client = getDbExec();
77
-
78
- const traceSpansCreateSql = `
79
- CREATE TABLE IF NOT EXISTS agent_trace_spans (
80
- id TEXT PRIMARY KEY,
81
- run_id TEXT NOT NULL,
82
- thread_id TEXT,
83
- user_id TEXT,
84
- parent_span_id TEXT,
85
- span_type TEXT NOT NULL,
86
- name TEXT NOT NULL,
87
- input_tokens ${intType()} NOT NULL DEFAULT 0,
88
- output_tokens ${intType()} NOT NULL DEFAULT 0,
89
- cache_read_tokens ${intType()} NOT NULL DEFAULT 0,
90
- cache_write_tokens ${intType()} NOT NULL DEFAULT 0,
91
- cost_cents_x100 ${intType()} NOT NULL DEFAULT 0,
92
- duration_ms ${intType()} NOT NULL DEFAULT 0,
93
- status TEXT NOT NULL DEFAULT 'success',
94
- error_message TEXT,
95
- metadata TEXT,
96
- created_at ${intType()} NOT NULL
97
- )
98
- `;
99
-
100
- const traceSummariesCreateSql = `
101
- CREATE TABLE IF NOT EXISTS agent_trace_summaries (
102
- run_id TEXT PRIMARY KEY,
103
- thread_id TEXT,
104
- user_id TEXT,
105
- total_spans ${intType()} NOT NULL DEFAULT 0,
106
- llm_calls ${intType()} NOT NULL DEFAULT 0,
107
- tool_calls ${intType()} NOT NULL DEFAULT 0,
108
- successful_tools ${intType()} NOT NULL DEFAULT 0,
109
- failed_tools ${intType()} NOT NULL DEFAULT 0,
110
- total_duration_ms ${intType()} NOT NULL DEFAULT 0,
111
- total_cost_cents_x100 ${intType()} NOT NULL DEFAULT 0,
112
- total_input_tokens ${intType()} NOT NULL DEFAULT 0,
113
- total_output_tokens ${intType()} NOT NULL DEFAULT 0,
114
- model TEXT NOT NULL DEFAULT '',
115
- created_at ${intType()} NOT NULL
116
- )
117
- `;
118
-
119
- const feedbackCreateSql = `
120
- CREATE TABLE IF NOT EXISTS agent_feedback (
121
- id TEXT PRIMARY KEY,
122
- run_id TEXT,
123
- thread_id TEXT,
124
- message_seq ${intType()},
125
- feedback_type TEXT NOT NULL,
126
- value TEXT NOT NULL DEFAULT '',
127
- user_id TEXT,
128
- created_at ${intType()} NOT NULL
129
- )
130
- `;
131
-
132
- const satisfactionScoresCreateSql = `
133
- CREATE TABLE IF NOT EXISTS agent_satisfaction_scores (
134
- id TEXT PRIMARY KEY,
135
- thread_id TEXT NOT NULL,
136
- user_id TEXT,
137
- frustration_score REAL NOT NULL DEFAULT 0,
138
- rephrasing_score REAL NOT NULL DEFAULT 0,
139
- abandonment_score REAL NOT NULL DEFAULT 0,
140
- sentiment_score REAL NOT NULL DEFAULT 0,
141
- length_trend_score REAL NOT NULL DEFAULT 0,
142
- computed_at ${intType()} NOT NULL
143
- )
144
- `;
145
-
146
- const evalsCreateSql = `
147
- CREATE TABLE IF NOT EXISTS agent_evals (
148
- id TEXT PRIMARY KEY,
149
- run_id TEXT NOT NULL,
150
- thread_id TEXT,
151
- user_id TEXT,
152
- eval_type TEXT NOT NULL,
153
- criteria TEXT NOT NULL,
154
- score REAL NOT NULL DEFAULT 0,
155
- reasoning TEXT,
156
- metadata TEXT,
157
- created_at ${intType()} NOT NULL
158
- )
159
- `;
160
-
161
- const evalDatasetsCreateSql = `
162
- CREATE TABLE IF NOT EXISTS agent_eval_datasets (
163
- id TEXT PRIMARY KEY,
164
- name TEXT NOT NULL,
165
- description TEXT NOT NULL DEFAULT '',
166
- entries TEXT NOT NULL DEFAULT '[]',
167
- created_at ${intType()} NOT NULL,
168
- updated_at ${intType()} NOT NULL
169
- )
170
- `;
171
-
172
- const experimentsCreateSql = `
173
- CREATE TABLE IF NOT EXISTS agent_experiments (
174
- id TEXT PRIMARY KEY,
175
- name TEXT NOT NULL,
176
- status TEXT NOT NULL DEFAULT 'draft',
177
- variants TEXT NOT NULL DEFAULT '[]',
178
- metrics TEXT NOT NULL DEFAULT '[]',
179
- assignment_level TEXT NOT NULL DEFAULT 'user',
180
- started_at ${intType()},
181
- ended_at ${intType()},
182
- created_at ${intType()} NOT NULL,
183
- owner_email TEXT
184
- )
185
- `;
186
-
187
- const experimentAssignmentsCreateSql = `
188
- CREATE TABLE IF NOT EXISTS agent_experiment_assignments (
189
- experiment_id TEXT NOT NULL,
190
- user_id TEXT NOT NULL,
191
- variant_id TEXT NOT NULL,
192
- assigned_at ${intType()} NOT NULL,
193
- PRIMARY KEY (experiment_id, user_id)
194
- )
195
- `;
196
-
197
- const experimentResultsCreateSql = `
198
- CREATE TABLE IF NOT EXISTS agent_experiment_results (
199
- id TEXT PRIMARY KEY,
200
- experiment_id TEXT NOT NULL,
201
- variant_id TEXT NOT NULL,
202
- metric TEXT NOT NULL,
203
- value REAL NOT NULL DEFAULT 0,
204
- sample_size ${intType()} NOT NULL DEFAULT 0,
205
- confidence_low REAL NOT NULL DEFAULT 0,
206
- confidence_high REAL NOT NULL DEFAULT 0,
207
- computed_at ${intType()} NOT NULL
208
- )
209
- `;
210
-
211
- if (isPostgres()) {
212
- // PG guard: probe → guarded DDL → re-probe; skips lock on already-migrated path
213
- await ensureTableExists("agent_trace_spans", traceSpansCreateSql);
214
- await ensureTableExists(
215
- "agent_trace_summaries",
216
- traceSummariesCreateSql,
217
- );
218
- await ensureTableExists("agent_feedback", feedbackCreateSql);
219
- await ensureTableExists(
220
- "agent_satisfaction_scores",
221
- satisfactionScoresCreateSql,
222
- );
223
- await ensureTableExists("agent_evals", evalsCreateSql);
224
- await ensureTableExists("agent_eval_datasets", evalDatasetsCreateSql);
225
- await ensureTableExists("agent_experiments", experimentsCreateSql);
226
- await ensureTableExists(
227
- "agent_experiment_assignments",
228
- experimentAssignmentsCreateSql,
229
- );
230
- await ensureTableExists(
231
- "agent_experiment_results",
232
- experimentResultsCreateSql,
233
- );
234
- await ensureColumnExists(
235
- "agent_experiments",
236
- "owner_email",
237
- `ALTER TABLE agent_experiments ADD COLUMN IF NOT EXISTS owner_email TEXT`,
238
- );
239
- for (const table of USER_SCOPED_TABLES) {
240
- await ensureColumnExists(
241
- table,
242
- "user_id",
243
- `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS user_id TEXT`,
244
- );
245
- }
246
- await ensureIndexExists(
247
- "idx_trace_spans_run",
248
- `CREATE INDEX IF NOT EXISTS idx_trace_spans_run ON agent_trace_spans (run_id)`,
249
- );
250
- await ensureIndexExists(
251
- "idx_trace_spans_thread",
252
- `CREATE INDEX IF NOT EXISTS idx_trace_spans_thread ON agent_trace_spans (thread_id)`,
253
- );
254
- await ensureIndexExists(
255
- "idx_trace_spans_created",
256
- `CREATE INDEX IF NOT EXISTS idx_trace_spans_created ON agent_trace_spans (created_at)`,
257
- );
258
- await ensureIndexExists(
259
- "idx_trace_summaries_created",
260
- `CREATE INDEX IF NOT EXISTS idx_trace_summaries_created ON agent_trace_summaries (created_at)`,
261
- );
262
- await ensureIndexExists(
263
- "idx_trace_summaries_user",
264
- `CREATE INDEX IF NOT EXISTS idx_trace_summaries_user ON agent_trace_summaries (user_id, created_at)`,
265
- );
266
- await ensureIndexExists(
267
- "idx_trace_spans_user",
268
- `CREATE INDEX IF NOT EXISTS idx_trace_spans_user ON agent_trace_spans (user_id)`,
269
- );
270
- await ensureIndexExists(
271
- "idx_feedback_thread",
272
- `CREATE INDEX IF NOT EXISTS idx_feedback_thread ON agent_feedback (thread_id)`,
273
- );
274
- await ensureIndexExists(
275
- "idx_feedback_created",
276
- `CREATE INDEX IF NOT EXISTS idx_feedback_created ON agent_feedback (created_at)`,
277
- );
278
- await ensureIndexExists(
279
- "idx_feedback_user",
280
- `CREATE INDEX IF NOT EXISTS idx_feedback_user ON agent_feedback (user_id, created_at)`,
281
- );
282
- await ensureIndexExists(
283
- "idx_satisfaction_thread",
284
- `CREATE INDEX IF NOT EXISTS idx_satisfaction_thread ON agent_satisfaction_scores (thread_id)`,
285
- );
286
- await ensureIndexExists(
287
- "idx_satisfaction_user",
288
- `CREATE INDEX IF NOT EXISTS idx_satisfaction_user ON agent_satisfaction_scores (user_id, computed_at)`,
289
- );
290
- await ensureIndexExists(
291
- "idx_evals_run",
292
- `CREATE INDEX IF NOT EXISTS idx_evals_run ON agent_evals (run_id)`,
293
- );
294
- await ensureIndexExists(
295
- "idx_evals_created",
296
- `CREATE INDEX IF NOT EXISTS idx_evals_created ON agent_evals (created_at)`,
297
- );
298
- await ensureIndexExists(
299
- "idx_evals_user",
300
- `CREATE INDEX IF NOT EXISTS idx_evals_user ON agent_evals (user_id, created_at)`,
301
- );
302
- await ensureIndexExists(
303
- "idx_experiment_results_exp",
304
- `CREATE INDEX IF NOT EXISTS idx_experiment_results_exp ON agent_experiment_results (experiment_id)`,
305
- );
306
- return;
307
- }
308
-
309
- // SQLite (local dev): no lock problem — keep the original behaviour.
310
- await retryOnDdlRace(() => client.execute(traceSpansCreateSql));
311
-
312
- await retryOnDdlRace(() => client.execute(traceSummariesCreateSql));
313
-
314
- await retryOnDdlRace(() => client.execute(feedbackCreateSql));
315
-
316
- await retryOnDdlRace(() => client.execute(satisfactionScoresCreateSql));
317
-
318
- await retryOnDdlRace(() => client.execute(evalsCreateSql));
319
-
320
- await retryOnDdlRace(() => client.execute(evalDatasetsCreateSql));
321
-
322
- await retryOnDdlRace(() => client.execute(experimentsCreateSql));
323
-
324
- // Additive migration for DBs created before the owner column shipped
325
- // (any pre-existing rows have NULL owner — see `updateExperiment` for
326
- // the migration semantics). Mutations on those rows fall back to the
327
- // standard authentication gate but cannot enforce per-owner scoping
328
- // until they're re-saved.
329
- try {
330
- await client.execute(
331
- `ALTER TABLE agent_experiments ADD COLUMN owner_email TEXT`,
332
- );
333
- } catch {
334
- // Column already exists — expected after first run.
335
- }
336
-
337
- await retryOnDdlRace(() =>
338
- client.execute(experimentAssignmentsCreateSql),
339
- );
340
-
341
- await retryOnDdlRace(() => client.execute(experimentResultsCreateSql));
342
-
343
- // Idempotent column upgrades for DBs created before per-user
344
- // isolation. SQLite has no `ADD COLUMN IF NOT EXISTS`; Postgres
345
- // surfaces "column ... already exists". `isDuplicateColumnError`
346
- // (from db/migrations.ts) recognizes both shapes.
347
- for (const table of USER_SCOPED_TABLES) {
348
- try {
349
- await client.execute(`ALTER TABLE ${table} ADD COLUMN user_id TEXT`);
350
- } catch (err) {
351
- if (isDuplicateColumnError(err)) continue;
352
- throw err;
353
- }
354
- }
355
-
356
- // Indexes for common query patterns
357
- const indexes = [
358
- `CREATE INDEX IF NOT EXISTS idx_trace_spans_run ON agent_trace_spans (run_id)`,
359
- `CREATE INDEX IF NOT EXISTS idx_trace_spans_thread ON agent_trace_spans (thread_id)`,
360
- `CREATE INDEX IF NOT EXISTS idx_trace_spans_created ON agent_trace_spans (created_at)`,
361
- `CREATE INDEX IF NOT EXISTS idx_trace_summaries_created ON agent_trace_summaries (created_at)`,
362
- `CREATE INDEX IF NOT EXISTS idx_trace_summaries_user ON agent_trace_summaries (user_id, created_at)`,
363
- `CREATE INDEX IF NOT EXISTS idx_trace_spans_user ON agent_trace_spans (user_id)`,
364
- `CREATE INDEX IF NOT EXISTS idx_feedback_thread ON agent_feedback (thread_id)`,
365
- `CREATE INDEX IF NOT EXISTS idx_feedback_created ON agent_feedback (created_at)`,
366
- `CREATE INDEX IF NOT EXISTS idx_feedback_user ON agent_feedback (user_id, created_at)`,
367
- `CREATE INDEX IF NOT EXISTS idx_satisfaction_thread ON agent_satisfaction_scores (thread_id)`,
368
- `CREATE INDEX IF NOT EXISTS idx_satisfaction_user ON agent_satisfaction_scores (user_id, computed_at)`,
369
- `CREATE INDEX IF NOT EXISTS idx_evals_run ON agent_evals (run_id)`,
370
- `CREATE INDEX IF NOT EXISTS idx_evals_created ON agent_evals (created_at)`,
371
- `CREATE INDEX IF NOT EXISTS idx_evals_user ON agent_evals (user_id, created_at)`,
372
- `CREATE INDEX IF NOT EXISTS idx_experiment_results_exp ON agent_experiment_results (experiment_id)`,
373
- ];
374
- for (const sql of indexes) {
375
- try {
376
- await client.execute(sql);
377
- } catch {
378
- // Index might already exist
379
- }
380
- }
381
- })().catch((err) => {
382
- _initPromise = undefined;
383
- throw err;
384
- });
385
- }
386
- return _initPromise;
387
- }
388
-
389
- // ─── Trace span CRUD ─────────────────────────────────────────────────
390
-
391
- export async function insertTraceSpan(span: TraceSpan): Promise<void> {
392
- await ensureObservabilityTables();
393
- const client = getDbExec();
394
- await client.execute({
395
- sql: `INSERT INTO agent_trace_spans
396
- (id, run_id, thread_id, user_id, parent_span_id, span_type, name,
397
- input_tokens, output_tokens, cache_read_tokens, cache_write_tokens,
398
- cost_cents_x100, duration_ms, status, error_message, metadata, created_at)
399
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
400
- args: [
401
- span.id,
402
- span.runId,
403
- span.threadId,
404
- span.userId,
405
- span.parentSpanId,
406
- span.spanType,
407
- span.name,
408
- span.inputTokens,
409
- span.outputTokens,
410
- span.cacheReadTokens,
411
- span.cacheWriteTokens,
412
- span.costCentsX100,
413
- span.durationMs,
414
- span.status,
415
- span.errorMessage,
416
- span.metadata ? JSON.stringify(span.metadata) : null,
417
- span.createdAt,
418
- ],
419
- });
420
- }
421
-
422
- export async function upsertTraceSummary(summary: TraceSummary): Promise<void> {
423
- await ensureObservabilityTables();
424
- const client = getDbExec();
425
- // user_id is intentionally NOT updated on conflict — once a run's
426
- // owner is recorded it shouldn't change under us.
427
- if (isPostgres()) {
428
- await client.execute({
429
- sql: `INSERT INTO agent_trace_summaries
430
- (run_id, thread_id, user_id, total_spans, llm_calls, tool_calls,
431
- successful_tools, failed_tools, total_duration_ms,
432
- total_cost_cents_x100, total_input_tokens, total_output_tokens,
433
- model, created_at)
434
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
435
- ON CONFLICT (run_id) DO UPDATE SET
436
- total_spans = EXCLUDED.total_spans,
437
- llm_calls = EXCLUDED.llm_calls,
438
- tool_calls = EXCLUDED.tool_calls,
439
- successful_tools = EXCLUDED.successful_tools,
440
- failed_tools = EXCLUDED.failed_tools,
441
- total_duration_ms = EXCLUDED.total_duration_ms,
442
- total_cost_cents_x100 = EXCLUDED.total_cost_cents_x100,
443
- total_input_tokens = EXCLUDED.total_input_tokens,
444
- total_output_tokens = EXCLUDED.total_output_tokens,
445
- model = EXCLUDED.model`,
446
- args: [
447
- summary.runId,
448
- summary.threadId,
449
- summary.userId,
450
- summary.totalSpans,
451
- summary.llmCalls,
452
- summary.toolCalls,
453
- summary.successfulTools,
454
- summary.failedTools,
455
- summary.totalDurationMs,
456
- summary.totalCostCentsX100,
457
- summary.totalInputTokens,
458
- summary.totalOutputTokens,
459
- summary.model,
460
- summary.createdAt,
461
- ],
462
- });
463
- } else {
464
- await client.execute({
465
- sql: `INSERT INTO agent_trace_summaries
466
- (run_id, thread_id, user_id, total_spans, llm_calls, tool_calls,
467
- successful_tools, failed_tools, total_duration_ms,
468
- total_cost_cents_x100, total_input_tokens, total_output_tokens,
469
- model, created_at)
470
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
471
- ON CONFLICT (run_id) DO UPDATE SET
472
- total_spans = EXCLUDED.total_spans,
473
- llm_calls = EXCLUDED.llm_calls,
474
- tool_calls = EXCLUDED.tool_calls,
475
- successful_tools = EXCLUDED.successful_tools,
476
- failed_tools = EXCLUDED.failed_tools,
477
- total_duration_ms = EXCLUDED.total_duration_ms,
478
- total_cost_cents_x100 = EXCLUDED.total_cost_cents_x100,
479
- total_input_tokens = EXCLUDED.total_input_tokens,
480
- total_output_tokens = EXCLUDED.total_output_tokens,
481
- model = EXCLUDED.model`,
482
- args: [
483
- summary.runId,
484
- summary.threadId,
485
- summary.userId,
486
- summary.totalSpans,
487
- summary.llmCalls,
488
- summary.toolCalls,
489
- summary.successfulTools,
490
- summary.failedTools,
491
- summary.totalDurationMs,
492
- summary.totalCostCentsX100,
493
- summary.totalInputTokens,
494
- summary.totalOutputTokens,
495
- summary.model,
496
- summary.createdAt,
497
- ],
498
- });
499
- }
500
- }
501
-
502
- /**
503
- * Purge trace spans, summaries, and eval results older than `cutoffMs`
504
- * (a Unix epoch in milliseconds — rows with `created_at < cutoffMs` are
505
- * deleted). Returns the per-table deletion counts. Satisfies the span
506
- * retention TTL noted in /tmp/security-audit/12-mcp-a2a-agent.md
507
- * (MEDIUM #14): trace metadata can hold sensitive tool inputs, so we
508
- * cap the storage horizon. Feedback rows are retained — they're
509
- * intentionally durable for product analytics. Experiments and
510
- * datasets are also retained because they are user-authored
511
- * configuration, not call telemetry.
512
- */
513
- export async function deleteOldTraceData(cutoffMs: number): Promise<{
514
- spans: number;
515
- summaries: number;
516
- evals: number;
517
- }> {
518
- await ensureObservabilityTables();
519
- const client = getDbExec();
520
- const cutoff = Math.floor(cutoffMs);
521
-
522
- const [spansResult, summariesResult, evalsResult] = await Promise.all([
523
- client.execute({
524
- sql: `DELETE FROM agent_trace_spans WHERE created_at < ?`,
525
- args: [cutoff],
526
- }),
527
- client.execute({
528
- sql: `DELETE FROM agent_trace_summaries WHERE created_at < ?`,
529
- args: [cutoff],
530
- }),
531
- client.execute({
532
- sql: `DELETE FROM agent_evals WHERE created_at < ?`,
533
- args: [cutoff],
534
- }),
535
- ]);
536
-
537
- return {
538
- spans: Number(spansResult.rowsAffected ?? 0),
539
- summaries: Number(summariesResult.rowsAffected ?? 0),
540
- evals: Number(evalsResult.rowsAffected ?? 0),
541
- };
542
- }
543
-
544
- export async function getTraceSpansForRun(
545
- runId: string,
546
- opts: { userId?: string } = {},
547
- ): Promise<TraceSpan[]> {
548
- await ensureObservabilityTables();
549
- const client = getDbExec();
550
- const { where, args } = withUserFilter("run_id = ?", [runId], opts.userId);
551
- const { rows } = await client.execute({
552
- sql: `SELECT * FROM agent_trace_spans WHERE ${where} ORDER BY created_at ASC`,
553
- args,
554
- });
555
- return (rows as any[]).map(rowToTraceSpan);
556
- }
557
-
558
- export async function getTraceSummaries(opts: {
559
- sinceMs?: number;
560
- limit?: number;
561
- userId?: string;
562
- }): Promise<TraceSummary[]> {
563
- await ensureObservabilityTables();
564
- const client = getDbExec();
565
- const sinceMs = opts.sinceMs ?? 0;
566
- const limit = opts.limit ?? 100;
567
- const { where, args } = withUserFilter(
568
- "created_at >= ?",
569
- [sinceMs],
570
- opts.userId,
571
- );
572
- const { rows } = await client.execute({
573
- sql: `SELECT * FROM agent_trace_summaries
574
- WHERE ${where}
575
- ORDER BY created_at DESC
576
- LIMIT ?`,
577
- args: [...args, limit],
578
- });
579
- return (rows as any[]).map(rowToTraceSummary);
580
- }
581
-
582
- export async function getTraceSummary(
583
- runId: string,
584
- opts: { userId?: string } = {},
585
- ): Promise<TraceSummary | null> {
586
- await ensureObservabilityTables();
587
- const client = getDbExec();
588
- const { where, args } = withUserFilter("run_id = ?", [runId], opts.userId);
589
- const { rows } = await client.execute({
590
- sql: `SELECT * FROM agent_trace_summaries WHERE ${where}`,
591
- args,
592
- });
593
- if (rows.length === 0) return null;
594
- return rowToTraceSummary(rows[0] as any);
595
- }
596
-
597
- // ─── Feedback CRUD ───────────────────────────────────────────────────
598
-
599
- export async function insertFeedback(entry: FeedbackEntry): Promise<void> {
600
- await ensureObservabilityTables();
601
- const client = getDbExec();
602
- await client.execute({
603
- sql: `INSERT INTO agent_feedback
604
- (id, run_id, thread_id, message_seq, feedback_type, value, user_id, created_at)
605
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
606
- args: [
607
- entry.id,
608
- entry.runId,
609
- entry.threadId,
610
- entry.messageSeq,
611
- entry.feedbackType,
612
- entry.value,
613
- entry.userId,
614
- entry.createdAt,
615
- ],
616
- });
617
- }
618
-
619
- export async function getFeedback(opts: {
620
- threadId?: string;
621
- sinceMs?: number;
622
- limit?: number;
623
- feedbackType?: string;
624
- userId?: string;
625
- }): Promise<FeedbackEntry[]> {
626
- await ensureObservabilityTables();
627
- const client = getDbExec();
628
- const conditions: string[] = [];
629
- const args: any[] = [];
630
- if (opts.threadId) {
631
- conditions.push("thread_id = ?");
632
- args.push(opts.threadId);
633
- }
634
- if (opts.sinceMs) {
635
- conditions.push("created_at >= ?");
636
- args.push(opts.sinceMs);
637
- }
638
- if (opts.feedbackType) {
639
- conditions.push("feedback_type = ?");
640
- args.push(opts.feedbackType);
641
- }
642
- if (opts.userId) {
643
- conditions.push("user_id = ?");
644
- args.push(opts.userId);
645
- }
646
- const where =
647
- conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
648
- const limit = opts.limit ?? 100;
649
- const { rows } = await client.execute({
650
- sql: `SELECT * FROM agent_feedback ${where} ORDER BY created_at DESC LIMIT ?`,
651
- args: [...args, limit],
652
- });
653
- return (rows as any[]).map(rowToFeedback);
654
- }
655
-
656
- export async function getFeedbackStats(
657
- sinceMs: number,
658
- opts: { userId?: string } = {},
659
- ): Promise<{
660
- total: number;
661
- thumbsUp: number;
662
- thumbsDown: number;
663
- categories: Record<string, number>;
664
- }> {
665
- await ensureObservabilityTables();
666
- const client = getDbExec();
667
- const { where, args } = withUserFilter(
668
- "created_at >= ?",
669
- [sinceMs],
670
- opts.userId,
671
- );
672
- const { rows } = await client.execute({
673
- sql: `SELECT feedback_type, value, COUNT(*) as cnt
674
- FROM agent_feedback WHERE ${where}
675
- GROUP BY feedback_type, value`,
676
- args,
677
- });
678
- let total = 0;
679
- let thumbsUp = 0;
680
- let thumbsDown = 0;
681
- const categories: Record<string, number> = {};
682
- for (const row of rows as any[]) {
683
- const cnt = Number(row.cnt);
684
- total += cnt;
685
- if (row.feedback_type === "thumbs_up") thumbsUp += cnt;
686
- else if (row.feedback_type === "thumbs_down") thumbsDown += cnt;
687
- else if (row.feedback_type === "category")
688
- categories[String(row.value)] = cnt;
689
- }
690
- return { total, thumbsUp, thumbsDown, categories };
691
- }
692
-
693
- // ─── Satisfaction scores CRUD ────────────────────────────────────────
694
-
695
- export async function upsertSatisfactionScore(
696
- score: SatisfactionScore,
697
- ): Promise<void> {
698
- await ensureObservabilityTables();
699
- const client = getDbExec();
700
- if (isPostgres()) {
701
- await client.execute({
702
- sql: `INSERT INTO agent_satisfaction_scores
703
- (id, thread_id, user_id, frustration_score, rephrasing_score,
704
- abandonment_score, sentiment_score, length_trend_score, computed_at)
705
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
706
- ON CONFLICT (id) DO UPDATE SET
707
- frustration_score = EXCLUDED.frustration_score,
708
- rephrasing_score = EXCLUDED.rephrasing_score,
709
- abandonment_score = EXCLUDED.abandonment_score,
710
- sentiment_score = EXCLUDED.sentiment_score,
711
- length_trend_score = EXCLUDED.length_trend_score,
712
- computed_at = EXCLUDED.computed_at`,
713
- args: [
714
- score.id,
715
- score.threadId,
716
- score.userId,
717
- score.frustrationScore,
718
- score.rephrasingScore,
719
- score.abandonmentScore,
720
- score.sentimentScore,
721
- score.lengthTrendScore,
722
- score.computedAt,
723
- ],
724
- });
725
- } else {
726
- await client.execute({
727
- sql: `INSERT INTO agent_satisfaction_scores
728
- (id, thread_id, user_id, frustration_score, rephrasing_score,
729
- abandonment_score, sentiment_score, length_trend_score, computed_at)
730
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
731
- ON CONFLICT (id) DO UPDATE SET
732
- frustration_score = EXCLUDED.frustration_score,
733
- rephrasing_score = EXCLUDED.rephrasing_score,
734
- abandonment_score = EXCLUDED.abandonment_score,
735
- sentiment_score = EXCLUDED.sentiment_score,
736
- length_trend_score = EXCLUDED.length_trend_score,
737
- computed_at = EXCLUDED.computed_at`,
738
- args: [
739
- score.id,
740
- score.threadId,
741
- score.userId,
742
- score.frustrationScore,
743
- score.rephrasingScore,
744
- score.abandonmentScore,
745
- score.sentimentScore,
746
- score.lengthTrendScore,
747
- score.computedAt,
748
- ],
749
- });
750
- }
751
- }
752
-
753
- export async function getSatisfactionScores(opts: {
754
- sinceMs?: number;
755
- limit?: number;
756
- minFrustration?: number;
757
- userId?: string;
758
- }): Promise<SatisfactionScore[]> {
759
- await ensureObservabilityTables();
760
- const client = getDbExec();
761
- const conditions: string[] = [];
762
- const args: any[] = [];
763
- if (opts.sinceMs) {
764
- conditions.push("computed_at >= ?");
765
- args.push(opts.sinceMs);
766
- }
767
- if (opts.minFrustration != null) {
768
- conditions.push("frustration_score >= ?");
769
- args.push(opts.minFrustration);
770
- }
771
- if (opts.userId) {
772
- conditions.push("user_id = ?");
773
- args.push(opts.userId);
774
- }
775
- const where =
776
- conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
777
- const { rows } = await client.execute({
778
- sql: `SELECT * FROM agent_satisfaction_scores ${where}
779
- ORDER BY computed_at DESC LIMIT ?`,
780
- args: [...args, opts.limit ?? 100],
781
- });
782
- return (rows as any[]).map(rowToSatisfaction);
783
- }
784
-
785
- // ─── Evals CRUD ──────────────────────────────────────────────────────
786
-
787
- export async function insertEvalResult(result: EvalResult): Promise<void> {
788
- await ensureObservabilityTables();
789
- const client = getDbExec();
790
- await client.execute({
791
- sql: `INSERT INTO agent_evals
792
- (id, run_id, thread_id, user_id, eval_type, criteria, score, reasoning, metadata, created_at)
793
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
794
- args: [
795
- result.id,
796
- result.runId,
797
- result.threadId,
798
- result.userId,
799
- result.evalType,
800
- result.criteria,
801
- result.score,
802
- result.reasoning,
803
- result.metadata ? JSON.stringify(result.metadata) : null,
804
- result.createdAt,
805
- ],
806
- });
807
- }
808
-
809
- export async function getEvalsForRun(
810
- runId: string,
811
- opts: { userId?: string } = {},
812
- ): Promise<EvalResult[]> {
813
- await ensureObservabilityTables();
814
- const client = getDbExec();
815
- const { where, args } = withUserFilter("run_id = ?", [runId], opts.userId);
816
- const { rows } = await client.execute({
817
- sql: `SELECT * FROM agent_evals WHERE ${where} ORDER BY created_at ASC`,
818
- args,
819
- });
820
- return (rows as any[]).map(rowToEval);
821
- }
822
-
823
- export async function getEvalStats(
824
- sinceMs: number,
825
- opts: { userId?: string } = {},
826
- ): Promise<{
827
- totalEvals: number;
828
- avgScore: number;
829
- byCriteria: Array<{ criteria: string; avgScore: number; count: number }>;
830
- }> {
831
- await ensureObservabilityTables();
832
- const client = getDbExec();
833
- const { where, args } = withUserFilter(
834
- "created_at >= ?",
835
- [sinceMs],
836
- opts.userId,
837
- );
838
- const { rows: totalRows } = await client.execute({
839
- sql: `SELECT COUNT(*) as cnt, AVG(score) as avg_score
840
- FROM agent_evals WHERE ${where}`,
841
- args,
842
- });
843
- const t = (totalRows[0] ?? {}) as Record<string, number | null>;
844
-
845
- const { rows: criteriaRows } = await client.execute({
846
- sql: `SELECT criteria, AVG(score) as avg_score, COUNT(*) as cnt
847
- FROM agent_evals WHERE ${where}
848
- GROUP BY criteria ORDER BY cnt DESC`,
849
- args,
850
- });
851
-
852
- return {
853
- totalEvals: Number(t.cnt ?? 0),
854
- avgScore: Number(t.avg_score ?? 0),
855
- byCriteria: (criteriaRows as any[]).map((r) => ({
856
- criteria: String(r.criteria),
857
- avgScore: Number(r.avg_score ?? 0),
858
- count: Number(r.cnt ?? 0),
859
- })),
860
- };
861
- }
862
-
863
- // ─── Eval datasets CRUD ──────────────────────────────────────────────
864
-
865
- export async function insertEvalDataset(dataset: EvalDataset): Promise<void> {
866
- await ensureObservabilityTables();
867
- const client = getDbExec();
868
- await client.execute({
869
- sql: `INSERT INTO agent_eval_datasets
870
- (id, name, description, entries, created_at, updated_at)
871
- VALUES (?, ?, ?, ?, ?, ?)`,
872
- args: [
873
- dataset.id,
874
- dataset.name,
875
- dataset.description,
876
- JSON.stringify(dataset.entries),
877
- dataset.createdAt,
878
- dataset.updatedAt,
879
- ],
880
- });
881
- }
882
-
883
- export async function listEvalDatasets(): Promise<EvalDataset[]> {
884
- await ensureObservabilityTables();
885
- const client = getDbExec();
886
- const { rows } = await client.execute(
887
- `SELECT * FROM agent_eval_datasets ORDER BY updated_at DESC`,
888
- );
889
- return (rows as any[]).map(rowToDataset);
890
- }
891
-
892
- export async function getEvalDataset(id: string): Promise<EvalDataset | null> {
893
- await ensureObservabilityTables();
894
- const client = getDbExec();
895
- const { rows } = await client.execute({
896
- sql: `SELECT * FROM agent_eval_datasets WHERE id = ?`,
897
- args: [id],
898
- });
899
- if (rows.length === 0) return null;
900
- return rowToDataset(rows[0] as any);
901
- }
902
-
903
- export async function updateEvalDataset(
904
- id: string,
905
- updates: Partial<Pick<EvalDataset, "name" | "description" | "entries">>,
906
- ): Promise<void> {
907
- await ensureObservabilityTables();
908
- const client = getDbExec();
909
- const sets: string[] = [];
910
- const args: any[] = [];
911
- if (updates.name !== undefined) {
912
- sets.push("name = ?");
913
- args.push(updates.name);
914
- }
915
- if (updates.description !== undefined) {
916
- sets.push("description = ?");
917
- args.push(updates.description);
918
- }
919
- if (updates.entries !== undefined) {
920
- sets.push("entries = ?");
921
- args.push(JSON.stringify(updates.entries));
922
- }
923
- sets.push("updated_at = ?");
924
- args.push(Date.now());
925
- args.push(id);
926
- await client.execute({
927
- sql: `UPDATE agent_eval_datasets SET ${sets.join(", ")} WHERE id = ?`,
928
- args,
929
- });
930
- }
931
-
932
- // ─── Experiments CRUD ────────────────────────────────────────────────
933
-
934
- export async function insertExperiment(exp: Experiment): Promise<void> {
935
- await ensureObservabilityTables();
936
- const client = getDbExec();
937
- await client.execute({
938
- sql: `INSERT INTO agent_experiments
939
- (id, name, status, variants, metrics, assignment_level,
940
- started_at, ended_at, created_at, owner_email)
941
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
942
- args: [
943
- exp.id,
944
- exp.name,
945
- exp.status,
946
- JSON.stringify(exp.variants),
947
- JSON.stringify(exp.metrics),
948
- exp.assignmentLevel,
949
- exp.startedAt,
950
- exp.endedAt,
951
- exp.createdAt,
952
- exp.ownerEmail ?? null,
953
- ],
954
- });
955
- }
956
-
957
- export async function updateExperiment(
958
- id: string,
959
- updates: Partial<
960
- Pick<Experiment, "name" | "status" | "variants" | "metrics" | "endedAt">
961
- >,
962
- ): Promise<void> {
963
- await ensureObservabilityTables();
964
- const client = getDbExec();
965
- const sets: string[] = [];
966
- const args: any[] = [];
967
- if (updates.name !== undefined) {
968
- sets.push("name = ?");
969
- args.push(updates.name);
970
- }
971
- if (updates.status !== undefined) {
972
- sets.push("status = ?");
973
- args.push(updates.status);
974
- if (updates.status === "running" && !updates.endedAt) {
975
- sets.push("started_at = ?");
976
- args.push(Date.now());
977
- }
978
- }
979
- if (updates.variants !== undefined) {
980
- sets.push("variants = ?");
981
- args.push(JSON.stringify(updates.variants));
982
- }
983
- if (updates.metrics !== undefined) {
984
- sets.push("metrics = ?");
985
- args.push(JSON.stringify(updates.metrics));
986
- }
987
- if (updates.endedAt !== undefined) {
988
- sets.push("ended_at = ?");
989
- args.push(updates.endedAt);
990
- }
991
- if (sets.length === 0) return;
992
- args.push(id);
993
- await client.execute({
994
- sql: `UPDATE agent_experiments SET ${sets.join(", ")} WHERE id = ?`,
995
- args,
996
- });
997
- }
998
-
999
- export async function listExperiments(): Promise<Experiment[]> {
1000
- await ensureObservabilityTables();
1001
- const client = getDbExec();
1002
- const { rows } = await client.execute(
1003
- `SELECT * FROM agent_experiments ORDER BY created_at DESC`,
1004
- );
1005
- return (rows as any[]).map(rowToExperiment);
1006
- }
1007
-
1008
- export async function getExperiment(id: string): Promise<Experiment | null> {
1009
- await ensureObservabilityTables();
1010
- const client = getDbExec();
1011
- const { rows } = await client.execute({
1012
- sql: `SELECT * FROM agent_experiments WHERE id = ?`,
1013
- args: [id],
1014
- });
1015
- if (rows.length === 0) return null;
1016
- return rowToExperiment(rows[0] as any);
1017
- }
1018
-
1019
- // ─── Experiment assignments CRUD ────────────────────────────────────
1020
-
1021
- export async function upsertAssignment(
1022
- assignment: ExperimentAssignment,
1023
- ): Promise<void> {
1024
- await ensureObservabilityTables();
1025
- const client = getDbExec();
1026
- if (isPostgres()) {
1027
- await client.execute({
1028
- sql: `INSERT INTO agent_experiment_assignments
1029
- (experiment_id, user_id, variant_id, assigned_at)
1030
- VALUES (?, ?, ?, ?)
1031
- ON CONFLICT (experiment_id, user_id) DO UPDATE SET
1032
- variant_id = EXCLUDED.variant_id,
1033
- assigned_at = EXCLUDED.assigned_at`,
1034
- args: [
1035
- assignment.experimentId,
1036
- assignment.userId,
1037
- assignment.variantId,
1038
- assignment.assignedAt,
1039
- ],
1040
- });
1041
- } else {
1042
- await client.execute({
1043
- sql: `INSERT OR REPLACE INTO agent_experiment_assignments
1044
- (experiment_id, user_id, variant_id, assigned_at)
1045
- VALUES (?, ?, ?, ?)`,
1046
- args: [
1047
- assignment.experimentId,
1048
- assignment.userId,
1049
- assignment.variantId,
1050
- assignment.assignedAt,
1051
- ],
1052
- });
1053
- }
1054
- }
1055
-
1056
- export async function getAssignment(
1057
- experimentId: string,
1058
- userId: string,
1059
- ): Promise<ExperimentAssignment | null> {
1060
- await ensureObservabilityTables();
1061
- const client = getDbExec();
1062
- const { rows } = await client.execute({
1063
- sql: `SELECT * FROM agent_experiment_assignments
1064
- WHERE experiment_id = ? AND user_id = ?`,
1065
- args: [experimentId, userId],
1066
- });
1067
- if (rows.length === 0) return null;
1068
- const r = rows[0] as any;
1069
- return {
1070
- experimentId: r.experiment_id,
1071
- userId: r.user_id,
1072
- variantId: r.variant_id,
1073
- assignedAt: Number(r.assigned_at),
1074
- };
1075
- }
1076
-
1077
- // ─── Experiment results CRUD ─────────────────────────────────────────
1078
-
1079
- export async function insertExperimentResult(
1080
- result: ExperimentMetricResult,
1081
- ): Promise<void> {
1082
- await ensureObservabilityTables();
1083
- const client = getDbExec();
1084
- await client.execute({
1085
- sql: `INSERT INTO agent_experiment_results
1086
- (id, experiment_id, variant_id, metric, value,
1087
- sample_size, confidence_low, confidence_high, computed_at)
1088
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1089
- args: [
1090
- result.id,
1091
- result.experimentId,
1092
- result.variantId,
1093
- result.metric,
1094
- result.value,
1095
- result.sampleSize,
1096
- result.confidenceLow,
1097
- result.confidenceHigh,
1098
- result.computedAt,
1099
- ],
1100
- });
1101
- }
1102
-
1103
- export async function getExperimentResults(
1104
- experimentId: string,
1105
- ): Promise<ExperimentMetricResult[]> {
1106
- await ensureObservabilityTables();
1107
- const client = getDbExec();
1108
- const { rows } = await client.execute({
1109
- sql: `SELECT * FROM agent_experiment_results
1110
- WHERE experiment_id = ?
1111
- ORDER BY computed_at DESC`,
1112
- args: [experimentId],
1113
- });
1114
- return (rows as any[]).map(rowToExperimentResult);
1115
- }
1116
-
1117
- // ─── Aggregate queries for dashboard ─────────────────────────────────
1118
-
1119
- export async function getObservabilityOverview(
1120
- sinceMs: number,
1121
- opts: { userId?: string } = {},
1122
- ): Promise<{
1123
- totalRuns: number;
1124
- totalCostCents: number;
1125
- avgDurationMs: number;
1126
- toolSuccessRate: number;
1127
- avgFrustrationScore: number;
1128
- thumbsUpRate: number;
1129
- avgEvalScore: number;
1130
- }> {
1131
- await ensureObservabilityTables();
1132
- const client = getDbExec();
1133
-
1134
- // Three of the four sub-queries time-key on `created_at`; satisfaction
1135
- // uses `computed_at`. Each gets its own `withUserFilter` invocation so
1136
- // the args array isn't aliased across calls (some drivers mutate args
1137
- // for prepared-statement caching).
1138
- const created = withUserFilter("created_at >= ?", [sinceMs], opts.userId);
1139
- const computed = withUserFilter("computed_at >= ?", [sinceMs], opts.userId);
1140
-
1141
- const [tracesResult, satisfactionResult, feedbackResult, evalsResult] =
1142
- await Promise.all([
1143
- client.execute({
1144
- sql: `SELECT
1145
- COUNT(*) as total_runs,
1146
- COALESCE(SUM(total_cost_cents_x100), 0) as total_cost,
1147
- COALESCE(AVG(total_duration_ms), 0) as avg_duration,
1148
- COALESCE(SUM(successful_tools), 0) as success_tools,
1149
- COALESCE(SUM(tool_calls), 0) as total_tools
1150
- FROM agent_trace_summaries WHERE ${created.where}`,
1151
- args: created.args,
1152
- }),
1153
- client.execute({
1154
- sql: `SELECT COALESCE(AVG(frustration_score), 0) as avg_frustration
1155
- FROM agent_satisfaction_scores WHERE ${computed.where}`,
1156
- args: computed.args,
1157
- }),
1158
- client.execute({
1159
- sql: `SELECT
1160
- COALESCE(SUM(CASE WHEN feedback_type = 'thumbs_up' THEN 1 ELSE 0 END), 0) as up,
1161
- COALESCE(SUM(CASE WHEN feedback_type IN ('thumbs_up', 'thumbs_down') THEN 1 ELSE 0 END), 0) as total
1162
- FROM agent_feedback WHERE ${created.where}`,
1163
- args: created.args,
1164
- }),
1165
- client.execute({
1166
- sql: `SELECT COALESCE(AVG(score), 0) as avg_score
1167
- FROM agent_evals WHERE ${created.where}`,
1168
- args: created.args,
1169
- }),
1170
- ]);
1171
-
1172
- const t = (tracesResult.rows[0] ?? {}) as Record<string, number | null>;
1173
- const s = (satisfactionResult.rows[0] ?? {}) as Record<string, number | null>;
1174
- const f = (feedbackResult.rows[0] ?? {}) as Record<string, number | null>;
1175
- const e = (evalsResult.rows[0] ?? {}) as Record<string, number | null>;
1176
-
1177
- const totalTools = Number(t.total_tools ?? 0);
1178
- const successTools = Number(t.success_tools ?? 0);
1179
- const feedbackTotal = Number(f.total ?? 0);
1180
- const feedbackUp = Number(f.up ?? 0);
1181
-
1182
- return {
1183
- totalRuns: Number(t.total_runs ?? 0),
1184
- totalCostCents: Number(t.total_cost ?? 0) / 100,
1185
- avgDurationMs: Number(t.avg_duration ?? 0),
1186
- toolSuccessRate: totalTools > 0 ? successTools / totalTools : 1,
1187
- avgFrustrationScore: Number(s.avg_frustration ?? 0),
1188
- thumbsUpRate: feedbackTotal > 0 ? feedbackUp / feedbackTotal : 0,
1189
- avgEvalScore: Number(e.avg_score ?? 0),
1190
- };
1191
- }
1192
-
1193
- // ─── Row mappers ─────────────────────────────────────────────────────
1194
-
1195
- function rowToTraceSpan(row: Record<string, any>): TraceSpan {
1196
- return {
1197
- id: String(row.id),
1198
- runId: String(row.run_id),
1199
- threadId: row.thread_id ? String(row.thread_id) : null,
1200
- userId: row.user_id ? String(row.user_id) : null,
1201
- parentSpanId: row.parent_span_id ? String(row.parent_span_id) : null,
1202
- spanType: row.span_type as TraceSpan["spanType"],
1203
- name: String(row.name),
1204
- inputTokens: Number(row.input_tokens ?? 0),
1205
- outputTokens: Number(row.output_tokens ?? 0),
1206
- cacheReadTokens: Number(row.cache_read_tokens ?? 0),
1207
- cacheWriteTokens: Number(row.cache_write_tokens ?? 0),
1208
- costCentsX100: Number(row.cost_cents_x100 ?? 0),
1209
- durationMs: Number(row.duration_ms ?? 0),
1210
- status: row.status as TraceSpan["status"],
1211
- errorMessage: row.error_message ? String(row.error_message) : null,
1212
- metadata: safeJsonParse(row.metadata, null),
1213
- createdAt: Number(row.created_at),
1214
- };
1215
- }
1216
-
1217
- function rowToTraceSummary(row: Record<string, any>): TraceSummary {
1218
- return {
1219
- runId: String(row.run_id),
1220
- threadId: row.thread_id ? String(row.thread_id) : null,
1221
- userId: row.user_id ? String(row.user_id) : null,
1222
- totalSpans: Number(row.total_spans ?? 0),
1223
- llmCalls: Number(row.llm_calls ?? 0),
1224
- toolCalls: Number(row.tool_calls ?? 0),
1225
- successfulTools: Number(row.successful_tools ?? 0),
1226
- failedTools: Number(row.failed_tools ?? 0),
1227
- totalDurationMs: Number(row.total_duration_ms ?? 0),
1228
- totalCostCentsX100: Number(row.total_cost_cents_x100 ?? 0),
1229
- totalInputTokens: Number(row.total_input_tokens ?? 0),
1230
- totalOutputTokens: Number(row.total_output_tokens ?? 0),
1231
- model: String(row.model ?? ""),
1232
- createdAt: Number(row.created_at),
1233
- };
1234
- }
1235
-
1236
- function rowToFeedback(row: Record<string, any>): FeedbackEntry {
1237
- return {
1238
- id: String(row.id),
1239
- runId: row.run_id ? String(row.run_id) : null,
1240
- threadId: row.thread_id ? String(row.thread_id) : null,
1241
- messageSeq: row.message_seq != null ? Number(row.message_seq) : null,
1242
- feedbackType: row.feedback_type as FeedbackEntry["feedbackType"],
1243
- value: String(row.value ?? ""),
1244
- userId: row.user_id ? String(row.user_id) : null,
1245
- createdAt: Number(row.created_at),
1246
- };
1247
- }
1248
-
1249
- function rowToSatisfaction(row: Record<string, any>): SatisfactionScore {
1250
- return {
1251
- id: String(row.id),
1252
- threadId: String(row.thread_id),
1253
- userId: row.user_id ? String(row.user_id) : null,
1254
- frustrationScore: Number(row.frustration_score ?? 0),
1255
- rephrasingScore: Number(row.rephrasing_score ?? 0),
1256
- abandonmentScore: Number(row.abandonment_score ?? 0),
1257
- sentimentScore: Number(row.sentiment_score ?? 0),
1258
- lengthTrendScore: Number(row.length_trend_score ?? 0),
1259
- computedAt: Number(row.computed_at),
1260
- };
1261
- }
1262
-
1263
- function rowToEval(row: Record<string, any>): EvalResult {
1264
- return {
1265
- id: String(row.id),
1266
- runId: String(row.run_id),
1267
- threadId: row.thread_id ? String(row.thread_id) : null,
1268
- userId: row.user_id ? String(row.user_id) : null,
1269
- evalType: row.eval_type as EvalResult["evalType"],
1270
- criteria: String(row.criteria),
1271
- score: Number(row.score ?? 0),
1272
- reasoning: row.reasoning ? String(row.reasoning) : null,
1273
- metadata: safeJsonParse(row.metadata, null),
1274
- createdAt: Number(row.created_at),
1275
- };
1276
- }
1277
-
1278
- function rowToDataset(row: Record<string, any>): EvalDataset {
1279
- return {
1280
- id: String(row.id),
1281
- name: String(row.name),
1282
- description: String(row.description ?? ""),
1283
- entries: safeJsonParse(row.entries, []),
1284
- createdAt: Number(row.created_at),
1285
- updatedAt: Number(row.updated_at),
1286
- };
1287
- }
1288
-
1289
- function rowToExperiment(row: Record<string, any>): Experiment {
1290
- return {
1291
- id: String(row.id),
1292
- name: String(row.name),
1293
- status: row.status as Experiment["status"],
1294
- variants: safeJsonParse(row.variants, []),
1295
- metrics: safeJsonParse(row.metrics, []),
1296
- assignmentLevel: (row.assignment_level as "user" | "session") ?? "user",
1297
- startedAt: row.started_at ? Number(row.started_at) : null,
1298
- endedAt: row.ended_at ? Number(row.ended_at) : null,
1299
- createdAt: Number(row.created_at),
1300
- ownerEmail:
1301
- typeof row.owner_email === "string" && row.owner_email
1302
- ? row.owner_email
1303
- : null,
1304
- };
1305
- }
1306
-
1307
- function rowToExperimentResult(
1308
- row: Record<string, any>,
1309
- ): ExperimentMetricResult {
1310
- return {
1311
- id: String(row.id),
1312
- experimentId: String(row.experiment_id),
1313
- variantId: String(row.variant_id),
1314
- metric: String(row.metric),
1315
- value: Number(row.value ?? 0),
1316
- sampleSize: Number(row.sample_size ?? 0),
1317
- confidenceLow: Number(row.confidence_low ?? 0),
1318
- confidenceHigh: Number(row.confidence_high ?? 0),
1319
- computedAt: Number(row.computed_at),
1320
- };
1321
- }
1
+ /**
2
+ * SQL persistence for the agent observability system.
3
+ *
4
+ * Creates and manages tables for traces, feedback, evals, experiments,
5
+ * and satisfaction scores. Follows the same raw-SQL pattern as
6
+ * run-store.ts and usage/store.ts — framework tables use getDbExec()
7
+ * rather than Drizzle ORM (which is for template-level schemas).
8
+ */
9
+ import {
10
+ getDbExec,
11
+ intType,
12
+ isPostgres,
13
+ retryOnDdlRace,
14
+ } from "../db/client.js";
15
+ import {
16
+ ensureTableExists,
17
+ ensureColumnExists,
18
+ ensureIndexExists,
19
+ } from "../db/ddl-guard.js";
20
+ import { isDuplicateColumnError } from "../db/migrations.js";
21
+ import { createInitMemo } from "../shared/init-memo.js";
22
+ import type {
23
+ TraceSpan,
24
+ TraceSummary,
25
+ FeedbackEntry,
26
+ SatisfactionScore,
27
+ EvalResult,
28
+ EvalDataset,
29
+ Experiment,
30
+ ExperimentAssignment,
31
+ ExperimentMetricResult,
32
+ } from "./types.js";
33
+
34
+ function safeJsonParse<T>(value: unknown, fallback: T): T {
35
+ if (!value) return fallback;
36
+ try {
37
+ return JSON.parse(String(value));
38
+ } catch {
39
+ return fallback;
40
+ }
41
+ }
42
+
43
+ // Tables whose rows are owned by an end user drives the boot-time
44
+ // user_id ALTER loop and the per-user composite indexes below. Every
45
+ // new user-owned observability table must be added here so the upgrade
46
+ // path and the per-user query plan stay in sync.
47
+ const USER_SCOPED_TABLES = [
48
+ "agent_trace_spans",
49
+ "agent_trace_summaries",
50
+ "agent_satisfaction_scores",
51
+ "agent_evals",
52
+ "agent_feedback",
53
+ ] as const;
54
+
55
+ /**
56
+ * Append an `AND user_id = ?` clause when a userId filter is requested.
57
+ * Returns the fully-bound WHERE clause + args ready to splice into the
58
+ * caller's SQL. Centralizes the pattern so tests can assert one shape.
59
+ */
60
+ function withUserFilter(
61
+ baseWhere: string,
62
+ baseArgs: any[],
63
+ userId: string | undefined,
64
+ ): { where: string; args: any[] } {
65
+ if (userId == null) return { where: baseWhere, args: baseArgs };
66
+ return {
67
+ where: `${baseWhere} AND user_id = ?`,
68
+ args: [...baseArgs, userId],
69
+ };
70
+ }
71
+
72
+ // workerd-safe memo: the raw `let _initPromise` pattern wedged every agent
73
+ // chat run on the unified Cloudflare runtime when an early-responding first
74
+ // request created (and froze) the init promise — see shared/init-memo.ts.
75
+ export const ensureObservabilityTables = createInitMemo(
76
+ async () => {
77
+ const client = getDbExec();
78
+
79
+ const traceSpansCreateSql = `
80
+ CREATE TABLE IF NOT EXISTS agent_trace_spans (
81
+ id TEXT PRIMARY KEY,
82
+ run_id TEXT NOT NULL,
83
+ thread_id TEXT,
84
+ user_id TEXT,
85
+ parent_span_id TEXT,
86
+ span_type TEXT NOT NULL,
87
+ name TEXT NOT NULL,
88
+ input_tokens ${intType()} NOT NULL DEFAULT 0,
89
+ output_tokens ${intType()} NOT NULL DEFAULT 0,
90
+ cache_read_tokens ${intType()} NOT NULL DEFAULT 0,
91
+ cache_write_tokens ${intType()} NOT NULL DEFAULT 0,
92
+ cost_cents_x100 ${intType()} NOT NULL DEFAULT 0,
93
+ duration_ms ${intType()} NOT NULL DEFAULT 0,
94
+ status TEXT NOT NULL DEFAULT 'success',
95
+ error_message TEXT,
96
+ metadata TEXT,
97
+ created_at ${intType()} NOT NULL
98
+ )
99
+ `;
100
+
101
+ const traceSummariesCreateSql = `
102
+ CREATE TABLE IF NOT EXISTS agent_trace_summaries (
103
+ run_id TEXT PRIMARY KEY,
104
+ thread_id TEXT,
105
+ user_id TEXT,
106
+ total_spans ${intType()} NOT NULL DEFAULT 0,
107
+ llm_calls ${intType()} NOT NULL DEFAULT 0,
108
+ tool_calls ${intType()} NOT NULL DEFAULT 0,
109
+ successful_tools ${intType()} NOT NULL DEFAULT 0,
110
+ failed_tools ${intType()} NOT NULL DEFAULT 0,
111
+ total_duration_ms ${intType()} NOT NULL DEFAULT 0,
112
+ total_cost_cents_x100 ${intType()} NOT NULL DEFAULT 0,
113
+ total_input_tokens ${intType()} NOT NULL DEFAULT 0,
114
+ total_output_tokens ${intType()} NOT NULL DEFAULT 0,
115
+ model TEXT NOT NULL DEFAULT '',
116
+ created_at ${intType()} NOT NULL
117
+ )
118
+ `;
119
+
120
+ const feedbackCreateSql = `
121
+ CREATE TABLE IF NOT EXISTS agent_feedback (
122
+ id TEXT PRIMARY KEY,
123
+ run_id TEXT,
124
+ thread_id TEXT,
125
+ message_seq ${intType()},
126
+ feedback_type TEXT NOT NULL,
127
+ value TEXT NOT NULL DEFAULT '',
128
+ user_id TEXT,
129
+ created_at ${intType()} NOT NULL
130
+ )
131
+ `;
132
+
133
+ const satisfactionScoresCreateSql = `
134
+ CREATE TABLE IF NOT EXISTS agent_satisfaction_scores (
135
+ id TEXT PRIMARY KEY,
136
+ thread_id TEXT NOT NULL,
137
+ user_id TEXT,
138
+ frustration_score REAL NOT NULL DEFAULT 0,
139
+ rephrasing_score REAL NOT NULL DEFAULT 0,
140
+ abandonment_score REAL NOT NULL DEFAULT 0,
141
+ sentiment_score REAL NOT NULL DEFAULT 0,
142
+ length_trend_score REAL NOT NULL DEFAULT 0,
143
+ computed_at ${intType()} NOT NULL
144
+ )
145
+ `;
146
+
147
+ const evalsCreateSql = `
148
+ CREATE TABLE IF NOT EXISTS agent_evals (
149
+ id TEXT PRIMARY KEY,
150
+ run_id TEXT NOT NULL,
151
+ thread_id TEXT,
152
+ user_id TEXT,
153
+ eval_type TEXT NOT NULL,
154
+ criteria TEXT NOT NULL,
155
+ score REAL NOT NULL DEFAULT 0,
156
+ reasoning TEXT,
157
+ metadata TEXT,
158
+ created_at ${intType()} NOT NULL
159
+ )
160
+ `;
161
+
162
+ const evalDatasetsCreateSql = `
163
+ CREATE TABLE IF NOT EXISTS agent_eval_datasets (
164
+ id TEXT PRIMARY KEY,
165
+ name TEXT NOT NULL,
166
+ description TEXT NOT NULL DEFAULT '',
167
+ entries TEXT NOT NULL DEFAULT '[]',
168
+ created_at ${intType()} NOT NULL,
169
+ updated_at ${intType()} NOT NULL
170
+ )
171
+ `;
172
+
173
+ const experimentsCreateSql = `
174
+ CREATE TABLE IF NOT EXISTS agent_experiments (
175
+ id TEXT PRIMARY KEY,
176
+ name TEXT NOT NULL,
177
+ status TEXT NOT NULL DEFAULT 'draft',
178
+ variants TEXT NOT NULL DEFAULT '[]',
179
+ metrics TEXT NOT NULL DEFAULT '[]',
180
+ assignment_level TEXT NOT NULL DEFAULT 'user',
181
+ started_at ${intType()},
182
+ ended_at ${intType()},
183
+ created_at ${intType()} NOT NULL,
184
+ owner_email TEXT
185
+ )
186
+ `;
187
+
188
+ const experimentAssignmentsCreateSql = `
189
+ CREATE TABLE IF NOT EXISTS agent_experiment_assignments (
190
+ experiment_id TEXT NOT NULL,
191
+ user_id TEXT NOT NULL,
192
+ variant_id TEXT NOT NULL,
193
+ assigned_at ${intType()} NOT NULL,
194
+ PRIMARY KEY (experiment_id, user_id)
195
+ )
196
+ `;
197
+
198
+ const experimentResultsCreateSql = `
199
+ CREATE TABLE IF NOT EXISTS agent_experiment_results (
200
+ id TEXT PRIMARY KEY,
201
+ experiment_id TEXT NOT NULL,
202
+ variant_id TEXT NOT NULL,
203
+ metric TEXT NOT NULL,
204
+ value REAL NOT NULL DEFAULT 0,
205
+ sample_size ${intType()} NOT NULL DEFAULT 0,
206
+ confidence_low REAL NOT NULL DEFAULT 0,
207
+ confidence_high REAL NOT NULL DEFAULT 0,
208
+ computed_at ${intType()} NOT NULL
209
+ )
210
+ `;
211
+
212
+ if (isPostgres()) {
213
+ // PG guard: probe → guarded DDL → re-probe; skips lock on already-migrated path
214
+ await ensureTableExists("agent_trace_spans", traceSpansCreateSql);
215
+ await ensureTableExists("agent_trace_summaries", traceSummariesCreateSql);
216
+ await ensureTableExists("agent_feedback", feedbackCreateSql);
217
+ await ensureTableExists(
218
+ "agent_satisfaction_scores",
219
+ satisfactionScoresCreateSql,
220
+ );
221
+ await ensureTableExists("agent_evals", evalsCreateSql);
222
+ await ensureTableExists("agent_eval_datasets", evalDatasetsCreateSql);
223
+ await ensureTableExists("agent_experiments", experimentsCreateSql);
224
+ await ensureTableExists(
225
+ "agent_experiment_assignments",
226
+ experimentAssignmentsCreateSql,
227
+ );
228
+ await ensureTableExists(
229
+ "agent_experiment_results",
230
+ experimentResultsCreateSql,
231
+ );
232
+ await ensureColumnExists(
233
+ "agent_experiments",
234
+ "owner_email",
235
+ `ALTER TABLE agent_experiments ADD COLUMN IF NOT EXISTS owner_email TEXT`,
236
+ );
237
+ for (const table of USER_SCOPED_TABLES) {
238
+ await ensureColumnExists(
239
+ table,
240
+ "user_id",
241
+ `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS user_id TEXT`,
242
+ );
243
+ }
244
+ await ensureIndexExists(
245
+ "idx_trace_spans_run",
246
+ `CREATE INDEX IF NOT EXISTS idx_trace_spans_run ON agent_trace_spans (run_id)`,
247
+ );
248
+ await ensureIndexExists(
249
+ "idx_trace_spans_thread",
250
+ `CREATE INDEX IF NOT EXISTS idx_trace_spans_thread ON agent_trace_spans (thread_id)`,
251
+ );
252
+ await ensureIndexExists(
253
+ "idx_trace_spans_created",
254
+ `CREATE INDEX IF NOT EXISTS idx_trace_spans_created ON agent_trace_spans (created_at)`,
255
+ );
256
+ await ensureIndexExists(
257
+ "idx_trace_summaries_created",
258
+ `CREATE INDEX IF NOT EXISTS idx_trace_summaries_created ON agent_trace_summaries (created_at)`,
259
+ );
260
+ await ensureIndexExists(
261
+ "idx_trace_summaries_user",
262
+ `CREATE INDEX IF NOT EXISTS idx_trace_summaries_user ON agent_trace_summaries (user_id, created_at)`,
263
+ );
264
+ await ensureIndexExists(
265
+ "idx_trace_spans_user",
266
+ `CREATE INDEX IF NOT EXISTS idx_trace_spans_user ON agent_trace_spans (user_id)`,
267
+ );
268
+ await ensureIndexExists(
269
+ "idx_feedback_thread",
270
+ `CREATE INDEX IF NOT EXISTS idx_feedback_thread ON agent_feedback (thread_id)`,
271
+ );
272
+ await ensureIndexExists(
273
+ "idx_feedback_created",
274
+ `CREATE INDEX IF NOT EXISTS idx_feedback_created ON agent_feedback (created_at)`,
275
+ );
276
+ await ensureIndexExists(
277
+ "idx_feedback_user",
278
+ `CREATE INDEX IF NOT EXISTS idx_feedback_user ON agent_feedback (user_id, created_at)`,
279
+ );
280
+ await ensureIndexExists(
281
+ "idx_satisfaction_thread",
282
+ `CREATE INDEX IF NOT EXISTS idx_satisfaction_thread ON agent_satisfaction_scores (thread_id)`,
283
+ );
284
+ await ensureIndexExists(
285
+ "idx_satisfaction_user",
286
+ `CREATE INDEX IF NOT EXISTS idx_satisfaction_user ON agent_satisfaction_scores (user_id, computed_at)`,
287
+ );
288
+ await ensureIndexExists(
289
+ "idx_evals_run",
290
+ `CREATE INDEX IF NOT EXISTS idx_evals_run ON agent_evals (run_id)`,
291
+ );
292
+ await ensureIndexExists(
293
+ "idx_evals_created",
294
+ `CREATE INDEX IF NOT EXISTS idx_evals_created ON agent_evals (created_at)`,
295
+ );
296
+ await ensureIndexExists(
297
+ "idx_evals_user",
298
+ `CREATE INDEX IF NOT EXISTS idx_evals_user ON agent_evals (user_id, created_at)`,
299
+ );
300
+ await ensureIndexExists(
301
+ "idx_experiment_results_exp",
302
+ `CREATE INDEX IF NOT EXISTS idx_experiment_results_exp ON agent_experiment_results (experiment_id)`,
303
+ );
304
+ return;
305
+ }
306
+
307
+ // SQLite (local dev): no lock problem — keep the original behaviour.
308
+ await retryOnDdlRace(() => client.execute(traceSpansCreateSql));
309
+
310
+ await retryOnDdlRace(() => client.execute(traceSummariesCreateSql));
311
+
312
+ await retryOnDdlRace(() => client.execute(feedbackCreateSql));
313
+
314
+ await retryOnDdlRace(() => client.execute(satisfactionScoresCreateSql));
315
+
316
+ await retryOnDdlRace(() => client.execute(evalsCreateSql));
317
+
318
+ await retryOnDdlRace(() => client.execute(evalDatasetsCreateSql));
319
+
320
+ await retryOnDdlRace(() => client.execute(experimentsCreateSql));
321
+
322
+ // Additive migration for DBs created before the owner column shipped
323
+ // (any pre-existing rows have NULL owner — see `updateExperiment` for
324
+ // the migration semantics). Mutations on those rows fall back to the
325
+ // standard authentication gate but cannot enforce per-owner scoping
326
+ // until they're re-saved.
327
+ try {
328
+ await client.execute(
329
+ `ALTER TABLE agent_experiments ADD COLUMN owner_email TEXT`,
330
+ );
331
+ } catch {
332
+ // Column already exists — expected after first run.
333
+ }
334
+
335
+ await retryOnDdlRace(() => client.execute(experimentAssignmentsCreateSql));
336
+
337
+ await retryOnDdlRace(() => client.execute(experimentResultsCreateSql));
338
+
339
+ // Idempotent column upgrades for DBs created before per-user
340
+ // isolation. SQLite has no `ADD COLUMN IF NOT EXISTS`; Postgres
341
+ // surfaces "column ... already exists". `isDuplicateColumnError`
342
+ // (from db/migrations.ts) recognizes both shapes.
343
+ for (const table of USER_SCOPED_TABLES) {
344
+ try {
345
+ await client.execute(`ALTER TABLE ${table} ADD COLUMN user_id TEXT`);
346
+ } catch (err) {
347
+ if (isDuplicateColumnError(err)) continue;
348
+ throw err;
349
+ }
350
+ }
351
+
352
+ // Indexes for common query patterns
353
+ const indexes = [
354
+ `CREATE INDEX IF NOT EXISTS idx_trace_spans_run ON agent_trace_spans (run_id)`,
355
+ `CREATE INDEX IF NOT EXISTS idx_trace_spans_thread ON agent_trace_spans (thread_id)`,
356
+ `CREATE INDEX IF NOT EXISTS idx_trace_spans_created ON agent_trace_spans (created_at)`,
357
+ `CREATE INDEX IF NOT EXISTS idx_trace_summaries_created ON agent_trace_summaries (created_at)`,
358
+ `CREATE INDEX IF NOT EXISTS idx_trace_summaries_user ON agent_trace_summaries (user_id, created_at)`,
359
+ `CREATE INDEX IF NOT EXISTS idx_trace_spans_user ON agent_trace_spans (user_id)`,
360
+ `CREATE INDEX IF NOT EXISTS idx_feedback_thread ON agent_feedback (thread_id)`,
361
+ `CREATE INDEX IF NOT EXISTS idx_feedback_created ON agent_feedback (created_at)`,
362
+ `CREATE INDEX IF NOT EXISTS idx_feedback_user ON agent_feedback (user_id, created_at)`,
363
+ `CREATE INDEX IF NOT EXISTS idx_satisfaction_thread ON agent_satisfaction_scores (thread_id)`,
364
+ `CREATE INDEX IF NOT EXISTS idx_satisfaction_user ON agent_satisfaction_scores (user_id, computed_at)`,
365
+ `CREATE INDEX IF NOT EXISTS idx_evals_run ON agent_evals (run_id)`,
366
+ `CREATE INDEX IF NOT EXISTS idx_evals_created ON agent_evals (created_at)`,
367
+ `CREATE INDEX IF NOT EXISTS idx_evals_user ON agent_evals (user_id, created_at)`,
368
+ `CREATE INDEX IF NOT EXISTS idx_experiment_results_exp ON agent_experiment_results (experiment_id)`,
369
+ ];
370
+ for (const sql of indexes) {
371
+ try {
372
+ await client.execute(sql);
373
+ } catch {
374
+ // Index might already exist
375
+ }
376
+ }
377
+ },
378
+ { label: "observability" },
379
+ );
380
+
381
+ // ─── Trace span CRUD ─────────────────────────────────────────────────
382
+
383
+ export async function insertTraceSpan(span: TraceSpan): Promise<void> {
384
+ await ensureObservabilityTables();
385
+ const client = getDbExec();
386
+ await client.execute({
387
+ sql: `INSERT INTO agent_trace_spans
388
+ (id, run_id, thread_id, user_id, parent_span_id, span_type, name,
389
+ input_tokens, output_tokens, cache_read_tokens, cache_write_tokens,
390
+ cost_cents_x100, duration_ms, status, error_message, metadata, created_at)
391
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
392
+ args: [
393
+ span.id,
394
+ span.runId,
395
+ span.threadId,
396
+ span.userId,
397
+ span.parentSpanId,
398
+ span.spanType,
399
+ span.name,
400
+ span.inputTokens,
401
+ span.outputTokens,
402
+ span.cacheReadTokens,
403
+ span.cacheWriteTokens,
404
+ span.costCentsX100,
405
+ span.durationMs,
406
+ span.status,
407
+ span.errorMessage,
408
+ span.metadata ? JSON.stringify(span.metadata) : null,
409
+ span.createdAt,
410
+ ],
411
+ });
412
+ }
413
+
414
+ export async function upsertTraceSummary(summary: TraceSummary): Promise<void> {
415
+ await ensureObservabilityTables();
416
+ const client = getDbExec();
417
+ // user_id is intentionally NOT updated on conflict — once a run's
418
+ // owner is recorded it shouldn't change under us.
419
+ if (isPostgres()) {
420
+ await client.execute({
421
+ sql: `INSERT INTO agent_trace_summaries
422
+ (run_id, thread_id, user_id, total_spans, llm_calls, tool_calls,
423
+ successful_tools, failed_tools, total_duration_ms,
424
+ total_cost_cents_x100, total_input_tokens, total_output_tokens,
425
+ model, created_at)
426
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
427
+ ON CONFLICT (run_id) DO UPDATE SET
428
+ total_spans = EXCLUDED.total_spans,
429
+ llm_calls = EXCLUDED.llm_calls,
430
+ tool_calls = EXCLUDED.tool_calls,
431
+ successful_tools = EXCLUDED.successful_tools,
432
+ failed_tools = EXCLUDED.failed_tools,
433
+ total_duration_ms = EXCLUDED.total_duration_ms,
434
+ total_cost_cents_x100 = EXCLUDED.total_cost_cents_x100,
435
+ total_input_tokens = EXCLUDED.total_input_tokens,
436
+ total_output_tokens = EXCLUDED.total_output_tokens,
437
+ model = EXCLUDED.model`,
438
+ args: [
439
+ summary.runId,
440
+ summary.threadId,
441
+ summary.userId,
442
+ summary.totalSpans,
443
+ summary.llmCalls,
444
+ summary.toolCalls,
445
+ summary.successfulTools,
446
+ summary.failedTools,
447
+ summary.totalDurationMs,
448
+ summary.totalCostCentsX100,
449
+ summary.totalInputTokens,
450
+ summary.totalOutputTokens,
451
+ summary.model,
452
+ summary.createdAt,
453
+ ],
454
+ });
455
+ } else {
456
+ await client.execute({
457
+ sql: `INSERT INTO agent_trace_summaries
458
+ (run_id, thread_id, user_id, total_spans, llm_calls, tool_calls,
459
+ successful_tools, failed_tools, total_duration_ms,
460
+ total_cost_cents_x100, total_input_tokens, total_output_tokens,
461
+ model, created_at)
462
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
463
+ ON CONFLICT (run_id) DO UPDATE SET
464
+ total_spans = EXCLUDED.total_spans,
465
+ llm_calls = EXCLUDED.llm_calls,
466
+ tool_calls = EXCLUDED.tool_calls,
467
+ successful_tools = EXCLUDED.successful_tools,
468
+ failed_tools = EXCLUDED.failed_tools,
469
+ total_duration_ms = EXCLUDED.total_duration_ms,
470
+ total_cost_cents_x100 = EXCLUDED.total_cost_cents_x100,
471
+ total_input_tokens = EXCLUDED.total_input_tokens,
472
+ total_output_tokens = EXCLUDED.total_output_tokens,
473
+ model = EXCLUDED.model`,
474
+ args: [
475
+ summary.runId,
476
+ summary.threadId,
477
+ summary.userId,
478
+ summary.totalSpans,
479
+ summary.llmCalls,
480
+ summary.toolCalls,
481
+ summary.successfulTools,
482
+ summary.failedTools,
483
+ summary.totalDurationMs,
484
+ summary.totalCostCentsX100,
485
+ summary.totalInputTokens,
486
+ summary.totalOutputTokens,
487
+ summary.model,
488
+ summary.createdAt,
489
+ ],
490
+ });
491
+ }
492
+ }
493
+
494
+ /**
495
+ * Purge trace spans, summaries, and eval results older than `cutoffMs`
496
+ * (a Unix epoch in milliseconds — rows with `created_at < cutoffMs` are
497
+ * deleted). Returns the per-table deletion counts. Satisfies the span
498
+ * retention TTL noted in /tmp/security-audit/12-mcp-a2a-agent.md
499
+ * (MEDIUM #14): trace metadata can hold sensitive tool inputs, so we
500
+ * cap the storage horizon. Feedback rows are retained — they're
501
+ * intentionally durable for product analytics. Experiments and
502
+ * datasets are also retained because they are user-authored
503
+ * configuration, not call telemetry.
504
+ */
505
+ export async function deleteOldTraceData(cutoffMs: number): Promise<{
506
+ spans: number;
507
+ summaries: number;
508
+ evals: number;
509
+ }> {
510
+ await ensureObservabilityTables();
511
+ const client = getDbExec();
512
+ const cutoff = Math.floor(cutoffMs);
513
+
514
+ const [spansResult, summariesResult, evalsResult] = await Promise.all([
515
+ client.execute({
516
+ sql: `DELETE FROM agent_trace_spans WHERE created_at < ?`,
517
+ args: [cutoff],
518
+ }),
519
+ client.execute({
520
+ sql: `DELETE FROM agent_trace_summaries WHERE created_at < ?`,
521
+ args: [cutoff],
522
+ }),
523
+ client.execute({
524
+ sql: `DELETE FROM agent_evals WHERE created_at < ?`,
525
+ args: [cutoff],
526
+ }),
527
+ ]);
528
+
529
+ return {
530
+ spans: Number(spansResult.rowsAffected ?? 0),
531
+ summaries: Number(summariesResult.rowsAffected ?? 0),
532
+ evals: Number(evalsResult.rowsAffected ?? 0),
533
+ };
534
+ }
535
+
536
+ export async function getTraceSpansForRun(
537
+ runId: string,
538
+ opts: { userId?: string } = {},
539
+ ): Promise<TraceSpan[]> {
540
+ await ensureObservabilityTables();
541
+ const client = getDbExec();
542
+ const { where, args } = withUserFilter("run_id = ?", [runId], opts.userId);
543
+ const { rows } = await client.execute({
544
+ sql: `SELECT * FROM agent_trace_spans WHERE ${where} ORDER BY created_at ASC`,
545
+ args,
546
+ });
547
+ return (rows as any[]).map(rowToTraceSpan);
548
+ }
549
+
550
+ export async function getTraceSummaries(opts: {
551
+ sinceMs?: number;
552
+ limit?: number;
553
+ userId?: string;
554
+ }): Promise<TraceSummary[]> {
555
+ await ensureObservabilityTables();
556
+ const client = getDbExec();
557
+ const sinceMs = opts.sinceMs ?? 0;
558
+ const limit = opts.limit ?? 100;
559
+ const { where, args } = withUserFilter(
560
+ "created_at >= ?",
561
+ [sinceMs],
562
+ opts.userId,
563
+ );
564
+ const { rows } = await client.execute({
565
+ sql: `SELECT * FROM agent_trace_summaries
566
+ WHERE ${where}
567
+ ORDER BY created_at DESC
568
+ LIMIT ?`,
569
+ args: [...args, limit],
570
+ });
571
+ return (rows as any[]).map(rowToTraceSummary);
572
+ }
573
+
574
+ export async function getTraceSummary(
575
+ runId: string,
576
+ opts: { userId?: string } = {},
577
+ ): Promise<TraceSummary | null> {
578
+ await ensureObservabilityTables();
579
+ const client = getDbExec();
580
+ const { where, args } = withUserFilter("run_id = ?", [runId], opts.userId);
581
+ const { rows } = await client.execute({
582
+ sql: `SELECT * FROM agent_trace_summaries WHERE ${where}`,
583
+ args,
584
+ });
585
+ if (rows.length === 0) return null;
586
+ return rowToTraceSummary(rows[0] as any);
587
+ }
588
+
589
+ // ─── Feedback CRUD ───────────────────────────────────────────────────
590
+
591
+ export async function insertFeedback(entry: FeedbackEntry): Promise<void> {
592
+ await ensureObservabilityTables();
593
+ const client = getDbExec();
594
+ await client.execute({
595
+ sql: `INSERT INTO agent_feedback
596
+ (id, run_id, thread_id, message_seq, feedback_type, value, user_id, created_at)
597
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
598
+ args: [
599
+ entry.id,
600
+ entry.runId,
601
+ entry.threadId,
602
+ entry.messageSeq,
603
+ entry.feedbackType,
604
+ entry.value,
605
+ entry.userId,
606
+ entry.createdAt,
607
+ ],
608
+ });
609
+ }
610
+
611
+ export async function getFeedback(opts: {
612
+ threadId?: string;
613
+ sinceMs?: number;
614
+ limit?: number;
615
+ feedbackType?: string;
616
+ userId?: string;
617
+ }): Promise<FeedbackEntry[]> {
618
+ await ensureObservabilityTables();
619
+ const client = getDbExec();
620
+ const conditions: string[] = [];
621
+ const args: any[] = [];
622
+ if (opts.threadId) {
623
+ conditions.push("thread_id = ?");
624
+ args.push(opts.threadId);
625
+ }
626
+ if (opts.sinceMs) {
627
+ conditions.push("created_at >= ?");
628
+ args.push(opts.sinceMs);
629
+ }
630
+ if (opts.feedbackType) {
631
+ conditions.push("feedback_type = ?");
632
+ args.push(opts.feedbackType);
633
+ }
634
+ if (opts.userId) {
635
+ conditions.push("user_id = ?");
636
+ args.push(opts.userId);
637
+ }
638
+ const where =
639
+ conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
640
+ const limit = opts.limit ?? 100;
641
+ const { rows } = await client.execute({
642
+ sql: `SELECT * FROM agent_feedback ${where} ORDER BY created_at DESC LIMIT ?`,
643
+ args: [...args, limit],
644
+ });
645
+ return (rows as any[]).map(rowToFeedback);
646
+ }
647
+
648
+ export async function getFeedbackStats(
649
+ sinceMs: number,
650
+ opts: { userId?: string } = {},
651
+ ): Promise<{
652
+ total: number;
653
+ thumbsUp: number;
654
+ thumbsDown: number;
655
+ categories: Record<string, number>;
656
+ }> {
657
+ await ensureObservabilityTables();
658
+ const client = getDbExec();
659
+ const { where, args } = withUserFilter(
660
+ "created_at >= ?",
661
+ [sinceMs],
662
+ opts.userId,
663
+ );
664
+ const { rows } = await client.execute({
665
+ sql: `SELECT feedback_type, value, COUNT(*) as cnt
666
+ FROM agent_feedback WHERE ${where}
667
+ GROUP BY feedback_type, value`,
668
+ args,
669
+ });
670
+ let total = 0;
671
+ let thumbsUp = 0;
672
+ let thumbsDown = 0;
673
+ const categories: Record<string, number> = {};
674
+ for (const row of rows as any[]) {
675
+ const cnt = Number(row.cnt);
676
+ total += cnt;
677
+ if (row.feedback_type === "thumbs_up") thumbsUp += cnt;
678
+ else if (row.feedback_type === "thumbs_down") thumbsDown += cnt;
679
+ else if (row.feedback_type === "category")
680
+ categories[String(row.value)] = cnt;
681
+ }
682
+ return { total, thumbsUp, thumbsDown, categories };
683
+ }
684
+
685
+ // ─── Satisfaction scores CRUD ────────────────────────────────────────
686
+
687
+ export async function upsertSatisfactionScore(
688
+ score: SatisfactionScore,
689
+ ): Promise<void> {
690
+ await ensureObservabilityTables();
691
+ const client = getDbExec();
692
+ if (isPostgres()) {
693
+ await client.execute({
694
+ sql: `INSERT INTO agent_satisfaction_scores
695
+ (id, thread_id, user_id, frustration_score, rephrasing_score,
696
+ abandonment_score, sentiment_score, length_trend_score, computed_at)
697
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
698
+ ON CONFLICT (id) DO UPDATE SET
699
+ frustration_score = EXCLUDED.frustration_score,
700
+ rephrasing_score = EXCLUDED.rephrasing_score,
701
+ abandonment_score = EXCLUDED.abandonment_score,
702
+ sentiment_score = EXCLUDED.sentiment_score,
703
+ length_trend_score = EXCLUDED.length_trend_score,
704
+ computed_at = EXCLUDED.computed_at`,
705
+ args: [
706
+ score.id,
707
+ score.threadId,
708
+ score.userId,
709
+ score.frustrationScore,
710
+ score.rephrasingScore,
711
+ score.abandonmentScore,
712
+ score.sentimentScore,
713
+ score.lengthTrendScore,
714
+ score.computedAt,
715
+ ],
716
+ });
717
+ } else {
718
+ await client.execute({
719
+ sql: `INSERT INTO agent_satisfaction_scores
720
+ (id, thread_id, user_id, frustration_score, rephrasing_score,
721
+ abandonment_score, sentiment_score, length_trend_score, computed_at)
722
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
723
+ ON CONFLICT (id) DO UPDATE SET
724
+ frustration_score = EXCLUDED.frustration_score,
725
+ rephrasing_score = EXCLUDED.rephrasing_score,
726
+ abandonment_score = EXCLUDED.abandonment_score,
727
+ sentiment_score = EXCLUDED.sentiment_score,
728
+ length_trend_score = EXCLUDED.length_trend_score,
729
+ computed_at = EXCLUDED.computed_at`,
730
+ args: [
731
+ score.id,
732
+ score.threadId,
733
+ score.userId,
734
+ score.frustrationScore,
735
+ score.rephrasingScore,
736
+ score.abandonmentScore,
737
+ score.sentimentScore,
738
+ score.lengthTrendScore,
739
+ score.computedAt,
740
+ ],
741
+ });
742
+ }
743
+ }
744
+
745
+ export async function getSatisfactionScores(opts: {
746
+ sinceMs?: number;
747
+ limit?: number;
748
+ minFrustration?: number;
749
+ userId?: string;
750
+ }): Promise<SatisfactionScore[]> {
751
+ await ensureObservabilityTables();
752
+ const client = getDbExec();
753
+ const conditions: string[] = [];
754
+ const args: any[] = [];
755
+ if (opts.sinceMs) {
756
+ conditions.push("computed_at >= ?");
757
+ args.push(opts.sinceMs);
758
+ }
759
+ if (opts.minFrustration != null) {
760
+ conditions.push("frustration_score >= ?");
761
+ args.push(opts.minFrustration);
762
+ }
763
+ if (opts.userId) {
764
+ conditions.push("user_id = ?");
765
+ args.push(opts.userId);
766
+ }
767
+ const where =
768
+ conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
769
+ const { rows } = await client.execute({
770
+ sql: `SELECT * FROM agent_satisfaction_scores ${where}
771
+ ORDER BY computed_at DESC LIMIT ?`,
772
+ args: [...args, opts.limit ?? 100],
773
+ });
774
+ return (rows as any[]).map(rowToSatisfaction);
775
+ }
776
+
777
+ // ─── Evals CRUD ──────────────────────────────────────────────────────
778
+
779
+ export async function insertEvalResult(result: EvalResult): Promise<void> {
780
+ await ensureObservabilityTables();
781
+ const client = getDbExec();
782
+ await client.execute({
783
+ sql: `INSERT INTO agent_evals
784
+ (id, run_id, thread_id, user_id, eval_type, criteria, score, reasoning, metadata, created_at)
785
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
786
+ args: [
787
+ result.id,
788
+ result.runId,
789
+ result.threadId,
790
+ result.userId,
791
+ result.evalType,
792
+ result.criteria,
793
+ result.score,
794
+ result.reasoning,
795
+ result.metadata ? JSON.stringify(result.metadata) : null,
796
+ result.createdAt,
797
+ ],
798
+ });
799
+ }
800
+
801
+ export async function getEvalsForRun(
802
+ runId: string,
803
+ opts: { userId?: string } = {},
804
+ ): Promise<EvalResult[]> {
805
+ await ensureObservabilityTables();
806
+ const client = getDbExec();
807
+ const { where, args } = withUserFilter("run_id = ?", [runId], opts.userId);
808
+ const { rows } = await client.execute({
809
+ sql: `SELECT * FROM agent_evals WHERE ${where} ORDER BY created_at ASC`,
810
+ args,
811
+ });
812
+ return (rows as any[]).map(rowToEval);
813
+ }
814
+
815
+ export async function getEvalStats(
816
+ sinceMs: number,
817
+ opts: { userId?: string } = {},
818
+ ): Promise<{
819
+ totalEvals: number;
820
+ avgScore: number;
821
+ byCriteria: Array<{ criteria: string; avgScore: number; count: number }>;
822
+ }> {
823
+ await ensureObservabilityTables();
824
+ const client = getDbExec();
825
+ const { where, args } = withUserFilter(
826
+ "created_at >= ?",
827
+ [sinceMs],
828
+ opts.userId,
829
+ );
830
+ const { rows: totalRows } = await client.execute({
831
+ sql: `SELECT COUNT(*) as cnt, AVG(score) as avg_score
832
+ FROM agent_evals WHERE ${where}`,
833
+ args,
834
+ });
835
+ const t = (totalRows[0] ?? {}) as Record<string, number | null>;
836
+
837
+ const { rows: criteriaRows } = await client.execute({
838
+ sql: `SELECT criteria, AVG(score) as avg_score, COUNT(*) as cnt
839
+ FROM agent_evals WHERE ${where}
840
+ GROUP BY criteria ORDER BY cnt DESC`,
841
+ args,
842
+ });
843
+
844
+ return {
845
+ totalEvals: Number(t.cnt ?? 0),
846
+ avgScore: Number(t.avg_score ?? 0),
847
+ byCriteria: (criteriaRows as any[]).map((r) => ({
848
+ criteria: String(r.criteria),
849
+ avgScore: Number(r.avg_score ?? 0),
850
+ count: Number(r.cnt ?? 0),
851
+ })),
852
+ };
853
+ }
854
+
855
+ // ─── Eval datasets CRUD ──────────────────────────────────────────────
856
+
857
+ export async function insertEvalDataset(dataset: EvalDataset): Promise<void> {
858
+ await ensureObservabilityTables();
859
+ const client = getDbExec();
860
+ await client.execute({
861
+ sql: `INSERT INTO agent_eval_datasets
862
+ (id, name, description, entries, created_at, updated_at)
863
+ VALUES (?, ?, ?, ?, ?, ?)`,
864
+ args: [
865
+ dataset.id,
866
+ dataset.name,
867
+ dataset.description,
868
+ JSON.stringify(dataset.entries),
869
+ dataset.createdAt,
870
+ dataset.updatedAt,
871
+ ],
872
+ });
873
+ }
874
+
875
+ export async function listEvalDatasets(): Promise<EvalDataset[]> {
876
+ await ensureObservabilityTables();
877
+ const client = getDbExec();
878
+ const { rows } = await client.execute(
879
+ `SELECT * FROM agent_eval_datasets ORDER BY updated_at DESC`,
880
+ );
881
+ return (rows as any[]).map(rowToDataset);
882
+ }
883
+
884
+ export async function getEvalDataset(id: string): Promise<EvalDataset | null> {
885
+ await ensureObservabilityTables();
886
+ const client = getDbExec();
887
+ const { rows } = await client.execute({
888
+ sql: `SELECT * FROM agent_eval_datasets WHERE id = ?`,
889
+ args: [id],
890
+ });
891
+ if (rows.length === 0) return null;
892
+ return rowToDataset(rows[0] as any);
893
+ }
894
+
895
+ export async function updateEvalDataset(
896
+ id: string,
897
+ updates: Partial<Pick<EvalDataset, "name" | "description" | "entries">>,
898
+ ): Promise<void> {
899
+ await ensureObservabilityTables();
900
+ const client = getDbExec();
901
+ const sets: string[] = [];
902
+ const args: any[] = [];
903
+ if (updates.name !== undefined) {
904
+ sets.push("name = ?");
905
+ args.push(updates.name);
906
+ }
907
+ if (updates.description !== undefined) {
908
+ sets.push("description = ?");
909
+ args.push(updates.description);
910
+ }
911
+ if (updates.entries !== undefined) {
912
+ sets.push("entries = ?");
913
+ args.push(JSON.stringify(updates.entries));
914
+ }
915
+ sets.push("updated_at = ?");
916
+ args.push(Date.now());
917
+ args.push(id);
918
+ await client.execute({
919
+ sql: `UPDATE agent_eval_datasets SET ${sets.join(", ")} WHERE id = ?`,
920
+ args,
921
+ });
922
+ }
923
+
924
+ // ─── Experiments CRUD ────────────────────────────────────────────────
925
+
926
+ export async function insertExperiment(exp: Experiment): Promise<void> {
927
+ await ensureObservabilityTables();
928
+ const client = getDbExec();
929
+ await client.execute({
930
+ sql: `INSERT INTO agent_experiments
931
+ (id, name, status, variants, metrics, assignment_level,
932
+ started_at, ended_at, created_at, owner_email)
933
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
934
+ args: [
935
+ exp.id,
936
+ exp.name,
937
+ exp.status,
938
+ JSON.stringify(exp.variants),
939
+ JSON.stringify(exp.metrics),
940
+ exp.assignmentLevel,
941
+ exp.startedAt,
942
+ exp.endedAt,
943
+ exp.createdAt,
944
+ exp.ownerEmail ?? null,
945
+ ],
946
+ });
947
+ }
948
+
949
+ export async function updateExperiment(
950
+ id: string,
951
+ updates: Partial<
952
+ Pick<Experiment, "name" | "status" | "variants" | "metrics" | "endedAt">
953
+ >,
954
+ ): Promise<void> {
955
+ await ensureObservabilityTables();
956
+ const client = getDbExec();
957
+ const sets: string[] = [];
958
+ const args: any[] = [];
959
+ if (updates.name !== undefined) {
960
+ sets.push("name = ?");
961
+ args.push(updates.name);
962
+ }
963
+ if (updates.status !== undefined) {
964
+ sets.push("status = ?");
965
+ args.push(updates.status);
966
+ if (updates.status === "running" && !updates.endedAt) {
967
+ sets.push("started_at = ?");
968
+ args.push(Date.now());
969
+ }
970
+ }
971
+ if (updates.variants !== undefined) {
972
+ sets.push("variants = ?");
973
+ args.push(JSON.stringify(updates.variants));
974
+ }
975
+ if (updates.metrics !== undefined) {
976
+ sets.push("metrics = ?");
977
+ args.push(JSON.stringify(updates.metrics));
978
+ }
979
+ if (updates.endedAt !== undefined) {
980
+ sets.push("ended_at = ?");
981
+ args.push(updates.endedAt);
982
+ }
983
+ if (sets.length === 0) return;
984
+ args.push(id);
985
+ await client.execute({
986
+ sql: `UPDATE agent_experiments SET ${sets.join(", ")} WHERE id = ?`,
987
+ args,
988
+ });
989
+ }
990
+
991
+ export async function listExperiments(): Promise<Experiment[]> {
992
+ await ensureObservabilityTables();
993
+ const client = getDbExec();
994
+ const { rows } = await client.execute(
995
+ `SELECT * FROM agent_experiments ORDER BY created_at DESC`,
996
+ );
997
+ return (rows as any[]).map(rowToExperiment);
998
+ }
999
+
1000
+ export async function getExperiment(id: string): Promise<Experiment | null> {
1001
+ await ensureObservabilityTables();
1002
+ const client = getDbExec();
1003
+ const { rows } = await client.execute({
1004
+ sql: `SELECT * FROM agent_experiments WHERE id = ?`,
1005
+ args: [id],
1006
+ });
1007
+ if (rows.length === 0) return null;
1008
+ return rowToExperiment(rows[0] as any);
1009
+ }
1010
+
1011
+ // ─── Experiment assignments CRUD ────────────────────────────────────
1012
+
1013
+ export async function upsertAssignment(
1014
+ assignment: ExperimentAssignment,
1015
+ ): Promise<void> {
1016
+ await ensureObservabilityTables();
1017
+ const client = getDbExec();
1018
+ if (isPostgres()) {
1019
+ await client.execute({
1020
+ sql: `INSERT INTO agent_experiment_assignments
1021
+ (experiment_id, user_id, variant_id, assigned_at)
1022
+ VALUES (?, ?, ?, ?)
1023
+ ON CONFLICT (experiment_id, user_id) DO UPDATE SET
1024
+ variant_id = EXCLUDED.variant_id,
1025
+ assigned_at = EXCLUDED.assigned_at`,
1026
+ args: [
1027
+ assignment.experimentId,
1028
+ assignment.userId,
1029
+ assignment.variantId,
1030
+ assignment.assignedAt,
1031
+ ],
1032
+ });
1033
+ } else {
1034
+ await client.execute({
1035
+ sql: `INSERT OR REPLACE INTO agent_experiment_assignments
1036
+ (experiment_id, user_id, variant_id, assigned_at)
1037
+ VALUES (?, ?, ?, ?)`,
1038
+ args: [
1039
+ assignment.experimentId,
1040
+ assignment.userId,
1041
+ assignment.variantId,
1042
+ assignment.assignedAt,
1043
+ ],
1044
+ });
1045
+ }
1046
+ }
1047
+
1048
+ export async function getAssignment(
1049
+ experimentId: string,
1050
+ userId: string,
1051
+ ): Promise<ExperimentAssignment | null> {
1052
+ await ensureObservabilityTables();
1053
+ const client = getDbExec();
1054
+ const { rows } = await client.execute({
1055
+ sql: `SELECT * FROM agent_experiment_assignments
1056
+ WHERE experiment_id = ? AND user_id = ?`,
1057
+ args: [experimentId, userId],
1058
+ });
1059
+ if (rows.length === 0) return null;
1060
+ const r = rows[0] as any;
1061
+ return {
1062
+ experimentId: r.experiment_id,
1063
+ userId: r.user_id,
1064
+ variantId: r.variant_id,
1065
+ assignedAt: Number(r.assigned_at),
1066
+ };
1067
+ }
1068
+
1069
+ // ─── Experiment results CRUD ─────────────────────────────────────────
1070
+
1071
+ export async function insertExperimentResult(
1072
+ result: ExperimentMetricResult,
1073
+ ): Promise<void> {
1074
+ await ensureObservabilityTables();
1075
+ const client = getDbExec();
1076
+ await client.execute({
1077
+ sql: `INSERT INTO agent_experiment_results
1078
+ (id, experiment_id, variant_id, metric, value,
1079
+ sample_size, confidence_low, confidence_high, computed_at)
1080
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1081
+ args: [
1082
+ result.id,
1083
+ result.experimentId,
1084
+ result.variantId,
1085
+ result.metric,
1086
+ result.value,
1087
+ result.sampleSize,
1088
+ result.confidenceLow,
1089
+ result.confidenceHigh,
1090
+ result.computedAt,
1091
+ ],
1092
+ });
1093
+ }
1094
+
1095
+ export async function getExperimentResults(
1096
+ experimentId: string,
1097
+ ): Promise<ExperimentMetricResult[]> {
1098
+ await ensureObservabilityTables();
1099
+ const client = getDbExec();
1100
+ const { rows } = await client.execute({
1101
+ sql: `SELECT * FROM agent_experiment_results
1102
+ WHERE experiment_id = ?
1103
+ ORDER BY computed_at DESC`,
1104
+ args: [experimentId],
1105
+ });
1106
+ return (rows as any[]).map(rowToExperimentResult);
1107
+ }
1108
+
1109
+ // ─── Aggregate queries for dashboard ─────────────────────────────────
1110
+
1111
+ export async function getObservabilityOverview(
1112
+ sinceMs: number,
1113
+ opts: { userId?: string } = {},
1114
+ ): Promise<{
1115
+ totalRuns: number;
1116
+ totalCostCents: number;
1117
+ avgDurationMs: number;
1118
+ toolSuccessRate: number;
1119
+ avgFrustrationScore: number;
1120
+ thumbsUpRate: number;
1121
+ avgEvalScore: number;
1122
+ }> {
1123
+ await ensureObservabilityTables();
1124
+ const client = getDbExec();
1125
+
1126
+ // Three of the four sub-queries time-key on `created_at`; satisfaction
1127
+ // uses `computed_at`. Each gets its own `withUserFilter` invocation so
1128
+ // the args array isn't aliased across calls (some drivers mutate args
1129
+ // for prepared-statement caching).
1130
+ const created = withUserFilter("created_at >= ?", [sinceMs], opts.userId);
1131
+ const computed = withUserFilter("computed_at >= ?", [sinceMs], opts.userId);
1132
+
1133
+ const [tracesResult, satisfactionResult, feedbackResult, evalsResult] =
1134
+ await Promise.all([
1135
+ client.execute({
1136
+ sql: `SELECT
1137
+ COUNT(*) as total_runs,
1138
+ COALESCE(SUM(total_cost_cents_x100), 0) as total_cost,
1139
+ COALESCE(AVG(total_duration_ms), 0) as avg_duration,
1140
+ COALESCE(SUM(successful_tools), 0) as success_tools,
1141
+ COALESCE(SUM(tool_calls), 0) as total_tools
1142
+ FROM agent_trace_summaries WHERE ${created.where}`,
1143
+ args: created.args,
1144
+ }),
1145
+ client.execute({
1146
+ sql: `SELECT COALESCE(AVG(frustration_score), 0) as avg_frustration
1147
+ FROM agent_satisfaction_scores WHERE ${computed.where}`,
1148
+ args: computed.args,
1149
+ }),
1150
+ client.execute({
1151
+ sql: `SELECT
1152
+ COALESCE(SUM(CASE WHEN feedback_type = 'thumbs_up' THEN 1 ELSE 0 END), 0) as up,
1153
+ COALESCE(SUM(CASE WHEN feedback_type IN ('thumbs_up', 'thumbs_down') THEN 1 ELSE 0 END), 0) as total
1154
+ FROM agent_feedback WHERE ${created.where}`,
1155
+ args: created.args,
1156
+ }),
1157
+ client.execute({
1158
+ sql: `SELECT COALESCE(AVG(score), 0) as avg_score
1159
+ FROM agent_evals WHERE ${created.where}`,
1160
+ args: created.args,
1161
+ }),
1162
+ ]);
1163
+
1164
+ const t = (tracesResult.rows[0] ?? {}) as Record<string, number | null>;
1165
+ const s = (satisfactionResult.rows[0] ?? {}) as Record<string, number | null>;
1166
+ const f = (feedbackResult.rows[0] ?? {}) as Record<string, number | null>;
1167
+ const e = (evalsResult.rows[0] ?? {}) as Record<string, number | null>;
1168
+
1169
+ const totalTools = Number(t.total_tools ?? 0);
1170
+ const successTools = Number(t.success_tools ?? 0);
1171
+ const feedbackTotal = Number(f.total ?? 0);
1172
+ const feedbackUp = Number(f.up ?? 0);
1173
+
1174
+ return {
1175
+ totalRuns: Number(t.total_runs ?? 0),
1176
+ totalCostCents: Number(t.total_cost ?? 0) / 100,
1177
+ avgDurationMs: Number(t.avg_duration ?? 0),
1178
+ toolSuccessRate: totalTools > 0 ? successTools / totalTools : 1,
1179
+ avgFrustrationScore: Number(s.avg_frustration ?? 0),
1180
+ thumbsUpRate: feedbackTotal > 0 ? feedbackUp / feedbackTotal : 0,
1181
+ avgEvalScore: Number(e.avg_score ?? 0),
1182
+ };
1183
+ }
1184
+
1185
+ // ─── Row mappers ─────────────────────────────────────────────────────
1186
+
1187
+ function rowToTraceSpan(row: Record<string, any>): TraceSpan {
1188
+ return {
1189
+ id: String(row.id),
1190
+ runId: String(row.run_id),
1191
+ threadId: row.thread_id ? String(row.thread_id) : null,
1192
+ userId: row.user_id ? String(row.user_id) : null,
1193
+ parentSpanId: row.parent_span_id ? String(row.parent_span_id) : null,
1194
+ spanType: row.span_type as TraceSpan["spanType"],
1195
+ name: String(row.name),
1196
+ inputTokens: Number(row.input_tokens ?? 0),
1197
+ outputTokens: Number(row.output_tokens ?? 0),
1198
+ cacheReadTokens: Number(row.cache_read_tokens ?? 0),
1199
+ cacheWriteTokens: Number(row.cache_write_tokens ?? 0),
1200
+ costCentsX100: Number(row.cost_cents_x100 ?? 0),
1201
+ durationMs: Number(row.duration_ms ?? 0),
1202
+ status: row.status as TraceSpan["status"],
1203
+ errorMessage: row.error_message ? String(row.error_message) : null,
1204
+ metadata: safeJsonParse(row.metadata, null),
1205
+ createdAt: Number(row.created_at),
1206
+ };
1207
+ }
1208
+
1209
+ function rowToTraceSummary(row: Record<string, any>): TraceSummary {
1210
+ return {
1211
+ runId: String(row.run_id),
1212
+ threadId: row.thread_id ? String(row.thread_id) : null,
1213
+ userId: row.user_id ? String(row.user_id) : null,
1214
+ totalSpans: Number(row.total_spans ?? 0),
1215
+ llmCalls: Number(row.llm_calls ?? 0),
1216
+ toolCalls: Number(row.tool_calls ?? 0),
1217
+ successfulTools: Number(row.successful_tools ?? 0),
1218
+ failedTools: Number(row.failed_tools ?? 0),
1219
+ totalDurationMs: Number(row.total_duration_ms ?? 0),
1220
+ totalCostCentsX100: Number(row.total_cost_cents_x100 ?? 0),
1221
+ totalInputTokens: Number(row.total_input_tokens ?? 0),
1222
+ totalOutputTokens: Number(row.total_output_tokens ?? 0),
1223
+ model: String(row.model ?? ""),
1224
+ createdAt: Number(row.created_at),
1225
+ };
1226
+ }
1227
+
1228
+ function rowToFeedback(row: Record<string, any>): FeedbackEntry {
1229
+ return {
1230
+ id: String(row.id),
1231
+ runId: row.run_id ? String(row.run_id) : null,
1232
+ threadId: row.thread_id ? String(row.thread_id) : null,
1233
+ messageSeq: row.message_seq != null ? Number(row.message_seq) : null,
1234
+ feedbackType: row.feedback_type as FeedbackEntry["feedbackType"],
1235
+ value: String(row.value ?? ""),
1236
+ userId: row.user_id ? String(row.user_id) : null,
1237
+ createdAt: Number(row.created_at),
1238
+ };
1239
+ }
1240
+
1241
+ function rowToSatisfaction(row: Record<string, any>): SatisfactionScore {
1242
+ return {
1243
+ id: String(row.id),
1244
+ threadId: String(row.thread_id),
1245
+ userId: row.user_id ? String(row.user_id) : null,
1246
+ frustrationScore: Number(row.frustration_score ?? 0),
1247
+ rephrasingScore: Number(row.rephrasing_score ?? 0),
1248
+ abandonmentScore: Number(row.abandonment_score ?? 0),
1249
+ sentimentScore: Number(row.sentiment_score ?? 0),
1250
+ lengthTrendScore: Number(row.length_trend_score ?? 0),
1251
+ computedAt: Number(row.computed_at),
1252
+ };
1253
+ }
1254
+
1255
+ function rowToEval(row: Record<string, any>): EvalResult {
1256
+ return {
1257
+ id: String(row.id),
1258
+ runId: String(row.run_id),
1259
+ threadId: row.thread_id ? String(row.thread_id) : null,
1260
+ userId: row.user_id ? String(row.user_id) : null,
1261
+ evalType: row.eval_type as EvalResult["evalType"],
1262
+ criteria: String(row.criteria),
1263
+ score: Number(row.score ?? 0),
1264
+ reasoning: row.reasoning ? String(row.reasoning) : null,
1265
+ metadata: safeJsonParse(row.metadata, null),
1266
+ createdAt: Number(row.created_at),
1267
+ };
1268
+ }
1269
+
1270
+ function rowToDataset(row: Record<string, any>): EvalDataset {
1271
+ return {
1272
+ id: String(row.id),
1273
+ name: String(row.name),
1274
+ description: String(row.description ?? ""),
1275
+ entries: safeJsonParse(row.entries, []),
1276
+ createdAt: Number(row.created_at),
1277
+ updatedAt: Number(row.updated_at),
1278
+ };
1279
+ }
1280
+
1281
+ function rowToExperiment(row: Record<string, any>): Experiment {
1282
+ return {
1283
+ id: String(row.id),
1284
+ name: String(row.name),
1285
+ status: row.status as Experiment["status"],
1286
+ variants: safeJsonParse(row.variants, []),
1287
+ metrics: safeJsonParse(row.metrics, []),
1288
+ assignmentLevel: (row.assignment_level as "user" | "session") ?? "user",
1289
+ startedAt: row.started_at ? Number(row.started_at) : null,
1290
+ endedAt: row.ended_at ? Number(row.ended_at) : null,
1291
+ createdAt: Number(row.created_at),
1292
+ ownerEmail:
1293
+ typeof row.owner_email === "string" && row.owner_email
1294
+ ? row.owner_email
1295
+ : null,
1296
+ };
1297
+ }
1298
+
1299
+ function rowToExperimentResult(
1300
+ row: Record<string, any>,
1301
+ ): ExperimentMetricResult {
1302
+ return {
1303
+ id: String(row.id),
1304
+ experimentId: String(row.experiment_id),
1305
+ variantId: String(row.variant_id),
1306
+ metric: String(row.metric),
1307
+ value: Number(row.value ?? 0),
1308
+ sampleSize: Number(row.sample_size ?? 0),
1309
+ confidenceLow: Number(row.confidence_low ?? 0),
1310
+ confidenceHigh: Number(row.confidence_high ?? 0),
1311
+ computedAt: Number(row.computed_at),
1312
+ };
1313
+ }