@obsunified/collector 2.0.1

package/src/lib/ai-store/calls.ts

@@ -0,0 +1,148 @@
+import type {
+	AICallRecord,
+	AICallsOverviewOptions,
+	AICallsOverviewResponse,
+} from "@obsunified/types";
+import { dialectFor, type SqlDb } from "../sql-db";
+import type { AICallRow, AICallSummaryRow } from "./types";
+import { clampInt } from "./types";
+
+export async function ingestAICallBatch(
+	db: SqlDb,
+	calls: AICallRecord[],
+): Promise<void> {
+	if (calls.length === 0) return;
+
+	const stmt = db.prepare(`
+      INSERT OR IGNORE INTO ai_calls (
+        project_id, call_id, trace_id, span_id, service_name, model_name, provider, call_type,
+        request_json, response_json, prompt_tokens, completion_tokens, total_cost_usd,
+        latency_ms, is_error, error_message, occurred_at, received_at, expires_at,
+        session_id, interaction_id
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+	const batch = calls.map((call) => {
+		if (!call.projectId)
+			throw new Error("AIStore.ingestBatch: call.projectId is required");
+		return stmt.bind(
+			call.projectId,
+			call.callId,
+			call.traceId,
+			call.spanId,
+			call.serviceName,
+			call.modelName,
+			call.provider,
+			call.callType,
+			call.requestJson,
+			call.responseJson,
+			call.promptTokens,
+			call.completionTokens,
+			call.totalCostUsd,
+			call.latencyMs,
+			call.isError ? 1 : 0,
+			call.errorMessage,
+			call.occurredAt,
+			call.receivedAt,
+			call.expiresAt,
+			call.sessionId ?? null,
+			call.interactionId ?? null,
+		);
+	});
+
+	await db.batch(batch);
+}
+
+export async function getAICallsOverview(
+	db: SqlDb,
+	options: AICallsOverviewOptions,
+): Promise<AICallsOverviewResponse> {
+	if (!options.projectId)
+		throw new Error("AIStore.getAICalls: projectId is required");
+	const dialect = dialectFor(db);
+	const hours = clampInt(options.hours, 1, 720, 24);
+	const limit = clampInt(options.limit, 1, 1000, 100);
+
+	let sql = `SELECT * FROM ai_calls WHERE project_id = ? AND received_at >= ${dialect.sinceHours("?")}`;
+	const params: unknown[] = [options.projectId, hours];
+
+	if (options.service) {
+		sql += ` AND service_name = ?`;
+		params.push(options.service);
+	}
+	if (options.model) {
+		sql += ` AND model_name = ?`;
+		params.push(options.model);
+	}
+	if (options.isError !== undefined) {
+		sql += ` AND is_error = ?`;
+		params.push(options.isError ? 1 : 0);
+	}
+	if (options.traceId) {
+		sql += ` AND trace_id = ?`;
+		params.push(options.traceId);
+	}
+
+	sql += ` ORDER BY received_at DESC LIMIT ?`;
+	params.push(limit);
+
+	const results = await db
+		.prepare(sql)
+		.bind(...params)
+		.all<AICallRow>();
+
+	const summarySql = `
+      SELECT
+        COUNT(*) as totalCalls,
+        SUM(total_cost_usd) as totalCostUsd,
+        SUM(prompt_tokens) as totalPromptTokens,
+        SUM(completion_tokens) as totalCompletionTokens,
+        SUM(is_error) as errorCalls
+      FROM ai_calls WHERE project_id = ? AND received_at >= ${dialect.sinceHours("?")}
+      ${options.service ? "AND service_name = ?" : ""}
+    `;
+	const summaryParams: unknown[] = [options.projectId, hours];
+	if (options.service) summaryParams.push(options.service);
+	const summaryResult =
+		(await db
+			.prepare(summarySql)
+			.bind(...summaryParams)
+			.first<AICallSummaryRow>()) || {};
+
+	const calls: AICallRecord[] = (results.results || []).map((row) => ({
+		projectId: row.project_id ?? "default",
+		callId: row.call_id,
+		traceId: row.trace_id,
+		spanId: row.span_id,
+		serviceName: row.service_name,
+		modelName: row.model_name,
+		provider: row.provider,
+		callType: row.call_type,
+		requestJson: row.request_json,
+		responseJson: row.response_json,
+		promptTokens: row.prompt_tokens,
+		completionTokens: row.completion_tokens,
+		totalCostUsd: row.total_cost_usd,
+		latencyMs: row.latency_ms,
+		isError: row.is_error === 1,
+		errorMessage: row.error_message,
+		occurredAt: row.occurred_at,
+		receivedAt: row.received_at,
+		expiresAt: row.expires_at,
+		sessionId: row.session_id ?? null,
+		interactionId: row.interaction_id ?? null,
+	}));
+
+	return {
+		calls,
+		summary: {
+			totalCalls: summaryResult.totalCalls || 0,
+			totalCostUsd: summaryResult.totalCostUsd || 0,
+			totalPromptTokens: summaryResult.totalPromptTokens || 0,
+			totalCompletionTokens: summaryResult.totalCompletionTokens || 0,
+			errorCalls: summaryResult.errorCalls || 0,
+		},
+		windowHours: hours,
+		timestamp: new Date().toISOString(),
+	};
+}

package/src/lib/ai-store/evaluations.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { mapEvaluationRows } from "./evaluations";
+
+describe("mapEvaluationRows", () => {
+	it("adds evidence references that pivot from AI evaluation to span and trace", () => {
+		const [evaluation] = mapEvaluationRows([
+			{
+				evaluation_id: "eval-1",
+				project_id: "default",
+				trace_id: "trace-1",
+				span_id: "span-1",
+				name: "groundedness",
+				score: 0.2,
+				label: "fail",
+				explanation: "unsupported answer",
+				source: "llm_judge",
+				metadata_json: "{}",
+				created_at: "2026-05-01T00:00:00.000Z",
+				expires_at: "2026-05-02T00:00:00.000Z",
+			},
+		]);
+
+		expect(evaluation.evidenceReferences).toEqual([
+			expect.objectContaining({
+				evidenceId: "ai-evaluation:eval-1",
+				entityKind: "eval",
+				entityId: "eval-1",
+				route: "#/evals/eval-1",
+				source: "ai_span_evaluations.llm_judge",
+				citations: [
+					{
+						label: "span span-1",
+						entityKind: "span",
+						entityId: "trace-1:span-1",
+						route: "#/traces/trace-1#span=span-1",
+					},
+					{
+						label: "trace trace-1",
+						entityKind: "trace",
+						entityId: "trace-1",
+						route: "#/traces/trace-1",
+					},
+				],
+			}),
+		]);
+	});
+});

package/src/lib/ai-store/evaluations.ts

@@ -0,0 +1,125 @@
+import type {
+	AIEvaluationRecord,
+	AIEvaluationSource,
+	AIEvaluationsListOptions,
+	AIEvaluationsListResponse,
+} from "@obsunified/types";
+import { EVIDENCE_REFERENCE_CONTRACT } from "@obsunified/types";
+import { aiEvaluationEvidenceReferences } from "../evidence-references";
+import { parseJsonRecord } from "../json";
+import type { SqlDb } from "../sql-db";
+import type { AIEvaluationRow, IngestEvaluation } from "./types";
+import { clampInt } from "./types";
+
+export async function ingestAIEvaluations(
+	db: SqlDb,
+	evaluations: IngestEvaluation[],
+): Promise<void> {
+	if (evaluations.length === 0) return;
+	const stmt = db.prepare(`
+      INSERT INTO ai_span_evaluations (
+        evaluation_id, project_id, trace_id, span_id, name,
+        score, label, explanation, source, metadata_json,
+        created_at, expires_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+	await db.batch(
+		evaluations.map((evaluation) =>
+			stmt.bind(
+				evaluation.evaluationId,
+				evaluation.projectId,
+				evaluation.traceId,
+				evaluation.spanId,
+				evaluation.name,
+				evaluation.score,
+				evaluation.label,
+				evaluation.explanation,
+				evaluation.source,
+				evaluation.metadataJson,
+				evaluation.createdAt,
+				evaluation.expiresAt,
+			),
+		),
+	);
+}
+
+export async function listAIEvaluations(
+	db: SqlDb,
+	options: AIEvaluationsListOptions,
+): Promise<AIEvaluationsListResponse> {
+	if (!options.projectId)
+		throw new Error("AIStore.listEvaluations: projectId is required");
+	const limit = clampInt(options.limit, 1, 1000, 200);
+
+	let sql = `SELECT * FROM ai_span_evaluations WHERE project_id = ?`;
+	const params: unknown[] = [options.projectId];
+
+	if (options.traceId) {
+		sql += ` AND trace_id = ?`;
+		params.push(options.traceId);
+	}
+	if (options.spanId) {
+		sql += ` AND span_id = ?`;
+		params.push(options.spanId);
+	}
+	if (options.name) {
+		sql += ` AND name = ?`;
+		params.push(options.name);
+	}
+
+	sql += ` ORDER BY created_at DESC LIMIT ?`;
+	params.push(limit);
+
+	const results = await db
+		.prepare(sql)
+		.bind(...params)
+		.all<AIEvaluationRow>();
+
+	return {
+		evaluations: mapEvaluationRows(results.results || []),
+		timestamp: new Date().toISOString(),
+	};
+}
+
+export async function getAIEvaluation(
+	db: SqlDb,
+	projectId: string,
+	evaluationId: string,
+): Promise<AIEvaluationRecord | null> {
+	if (!projectId)
+		throw new Error("AIStore.getEvaluation: projectId is required");
+	const row = await db
+		.prepare(
+			`SELECT * FROM ai_span_evaluations
+			 WHERE project_id = ? AND evaluation_id = ?
+			 LIMIT 1`,
+		)
+		.bind(projectId, evaluationId)
+		.first<AIEvaluationRow>();
+
+	return row ? mapEvaluationRows([row])[0] : null;
+}
+
+export function mapEvaluationRows(
+	rows: AIEvaluationRow[],
+): AIEvaluationRecord[] {
+	return rows.map((row) => {
+		const evaluation: AIEvaluationRecord = {
+			evaluationId: row.evaluation_id,
+			projectId: row.project_id,
+			traceId: row.trace_id,
+			spanId: row.span_id,
+			name: row.name,
+			score: row.score,
+			label: row.label,
+			explanation: row.explanation,
+			source: row.source as AIEvaluationSource,
+			metadata: parseJsonRecord(row.metadata_json),
+			createdAt: row.created_at,
+			expiresAt: row.expires_at,
+		};
+		evaluation.evidenceReferences = aiEvaluationEvidenceReferences(evaluation);
+		evaluation.evidenceContract = EVIDENCE_REFERENCE_CONTRACT;
+		return evaluation;
+	});
+}

package/src/lib/ai-store/helpers.ts

@@ -0,0 +1,37 @@
+import type { JsonValue } from "@obsunified/types";
+import { computeCost } from "../ai-pricing";
+
+export const attrNum = (
+	attrs: Record<string, JsonValue>,
+	key: string,
+): number | null => {
+	const v = attrs[key];
+	return typeof v === "number" && Number.isFinite(v) ? v : null;
+};
+
+export const attrStr = (
+	attrs: Record<string, JsonValue>,
+	key: string,
+): string | null => {
+	const v = attrs[key];
+	return typeof v === "string" && v.length > 0 ? v : null;
+};
+
+/**
+ * Enrich a span's attributes with a computed `llm.cost.total_usd` when the
+ * span has token counts but no reported cost. Mutates the passed attrs
+ * object and returns the final cost (or null).
+ */
+export const enrichCost = (attrs: Record<string, JsonValue>): number | null => {
+	const existing = attrNum(attrs, "llm.cost.total_usd");
+	if (existing !== null) return existing;
+	const model = attrStr(attrs, "llm.model_name");
+	const prompt = attrNum(attrs, "llm.token_count.prompt");
+	const completion = attrNum(attrs, "llm.token_count.completion");
+	if (prompt === null && completion === null) return null;
+	const cost = computeCost(model, prompt, completion);
+	if (cost === null) return null;
+	attrs["llm.cost.total_usd"] = cost;
+	attrs["llm.cost.computed"] = true;
+	return cost;
+};

package/src/lib/ai-store/retention.ts

@@ -0,0 +1,17 @@
+import { dialectFor, type SqlDb } from "../sql-db";
+
+export async function purgeExpiredAIData(db: SqlDb): Promise<number> {
+	const dialect = dialectFor(db);
+	const { meta } = await db
+		.prepare(`DELETE FROM ai_calls WHERE expires_at < ${dialect.now()}`)
+		.run();
+	const { meta: payloadMeta } = await db
+		.prepare(`DELETE FROM ai_span_payloads WHERE expires_at < ${dialect.now()}`)
+		.run();
+	const { meta: evalMeta } = await db
+		.prepare(
+			`DELETE FROM ai_span_evaluations WHERE expires_at < ${dialect.now()}`,
+		)
+		.run();
+	return meta.changes + (payloadMeta?.changes ?? 0) + (evalMeta?.changes ?? 0);
+}

package/src/lib/ai-store/sessions.ts

@@ -0,0 +1,277 @@
+import type {
+	AISessionDetailResponse,
+	AISessionSummary,
+	AISessionsListOptions,
+	AISessionsListResponse,
+	AISpanRecord,
+	JsonValue,
+} from "@obsunified/types";
+import { parseJsonRecord } from "../json";
+import { dialectFor, type SqlDb } from "../sql-db";
+import { mapEvaluationRows } from "./evaluations";
+import { attrNum, enrichCost } from "./helpers";
+import type {
+	AIEvaluationRow,
+	AISessionPreviewRow,
+	AISessionRow,
+	AISpanRow,
+} from "./types";
+import { clampInt } from "./types";
+
+export async function listAISessions(
+	db: SqlDb,
+	options: AISessionsListOptions,
+): Promise<AISessionsListResponse> {
+	if (!options.projectId)
+		throw new Error("AIStore.listSessions: projectId is required");
+	const dialect = dialectFor(db);
+	const hours = clampInt(options.hours, 1, 720, 24);
+	const limit = clampInt(options.limit, 1, 1000, 100);
+
+	let sql = `
+      SELECT
+        p.session_id          AS session_id,
+        MAX(p.user_id)        AS user_id,
+        COUNT(*)              AS span_count,
+        SUM(CASE WHEN p.span_kind = 'LLM' THEN 1 ELSE 0 END) AS llm_span_count,
+        SUM(CASE WHEN s.status_code = 2 THEN 1 ELSE 0 END)   AS error_count,
+        SUM(CAST(${dialect.jsonText("s.attributes_json", '$."llm.token_count.prompt"')} AS REAL))     AS prompt_tokens,
+        SUM(CAST(${dialect.jsonText("s.attributes_json", '$."llm.token_count.completion"')} AS REAL)) AS completion_tokens,
+        SUM(CAST(${dialect.jsonText("s.attributes_json", '$."llm.cost.total_usd"')} AS REAL))         AS cost_usd,
+        MIN(s.start_time)     AS first_span_at,
+        MAX(s.start_time)     AS last_span_at,
+        COUNT(DISTINCT s.trace_id) AS trace_count
+      FROM ai_span_payloads p
+      INNER JOIN telemetry_spans s
+        ON s.trace_id = p.trace_id AND s.span_id = p.span_id
+      WHERE p.project_id = ?
+        AND p.session_id IS NOT NULL
+        AND p.received_at >= ${dialect.sinceHours("?")}
+    `;
+	const params: unknown[] = [options.projectId, hours];
+	if (options.userId) {
+		sql += ` AND p.user_id = ?`;
+		params.push(options.userId);
+	}
+	sql += ` GROUP BY p.session_id ORDER BY last_span_at DESC LIMIT ?`;
+	params.push(limit);
+
+	const results = await db
+		.prepare(sql)
+		.bind(...params)
+		.all<AISessionRow>();
+	const sessionRows = results.results || [];
+	const previewBySession = await loadSessionPreviews(
+		db,
+		dialect,
+		options.projectId,
+		hours,
+		sessionRows.map((row) => row.session_id),
+	);
+
+	const sessions: AISessionSummary[] = sessionRows.map((row) => ({
+		sessionId: row.session_id,
+		userId: row.user_id ?? null,
+		spanCount: row.span_count ?? 0,
+		llmSpanCount: row.llm_span_count ?? 0,
+		errorCount: row.error_count ?? 0,
+		totalPromptTokens: row.prompt_tokens ?? 0,
+		totalCompletionTokens: row.completion_tokens ?? 0,
+		totalCostUsd: row.cost_usd ?? 0,
+		firstSpanAt: row.first_span_at,
+		lastSpanAt: row.last_span_at,
+		traceCount: row.trace_count ?? 0,
+		lastInputPreview: previewBySession.get(row.session_id) ?? null,
+	}));
+
+	return {
+		sessions,
+		windowHours: hours,
+		timestamp: new Date().toISOString(),
+	};
+}
+
+export async function getAISessionDetail(
+	db: SqlDb,
+	projectId: string,
+	sessionId: string,
+): Promise<AISessionDetailResponse> {
+	if (!projectId) throw new Error("AIStore.getSession: projectId is required");
+	if (!sessionId) throw new Error("AIStore.getSession: sessionId is required");
+
+	const sql = `
+      SELECT
+        s.trace_id            AS trace_id,
+        s.span_id             AS span_id,
+        s.parent_span_id      AS parent_span_id,
+        s.service_name        AS service_name,
+        s.span_name           AS span_name,
+        s.status_code         AS status_code,
+        s.status_message      AS status_message,
+        s.start_time          AS start_time,
+        s.end_time            AS end_time,
+        s.duration_ms         AS duration_ms,
+        s.attributes_json     AS attributes_json,
+        p.span_kind           AS span_kind,
+        p.input_json          AS input_json,
+        p.output_json         AS output_json,
+        p.user_id             AS user_id
+      FROM ai_span_payloads p
+      INNER JOIN telemetry_spans s
+        ON s.trace_id = p.trace_id AND s.span_id = p.span_id
+      WHERE p.project_id = ? AND p.session_id = ?
+      ORDER BY s.start_time ASC
+      LIMIT 1000
+    `;
+	const results = await db
+		.prepare(sql)
+		.bind(projectId, sessionId)
+		.all<AISpanRow>();
+
+	const state = {
+		userId: null as string | null,
+		totalPromptTokens: 0,
+		totalCompletionTokens: 0,
+		totalCostUsd: 0,
+		errorCount: 0,
+		firstSpanAt: null as string | null,
+		lastSpanAt: null as string | null,
+	};
+
+	const spans: AISpanRecord[] = (results.results || []).map((row) =>
+		mapSessionSpan(row, state),
+	);
+	const evaluations = await loadSessionEvaluations(db, projectId, spans);
+
+	return {
+		sessionId,
+		userId: state.userId,
+		spans,
+		evaluations,
+		summary: {
+			spanCount: spans.length,
+			totalPromptTokens: state.totalPromptTokens,
+			totalCompletionTokens: state.totalCompletionTokens,
+			totalCostUsd: state.totalCostUsd,
+			errorCount: state.errorCount,
+			firstSpanAt: state.firstSpanAt,
+			lastSpanAt: state.lastSpanAt,
+		},
+		timestamp: new Date().toISOString(),
+	};
+}
+
+async function loadSessionPreviews(
+	db: SqlDb,
+	dialect: ReturnType<typeof dialectFor>,
+	projectId: string,
+	hours: number,
+	sessionIds: string[],
+) {
+	const previewBySession = new Map<string, string>();
+	if (sessionIds.length === 0) return previewBySession;
+
+	const previewRows = await db
+		.prepare(
+			`
+      SELECT p.session_id, p.input_json, s.start_time
+      FROM ai_span_payloads p
+      INNER JOIN telemetry_spans s
+        ON s.trace_id = p.trace_id AND s.span_id = p.span_id
+      WHERE p.project_id = ?
+        AND p.session_id IS NOT NULL
+        AND p.session_id IN (${sessionIds.map(() => "?").join(", ")})
+        AND p.received_at >= ${dialect.sinceHours("?")}
+        AND p.input_json IS NOT NULL
+      ORDER BY s.start_time DESC
+      LIMIT ?
+    `,
+		)
+		.bind(projectId, ...sessionIds, hours, sessionIds.length * 5)
+		.all<AISessionPreviewRow>();
+
+	for (const row of previewRows.results || []) {
+		if (!previewBySession.has(row.session_id)) {
+			const raw = typeof row.input_json === "string" ? row.input_json : "";
+			previewBySession.set(
+				row.session_id,
+				raw.length > 200 ? `${raw.slice(0, 200)}…` : raw,
+			);
+		}
+	}
+
+	return previewBySession;
+}
+
+function mapSessionSpan(
+	row: AISpanRow,
+	state: {
+		userId: string | null;
+		totalPromptTokens: number;
+		totalCompletionTokens: number;
+		totalCostUsd: number;
+		errorCount: number;
+		firstSpanAt: string | null;
+		lastSpanAt: string | null;
+	},
+): AISpanRecord {
+	const attrs = parseJsonRecord(row.attributes_json) as Record<
+		string,
+		JsonValue
+	>;
+	const cost = enrichCost(attrs);
+	if (!state.userId && row.user_id) state.userId = row.user_id;
+	state.totalPromptTokens += attrNum(attrs, "llm.token_count.prompt") ?? 0;
+	state.totalCompletionTokens +=
+		attrNum(attrs, "llm.token_count.completion") ?? 0;
+	if (cost !== null) state.totalCostUsd += cost;
+	if (row.status_code === 2) state.errorCount++;
+	if (!state.firstSpanAt || row.start_time < state.firstSpanAt) {
+		state.firstSpanAt = row.start_time;
+	}
+	if (!state.lastSpanAt || row.start_time > state.lastSpanAt) {
+		state.lastSpanAt = row.start_time;
+	}
+
+	return {
+		traceId: row.trace_id,
+		spanId: row.span_id,
+		parentSpanId: row.parent_span_id,
+		serviceName: row.service_name,
+		spanName: row.span_name,
+		spanKind: row.span_kind,
+		statusCode: row.status_code ?? 0,
+		statusMessage: row.status_message,
+		startTime: row.start_time,
+		endTime: row.end_time ?? row.start_time,
+		durationMs: row.duration_ms ?? 0,
+		attributes: attrs,
+		inputJson: row.input_json,
+		outputJson: row.output_json,
+	};
+}
+
+async function loadSessionEvaluations(
+	db: SqlDb,
+	projectId: string,
+	spans: AISpanRecord[],
+) {
+	if (spans.length === 0) return [];
+
+	const placeholders = spans.map(() => "(?, ?)").join(", ");
+	const bindings: unknown[] = [projectId];
+	for (const span of spans) {
+		bindings.push(span.traceId, span.spanId);
+	}
+	const sql = `
+        SELECT * FROM ai_span_evaluations
+        WHERE project_id = ?
+          AND (trace_id, span_id) IN (${placeholders})
+        ORDER BY created_at DESC
+      `;
+	const results = await db
+		.prepare(sql)
+		.bind(...bindings)
+		.all<AIEvaluationRow>();
+	return mapEvaluationRows(results.results || []);
+}