@obsunified/collector 2.0.1

package/src/lib/store/overview.ts

@@ -0,0 +1,127 @@
+import type {
+	TelemetryOverviewOptions,
+	TelemetryOverviewResponse,
+} from "@obsunified/types";
+import type { SqlDb } from "../sql-db";
+import {
+	average,
+	cutoffIso,
+	normalizeService,
+	type ParsedSpan,
+	percentile,
+} from "./helpers";
+import { fetchSpansForTraceIds, selectTraceCandidates } from "./trace-queries";
+
+export async function getTelemetryOverview(
+	db: SqlDb,
+	options: TelemetryOverviewOptions,
+): Promise<TelemetryOverviewResponse> {
+	if (!options.projectId)
+		throw new Error("TelemetryStore.getOverview: projectId is required");
+	const cutoff = cutoffIso(options.hours);
+	const traceLimit = options.limit ?? 30;
+	const candidates = await selectTraceCandidates(db, {
+		projectId: options.projectId,
+		cutoff,
+		service: options.service,
+		search: options.search,
+		status: options.status,
+		limit: traceLimit,
+	});
+	const candidateOrder = new Map(
+		candidates.map((candidate, index) => [candidate.trace_id, index]),
+	);
+	const grouped = new Map<string, ParsedSpan[]>();
+	for (const parsed of await fetchSpansForTraceIds(
+		db,
+		options.projectId,
+		candidates.map((candidate) => candidate.trace_id),
+	)) {
+		const traceSpans = grouped.get(parsed.traceId) ?? [];
+		traceSpans.push(parsed);
+		grouped.set(parsed.traceId, traceSpans);
+	}
+
+	const traces = Array.from(grouped.values())
+		.map((traceSpans) => {
+			const sorted = traceSpans.sort((left, right) =>
+				left.startTime.localeCompare(right.startTime),
+			);
+			const root = sorted.find((span) => !span.parentSpanId) ?? sorted[0];
+			return {
+				traceId: root.traceId,
+				serviceName: normalizeService(root.serviceName),
+				spanName: root.spanName,
+				statusCode: traceSpans.some((span) => span.statusCode === 2)
+					? 2
+					: root.statusCode,
+				statusMessage:
+					traceSpans.find((span) => span.statusCode === 2)?.statusMessage ??
+					root.statusMessage,
+				startTime: root.startTime,
+				endTime: root.endTime,
+				durationMs: root.durationMs,
+				receivedAt: root.receivedAt,
+				spanCount: traceSpans.length,
+				errorSpanCount: traceSpans.filter((span) => span.statusCode === 2)
+					.length,
+			};
+		})
+		.sort((left, right) => right.receivedAt.localeCompare(left.receivedAt));
+	traces.sort(
+		(left, right) =>
+			(candidateOrder.get(left.traceId) ?? Number.MAX_SAFE_INTEGER) -
+			(candidateOrder.get(right.traceId) ?? Number.MAX_SAFE_INTEGER),
+	);
+
+	const durations = traces.map((trace) => trace.durationMs);
+	const errorTraces = traces.filter((trace) => trace.errorSpanCount > 0);
+	const serviceMap = new Map<
+		string,
+		{ traceCount: number; errorTraceCount: number; durations: number[] }
+	>();
+	for (const trace of traces) {
+		const entry = serviceMap.get(trace.serviceName) ?? {
+			traceCount: 0,
+			errorTraceCount: 0,
+			durations: [],
+		};
+		entry.traceCount += 1;
+		entry.errorTraceCount += trace.errorSpanCount > 0 ? 1 : 0;
+		entry.durations.push(trace.durationMs);
+		serviceMap.set(trace.serviceName, entry);
+	}
+
+	return {
+		summary: {
+			totalTraces: traces.length,
+			errorTraces: errorTraces.length,
+			successTraces: traces.length - errorTraces.length,
+			errorRate: traces.length > 0 ? errorTraces.length / traces.length : 0,
+			averageDurationMs: Math.round(average(durations)),
+			p95DurationMs: Math.round(percentile(durations, 0.95)),
+		},
+		services: Array.from(serviceMap.entries())
+			.map(([serviceName, entry]) => ({
+				serviceName,
+				traceCount: entry.traceCount,
+				errorTraceCount: entry.errorTraceCount,
+				errorRate:
+					entry.traceCount > 0 ? entry.errorTraceCount / entry.traceCount : 0,
+				averageDurationMs: Math.round(average(entry.durations)),
+				maxDurationMs: Math.max(...entry.durations, 0),
+			}))
+			.sort(
+				(left, right) =>
+					right.traceCount - left.traceCount ||
+					left.serviceName.localeCompare(right.serviceName),
+			),
+		traces: traces.slice(0, options.limit ?? 30),
+		windowHours: options.hours,
+		filters: {
+			service: options.service ?? "all",
+			status: options.status ?? "all",
+		},
+		timestamp: new Date().toISOString(),
+	};
+}

package/src/lib/store/retention.ts

@@ -0,0 +1,10 @@
+import type { SqlDb } from "../sql-db";
+
+export async function purgeExpiredTelemetry(db: SqlDb): Promise<number> {
+	const now = new Date().toISOString();
+	const result = await db
+		.prepare("DELETE FROM telemetry_spans WHERE expires_at <= ?")
+		.bind(now)
+		.run();
+	return result.meta?.changes ?? 0;
+}

package/src/lib/store/service-map.ts

@@ -0,0 +1,384 @@
+import { dialectFor, type SqlDb } from "../sql-db";
+import { cutoffIso, percentile } from "./helpers";
+
+export interface ServiceMapOptions {
+	projectId: string;
+	hours: number;
+	source?: "all" | "sdk" | "ebpf";
+}
+
+export interface ServiceOperationsOptions {
+	projectId: string;
+	service: string;
+	hours: number;
+}
+
+export async function getTelemetryServiceMap(
+	db: SqlDb,
+	options: ServiceMapOptions,
+) {
+	if (!options.projectId)
+		throw new Error("TelemetryStore.getServiceMap: projectId is required");
+	const cutoff = cutoffIso(options.hours);
+	const dialect = dialectFor(db);
+	const source = options.source ?? "all";
+
+	// RFC 0009 — eBPF-derived agents identify themselves via
+	// resource_attribute `telemetry.sdk.name`. The set is small and
+	// stable enough to inline; expand as new agents land.
+	const EBPF_SDK_NAMES = new Set(["beyla", "otel-ebpf-profiler"]);
+	const sourceClause =
+		source === "ebpf"
+			? ` AND telemetry_sdk_name IN (${Array.from(EBPF_SDK_NAMES)
+					.map(() => "?")
+					.join(",")})`
+			: source === "sdk"
+				? ` AND (telemetry_sdk_name IS NULL OR telemetry_sdk_name NOT IN (${Array.from(
+						EBPF_SDK_NAMES,
+					)
+						.map(() => "?")
+						.join(",")}))`
+				: "";
+	const sourceBinds: unknown[] =
+		source === "all" ? [] : Array.from(EBPF_SDK_NAMES);
+
+	const nodesResult = await db
+		.prepare(
+			`SELECT
+					service_name,
+					COUNT(*) AS span_count,
+					SUM(CASE WHEN status_code = 2 THEN 1 ELSE 0 END) AS error_count,
+					COUNT(DISTINCT trace_id) AS trace_count
+				FROM telemetry_spans
+				WHERE project_id = ? AND received_at >= ? AND service_name IS NOT NULL${sourceClause}
+				GROUP BY service_name`,
+		)
+		.bind(options.projectId, cutoff, ...sourceBinds)
+		.all<{
+			service_name: string;
+			span_count: number;
+			error_count: number;
+			trace_count: number;
+		}>();
+
+	// Edges come from two relationships:
+	//   1. Synchronous: child span's parent_span_id points at the parent's
+	//      span_id within the same trace, and they're in different services.
+	//   2. Asynchronous: child span carries a `link` (in links_json) back
+	//      to a producer span in another service — this is how OpenTelemetry
+	//      represents Kafka / RabbitMQ / SQS / Pub/Sub dispatch, because
+	//      consumers can process messages long after producers close their
+	//      span and a single consume can correspond to many produces.
+	// We UNION ALL both — duplicates are accumulated as separate calls,
+	// which matches the synchronous semantics (each child = one call).
+	// RFC 0009 — applying the source filter to *child* spans for sync
+	// edges and to *consumer* spans for async edges. The producer
+	// side is left unfiltered (an SDK-instrumented service that
+	// publishes to a Kafka topic still belongs in the eBPF view if
+	// the consumer is Beyla-instrumented; the edge represents the
+	// kernel-observed call, not the producer's instrumentation).
+	const childSourceClause =
+		source === "ebpf"
+			? ` AND c.telemetry_sdk_name IN (${Array.from(EBPF_SDK_NAMES)
+					.map(() => "?")
+					.join(",")})`
+			: source === "sdk"
+				? ` AND (c.telemetry_sdk_name IS NULL OR c.telemetry_sdk_name NOT IN (${Array.from(
+						EBPF_SDK_NAMES,
+					)
+						.map(() => "?")
+						.join(",")}))`
+				: "";
+	const consumerSourceClause =
+		source === "ebpf"
+			? ` AND telemetry_sdk_name IN (${Array.from(EBPF_SDK_NAMES)
+					.map(() => "?")
+					.join(",")})`
+			: source === "sdk"
+				? ` AND (telemetry_sdk_name IS NULL OR telemetry_sdk_name NOT IN (${Array.from(
+						EBPF_SDK_NAMES,
+					)
+						.map(() => "?")
+						.join(",")}))`
+				: "";
+	const linkEdgesSql =
+		dialect.name === "postgres"
+			? `link_edges AS (
+					SELECT
+						producer.service_name AS source,
+						consumer.service_name AS target,
+						consumer.status_code AS status_code,
+						consumer.duration_ms AS duration_ms,
+						consumer.received_at AS received_at,
+						CASE WHEN consumer.telemetry_sdk_name IN ('beyla', 'otel-ebpf-profiler') THEN 1 ELSE 0 END AS is_ebpf
+					FROM (
+						SELECT trace_id, span_id, project_id, service_name,
+							status_code, duration_ms, received_at, links_json, telemetry_sdk_name
+						FROM telemetry_spans
+						WHERE project_id = ?
+							AND received_at >= ?
+							AND links_json IS NOT NULL
+							AND links_json != '[]'${consumerSourceClause}
+					) consumer
+					CROSS JOIN LATERAL jsonb_array_elements(consumer.links_json::jsonb) link(value)
+					JOIN telemetry_spans producer
+						ON producer.trace_id = link.value ->> 'traceId'
+						AND producer.span_id = link.value ->> 'spanId'
+						AND producer.project_id = consumer.project_id
+					WHERE producer.received_at >= ?
+						AND consumer.service_name IS NOT NULL
+						AND producer.service_name IS NOT NULL
+						AND producer.service_name != consumer.service_name
+				)`
+			: `link_edges AS (
+					SELECT
+						producer.service_name AS source,
+						consumer.service_name AS target,
+						consumer.status_code AS status_code,
+						consumer.duration_ms AS duration_ms,
+						consumer.received_at AS received_at,
+						CASE WHEN consumer.telemetry_sdk_name IN ('beyla', 'otel-ebpf-profiler') THEN 1 ELSE 0 END AS is_ebpf
+					-- Pre-filter consumers in a subquery so json_each is only
+					-- invoked on spans that actually carry links. Without this
+					-- gate the cross product fans out across every span in the
+					-- window and the query times out on tens of thousands of
+					-- rows.
+					FROM (
+						SELECT trace_id, span_id, project_id, service_name,
+							status_code, duration_ms, received_at, links_json, telemetry_sdk_name
+						FROM telemetry_spans
+						WHERE project_id = ?
+							AND received_at >= ?
+							AND links_json IS NOT NULL
+							AND links_json != '[]'${consumerSourceClause}
+					) consumer,
+						json_each(consumer.links_json) link
+					-- Match on (trace_id, span_id) so the lookup hits the
+					-- composite PRIMARY KEY index. span_id alone has no index
+					-- (only the composite PK), so a span_id-only join would
+					-- full-scan per link.
+					JOIN telemetry_spans producer
+						ON producer.trace_id = json_extract(link.value, '$.traceId')
+						AND producer.span_id = json_extract(link.value, '$.spanId')
+						AND producer.project_id = consumer.project_id
+					WHERE producer.received_at >= ?
+						AND consumer.service_name IS NOT NULL
+						AND producer.service_name IS NOT NULL
+						AND producer.service_name != consumer.service_name
+				)`;
+
+	const edgeRowsResult = await db
+		.prepare(
+			`WITH parent_child_edges AS (
+					SELECT
+						p.service_name AS source,
+						c.service_name AS target,
+						c.status_code AS status_code,
+						c.duration_ms AS duration_ms,
+						c.received_at AS received_at,
+						CASE WHEN c.telemetry_sdk_name IN ('beyla', 'otel-ebpf-profiler') THEN 1 ELSE 0 END AS is_ebpf
+					FROM telemetry_spans p
+					JOIN telemetry_spans c
+						ON c.parent_span_id = p.span_id
+						AND c.trace_id = p.trace_id
+						AND c.project_id = p.project_id
+					WHERE p.project_id = ?
+						AND p.received_at >= ?
+						AND c.received_at >= ?
+						AND p.service_name IS NOT NULL
+						AND c.service_name IS NOT NULL
+						AND p.service_name != c.service_name${childSourceClause}
+				),
+				${linkEdgesSql}
+				SELECT * FROM parent_child_edges
+				UNION ALL
+				SELECT * FROM link_edges
+				ORDER BY received_at DESC
+				LIMIT 50000`,
+		)
+		.bind(
+			options.projectId,
+			cutoff,
+			cutoff,
+			...sourceBinds,
+			options.projectId,
+			cutoff,
+			...sourceBinds,
+			cutoff,
+		)
+		.all<{
+			source: string;
+			target: string;
+			status_code: number;
+			duration_ms: number;
+			is_ebpf: number;
+		}>();
+
+	interface EdgeAcc {
+		source: string;
+		target: string;
+		calls: number;
+		errors: number;
+		durations: number[];
+		isEbpf: boolean;
+	}
+	const edgeMap = new Map<string, EdgeAcc>();
+	for (const row of edgeRowsResult.results ?? []) {
+		const key = `${row.source}|${row.target}`;
+		let acc = edgeMap.get(key);
+		if (!acc) {
+			acc = {
+				source: row.source,
+				target: row.target,
+				calls: 0,
+				errors: 0,
+				durations: [],
+				isEbpf: false,
+			};
+			edgeMap.set(key, acc);
+		}
+		acc.calls += 1;
+		if (row.status_code === 2) acc.errors += 1;
+		acc.durations.push(row.duration_ms ?? 0);
+		if (row.is_ebpf === 1) acc.isEbpf = true;
+	}
+
+	const windowSeconds = Math.max(1, options.hours * 3600);
+	const edges = Array.from(edgeMap.values()).map((e) => ({
+		source: e.source,
+		target: e.target,
+		calls: e.calls,
+		errors: e.errors,
+		errorRate: e.calls > 0 ? e.errors / e.calls : 0,
+		p50DurationMs: percentile(e.durations, 0.5),
+		p95DurationMs: percentile(e.durations, 0.95),
+		rps: e.calls / windowSeconds,
+		isEbpf: e.isEbpf,
+	}));
+
+	const nodes = (nodesResult.results ?? []).map((r) => ({
+		service: r.service_name,
+		spanCount: r.span_count,
+		errorCount: r.error_count,
+		traceCount: r.trace_count,
+		errorRate: r.span_count > 0 ? r.error_count / r.span_count : 0,
+	}));
+
+	return { nodes, edges };
+}
+
+export async function getTelemetryServiceOperations(
+	db: SqlDb,
+	options: ServiceOperationsOptions,
+) {
+	if (!options.projectId)
+		throw new Error(
+			"TelemetryStore.getServiceOperations: projectId is required",
+		);
+	const cutoff = cutoffIso(options.hours);
+
+	// Top operations — group by span_name, accumulate durations in JS.
+	const opsResult = await db
+		.prepare(
+			// One row per span; calls/errors/percentiles are aggregated per
+			// span_name in JS below. Do NOT reintroduce COUNT()/SUM() without
+			// a GROUP BY — that collapses the result to a single row in SQLite.
+			`SELECT
+					span_name,
+					status_code,
+					duration_ms
+				FROM telemetry_spans
+				WHERE project_id = ?
+					AND service_name = ?
+					AND received_at >= ?
+				ORDER BY received_at DESC
+				LIMIT 20000`,
+		)
+		.bind(options.projectId, options.service, cutoff)
+		.all<{
+			span_name: string;
+			status_code: number | null;
+			duration_ms: number;
+		}>();
+
+	interface OpAcc {
+		calls: number;
+		errors: number;
+		durations: number[];
+	}
+	const opMap = new Map<string, OpAcc>();
+	let totalSpans = 0;
+	let totalErrors = 0;
+	for (const row of opsResult.results ?? []) {
+		const isError = row.status_code === 2;
+		totalSpans += 1;
+		if (isError) totalErrors += 1;
+		let acc = opMap.get(row.span_name);
+		if (!acc) {
+			acc = { calls: 0, errors: 0, durations: [] };
+			opMap.set(row.span_name, acc);
+		}
+		acc.calls += 1;
+		if (isError) acc.errors += 1;
+		acc.durations.push(row.duration_ms ?? 0);
+	}
+	const operations = Array.from(opMap.entries())
+		.map(([spanName, acc]) => ({
+			spanName,
+			calls: acc.calls,
+			errors: acc.errors,
+			errorRate: acc.calls > 0 ? acc.errors / acc.calls : 0,
+			p50DurationMs: percentile(acc.durations, 0.5),
+			p95DurationMs: percentile(acc.durations, 0.95),
+		}))
+		.sort((l, r) => r.calls - l.calls)
+		.slice(0, 12);
+
+	// Recent error spans for triage.
+	const errorRows = await db
+		.prepare(
+			`SELECT trace_id, span_id, span_name, status_message, duration_ms, start_time
+				FROM telemetry_spans
+				WHERE project_id = ?
+					AND service_name = ?
+					AND received_at >= ?
+					AND status_code = 2
+				ORDER BY received_at DESC
+				LIMIT 10`,
+		)
+		.bind(options.projectId, options.service, cutoff)
+		.all<{
+			trace_id: string;
+			span_id: string;
+			span_name: string;
+			status_message: string | null;
+			duration_ms: number;
+			start_time: string;
+		}>();
+
+	// Distinct trace count for this service.
+	const traceCountRow = await db
+		.prepare(
+			`SELECT COUNT(DISTINCT trace_id) AS trace_count
+				FROM telemetry_spans
+				WHERE project_id = ? AND service_name = ? AND received_at >= ?`,
+		)
+		.bind(options.projectId, options.service, cutoff)
+		.first<{ trace_count: number }>();
+
+	return {
+		service: options.service,
+		spanCount: totalSpans,
+		traceCount: traceCountRow?.trace_count ?? 0,
+		errorCount: totalErrors,
+		operations,
+		recentErrors: (errorRows.results ?? []).map((r) => ({
+			traceId: r.trace_id,
+			spanId: r.span_id,
+			spanName: r.span_name,
+			statusMessage: r.status_message,
+			durationMs: r.duration_ms,
+			startTime: r.start_time,
+		})),
+	};
+}

package/src/lib/store/trace-detail.bench.ts

@@ -0,0 +1,76 @@
+import type { TelemetrySpanDetail } from "@obsunified/types";
+import { bench, describe } from "vitest";
+import { buildTraceInstrumentationGaps } from "./trace-detail";
+
+/**
+ * Q1 benchmark — read-time gap computation cost vs trace size.
+ *
+ * Since PR #21, trace instrumentation gaps are computed lazily at read time
+ * from a trace's spans (`buildTraceInstrumentationGaps`) instead of being
+ * materialized on ingest. This benchmark measures that pure-compute cost as a
+ * function of spans-per-trace, to answer: is read-time computation fast enough
+ * to serve on a trace-detail view without a cache, even for very large traces?
+ *
+ * This measures the trace-detail-page hot path, where spans are already loaded
+ * in memory and passed straight into the gap computation. The standalone
+ * `/internal/.../gaps` endpoint additionally pays a project-scoped span SELECT
+ * + JSON parse; that DB-inclusive end-to-end cost is captured separately by the
+ * read-path child-span instrumentation (the other half of "Q1 = Both"), not
+ * here.
+ *
+ * Run: `pnpm --filter @obsunified/collector bench` (excluded from `pnpm test`).
+ */
+
+const SIZES = [10, 100, 500, 1_000, 5_000, 10_000];
+
+// Deterministic synthetic trace generator (no Math.random — reproducible runs).
+// Builds a tree with branching factor 2 and durations that taper with depth so
+// the gap heuristic does real work (self-time/ratio math + a child reduce for
+// every span). Returns fully-formed TelemetrySpanDetail objects so we exercise
+// the exact code path the read handler uses.
+function makeTrace(spanCount: number): {
+	spans: TelemetrySpanDetail[];
+	rootDurationMs: number;
+} {
+	const BRANCH = 2;
+	const baseStart = Date.parse("2026-06-01T00:00:00.000Z");
+	const spans: TelemetrySpanDetail[] = [];
+
+	for (let i = 0; i < spanCount; i++) {
+		const parentIndex = i === 0 ? null : Math.floor((i - 1) / BRANCH);
+		const depth = i === 0 ? 0 : Math.floor(Math.log2(i + 1));
+		// Duration tapers with depth; kept > 0 so ratio math runs.
+		const durationMs = Math.max(1, 2_000 - depth * 150);
+		const startMs = baseStart + i; // monotonic, distinct
+		spans.push({
+			traceId: "bench-trace",
+			spanId: `s${i}`,
+			parentSpanId: parentIndex === null ? null : `s${parentIndex}`,
+			serviceName: `svc-${i % 8}`,
+			scopeName: null,
+			scopeVersion: null,
+			spanName: `op.${i % 32}`,
+			spanKind: 2,
+			statusCode: 1,
+			statusMessage: null,
+			startTime: new Date(startMs).toISOString(),
+			endTime: new Date(startMs + durationMs).toISOString(),
+			durationMs,
+			attributes: { "http.method": "GET", "http.route": `/r/${i % 16}` },
+			resourceAttributes: { "service.name": `svc-${i % 8}` },
+			events: [],
+			links: [],
+		});
+	}
+
+	return { spans, rootDurationMs: spans[0]?.durationMs ?? 0 };
+}
+
+describe("buildTraceInstrumentationGaps (read-time gap compute)", () => {
+	for (const size of SIZES) {
+		const { spans, rootDurationMs } = makeTrace(size);
+		bench(`${size} spans`, () => {
+			buildTraceInstrumentationGaps("bench-trace", spans, rootDurationMs);
+		});
+	}
+});