@kitlangton/motel 0.2.1 → 0.2.5

package/src/services/TelemetryStore.ts

@@ -7,6 +7,9 @@ import type { AiCallDetail, AiCallSummary, FacetItem, LogItem, SpanItem, StatsIt
 import { AI_ATTR_MAP, AI_FTS_KEYS, AI_TEXT_SEARCH_KEYS, truncatePreview } from "../domain.js"
 import { attributeMap, nanosToMilliseconds, parseAnyValue, spanKindLabel, spanStatusLabel, stringifyValue, type OtlpLogExportRequest, type OtlpTraceExportRequest } from "../otlp.js"
 
+const isSqliteLockError = (error: unknown) =>
+	error instanceof Error && /(database is locked|database table is locked|SQLITE_BUSY)/i.test(error.message)
+
 interface SpanRow {
 	readonly trace_id: string
 	readonly span_id: string
@@ -502,108 +505,110 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				PRAGMA mmap_size = 268435456;
 			`)
 		} else {
-		db.exec(`
-			PRAGMA journal_mode = WAL;
-			PRAGMA synchronous = NORMAL;
-			PRAGMA temp_store = MEMORY;
-			-- Longer busy timeout: the ingest worker holds the write lock
-			-- for up to a few seconds during big OTLP batches, and the main
-			-- daemon's retention passes can do the same. 15s gives either
-			-- side enough slack to serialise instead of erroring.
-			PRAGMA busy_timeout = 15000;
-			-- WAL checkpoint automatically when it grows past ~16MB. Without
-			-- this the WAL happily runs into the hundreds of MB and queries
-			-- start paying the cost of walking the WAL on every read.
-			PRAGMA wal_autocheckpoint = 4000;
-			-- Bump cache above the 2MB default. 64MB fits most hot index pages
-			-- (trace_summaries, spans, span_attributes indexes) in RAM even on
-			-- multi-GB databases, cutting cold-read latency meaningfully on
-			-- picker / search queries that sweep the index.
-			PRAGMA cache_size = -65536;
-			-- Let SQLite memory-map the first 256MB of the file. This is a
-			-- cheap way to avoid read() syscalls on hot pages and lets the OS
-			-- page cache serve index lookups directly. Safe on macOS and Linux;
-			-- SQLite silently caps at actual file size for smaller DBs.
-			PRAGMA mmap_size = 268435456;
-
-			CREATE TABLE IF NOT EXISTS spans (
-				trace_id TEXT NOT NULL,
-				span_id TEXT NOT NULL,
-				parent_span_id TEXT,
-				service_name TEXT NOT NULL,
-				scope_name TEXT,
-				operation_name TEXT NOT NULL,
-				kind TEXT,
-				start_time_ms INTEGER NOT NULL,
-				end_time_ms INTEGER NOT NULL,
-				duration_ms REAL NOT NULL,
-				status TEXT NOT NULL,
-				attributes_json TEXT NOT NULL,
-				resource_json TEXT NOT NULL,
-				events_json TEXT NOT NULL,
-				PRIMARY KEY (trace_id, span_id)
-			);
-
-			CREATE INDEX IF NOT EXISTS idx_spans_service_time ON spans(service_name, start_time_ms DESC);
-			CREATE INDEX IF NOT EXISTS idx_spans_trace_time ON spans(trace_id, start_time_ms ASC);
-			CREATE INDEX IF NOT EXISTS idx_spans_span_id ON spans(span_id);
-			CREATE INDEX IF NOT EXISTS idx_spans_status_time ON spans(status, start_time_ms DESC);
-
-			CREATE TABLE IF NOT EXISTS logs (
-				id INTEGER PRIMARY KEY AUTOINCREMENT,
-				trace_id TEXT,
-				span_id TEXT,
-				service_name TEXT NOT NULL,
-				scope_name TEXT,
-				severity_text TEXT NOT NULL,
-				timestamp_ms INTEGER NOT NULL,
-				body TEXT NOT NULL,
-				attributes_json TEXT NOT NULL,
-				resource_json TEXT NOT NULL
-			);
-
-			CREATE INDEX IF NOT EXISTS idx_logs_service_time ON logs(service_name, timestamp_ms DESC);
-			CREATE INDEX IF NOT EXISTS idx_logs_trace_time ON logs(trace_id, timestamp_ms DESC);
-			CREATE INDEX IF NOT EXISTS idx_logs_span_time ON logs(span_id, timestamp_ms DESC);
-			CREATE INDEX IF NOT EXISTS idx_logs_severity_time ON logs(severity_text, timestamp_ms DESC);
-
-			CREATE TABLE IF NOT EXISTS trace_summaries (
-				trace_id TEXT PRIMARY KEY,
-				service_name TEXT NOT NULL,
-				root_operation_name TEXT NOT NULL,
-				started_at_ms INTEGER NOT NULL,
-				ended_at_ms INTEGER NOT NULL,
-				active_span_count INTEGER NOT NULL DEFAULT 0,
-				duration_ms REAL NOT NULL,
-				span_count INTEGER NOT NULL,
-				error_count INTEGER NOT NULL
-			);
-
-			CREATE INDEX IF NOT EXISTS idx_trace_summaries_started_at ON trace_summaries(started_at_ms DESC, trace_id DESC);
-			CREATE INDEX IF NOT EXISTS idx_trace_summaries_service_started_at ON trace_summaries(service_name, started_at_ms DESC, trace_id DESC);
-			CREATE INDEX IF NOT EXISTS idx_trace_summaries_duration ON trace_summaries(duration_ms DESC);
-
-			CREATE TABLE IF NOT EXISTS span_attributes (
-				trace_id TEXT NOT NULL,
-				span_id TEXT NOT NULL,
-				key TEXT NOT NULL,
-				value TEXT NOT NULL,
-				PRIMARY KEY (trace_id, span_id, key)
-			);
-
-			CREATE INDEX IF NOT EXISTS idx_span_attributes_key_value ON span_attributes(key, value, trace_id, span_id);
-			CREATE INDEX IF NOT EXISTS idx_span_attributes_trace_span ON span_attributes(trace_id, span_id);
-
-			CREATE TABLE IF NOT EXISTS log_attributes (
-				log_id INTEGER NOT NULL,
-				key TEXT NOT NULL,
-				value TEXT NOT NULL,
-				PRIMARY KEY (log_id, key)
-			);
-
-			CREATE INDEX IF NOT EXISTS idx_log_attributes_key_value ON log_attributes(key, value, log_id);
-			CREATE INDEX IF NOT EXISTS idx_log_attributes_log_id ON log_attributes(log_id);
-		`)
+			db.exec(`
+				-- Bump cache above the 2MB default. 64MB fits most hot index pages
+				-- (trace_summaries, spans, span_attributes indexes) in RAM even on
+				-- multi-GB databases, cutting cold-read latency meaningfully on
+				-- picker / search queries that sweep the index.
+				PRAGMA cache_size = -65536;
+				-- Let SQLite memory-map the first 256MB of the file. This is a
+				-- cheap way to avoid read() syscalls on hot pages and lets the OS
+				-- page cache serve index lookups directly. Safe on macOS and Linux;
+				-- SQLite silently caps at actual file size for smaller DBs.
+				PRAGMA mmap_size = 268435456;
+			`)
+			try {
+				db.exec(`
+					PRAGMA journal_mode = WAL;
+					PRAGMA synchronous = NORMAL;
+					PRAGMA temp_store = MEMORY;
+					-- WAL checkpoint automatically when it grows past ~16MB. Without
+					-- this the WAL happily runs into the hundreds of MB and queries
+					-- start paying the cost of walking the WAL on every read.
+					PRAGMA wal_autocheckpoint = 4000;
+
+					CREATE TABLE IF NOT EXISTS spans (
+						trace_id TEXT NOT NULL,
+						span_id TEXT NOT NULL,
+						parent_span_id TEXT,
+						service_name TEXT NOT NULL,
+						scope_name TEXT,
+						operation_name TEXT NOT NULL,
+						kind TEXT,
+						start_time_ms INTEGER NOT NULL,
+						end_time_ms INTEGER NOT NULL,
+						duration_ms REAL NOT NULL,
+						status TEXT NOT NULL,
+						attributes_json TEXT NOT NULL,
+						resource_json TEXT NOT NULL,
+						events_json TEXT NOT NULL,
+						PRIMARY KEY (trace_id, span_id)
+					);
+
+					CREATE INDEX IF NOT EXISTS idx_spans_service_time ON spans(service_name, start_time_ms DESC);
+					CREATE INDEX IF NOT EXISTS idx_spans_trace_time ON spans(trace_id, start_time_ms ASC);
+					CREATE INDEX IF NOT EXISTS idx_spans_span_id ON spans(span_id);
+					CREATE INDEX IF NOT EXISTS idx_spans_status_time ON spans(status, start_time_ms DESC);
+
+					CREATE TABLE IF NOT EXISTS logs (
+						id INTEGER PRIMARY KEY AUTOINCREMENT,
+						trace_id TEXT,
+						span_id TEXT,
+						service_name TEXT NOT NULL,
+						scope_name TEXT,
+						severity_text TEXT NOT NULL,
+						timestamp_ms INTEGER NOT NULL,
+						body TEXT NOT NULL,
+						attributes_json TEXT NOT NULL,
+						resource_json TEXT NOT NULL
+					);
+
+					CREATE INDEX IF NOT EXISTS idx_logs_service_time ON logs(service_name, timestamp_ms DESC);
+					CREATE INDEX IF NOT EXISTS idx_logs_trace_time ON logs(trace_id, timestamp_ms DESC);
+					CREATE INDEX IF NOT EXISTS idx_logs_span_time ON logs(span_id, timestamp_ms DESC);
+					CREATE INDEX IF NOT EXISTS idx_logs_severity_time ON logs(severity_text, timestamp_ms DESC);
+
+					CREATE TABLE IF NOT EXISTS trace_summaries (
+						trace_id TEXT PRIMARY KEY,
+						service_name TEXT NOT NULL,
+						root_operation_name TEXT NOT NULL,
+						started_at_ms INTEGER NOT NULL,
+						ended_at_ms INTEGER NOT NULL,
+						active_span_count INTEGER NOT NULL DEFAULT 0,
+						duration_ms REAL NOT NULL,
+						span_count INTEGER NOT NULL,
+						error_count INTEGER NOT NULL
+					);
+
+					CREATE INDEX IF NOT EXISTS idx_trace_summaries_started_at ON trace_summaries(started_at_ms DESC, trace_id DESC);
+					CREATE INDEX IF NOT EXISTS idx_trace_summaries_service_started_at ON trace_summaries(service_name, started_at_ms DESC, trace_id DESC);
+					CREATE INDEX IF NOT EXISTS idx_trace_summaries_duration ON trace_summaries(duration_ms DESC);
+
+					CREATE TABLE IF NOT EXISTS span_attributes (
+						trace_id TEXT NOT NULL,
+						span_id TEXT NOT NULL,
+						key TEXT NOT NULL,
+						value TEXT NOT NULL,
+						PRIMARY KEY (trace_id, span_id, key)
+					);
+
+					CREATE INDEX IF NOT EXISTS idx_span_attributes_key_value ON span_attributes(key, value, trace_id, span_id);
+					CREATE INDEX IF NOT EXISTS idx_span_attributes_trace_span ON span_attributes(trace_id, span_id);
+
+					CREATE TABLE IF NOT EXISTS log_attributes (
+						log_id INTEGER NOT NULL,
+						key TEXT NOT NULL,
+						value TEXT NOT NULL,
+						PRIMARY KEY (log_id, key)
+					);
+
+					CREATE INDEX IF NOT EXISTS idx_log_attributes_key_value ON log_attributes(key, value, log_id);
+					CREATE INDEX IF NOT EXISTS idx_log_attributes_log_id ON log_attributes(log_id);
+				`)
+			} catch (err) {
+				if (!isSqliteLockError(err)) throw err
+				console.warn(`motel: writer bootstrap skipped during startup: ${(err as Error).message}`)
+			}
 		}
 
 		// Tables detected at runtime. For writer connections these flags are
@@ -726,6 +731,12 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			// ANALYZE / optimize failures are never fatal — queries still work,
 			// they just run with default row estimates.
 		}
+			// Longer busy timeout: the ingest worker holds the write lock for up
+			// to a few seconds during big OTLP batches, and the daemon's retention
+			// passes can do the same. Apply this AFTER startup maintenance so
+			// lock-conflicted bootstrap steps fail fast instead of stalling health
+			// for the full 15s timeout.
+			try { db.exec(`PRAGMA busy_timeout = 15000;`) } catch { /* ignore */ }
 		} // end: if (!opts.readonly) writer init
 
 		const insertSpan = db.query(`
@@ -774,21 +785,94 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			GROUP BY trace_id
 		`)
 
-		// One-time full rebuild of the trace_summaries table at open so
-		// any drift from interrupted ingests gets reconciled. Writer-only
-		// because the DELETE + INSERT would fail on a readonly connection
-		// (and would fight the daemon's writer for the lock anyway).
-		if (!opts.readonly) {
-			db.query(`DELETE FROM trace_summaries`).run()
-			rebuildTraceSummaries.run()
-		}
+		const reconcileTraceSummaries = Effect.sync(() => {
+			try {
+				db.query(`DELETE FROM trace_summaries`).run()
+				rebuildTraceSummaries.run()
+			} catch (err) {
+				if (!isSqliteLockError(err)) throw err
+				console.warn(`motel: trace summary rebuild skipped during startup: ${(err as Error).message}`)
+			}
+		})
 
 		const deleteSpanAttributes = db.query(`DELETE FROM span_attributes WHERE trace_id = ? AND span_id = ?`)
 		const insertSpanAttribute = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES (?, ?, ?, ?)`)
+		const spanAttributeInsertManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const insertSpanAttributesMany = (traceId: string, spanId: string, attributes: Readonly<Record<string, string>>) => {
+			const entries = Object.entries(attributes)
+			if (entries.length === 0) return
+			if (entries.length === 1) {
+				const [key, value] = entries[0]!
+				insertSpanAttribute.run(traceId, spanId, key, value)
+				return
+			}
+			let query = spanAttributeInsertManyByCount.get(entries.length)
+			if (!query) {
+				query = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES ${entries.map(() => "(?, ?, ?, ?)").join(", ")}`)
+				spanAttributeInsertManyByCount.set(entries.length, query)
+			}
+			query.run(...entries.flatMap(([key, value]) => [traceId, spanId, key, value]))
+		}
 		const deleteSpanOperationSearch = db.query(`DELETE FROM span_operation_fts WHERE trace_id = ? AND span_id = ?`)
 		const insertSpanOperationSearch = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES (?, ?, ?)`)
+		const deleteSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const insertSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const updateSpanOperationSearchMany = (operations: ReadonlyArray<readonly [string, string, string]>) => {
+			if (operations.length === 0) return
+			if (operations.length === 1) {
+				const [traceId, spanId, operationName] = operations[0]!
+				deleteSpanOperationSearch.run(traceId, spanId)
+				insertSpanOperationSearch.run(traceId, spanId, operationName)
+				return
+			}
+
+			let deleteQuery = deleteSpanOperationSearchManyByCount.get(operations.length)
+			if (!deleteQuery) {
+				deleteQuery = db.query(`DELETE FROM span_operation_fts WHERE ${operations.map(() => "(trace_id = ? AND span_id = ?)").join(" OR ")}`)
+				deleteSpanOperationSearchManyByCount.set(operations.length, deleteQuery)
+			}
+			deleteQuery.run(...operations.flatMap(([traceId, spanId]) => [traceId, spanId]))
+
+			let insertQuery = insertSpanOperationSearchManyByCount.get(operations.length)
+			if (!insertQuery) {
+				insertQuery = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES ${operations.map(() => "(?, ?, ?)").join(", ")}`)
+				insertSpanOperationSearchManyByCount.set(operations.length, insertQuery)
+			}
+			insertQuery.run(...operations.flatMap(([traceId, spanId, operationName]) => [traceId, spanId, operationName]))
+		}
 		const insertLogAttribute = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES (?, ?, ?)`)
+		const logAttributeInsertManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const insertLogAttributesMany = (logId: number, attributes: Readonly<Record<string, string>>) => {
+			const entries = Object.entries(attributes)
+			if (entries.length === 0) return
+			if (entries.length === 1) {
+				const [key, value] = entries[0]!
+				insertLogAttribute.run(logId, key, value)
+				return
+			}
+			let query = logAttributeInsertManyByCount.get(entries.length)
+			if (!query) {
+				query = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES ${entries.map(() => "(?, ?, ?)").join(", ")}`)
+				logAttributeInsertManyByCount.set(entries.length, query)
+			}
+			query.run(...entries.flatMap(([key, value]) => [logId, key, value]))
+		}
 		const insertLogBodySearch = db.query(`INSERT INTO log_body_fts (log_id, body) VALUES (?, ?)`)
+		const insertLogBodySearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const insertLogBodySearchMany = (entries: ReadonlyArray<readonly [string, string]>) => {
+			if (entries.length === 0) return
+			if (entries.length === 1) {
+				const [logId, body] = entries[0]!
+				insertLogBodySearch.run(logId, body)
+				return
+			}
+			let query = insertLogBodySearchManyByCount.get(entries.length)
+			if (!query) {
+				query = db.query(`INSERT INTO log_body_fts (log_id, body) VALUES ${entries.map(() => "(?, ?)").join(", ")}`)
+				insertLogBodySearchManyByCount.set(entries.length, query)
+			}
+			query.run(...entries.flatMap(([logId, body]) => [logId, body]))
+		}
 
 		const maxDbSizeBytes = config.otel.maxDbSizeMb * 1024 * 1024
 
@@ -887,6 +971,12 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 		// daemon). The ingest worker and TUI skip it to avoid two writers
 		// competing for the write lock with overlapping DELETE passes.
 		if (opts.runRetention) {
+			// Reconcile any summary drift from interrupted ingests, but do it
+			// after the server becomes healthy. Running this synchronously at
+			// open can sit behind another writer's lock for ~15s and make the
+			// daemon look hung even though the port is already bound.
+			yield* Effect.forkScoped(reconcileTraceSummaries)
+
 			// Enable incremental vacuum so retention can reclaim freed
 			// pages over time instead of needing a stop-the-world VACUUM.
 			// Idempotent: repeat calls after the first are no-ops.
@@ -943,6 +1033,7 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				let insertedSpans = 0
 				const transaction = db.transaction((request: OtlpTraceExportRequest) => {
 					const touchedTraceIds = new Set<string>()
+					const touchedOperations: Array<readonly [string, string, string]> = []
 					for (const resourceSpans of request.resourceSpans ?? []) {
 						const resourceAttributes = attributeMap(resourceSpans.resource?.attributes)
 						const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
@@ -978,20 +1069,21 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 									JSON.stringify(events),
 								)
 								deleteSpanAttributes.run(span.traceId, span.spanId)
-								for (const [key, value] of Object.entries(mergedAttributes)) {
-									insertSpanAttribute.run(span.traceId, span.spanId, key, value)
-								}
-								try {
-									deleteSpanOperationSearch.run(span.traceId, span.spanId)
-									insertSpanOperationSearch.run(span.traceId, span.spanId, span.name ?? "unknown")
-								} catch {
-									// FTS is optional.
-								}
+								insertSpanAttributesMany(span.traceId, span.spanId, mergedAttributes)
+								touchedOperations.push([span.traceId, span.spanId, span.name ?? "unknown"])
 								touchedTraceIds.add(span.traceId)
 								insertedSpans += 1
 							}
 						}
 					}
+					try {
+						const BATCH_SIZE = 500
+						for (let offset = 0; offset < touchedOperations.length; offset += BATCH_SIZE) {
+							updateSpanOperationSearchMany(touchedOperations.slice(offset, offset + BATCH_SIZE))
+						}
+					} catch {
+						// FTS is optional.
+					}
 					for (const traceId of touchedTraceIds) {
 						upsertTraceSummary.run(traceId)
 					}
@@ -1006,6 +1098,7 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			return yield* Effect.sync(() => {
 				let insertedLogs = 0
 				const transaction = db.transaction((request: OtlpLogExportRequest) => {
+					const touchedLogBodies: Array<readonly [string, string]> = []
 					for (const resourceLogs of request.resourceLogs ?? []) {
 						const resourceAttributes = attributeMap(resourceLogs.resource?.attributes)
 						const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
@@ -1030,18 +1123,20 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 									JSON.stringify(resourceAttributes),
 								)
 								const logId = Number((result as { lastInsertRowid: number | bigint }).lastInsertRowid)
-								for (const [key, value] of Object.entries(mergedAttributes)) {
-									insertLogAttribute.run(logId, key, value)
-								}
-								try {
-									insertLogBodySearch.run(String(logId), body)
-								} catch {
-									// FTS is optional.
-								}
+								insertLogAttributesMany(logId, mergedAttributes)
+								touchedLogBodies.push([String(logId), body])
 								insertedLogs += 1
 							}
 						}
 					}
+					try {
+						const BATCH_SIZE = 500
+						for (let offset = 0; offset < touchedLogBodies.length; offset += BATCH_SIZE) {
+							insertLogBodySearchMany(touchedLogBodies.slice(offset, offset + BATCH_SIZE))
+						}
+					} catch {
+						// FTS is optional.
+					}
 				})
 
 				transaction(payload)
@@ -1264,6 +1359,8 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			const candidateLimit = hasContainsFilters ? Math.max(limit * 20, 500) : Math.max(limit * 10, 200)
 
 			return yield* Effect.sync(() => {
+				let fromSql = "FROM spans AS s"
+				const joinParams: Array<string | number> = []
 				const clauses: string[] = ["s.start_time_ms >= ?"]
 				const params: Array<string | number> = [cutoff]
 
@@ -1278,8 +1375,8 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				if (input.operation) {
 					const ftsQuery = toFtsMatchQuery(input.operation)
 					if (hasFts && ftsQuery) {
-						clauses.push("EXISTS (SELECT 1 FROM span_operation_fts WHERE span_operation_fts.trace_id = s.trace_id AND span_operation_fts.span_id = s.span_id AND span_operation_fts MATCH ?)")
-						params.push(ftsQuery)
+						fromSql += ` INNER JOIN (SELECT trace_id, span_id FROM span_operation_fts WHERE span_operation_fts MATCH ?) AS span_operation_match ON span_operation_match.trace_id = s.trace_id AND span_operation_match.span_id = s.span_id`
+						joinParams.push(ftsQuery)
 					} else {
 						clauses.push("s.operation_name LIKE ? COLLATE NOCASE")
 						params.push(`%${input.operation}%`)
@@ -1303,42 +1400,90 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				}
 
 				const rows = db.query(`
-					SELECT trace_id, span_id
-					FROM spans AS s
+					SELECT *
+					${fromSql}
 					WHERE ${clauses.join(" AND ")}
 					ORDER BY s.start_time_ms DESC
 					LIMIT ?
-				`).all(...params, candidateLimit) as Array<{ trace_id: string; span_id: string }>
+				`).all(...joinParams, ...params, candidateLimit) as SpanRow[]
 
 				const traceIds = [...new Set(rows.map((row) => row.trace_id))]
 				if (traceIds.length === 0) return [] as readonly SpanItem[]
 
+				const keyOf = (traceId: string, spanId: string) => `${traceId}:${spanId}`
+				const spanContextById = new Map<string, { readonly parentSpanId: string | null; readonly operationName: string }>()
+				for (const row of rows) {
+					spanContextById.set(keyOf(row.trace_id, row.span_id), {
+						parentSpanId: row.parent_span_id,
+						operationName: row.operation_name,
+					})
+				}
+
 				const placeholders = traceIds.map(() => "?").join(", ")
-				const spanRows = db.query(`
-					SELECT * FROM spans
-					WHERE trace_id IN (${placeholders})
+				const rootRows = db.query(`
+					SELECT trace_id, operation_name
+					FROM spans
+					WHERE trace_id IN (${placeholders}) AND parent_span_id IS NULL
 					ORDER BY start_time_ms ASC
-				`).all(...traceIds) as SpanRow[]
-
-				const grouped = new Map<string, SpanRow[]>()
-				for (const row of spanRows) {
-					const group = grouped.get(row.trace_id) ?? []
-					group.push(row)
-					grouped.set(row.trace_id, group)
+				`).all(...traceIds) as Array<{ trace_id: string; operation_name: string }>
+				const rootOperationByTraceId = new Map<string, string>()
+				for (const row of rootRows) {
+					if (!rootOperationByTraceId.has(row.trace_id)) {
+						rootOperationByTraceId.set(row.trace_id, row.operation_name)
+					}
 				}
 
-				const itemById = new Map<string, SpanItem>()
-				for (const traceId of traceIds) {
-					const traceSpanRows = grouped.get(traceId)
-					if (!traceSpanRows) continue
-					for (const item of buildSpanItems(traceId, traceSpanRows)) {
-						itemById.set(`${item.traceId}:${item.span.spanId}`, item)
+				const spanContextLookup = db.query(`
+					SELECT parent_span_id, operation_name
+					FROM spans
+					WHERE trace_id = ? AND span_id = ?
+				`)
+
+				const getSpanContext = (traceId: string, spanId: string) => {
+					const key = keyOf(traceId, spanId)
+					const cached = spanContextById.get(key)
+					if (cached !== undefined) return cached
+					const row = spanContextLookup.get(traceId, spanId) as { parent_span_id: string | null; operation_name: string } | null
+					if (!row) return null
+					const value = {
+						parentSpanId: row.parent_span_id,
+						operationName: row.operation_name,
 					}
+					spanContextById.set(key, value)
+					return value
+				}
+
+				const depthById = new Map<string, number>()
+				const getDepth = (traceId: string, spanId: string, visiting = new Set<string>()): number => {
+					const key = keyOf(traceId, spanId)
+					const cached = depthById.get(key)
+					if (cached !== undefined) return cached
+					if (visiting.has(key)) return 0
+					visiting.add(key)
+					const context = getSpanContext(traceId, spanId)
+					const depth = context?.parentSpanId ? getDepth(traceId, context.parentSpanId, visiting) + 1 : 0
+					depthById.set(key, depth)
+					return depth
 				}
 
 				return rows
-					.map((row) => itemById.get(`${row.trace_id}:${row.span_id}`))
-					.filter((item): item is SpanItem => item !== undefined)
+					.map((row) => {
+						const parentContext = row.parent_span_id ? getSpanContext(row.trace_id, row.parent_span_id) : null
+						const parsedSpan = parseSpanRow(row)
+						const span = {
+							...parsedSpan,
+							depth: getDepth(row.trace_id, row.span_id),
+							warnings: row.parent_span_id && !parentContext
+								? [`missing span ${row.parent_span_id} (1 child)`]
+								: parsedSpan.warnings,
+						}
+						return {
+							traceId: row.trace_id,
+							rootOperationName: rootOperationByTraceId.get(row.trace_id) ?? span.operationName,
+							parentOperationName: parentContext?.operationName ?? null,
+							span,
+						} satisfies SpanItem
+					})
 					.filter((item) => {
 						if (input.parentOperation) {
 							const needle = input.parentOperation.toLowerCase()
@@ -1648,7 +1793,6 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 		})
 
 		const listFacets = Effect.fn("motel/TelemetryStore.listFacets")(function* (input: FacetSearch) {
-
 			const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
 			const limit = input.limit ?? 20
 
@@ -1738,21 +1882,26 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 						// FACET_VALUE_MAX_LEN. For opencode this hides `ai.prompt`,
 						// `ai.prompt.messages`, and `ai.prompt.tools` — which are 1-6MB text
 						// blobs that you'd never want to filter by exact match anyway. The
-						// WHERE clause lets SQLite skip reading those pages from disk, taking
-						// the picker open time from ~1.2s to ~370ms on a 2GB database.
+						// WHERE clause lets SQLite skip reading those pages from disk. We also
+						// dedupe to one (trace, key, value) row before grouping so repeated
+						// span-level duplicates don't blow up the temp B-trees used for the
+						// picker ranking query.
 						const params: Array<string | number> = [FACET_VALUE_MAX_LEN, cutoff]
 						if (input.serviceName) params.push(input.serviceName)
 						params.push(limit)
 						const rows = db.query(`
-							SELECT sa.key AS value,
-							       COUNT(DISTINCT sa.trace_id) AS count,
-							       COUNT(DISTINCT sa.value) AS distinct_values
-							FROM span_attributes sa
-							JOIN spans s ON s.trace_id = sa.trace_id AND s.span_id = sa.span_id
-							WHERE LENGTH(sa.value) < ?
-							  AND s.start_time_ms >= ?
-							${input.serviceName ? "AND s.service_name = ?" : ""}
-							GROUP BY sa.key
+							SELECT scoped.key AS value,
+							       COUNT(DISTINCT scoped.trace_id) AS count,
+							       COUNT(DISTINCT scoped.value) AS distinct_values
+							FROM (
+								SELECT DISTINCT sa.trace_id, sa.key, sa.value
+								FROM span_attributes sa
+								JOIN trace_summaries ts ON ts.trace_id = sa.trace_id
+								WHERE LENGTH(sa.value) < ?
+								  AND ts.started_at_ms >= ?
+								  ${input.serviceName ? "AND ts.service_name = ?" : ""}
+							) AS scoped
+							GROUP BY scoped.key
 							ORDER BY (CASE WHEN distinct_values = 1 THEN 1 ELSE 0 END) ASC,
 							         distinct_values DESC,
 							         count DESC,

package/src/services/TraceQueryService.ts

@@ -6,9 +6,9 @@ export class TraceQueryService extends Context.Service<
 	TraceQueryService,
 	{
 		readonly listServices: Effect.Effect<readonly string[], Error>
-		readonly listRecentTraces: (serviceName: string, options?: { readonly lookbackMinutes?: number; readonly limit?: number }) => Effect.Effect<readonly TraceItem[], Error>
-		readonly listTraceSummaries: (serviceName: string, options?: { readonly lookbackMinutes?: number; readonly limit?: number }) => Effect.Effect<readonly TraceSummaryItem[], Error>
-		readonly searchTraceSummaries: (input: { readonly serviceName?: string | null; readonly operation?: string | null; readonly status?: "ok" | "error" | null; readonly minDurationMs?: number | null; readonly lookbackMinutes?: number; readonly limit?: number; readonly attributeFilters?: Readonly<Record<string, string>>; readonly aiText?: string | null }) => Effect.Effect<readonly TraceSummaryItem[], Error>
+		readonly listRecentTraces: (serviceName: string, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceItem[], Error>
+		readonly listTraceSummaries: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceSummaryItem[], Error>
+		readonly searchTraceSummaries: (input: { readonly serviceName?: string | null; readonly operation?: string | null; readonly status?: "ok" | "error" | null; readonly minDurationMs?: number | null; readonly lookbackMinutes?: number; readonly limit?: number; readonly attributeFilters?: Readonly<Record<string, string>>; readonly aiText?: string | null; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceSummaryItem[], Error>
 		readonly listFacets: (input: { readonly type: "traces" | "logs"; readonly field: string; readonly serviceName?: string | null; readonly key?: string | null; readonly lookbackMinutes?: number; readonly limit?: number }) => Effect.Effect<readonly { readonly value: string; readonly count: number }[], Error>
 		readonly searchTraces: (input: { readonly serviceName?: string | null; readonly operation?: string | null; readonly status?: "ok" | "error" | null; readonly minDurationMs?: number | null; readonly lookbackMinutes?: number; readonly limit?: number; readonly attributeFilters?: Readonly<Record<string, string>> }) => Effect.Effect<readonly TraceItem[], Error>
 		readonly traceStats: (input: { readonly groupBy: string; readonly agg: "count" | "avg_duration" | "p95_duration" | "error_rate"; readonly serviceName?: string | null; readonly operation?: string | null; readonly status?: "ok" | "error" | null; readonly minDurationMs?: number | null; readonly lookbackMinutes?: number; readonly limit?: number; readonly attributeFilters?: Readonly<Record<string, string>> }) => Effect.Effect<readonly { readonly group: string; readonly value: number; readonly count: number }[], Error>
@@ -16,7 +16,7 @@ export class TraceQueryService extends Context.Service<
 		readonly getSpan: (spanId: string) => Effect.Effect<SpanItem | null, Error>
 		readonly getAiCall: (spanId: string) => Effect.Effect<AiCallDetail | null, Error>
 		readonly listTraceSpans: (traceId: string) => Effect.Effect<readonly SpanItem[], Error>
-		readonly searchSpans: (input: { readonly serviceName?: string | null; readonly operation?: string | null; readonly parentOperation?: string | null; readonly status?: "ok" | "error" | null; readonly lookbackMinutes?: number; readonly limit?: number; readonly attributeFilters?: Readonly<Record<string, string>> }) => Effect.Effect<readonly SpanItem[], Error>
+		readonly searchSpans: (input: { readonly serviceName?: string | null; readonly traceId?: string | null; readonly operation?: string | null; readonly parentOperation?: string | null; readonly status?: "ok" | "error" | null; readonly lookbackMinutes?: number; readonly limit?: number; readonly attributeFilters?: Readonly<Record<string, string>>; readonly attributeContainsFilters?: Readonly<Record<string, string>> }) => Effect.Effect<readonly SpanItem[], Error>
 	}
 >()("motel/TraceQueryService") {}
 
@@ -31,7 +31,7 @@ export const TraceQueryServiceLive = Layer.effect(
 			return services
 		})()
 
-		const listRecentTraces = Effect.fn("motel/TraceQueryService.listRecentTraces")(function* (serviceName: string, options?: { readonly lookbackMinutes?: number; readonly limit?: number }) {
+		const listRecentTraces = Effect.fn("motel/TraceQueryService.listRecentTraces")(function* (serviceName: string, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) {
 			yield* Effect.annotateCurrentSpan({
 				"trace.service_name": serviceName,
 			})
@@ -40,7 +40,7 @@ export const TraceQueryServiceLive = Layer.effect(
 			return traces
 		})
 
-		const listTraceSummaries = Effect.fn("motel/TraceQueryService.listTraceSummaries")(function* (serviceName: string, options?: { readonly lookbackMinutes?: number; readonly limit?: number }) {
+		const listTraceSummaries = Effect.fn("motel/TraceQueryService.listTraceSummaries")(function* (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) {
 			yield* Effect.annotateCurrentSpan({
 				"trace.service_name": serviceName,
 			})