@kitlangton/motel 0.2.4 → 0.2.6

package/src/services/TelemetryStore.ts

@@ -1,11 +1,11 @@
 import { Database } from "bun:sqlite"
-import { mkdirSync } from "node:fs"
+import * as BunFileSystem from "@effect/platform-bun/BunFileSystem"
 import { dirname } from "node:path"
-import { Clock, Effect, Layer, Schedule, Context } from "effect"
+import { Cause, Clock, Effect, FileSystem, Layer, Schedule, Context } from "effect"
 import { config } from "../config.js"
 import type { AiCallDetail, AiCallSummary, FacetItem, LogItem, SpanItem, StatsItem, TraceItem, TraceSummaryItem, TraceSpanEvent, TraceSpanItem } from "../domain.js"
 import { AI_ATTR_MAP, AI_FTS_KEYS, AI_TEXT_SEARCH_KEYS, truncatePreview } from "../domain.js"
-import { attributeMap, nanosToMilliseconds, parseAnyValue, spanKindLabel, spanStatusLabel, stringifyValue, type OtlpLogExportRequest, type OtlpTraceExportRequest } from "../otlp.js"
+import { attributeMap, nanosToMilliseconds, normalizeOtlpBinaryId, parseAnyValue, spanKindLabel, spanStatusLabel, stringifyValue, type OtlpLogExportRequest, type OtlpTraceExportRequest } from "../otlp.js"
 
 const isSqliteLockError = (error: unknown) =>
 	error instanceof Error && /(database is locked|database table is locked|SQLITE_BUSY)/i.test(error.message)
@@ -196,18 +196,40 @@ const TRACE_SUMMARY_SELECT_SQL = `
 	FROM spans
 `
 
+// Memoize small repeated JSON records. Resource attributes are the primary
+// beneficiary because many spans share the same serialized value; compact
+// repeated span attributes also benefit while large unique payloads bypass
+// the cache to keep memory bounded for long-running daemons.
+const RECORD_PARSE_CACHE_MAX_VALUE_LEN = 1024
+const RECORD_PARSE_CACHE_LIMIT = 256
+const recordParseCache = new Map<string, Record<string, string>>()
+const EMPTY_RECORD: Record<string, string> = {}
+
 const parseRecord = (value: string): Record<string, string> => {
+	if (value === "" || value === "{}") return EMPTY_RECORD
+	const cacheable = value.length <= RECORD_PARSE_CACHE_MAX_VALUE_LEN
+	if (cacheable) {
+		const cached = recordParseCache.get(value)
+		if (cached !== undefined) return cached
+	}
+	let parsed: Record<string, string>
 	try {
-		const parsed = JSON.parse(value) as Record<string, unknown>
-		return Object.fromEntries(Object.entries(parsed).map(([key, entry]) => [key, stringifyValue(entry)]))
+		const json = JSON.parse(value) as Record<string, unknown>
+		parsed = Object.fromEntries(Object.entries(json).map(([key, entry]) => [key, stringifyValue(entry)]))
 	} catch {
-		return {}
+		parsed = EMPTY_RECORD
 	}
+	if (cacheable && recordParseCache.size < RECORD_PARSE_CACHE_LIMIT) {
+		recordParseCache.set(value, parsed)
+	}
+	return parsed
 }
 
 const parseEvents = (value: string): readonly TraceSpanEvent[] => {
+	if (value === "" || value === "[]") return []
 	try {
 		const parsed = JSON.parse(value) as Array<{ name: string; timestamp: number; attributes: Record<string, string> }>
+		if (parsed.length === 0) return []
 		return parsed.map((event) => ({
 			name: event.name,
 			timestamp: new Date(event.timestamp),
@@ -423,29 +445,41 @@ const buildContainsAttributeMatchSubquery = (
 	}
 }
 
+// Read-only surface of the telemetry store. Pulled out so a readonly
+// SQLite connection (TUI / HTTP query handlers) can be expressed as a
+// distinct service identifier from the writer, without re-declaring
+// every query in a wrapper layer. The writer's value still satisfies
+// this shape — TelemetryStoreLive can provide both identifiers from
+// one underlying object if needed.
+export interface TelemetryStoreReader {
+	readonly listServices: Effect.Effect<readonly string[], Error>
+	readonly listRecentTraces: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceItem[], Error>
+	readonly listTraceSummaries: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceSummaryItem[], Error>
+	readonly searchTraces: (input: TraceSearch) => Effect.Effect<readonly TraceItem[], Error>
+	readonly searchTraceSummaries: (input: TraceSearch) => Effect.Effect<readonly TraceSummaryItem[], Error>
+	readonly traceStats: (input: TraceStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
+	readonly getTrace: (traceId: string) => Effect.Effect<TraceItem | null, Error>
+	readonly getSpan: (spanId: string) => Effect.Effect<SpanItem | null, Error>
+	readonly listTraceSpans: (traceId: string) => Effect.Effect<readonly SpanItem[], Error>
+	readonly searchSpans: (input: SpanSearch) => Effect.Effect<readonly SpanItem[], Error>
+	readonly searchLogs: (input: LogSearch) => Effect.Effect<readonly LogItem[], Error>
+	readonly logStats: (input: LogStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
+	readonly listFacets: (input: FacetSearch) => Effect.Effect<readonly FacetItem[], Error>
+	readonly listRecentLogs: (serviceName: string) => Effect.Effect<readonly LogItem[], Error>
+	readonly listTraceLogs: (traceId: string) => Effect.Effect<readonly LogItem[], Error>
+	readonly searchAiCalls: (input: AiCallSearch) => Effect.Effect<readonly AiCallSummary[], Error>
+	readonly getAiCall: (spanId: string) => Effect.Effect<AiCallDetail | null, Error>
+	readonly aiCallStats: (input: AiCallStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
+}
+
+export class TelemetryStoreReadonly extends Context.Service<TelemetryStoreReadonly, TelemetryStoreReader>()("motel/TelemetryStoreReadonly") {}
+
 export class TelemetryStore extends Context.Service<
 	TelemetryStore,
-	{
+	TelemetryStoreReader & {
 		readonly ingestTraces: (payload: OtlpTraceExportRequest) => Effect.Effect<{ readonly insertedSpans: number }, Error>
 		readonly ingestLogs: (payload: OtlpLogExportRequest) => Effect.Effect<{ readonly insertedLogs: number }, Error>
-		readonly listServices: Effect.Effect<readonly string[], Error>
-		readonly listRecentTraces: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceItem[], Error>
-		readonly listTraceSummaries: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceSummaryItem[], Error>
-		readonly searchTraces: (input: TraceSearch) => Effect.Effect<readonly TraceItem[], Error>
-		readonly searchTraceSummaries: (input: TraceSearch) => Effect.Effect<readonly TraceSummaryItem[], Error>
-		readonly traceStats: (input: TraceStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
-		readonly getTrace: (traceId: string) => Effect.Effect<TraceItem | null, Error>
-		readonly getSpan: (spanId: string) => Effect.Effect<SpanItem | null, Error>
-		readonly listTraceSpans: (traceId: string) => Effect.Effect<readonly SpanItem[], Error>
-		readonly searchSpans: (input: SpanSearch) => Effect.Effect<readonly SpanItem[], Error>
-		readonly searchLogs: (input: LogSearch) => Effect.Effect<readonly LogItem[], Error>
-		readonly logStats: (input: LogStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
-		readonly listFacets: (input: FacetSearch) => Effect.Effect<readonly FacetItem[], Error>
-		readonly listRecentLogs: (serviceName: string) => Effect.Effect<readonly LogItem[], Error>
-		readonly listTraceLogs: (traceId: string) => Effect.Effect<readonly LogItem[], Error>
-		readonly searchAiCalls: (input: AiCallSearch) => Effect.Effect<readonly AiCallSummary[], Error>
-		readonly getAiCall: (spanId: string) => Effect.Effect<AiCallDetail | null, Error>
-		readonly aiCallStats: (input: AiCallStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
+		readonly runRetentionNow: Effect.Effect<void, Error>
 	}
 >()("motel/TelemetryStore") {}
 
@@ -462,18 +496,17 @@ export class TelemetryStore extends Context.Service<
  *
  * - `runRetention` — fork the background cleanup loop (age + size cap
  *   eviction, WAL checkpoint). Only one process should own this at a
- *   time. Currently the main daemon (localServer) does; the ingest
- *   worker and the TUI skip it.
+ *   time. The ingest worker owns it; the HTTP thread and TUI skip it.
  */
 export interface TelemetryStoreOptions {
 	readonly readonly: boolean
 	readonly runRetention: boolean
 }
 
-export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.effect(
-	TelemetryStore,
+const makeTelemetryStoreEffect = (opts: TelemetryStoreOptions) =>
 	Effect.gen(function* () {
-		mkdirSync(dirname(config.otel.databasePath), { recursive: true })
+		const fileSystem = yield* FileSystem.FileSystem
+		yield* fileSystem.makeDirectory(dirname(config.otel.databasePath), { recursive: true })
 		const db = yield* Effect.acquireRelease(
 			Effect.sync(() => new Database(config.otel.databasePath, {
 				create: !opts.readonly,
@@ -517,6 +550,13 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				-- SQLite silently caps at actual file size for smaller DBs.
 				PRAGMA mmap_size = 268435456;
 			`)
+			// auto_vacuum is a header-level setting: it only takes effect on
+			// an empty DB, or on the next VACUUM after a change. Setting it
+			// here, BEFORE the first CREATE TABLE, is the only path that
+			// makes incremental_vacuum work without a full VACUUM. For
+			// existing DBs that predate this setting keep their current mode;
+			// Motel never performs a surprise full-file VACUUM at startup.
+			try { db.exec(`PRAGMA auto_vacuum = INCREMENTAL;`) } catch { /* ignore */ }
 			try {
 				db.exec(`
 					PRAGMA journal_mode = WAL;
@@ -526,6 +566,13 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 					-- this the WAL happily runs into the hundreds of MB and queries
 					-- start paying the cost of walking the WAL on every read.
 					PRAGMA wal_autocheckpoint = 4000;
+					-- Hard floor for the WAL file. Auto-checkpoint controls *when*
+					-- pages move out of the WAL; size_limit controls how much the
+					-- WAL file is allowed to grow on disk. 128MB is generous enough
+					-- to absorb a long write burst without blocking on truncation,
+					-- tight enough that a wedged retention loop can't hide a 20GB
+					-- WAL the way a default no-limit configuration can.
+					PRAGMA journal_size_limit = 134217728;
 
 					CREATE TABLE IF NOT EXISTS spans (
 						trace_id TEXT NOT NULL,
@@ -604,6 +651,11 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 
 					CREATE INDEX IF NOT EXISTS idx_log_attributes_key_value ON log_attributes(key, value, log_id);
 					CREATE INDEX IF NOT EXISTS idx_log_attributes_log_id ON log_attributes(log_id);
+
+					CREATE TABLE IF NOT EXISTS motel_maintenance (
+						key TEXT PRIMARY KEY,
+						value TEXT NOT NULL
+					);
 				`)
 			} catch (err) {
 				if (!isSqliteLockError(err)) throw err
@@ -624,7 +676,8 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			} catch { hasFts = false }
 			try {
 				const row = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='span_attr_fts'`).get()
-				hasAttrFts = row !== null
+				const backfill = db.query(`SELECT value FROM motel_maintenance WHERE key = 'span_attr_fts_v1'`).get() as { value: string } | null
+				hasAttrFts = row !== null && backfill?.value === "complete"
 			} catch { hasAttrFts = false }
 		}
 
@@ -723,10 +776,6 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 		// pay 3-4s on cold open instead of 400ms.
 		try {
 			db.exec(`PRAGMA analysis_limit = 1000; PRAGMA optimize;`)
-			// First-time databases won't have sqlite_stat1 until we run a
-			// real ANALYZE. Force it once if stats haven't been collected.
-			const hasStats = db.query(`SELECT 1 FROM sqlite_master WHERE name = 'sqlite_stat1' LIMIT 1`).get() !== null
-			if (!hasStats) db.exec(`ANALYZE;`)
 		} catch {
 			// ANALYZE / optimize failures are never fatal — queries still work,
 			// they just run with default row estimates.
@@ -777,33 +826,157 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			)
 		`)
 
-		const rebuildTraceSummaries = db.query(`
-			INSERT INTO trace_summaries (
-				trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
-			)
-			${TRACE_SUMMARY_SELECT_SQL}
-			GROUP BY trace_id
-		`)
-
 		const reconcileTraceSummaries = Effect.sync(() => {
-			try {
-				db.query(`DELETE FROM trace_summaries`).run()
-				rebuildTraceSummaries.run()
-			} catch (err) {
-				if (!isSqliteLockError(err)) throw err
-				console.warn(`motel: trace summary rebuild skipped during startup: ${(err as Error).message}`)
+			const marker = db.query(`SELECT value FROM motel_maintenance WHERE key = 'trace_summary_cursor'`).get() as { value: string } | null
+			const cursor = Number(marker?.value ?? 0)
+			const rows = db.query(`SELECT rowid, trace_id FROM spans WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`).all(cursor, config.otel.retentionTraceBatch) as Array<{ rowid: number; trace_id: string }>
+			if (rows.length === 0) {
+				db.query(`INSERT OR REPLACE INTO motel_maintenance(key, value) VALUES ('trace_summary_cursor', '0')`).run()
+				return
 			}
+			const transaction = db.transaction(() => {
+				for (const traceId of new Set(rows.map((row) => row.trace_id))) upsertTraceSummary.run(traceId)
+				db.query(`INSERT OR REPLACE INTO motel_maintenance(key, value) VALUES ('trace_summary_cursor', ?)`).run(String(rows.at(-1)!.rowid))
+			})
+			transaction()
 		})
 
 		const deleteSpanAttributes = db.query(`DELETE FROM span_attributes WHERE trace_id = ? AND span_id = ?`)
 		const insertSpanAttribute = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES (?, ?, ?, ?)`)
+		const spanAttributeInsertManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const insertSpanAttributesMany = (traceId: string, spanId: string, attributes: Readonly<Record<string, string>>) => {
+			const entries = Object.entries(attributes)
+			if (entries.length === 0) return
+			if (entries.length === 1) {
+				const [key, value] = entries[0]!
+				insertSpanAttribute.run(traceId, spanId, key, value)
+				return
+			}
+			let query = spanAttributeInsertManyByCount.get(entries.length)
+			if (!query) {
+				query = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES ${entries.map(() => "(?, ?, ?, ?)").join(", ")}`)
+				spanAttributeInsertManyByCount.set(entries.length, query)
+			}
+			query.run(...entries.flatMap(([key, value]) => [traceId, spanId, key, value]))
+		}
 		const deleteSpanOperationSearch = db.query(`DELETE FROM span_operation_fts WHERE trace_id = ? AND span_id = ?`)
 		const insertSpanOperationSearch = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES (?, ?, ?)`)
+		const deleteSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const insertSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const updateSpanOperationSearchMany = (operations: ReadonlyArray<readonly [string, string, string]>) => {
+			if (operations.length === 0) return
+			if (operations.length === 1) {
+				const [traceId, spanId, operationName] = operations[0]!
+				deleteSpanOperationSearch.run(traceId, spanId)
+				insertSpanOperationSearch.run(traceId, spanId, operationName)
+				return
+			}
+
+			let deleteQuery = deleteSpanOperationSearchManyByCount.get(operations.length)
+			if (!deleteQuery) {
+				deleteQuery = db.query(`DELETE FROM span_operation_fts WHERE ${operations.map(() => "(trace_id = ? AND span_id = ?)").join(" OR ")}`)
+				deleteSpanOperationSearchManyByCount.set(operations.length, deleteQuery)
+			}
+			deleteQuery.run(...operations.flatMap(([traceId, spanId]) => [traceId, spanId]))
+
+			let insertQuery = insertSpanOperationSearchManyByCount.get(operations.length)
+			if (!insertQuery) {
+				insertQuery = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES ${operations.map(() => "(?, ?, ?)").join(", ")}`)
+				insertSpanOperationSearchManyByCount.set(operations.length, insertQuery)
+			}
+			insertQuery.run(...operations.flatMap(([traceId, spanId, operationName]) => [traceId, spanId, operationName]))
+		}
 		const insertLogAttribute = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES (?, ?, ?)`)
+		const logAttributeInsertManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const insertLogAttributesMany = (logId: number, attributes: Readonly<Record<string, string>>) => {
+			const entries = Object.entries(attributes)
+			if (entries.length === 0) return
+			if (entries.length === 1) {
+				const [key, value] = entries[0]!
+				insertLogAttribute.run(logId, key, value)
+				return
+			}
+			let query = logAttributeInsertManyByCount.get(entries.length)
+			if (!query) {
+				query = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES ${entries.map(() => "(?, ?, ?)").join(", ")}`)
+				logAttributeInsertManyByCount.set(entries.length, query)
+			}
+			query.run(...entries.flatMap(([key, value]) => [logId, key, value]))
+		}
 		const insertLogBodySearch = db.query(`INSERT INTO log_body_fts (log_id, body) VALUES (?, ?)`)
+		const insertLogBodySearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
+		const insertLogBodySearchMany = (entries: ReadonlyArray<readonly [string, string]>) => {
+			if (entries.length === 0) return
+			if (entries.length === 1) {
+				const [logId, body] = entries[0]!
+				insertLogBodySearch.run(logId, body)
+				return
+			}
+			let query = insertLogBodySearchManyByCount.get(entries.length)
+			if (!query) {
+				query = db.query(`INSERT INTO log_body_fts (log_id, body) VALUES ${entries.map(() => "(?, ?)").join(", ")}`)
+				insertLogBodySearchManyByCount.set(entries.length, query)
+			}
+			query.run(...entries.flatMap(([logId, body]) => [logId, body]))
+		}
 
 		const maxDbSizeBytes = config.otel.maxDbSizeMb * 1024 * 1024
 
+		// Freelist-ratio thresholds for the adaptive reclaim loop. Below the
+		// LOW threshold there's nothing worth doing; above HIGH we are in the
+		// 17GB-DB-with-10GB-freelist failure mode and need to reclaim aggressively
+		// even if it costs writer-lock time.
+		const FREELIST_LOW_RATIO = 0.05
+		const FREELIST_MID_RATIO = 0.20
+		const FREELIST_HIGH_RATIO = 0.50
+		const VACUUM_PAGES_NORMAL = 2000     // ~8MB/pass
+		const VACUUM_PAGES_BUSY = 20000      // ~80MB/pass — used when freelist > 20%
+		const VACUUM_PAGES_PANIC = 50000     // ~200MB/pass — only when ratio > 50%
+
+		const ftsTableNames = ["span_attr_fts", "log_body_fts", "span_operation_fts"] as const
+
+		const incrementalFtsMerge = (pages: number) => {
+			// FTS5 segment merges drop tombstone rows that DELETE leaves behind.
+			// Without periodic merges, deleted FTS rows stay on disk indefinitely
+			// — a major source of freelist pages on a heavy-deletion workload.
+			// `merge=N` is a bounded, online operation: it merges at most N
+			// pages of work and returns. Per FTS5 docs, missing tables silently
+			// throw; we swallow because not every DB has every FTS table.
+			for (const name of ftsTableNames) {
+				try { db.query(`INSERT INTO ${name}(${name}) VALUES (?)`).run(`merge=${pages}`) } catch { /* table absent or older schema */ }
+			}
+		}
+
+		const reclaimSpace = Effect.fn("motel/TelemetryStore.reclaimSpace")(function* () {
+			yield* Effect.sync(() => {
+				const pageCount = (db.query(`PRAGMA page_count`).get() as { page_count: number }).page_count
+				const freePages = (db.query(`PRAGMA freelist_count`).get() as { freelist_count: number }).freelist_count
+				if (pageCount === 0) return
+				const ratio = freePages / pageCount
+				if (ratio < FREELIST_LOW_RATIO) return
+
+				// Adaptive vacuum sizing — fixed 2000 pages/min could not keep
+				// up with sustained deletions, leaking 10GB of freelist over
+				// time. Scale the per-pass work to the size of the backlog so
+				// we stay roughly proportional to the deficit.
+				const pages =
+					ratio >= FREELIST_HIGH_RATIO ? VACUUM_PAGES_PANIC :
+					ratio >= FREELIST_MID_RATIO ? VACUUM_PAGES_BUSY :
+					VACUUM_PAGES_NORMAL
+
+				try { db.exec(`PRAGMA incremental_vacuum(${pages});`) } catch { /* ignore */ }
+
+				// In WAL mode incremental_vacuum only moves pages — the file
+				// shrinks on the next checkpoint. PASSIVE silently skips when
+				// readers are active (the failure mode the agent's research
+				// flagged: checkpoint starvation). Use RESTART normally and
+				// TRUNCATE in panic mode to physically shrink the WAL when it
+				// has grown.
+				const mode = ratio >= FREELIST_HIGH_RATIO ? "TRUNCATE" : "RESTART"
+				try { db.exec(`PRAGMA wal_checkpoint(${mode});`) } catch { /* ignore */ }
+			})
+		})
+
 		const cleanupExpired = Effect.fn("motel/TelemetryStore.cleanupExpired")(function* () {
 			const now = yield* Clock.currentTimeMillis
 
@@ -819,12 +992,12 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 
 				// Time-based: completed traces whose last span ended before cutoff.
 				const timeExpired = db.query(
-					`SELECT trace_id FROM trace_summaries WHERE active_span_count = 0 AND ended_at_ms > 0 AND ended_at_ms < ?`,
-				).all(cutoff) as readonly { trace_id: string }[]
+					`SELECT trace_id FROM trace_summaries WHERE active_span_count = 0 AND ended_at_ms > 0 AND ended_at_ms < ? ORDER BY ended_at_ms ASC LIMIT ?`,
+				).all(cutoff, config.otel.retentionTraceBatch) as readonly { trace_id: string }[]
 				for (const row of timeExpired) toEvict.add(row.trace_id)
 
-				// Size-based: if actual data exceeds cap, drop oldest 20% of the
-				// remaining completed traces. `(page_count - freelist_count)`
+				// Size-based: if actual data exceeds the target, drop one bounded
+				// batch of the oldest completed traces. `(page_count - freelist_count)`
 				// ignores freed-but-not-vacuumed pages so a large freelist doesn't
 				// trigger a deletion death spiral.
 				const pageCount = (db.query(`PRAGMA page_count`).get() as { page_count: number }).page_count
@@ -832,22 +1005,22 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				const pageSize = (db.query(`PRAGMA page_size`).get() as { page_size: number }).page_size
 				const dbSize = (pageCount - freePages) * pageSize
 				if (dbSize > maxDbSizeBytes) {
-					const completedCount = (db.query(
-						`SELECT COUNT(*) AS c FROM trace_summaries WHERE active_span_count = 0`,
-					).get() as { c: number }).c
-					const traceCutCount = Math.max(1, Math.floor(completedCount * 0.2))
 					const oldest = db.query(
 						`SELECT trace_id FROM trace_summaries WHERE active_span_count = 0 ORDER BY started_at_ms ASC LIMIT ?`,
-					).all(traceCutCount) as readonly { trace_id: string }[]
+					).all(config.otel.retentionTraceBatch) as readonly { trace_id: string }[]
 					// Set.add dedupes overlap with the time-expired batch above.
 					for (const row of oldest) toEvict.add(row.trace_id)
 				}
 
-				// Always prune orphan logs (no trace_id) by timestamp — they're
-				// not covered by trace eviction.
-				db.query(`DELETE FROM logs WHERE trace_id IS NULL AND timestamp_ms < ?`).run(cutoff)
-
-				if (toEvict.size === 0) return
+				// Logs have their own retention boundary. A correlated log may refer
+				// to a trace that was sampled elsewhere or never reached Motel, so
+				// tying log eviction to trace_summaries lets those rows grow forever.
+				const expiredLogs = db.query(`DELETE FROM logs WHERE id IN (SELECT id FROM logs WHERE timestamp_ms < ? ORDER BY timestamp_ms ASC LIMIT ?)`).run(cutoff, config.otel.retentionLogBatch)
+				let deletedLogs = Number(expiredLogs.changes) > 0
+				if (dbSize > maxDbSizeBytes) {
+					const oversizedLogs = db.query(`DELETE FROM logs WHERE id IN (SELECT id FROM logs ORDER BY timestamp_ms ASC LIMIT ?)`).run(config.otel.retentionLogBatch)
+					deletedLogs = deletedLogs || Number(oversizedLogs.changes) > 0
+				}
 
 				// Batch the trace-id list so the IN placeholders stay under
 				// SQLite's default limit (~999). Each batch wipes every row
@@ -870,48 +1043,54 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 
 				// Log-side orphans (log_attributes + FTS) are keyed by log.id,
 				// so prune what no longer has a parent log row.
-				db.query(`DELETE FROM log_attributes WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = log_attributes.log_id)`).run()
+				const orphanAttributes = db.query(`DELETE FROM log_attributes WHERE rowid IN (SELECT log_attributes.rowid FROM log_attributes WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = log_attributes.log_id) LIMIT ?)`).run(config.otel.retentionLogBatch)
+				let deletedOrphans = Number(orphanAttributes.changes) > 0
 				try {
-					db.query(`DELETE FROM log_body_fts WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = CAST(log_body_fts.log_id AS INTEGER))`).run()
+					const orphanFts = db.query(`DELETE FROM log_body_fts WHERE rowid IN (SELECT rowid FROM log_body_fts WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = CAST(log_body_fts.log_id AS INTEGER)) LIMIT ?)`).run(config.otel.retentionLogBatch)
+					deletedOrphans = deletedOrphans || Number(orphanFts.changes) > 0
 				} catch {
 					// FTS table may not exist on old DBs.
 				}
 
-				// Truncate the WAL after a big delete pass. Without this the
-				// WAL keeps growing (observed: 640MB) because wal_autocheckpoint
-				// only triggers when WAL pages exceed the threshold during
-				// writes — a retention pass that evicts millions of rows can
-				// blow far past that before the auto-checkpoint fires. Using
-				// PASSIVE so active readers aren't interrupted; if the WAL
-				// can't be fully reclaimed right now, we'll try again next
-				// cycle.
-				try { db.exec(`PRAGMA wal_checkpoint(PASSIVE);`) } catch { /* ignore */ }
-
-				// Incremental vacuum reclaims some of the freed pages back
-				// to the OS so the file size actually shrinks over time
-				// instead of just growing the freelist. Bounded to 2000
-				// pages per pass (≈8MB) to avoid a long-running transaction.
-				try { db.exec(`PRAGMA incremental_vacuum(2000);`) } catch { /* ignore */ }
+				// Checkpoint after a big delete pass so the freed pages land
+				// in the main DB file and become eligible for incremental
+				// vacuum. Use RESTART (not PASSIVE): PASSIVE silently no-ops
+				// when readers are active, which is the documented mechanism
+				// behind WAL/freelist starvation when ingest is busy.
+				if (toEvict.size === 0 && !deletedLogs && !deletedOrphans) return
+				try { db.exec(`PRAGMA wal_checkpoint(RESTART);`) } catch { /* ignore */ }
+
+				// Incremental FTS5 merge — DELETE on an FTS5-indexed row
+				// leaves a tombstone in the segment tree that only `merge`
+				// reclaims. Skipping this is the second compounding cause
+				// (after fixed-size vacuum) of the slow freelist accretion
+				// that took the DB to 17GB. 100 pages of merge work per
+				// retention tick is bounded and runs in milliseconds.
+				incrementalFtsMerge(100)
+
+				// Actual page reclamation lives in `reclaimSpace`, which
+				// runs on its own faster cadence so the file shrinks even
+				// when no traces are evicted in a given retention tick (e.g.
+				// after a large historical eviction has already happened).
 			})
 		})
 
-		// Retention only runs in processes that opt in (currently the main
-		// daemon). The ingest worker and TUI skip it to avoid two writers
-		// competing for the write lock with overlapping DELETE passes.
+		// Retention only runs in the ingest worker so maintenance never blocks
+		// the HTTP event loop and no second writer duplicates cleanup work.
 		if (opts.runRetention) {
-			// Reconcile any summary drift from interrupted ingests, but do it
-			// after the server becomes healthy. Running this synchronously at
-			// open can sit behind another writer's lock for ~15s and make the
-			// daemon look hung even though the port is already bound.
-			yield* Effect.forkScoped(reconcileTraceSummaries)
-
-			// Enable incremental vacuum so retention can reclaim freed
-			// pages over time instead of needing a stop-the-world VACUUM.
-			// Idempotent: repeat calls after the first are no-ops.
-			try { db.exec(`PRAGMA auto_vacuum = INCREMENTAL;`) } catch { /* ignore */ }
-
-			// Run cleanup every 60 seconds in the background, tied to the layer's scope
-			yield* Effect.forkScoped(Effect.repeat(cleanupExpired(), Schedule.spaced("60 seconds")))
+			// Cleanup runs on the telemetry worker, never the HTTP event loop.
+			yield* Effect.forkScoped(Effect.repeat(
+				Effect.andThen(reconcileTraceSummaries, cleanupExpired()).pipe(Effect.catchCause((cause) => Effect.logWarning(`motel: maintenance pass failed: ${Cause.pretty(cause)}`))),
+				Schedule.spaced(`${config.otel.retentionIntervalSeconds} seconds`),
+			))
+
+			// Page reclamation runs on a separate, faster cadence (10s) and
+			// is independent of the eviction loop. The reason: a single sweep
+			// at 60s intervals can move only ~8MB of pages before the next
+			// burst of inserts grows the freelist again. Decoupling lets us
+			// catch up adaptively (see VACUUM_PAGES_BUSY/PANIC) without
+			// changing the cost of the heavier delete sweep.
+			yield* Effect.forkScoped(Effect.repeat(reclaimSpace(), Schedule.spaced("10 seconds")))
 
 			// Periodically refresh query planner stats. `PRAGMA optimize` is a
 			// no-op when nothing has changed, so this is essentially free on idle
@@ -924,35 +1103,48 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			yield* Effect.forkScoped(Effect.repeat(refreshPlannerStats, Schedule.spaced("15 minutes")))
 		}
 
-		// One-time backfill for existing DBs: if span_attr_fts is empty but
-		// span_attributes has rows with AI_FTS_KEYS, populate the index.
-		// Runs forked so server startup isn't blocked; queries hitting the
-		// FTS will just return empty until the fill lands. On a 2 GB DB with
-		// ~400 matching rows this takes ~3-8 seconds. Writer-only because
-		// it does INSERT INTO ... — readonly connections would error.
+		// Incrementally rebuild historical AI attributes in bounded batches.
+		// Queries fall back to LIKE until the persistent marker is complete.
 		if (hasAttrFts && !opts.readonly) {
-			const backfillAttrFts = Effect.sync(() => {
+			const backfillAttrFtsBatch = Effect.sync(() => {
 				try {
-					const ftsCount = (db.query(`SELECT COUNT(*) AS c FROM span_attr_fts`).get() as { c: number }).c
-					if (ftsCount > 0) return
 					const keyList = AI_FTS_KEYS.map((k) => `'${k.replace(/'/g, "''")}'`).join(", ")
-					const attrCount = (db.query(
-						`SELECT COUNT(*) AS c FROM span_attributes WHERE key IN (${keyList})`,
-					).get() as { c: number }).c
-					if (attrCount === 0) return
-					// Single INSERT..SELECT is atomic and fast; FTS5 batches
-					// its internal segment writes. No transaction wrapper
-					// needed — it runs as one statement.
-					db.exec(`
-						INSERT INTO span_attr_fts(rowid, value)
-						SELECT rowid, value FROM span_attributes WHERE key IN (${keyList})
-					`)
+					const marker = db.query(`SELECT value FROM motel_maintenance WHERE key = 'span_attr_fts_v1'`).get() as { value: string } | null
+					if (marker?.value === "complete") return false
+					let cursor = 0
+					let maxRowId = 0
+					if (marker) {
+						[cursor, maxRowId] = marker.value.split(":").map(Number)
+					} else {
+						maxRowId = (db.query(`SELECT COALESCE(MAX(rowid), 0) AS value FROM span_attributes`).get() as { value: number }).value
+						db.query(`INSERT INTO span_attr_fts(span_attr_fts) VALUES ('delete-all')`).run()
+						db.query(`INSERT OR REPLACE INTO motel_maintenance(key, value) VALUES ('span_attr_fts_v1', ?)`).run(`0:${maxRowId}`)
+					}
+					const rows = db.query(`SELECT rowid, value FROM span_attributes WHERE key IN (${keyList}) AND rowid > ? AND rowid <= ? ORDER BY rowid ASC LIMIT 500`).all(cursor, maxRowId) as Array<{ rowid: number; value: string }>
+					if (rows.length === 0) {
+						db.query(`UPDATE motel_maintenance SET value = 'complete' WHERE key = 'span_attr_fts_v1'`).run()
+						hasAttrFts = true
+						return false
+					}
+					const insert = db.query(`INSERT INTO span_attr_fts(rowid, value) VALUES (?, ?)`)
+					const transaction = db.transaction(() => {
+						for (const row of rows) insert.run(row.rowid, row.value)
+						db.query(`UPDATE motel_maintenance SET value = ? WHERE key = 'span_attr_fts_v1'`).run(`${rows.at(-1)!.rowid}:${maxRowId}`)
+					})
+					transaction()
+					return true
 				} catch {
 					// Backfill failure is never fatal — new ingests still
 					// populate FTS via the trigger, and queries fall back to
 					// LIKE when FTS lookups return empty.
+					return true
 				}
 			})
+			const backfillAttrFts: Effect.Effect<void> = Effect.suspend(() =>
+				Effect.flatMap(backfillAttrFtsBatch, (pending) =>
+					pending ? Effect.andThen(Effect.sleep("100 millis"), backfillAttrFts) : Effect.void,
+				),
+			)
 			yield* Effect.forkScoped(backfillAttrFts)
 		}
 
@@ -961,6 +1153,7 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				let insertedSpans = 0
 				const transaction = db.transaction((request: OtlpTraceExportRequest) => {
 					const touchedTraceIds = new Set<string>()
+					const touchedOperations: Array<readonly [string, string, string]> = []
 					for (const resourceSpans of request.resourceSpans ?? []) {
 						const resourceAttributes = attributeMap(resourceSpans.resource?.attributes)
 						const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
@@ -969,6 +1162,10 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 							const scopeName = scopeSpans.scope?.name ?? null
 
 							for (const span of scopeSpans.spans ?? []) {
+								const traceId = normalizeOtlpBinaryId(span.traceId, 16)
+								const spanId = normalizeOtlpBinaryId(span.spanId, 8)
+								if (!traceId || !spanId) continue
+								const parentSpanId = normalizeOtlpBinaryId(span.parentSpanId, 8)
 								const spanAttributes = attributeMap(span.attributes)
 								const mergedAttributes = { ...resourceAttributes, ...spanAttributes }
 								const startTimeMs = nanosToMilliseconds(span.startTimeUnixNano)
@@ -980,9 +1177,9 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 								}))
 
 								insertSpan.run(
-									span.traceId,
-									span.spanId,
-									span.parentSpanId ?? null,
+									traceId,
+									spanId,
+									parentSpanId,
 									serviceName,
 									scopeName,
 									span.name ?? "unknown",
@@ -995,21 +1192,22 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 									JSON.stringify(resourceAttributes),
 									JSON.stringify(events),
 								)
-								deleteSpanAttributes.run(span.traceId, span.spanId)
-								for (const [key, value] of Object.entries(mergedAttributes)) {
-									insertSpanAttribute.run(span.traceId, span.spanId, key, value)
-								}
-								try {
-									deleteSpanOperationSearch.run(span.traceId, span.spanId)
-									insertSpanOperationSearch.run(span.traceId, span.spanId, span.name ?? "unknown")
-								} catch {
-									// FTS is optional.
-								}
-								touchedTraceIds.add(span.traceId)
+								deleteSpanAttributes.run(traceId, spanId)
+								insertSpanAttributesMany(traceId, spanId, mergedAttributes)
+								touchedOperations.push([traceId, spanId, span.name ?? "unknown"])
+								touchedTraceIds.add(traceId)
 								insertedSpans += 1
 							}
 						}
 					}
+					try {
+						const BATCH_SIZE = 500
+						for (let offset = 0; offset < touchedOperations.length; offset += BATCH_SIZE) {
+							updateSpanOperationSearchMany(touchedOperations.slice(offset, offset + BATCH_SIZE))
+						}
+					} catch {
+						// FTS is optional.
+					}
 					for (const traceId of touchedTraceIds) {
 						upsertTraceSummary.run(traceId)
 					}
@@ -1024,6 +1222,7 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			return yield* Effect.sync(() => {
 				let insertedLogs = 0
 				const transaction = db.transaction((request: OtlpLogExportRequest) => {
+					const touchedLogBodies: Array<readonly [string, string]> = []
 					for (const resourceLogs of request.resourceLogs ?? []) {
 						const resourceAttributes = attributeMap(resourceLogs.resource?.attributes)
 						const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
@@ -1036,9 +1235,11 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 								const mergedAttributes = { ...resourceAttributes, ...attributes }
 								const timestampMs = nanosToMilliseconds(record.timeUnixNano ?? record.observedTimeUnixNano)
 								const body = stringifyValue(parseAnyValue(record.body))
+								const rawTraceId = attributes.traceId || attributes.trace_id || record.traceId || null
+								const rawSpanId = attributes.spanId || attributes.span_id || record.spanId || null
 								const result = insertLog.run(
-									attributes.traceId || attributes.trace_id || record.traceId || null,
-									attributes.spanId || attributes.span_id || record.spanId || null,
+									normalizeOtlpBinaryId(rawTraceId, 16),
+									normalizeOtlpBinaryId(rawSpanId, 8),
 									serviceName,
 									scopeName,
 									record.severityText ?? "INFO",
@@ -1048,18 +1249,20 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 									JSON.stringify(resourceAttributes),
 								)
 								const logId = Number((result as { lastInsertRowid: number | bigint }).lastInsertRowid)
-								for (const [key, value] of Object.entries(mergedAttributes)) {
-									insertLogAttribute.run(logId, key, value)
-								}
-								try {
-									insertLogBodySearch.run(String(logId), body)
-								} catch {
-									// FTS is optional.
-								}
+								insertLogAttributesMany(logId, mergedAttributes)
+								touchedLogBodies.push([String(logId), body])
 								insertedLogs += 1
 							}
 						}
 					}
+					try {
+						const BATCH_SIZE = 500
+						for (let offset = 0; offset < touchedLogBodies.length; offset += BATCH_SIZE) {
+							insertLogBodySearchMany(touchedLogBodies.slice(offset, offset + BATCH_SIZE))
+						}
+					} catch {
+						// FTS is optional.
+					}
 				})
 
 				transaction(payload)
@@ -1068,9 +1271,11 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 		})
 
 		const listServices = Effect.fn("motel/TelemetryStore.listServices")(function* () {
-
 			const cutoff = (yield* Clock.currentTimeMillis) - config.otel.traceLookbackMinutes * 60 * 1000
-			return yield* Effect.sync(() => {
+			const services = yield* Effect.sync(() => {
+				// Discover recent activity from span rows, not trace starts: a
+				// long-running trace can emit a current child after its root ages
+				// outside the lookback window.
 				const rows = db.query(`
 					SELECT service_name FROM spans WHERE start_time_ms >= ?
 					UNION
@@ -1079,6 +1284,8 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				`).all(cutoff, cutoff) as Array<{ service_name: string }>
 				return rows.map((row) => row.service_name)
 			})
+			yield* Effect.annotateCurrentSpan("trace.service_count", services.length)
+			return services
 		})()
 
 		const loadTracesByIds = (traceIds: readonly string[]) => {
@@ -1104,15 +1311,19 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 		}
 
 		const listRecentTraces = Effect.fn("motel/TelemetryStore.listRecentTraces")(function* (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number }) {
+			yield* Effect.annotateCurrentSpan("trace.service_name", serviceName ?? "all")
 			const summaries = yield* listTraceSummaries(serviceName, options)
-			return yield* Effect.sync(() => loadTracesByIds(summaries.map((summary) => summary.traceId)))
+			const traces = yield* Effect.sync(() => loadTracesByIds(summaries.map((summary) => summary.traceId)))
+			yield* Effect.annotateCurrentSpan("trace.result_count", traces.length)
+			return traces
 		})
 
 		const listTraceSummaries = Effect.fn("motel/TelemetryStore.listTraceSummaries")(function* (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) {
+			yield* Effect.annotateCurrentSpan("trace.service_name", serviceName ?? "all")
 			const cutoff = (yield* Clock.currentTimeMillis) - (options?.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
 			const limit = options?.limit ?? config.otel.traceFetchLimit
 
-			return yield* Effect.sync(() => {
+			const summaries = yield* Effect.sync(() => {
 				const clauses = ["started_at_ms >= ?"]
 				const params: Array<string | number> = [cutoff]
 
@@ -1134,6 +1345,8 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 					LIMIT ?
 				`).all(...params, limit) as TraceSummaryRow[]
 			}).pipe(Effect.map((rows) => rows.map(parseSummaryRow)))
+			yield* Effect.annotateCurrentSpan("trace.result_count", summaries.length)
+			return summaries
 		})
 
 		const searchTraceSummaries = Effect.fn("motel/TelemetryStore.searchTraceSummaries")(function* (input: TraceSearch) {
@@ -1212,6 +1425,7 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 		})
 
 		const getTrace = Effect.fn("motel/TelemetryStore.getTrace")(function* (traceId: string) {
+			yield* Effect.annotateCurrentSpan("trace.trace_id", traceId)
 			return yield* Effect.sync(() => {
 				const rows = db.query(`
 					SELECT * FROM spans WHERE trace_id = ? ORDER BY start_time_ms ASC
@@ -1221,6 +1435,7 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 		})
 
 		const getSpan = Effect.fn("motel/TelemetryStore.getSpan")(function* (spanId: string) {
+			yield* Effect.annotateCurrentSpan("trace.span_id", spanId)
 			return yield* Effect.sync(() => {
 				// Fetch only the target span row (uses idx_spans_span_id)
 				const spanRow = db.query(`SELECT * FROM spans WHERE span_id = ? LIMIT 1`).get(spanId) as SpanRow | null
@@ -1228,7 +1443,28 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 
 				const traceId = spanRow.trace_id
 
-				// Get root operation name (indexed by trace_id)
+				// Walk the parent chain in one recursive CTE instead of one query
+				// per hop. Root context remains the earliest root in the trace,
+				// matching full trace hydration even when input has multiple roots.
+				let parentOperationName: string | null = null
+				let depth = 0
+				if (spanRow.parent_span_id) {
+					const ancestors = db.query(`
+						WITH RECURSIVE ancestors(span_id, parent_span_id, operation_name, hop) AS (
+							SELECT span_id, parent_span_id, operation_name, 1
+							FROM spans WHERE trace_id = ? AND span_id = ?
+							UNION ALL
+							SELECT s.span_id, s.parent_span_id, s.operation_name, a.hop + 1
+							FROM ancestors a
+							JOIN spans s ON s.trace_id = ? AND s.span_id = a.parent_span_id
+						)
+						SELECT span_id, parent_span_id, operation_name, hop FROM ancestors ORDER BY hop ASC
+					`).all(traceId, spanRow.parent_span_id, traceId) as Array<{ span_id: string; parent_span_id: string | null; operation_name: string; hop: number }>
+
+					parentOperationName = ancestors[0]?.operation_name ?? null
+					depth = ancestors.length
+				}
+
 				const rootRow = db.query(`
 					SELECT operation_name FROM spans
 					WHERE trace_id = ? AND parent_span_id IS NULL
@@ -1236,28 +1472,6 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				`).get(traceId) as { operation_name: string } | null
 				const rootOperationName = rootRow?.operation_name ?? "unknown"
 
-				// Get parent operation name if span has a parent (PK lookup)
-				let parentOperationName: string | null = null
-				if (spanRow.parent_span_id) {
-					const parentRow = db.query(`
-						SELECT operation_name FROM spans
-						WHERE trace_id = ? AND span_id = ?
-					`).get(traceId, spanRow.parent_span_id) as { operation_name: string } | null
-					parentOperationName = parentRow?.operation_name ?? null
-				}
-
-				// Compute depth by walking up parent chain (typically 3-5 hops)
-				let depth = 0
-				let currentParentId = spanRow.parent_span_id
-				while (currentParentId) {
-					const parentRow = db.query(`
-						SELECT parent_span_id FROM spans WHERE trace_id = ? AND span_id = ?
-					`).get(traceId, currentParentId) as { parent_span_id: string | null } | null
-					if (!parentRow) break
-					depth++
-					currentParentId = parentRow.parent_span_id
-				}
-
 				const parsed = parseSpanRow(spanRow)
 				return {
 					traceId,
@@ -1279,9 +1493,24 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
 			const limit = input.limit ?? 100
 			const hasContainsFilters = Object.keys(input.attributeContainsFilters ?? {}).length > 0
-			const candidateLimit = hasContainsFilters ? Math.max(limit * 20, 500) : Math.max(limit * 10, 200)
+			// Only over-fetch when post-filtering will discard rows. Without
+			// a parentOperation filter the SQL `LIMIT` already returns the
+			// final set, and over-fetching just makes us parse JSON blobs
+			// for rows we'll throw away.
+			const needsPostFilter = !!input.parentOperation
+			const candidateLimit = !needsPostFilter
+				? limit
+				: hasContainsFilters
+					? Math.max(limit * 20, 500)
+					: Math.max(limit * 10, 200)
 
 			return yield* Effect.sync(() => {
+				// First pass: fetch only the columns needed to filter and
+				// to drive the parent-context lookup. Parsing the heavy
+				// `*_json` blobs is deferred until after we've sliced down
+				// to the final `limit`.
+				let fromSql = "FROM spans AS s"
+				const joinParams: Array<string | number> = []
 				const clauses: string[] = ["s.start_time_ms >= ?"]
 				const params: Array<string | number> = [cutoff]
 
@@ -1296,8 +1525,8 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 				if (input.operation) {
 					const ftsQuery = toFtsMatchQuery(input.operation)
 					if (hasFts && ftsQuery) {
-						clauses.push("EXISTS (SELECT 1 FROM span_operation_fts WHERE span_operation_fts.trace_id = s.trace_id AND span_operation_fts.span_id = s.span_id AND span_operation_fts MATCH ?)")
-						params.push(ftsQuery)
+						fromSql += ` INNER JOIN (SELECT trace_id, span_id FROM span_operation_fts WHERE span_operation_fts MATCH ?) AS span_operation_match ON span_operation_match.trace_id = s.trace_id AND span_operation_match.span_id = s.span_id`
+						joinParams.push(ftsQuery)
 					} else {
 						clauses.push("s.operation_name LIKE ? COLLATE NOCASE")
 						params.push(`%${input.operation}%`)
@@ -1320,51 +1549,111 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 					params.push(...containsAttrMatch.params)
 				}
 
-				const rows = db.query(`
-					SELECT trace_id, span_id
-					FROM spans AS s
+				const candidateRows = db.query(`
+					SELECT s.trace_id, s.span_id, s.parent_span_id, s.operation_name, s.start_time_ms
+					${fromSql}
 					WHERE ${clauses.join(" AND ")}
 					ORDER BY s.start_time_ms DESC
 					LIMIT ?
-				`).all(...params, candidateLimit) as Array<{ trace_id: string; span_id: string }>
+				`).all(...joinParams, ...params, candidateLimit) as Array<{ trace_id: string; span_id: string; parent_span_id: string | null; operation_name: string; start_time_ms: number }>
 
-				const traceIds = [...new Set(rows.map((row) => row.trace_id))]
+				const traceIds = [...new Set(candidateRows.map((row) => row.trace_id))]
 				if (traceIds.length === 0) return [] as readonly SpanItem[]
 
+				const keyOf = (traceId: string, spanId: string) => `${traceId}:${spanId}`
+				const spanContextById = new Map<string, { readonly parentSpanId: string | null; readonly operationName: string }>()
+
+				// Bulk-prefetch parent metadata for every span in every trace
+				// touched by the candidate set. One indexed scan per trace_id
+				// is much cheaper than a per-span lookup loop while computing
+				// depth, and we get the trace-root lookup in the same pass.
 				const placeholders = traceIds.map(() => "?").join(", ")
-				const spanRows = db.query(`
-					SELECT * FROM spans
+				const allSpanRows = db.query(`
+					SELECT trace_id, span_id, parent_span_id, operation_name, start_time_ms
+					FROM spans
 					WHERE trace_id IN (${placeholders})
-					ORDER BY start_time_ms ASC
-				`).all(...traceIds) as SpanRow[]
+				`).all(...traceIds) as Array<{ trace_id: string; span_id: string; parent_span_id: string | null; operation_name: string; start_time_ms: number }>
 
-				const grouped = new Map<string, SpanRow[]>()
-				for (const row of spanRows) {
-					const group = grouped.get(row.trace_id) ?? []
-					group.push(row)
-					grouped.set(row.trace_id, group)
+				const rootOperationByTraceId = new Map<string, { operationName: string; startTimeMs: number }>()
+				for (const row of allSpanRows) {
+					spanContextById.set(keyOf(row.trace_id, row.span_id), {
+						parentSpanId: row.parent_span_id,
+						operationName: row.operation_name,
+					})
+					if (row.parent_span_id === null) {
+						const existing = rootOperationByTraceId.get(row.trace_id)
+						if (!existing || row.start_time_ms < existing.startTimeMs) {
+							rootOperationByTraceId.set(row.trace_id, { operationName: row.operation_name, startTimeMs: row.start_time_ms })
+						}
+					}
 				}
 
-				const itemById = new Map<string, SpanItem>()
-				for (const traceId of traceIds) {
-					const traceSpanRows = grouped.get(traceId)
-					if (!traceSpanRows) continue
-					for (const item of buildSpanItems(traceId, traceSpanRows)) {
-						itemById.set(`${item.traceId}:${item.span.spanId}`, item)
+				const getSpanContext = (traceId: string, spanId: string) => spanContextById.get(keyOf(traceId, spanId)) ?? null
+
+				const depthById = new Map<string, number>()
+				const getDepth = (traceId: string, spanId: string, visiting = new Set<string>()): number => {
+					const key = keyOf(traceId, spanId)
+					const cached = depthById.get(key)
+					if (cached !== undefined) return cached
+					if (visiting.has(key)) return 0
+					visiting.add(key)
+					const context = getSpanContext(traceId, spanId)
+					const depth = context?.parentSpanId ? getDepth(traceId, context.parentSpanId, visiting) + 1 : 0
+					depthById.set(key, depth)
+					return depth
+				}
+
+				// Apply parentOperation post-filter on the lite candidate set
+				// (cheap — string compare against cached parent op) and then
+				// slice down to the final result size before parsing any JSON.
+				const parentOperationNeedle = input.parentOperation?.toLowerCase() ?? null
+				const filteredLite: typeof candidateRows = []
+				for (const row of candidateRows) {
+					if (parentOperationNeedle) {
+						const parent = row.parent_span_id ? getSpanContext(row.trace_id, row.parent_span_id) : null
+						if (!parent?.operationName.toLowerCase().includes(parentOperationNeedle)) continue
 					}
+					filteredLite.push(row)
+					if (filteredLite.length >= limit) break
 				}
 
-				return rows
-					.map((row) => itemById.get(`${row.trace_id}:${row.span_id}`))
-					.filter((item): item is SpanItem => item !== undefined)
-					.filter((item) => {
-						if (input.parentOperation) {
-							const needle = input.parentOperation.toLowerCase()
-							if (!item.parentOperationName?.toLowerCase().includes(needle)) return false
-						}
-						return true
+				if (filteredLite.length === 0) return [] as readonly SpanItem[]
+
+				// Hydrate only the kept rows: one batched fetch of the full
+				// SpanRow (with resource_json / attributes_json / events_json)
+				// using SQLite's row-value `IN` syntax, then parseSpanRow per
+				// kept row. Result order follows `filteredLite` so the caller
+				// sees the same ordering the candidate scan produced.
+				const keptValues = filteredLite.map(() => "(?, ?)").join(", ")
+				const fullRows = db.query(`
+					SELECT * FROM spans WHERE (trace_id, span_id) IN (VALUES ${keptValues})
+				`).all(...filteredLite.flatMap((row) => [row.trace_id, row.span_id])) as SpanRow[]
+				const fullRowByKey = new Map<string, SpanRow>()
+				for (const row of fullRows) {
+					fullRowByKey.set(keyOf(row.trace_id, row.span_id), row)
+				}
+
+				const items: SpanItem[] = []
+				for (const lite of filteredLite) {
+					const row = fullRowByKey.get(keyOf(lite.trace_id, lite.span_id))
+					if (!row) continue
+					const parentContext = row.parent_span_id ? getSpanContext(row.trace_id, row.parent_span_id) : null
+					const parsedSpan = parseSpanRow(row)
+					const span = {
+						...parsedSpan,
+						depth: getDepth(row.trace_id, row.span_id),
+						warnings: row.parent_span_id && !parentContext
+							? [`missing span ${row.parent_span_id} (1 child)`]
+							: parsedSpan.warnings,
+					}
+					items.push({
+						traceId: row.trace_id,
+						rootOperationName: rootOperationByTraceId.get(row.trace_id)?.operationName ?? span.operationName,
+						parentOperationName: parentContext?.operationName ?? null,
+						span,
 					})
-					.slice(0, limit)
+				}
+				return items
 			})
 		})
 
@@ -1662,11 +1951,13 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 		})
 
 		const listRecentLogs = Effect.fn("motel/TelemetryStore.listRecentLogs")(function* (serviceName: string) {
-			return yield* searchLogs({ serviceName, limit: config.otel.logFetchLimit })
+			yield* Effect.annotateCurrentSpan("log.service_name", serviceName)
+			const logs = yield* searchLogs({ serviceName, limit: config.otel.logFetchLimit })
+			yield* Effect.annotateCurrentSpan("log.result_count", logs.length)
+			return logs
 		})
 
 		const listFacets = Effect.fn("motel/TelemetryStore.listFacets")(function* (input: FacetSearch) {
-
 			const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
 			const limit = input.limit ?? 20
 
@@ -1756,21 +2047,30 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 						// FACET_VALUE_MAX_LEN. For opencode this hides `ai.prompt`,
 						// `ai.prompt.messages`, and `ai.prompt.tools` — which are 1-6MB text
 						// blobs that you'd never want to filter by exact match anyway. The
-						// WHERE clause lets SQLite skip reading those pages from disk, taking
-						// the picker open time from ~1.2s to ~370ms on a 2GB database.
-						const params: Array<string | number> = [FACET_VALUE_MAX_LEN, cutoff]
-						if (input.serviceName) params.push(input.serviceName)
-						params.push(limit)
+						// WHERE clause lets SQLite skip reading those pages from disk.
+						// COUNT(DISTINCT ...) does its own per-group dedup via a temp B-tree,
+						// so the outer query needs no DISTINCT subquery in front of it. We
+						// pre-filter trace_ids through trace_summaries (an indexed lookup) so
+						// the planner can use a SEMI JOIN against the small in-window set
+						// instead of joining every span_attributes row to trace_summaries.
+						const params: Array<string | number> = []
+						let traceFilter: string
+						if (input.serviceName) {
+							traceFilter = `(SELECT trace_id FROM trace_summaries WHERE started_at_ms >= ? AND service_name = ?)`
+							params.push(cutoff, input.serviceName)
+						} else {
+							traceFilter = `(SELECT trace_id FROM trace_summaries WHERE started_at_ms >= ?)`
+							params.push(cutoff)
+						}
+						params.push(FACET_VALUE_MAX_LEN, limit)
 						const rows = db.query(`
-							SELECT sa.key AS value,
-							       COUNT(DISTINCT sa.trace_id) AS count,
-							       COUNT(DISTINCT sa.value) AS distinct_values
-							FROM span_attributes sa
-							JOIN spans s ON s.trace_id = sa.trace_id AND s.span_id = sa.span_id
-							WHERE LENGTH(sa.value) < ?
-							  AND s.start_time_ms >= ?
-							${input.serviceName ? "AND s.service_name = ?" : ""}
-							GROUP BY sa.key
+							SELECT key AS value,
+							       COUNT(DISTINCT trace_id) AS count,
+							       COUNT(DISTINCT value) AS distinct_values
+							FROM span_attributes
+							WHERE trace_id IN ${traceFilter}
+							  AND LENGTH(value) < ?
+							GROUP BY key
 							ORDER BY (CASE WHEN distinct_values = 1 THEN 1 ELSE 0 END) ASC,
 							         distinct_values DESC,
 							         count DESC,
@@ -1807,7 +2107,10 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 		})
 
 		const listTraceLogs = Effect.fn("motel/TelemetryStore.listTraceLogs")(function* (traceId: string) {
-			return yield* searchLogs({ traceId, limit: config.otel.logFetchLimit })
+			yield* Effect.annotateCurrentSpan("log.trace_id", traceId)
+			const logs = yield* searchLogs({ traceId, limit: config.otel.logFetchLimit })
+			yield* Effect.annotateCurrentSpan("log.result_count", logs.length)
+			return logs
 		})
 
 		// ---------------------------------------------------------------------------
@@ -2212,28 +2515,40 @@ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.ef
 			searchAiCalls,
 			getAiCall,
 			aiCallStats,
+			runRetentionNow: cleanupExpired(),
 		})
-	}),
-)
+	})
+
+/** Compatibility factory for callers constructing a writer/query-capable store layer. */
+export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) =>
+	Layer.effect(TelemetryStore, makeTelemetryStoreEffect(opts)).pipe(Layer.provide(BunFileSystem.layer))
 
 /**
- * Default writer instance: the main daemon uses this. Owns schema
- * migrations, FTS backfill, and the retention loop.
+ * Default writer runtime used by tests and direct store consumers.
  */
 export const TelemetryStoreLive = makeTelemetryStoreLayer({ readonly: false, runRetention: true })
 
 /**
- * Writer instance that SKIPS retention. The ingest worker uses this
- * so the daemon and the worker aren't both running DELETE passes at
- * the same time (they'd just serialise behind the write lock and
- * duplicate work).
+ * The ingest worker's writer. It is the managed daemon's sole owner of
+ * schema migrations, FTS backfill, retention, and page reclamation.
  */
-export const TelemetryStoreWorkerLive = makeTelemetryStoreLayer({ readonly: false, runRetention: false })
+export const TelemetryStoreWorkerLive = TelemetryStoreLive
 
 /**
- * Read-only instance for query-only processes (currently the TUI).
- * Skips every DDL/DML statement at startup so the connection can be
- * opened while a writer is mid-transaction without racing for the
- * write lock. Writes through the service interface will throw.
+ * Read-only instance for query-only processes (currently the TUI and
+ * HTTP query handlers). Skips every DDL/DML statement at startup so
+ * the connection can be opened while a writer is mid-transaction
+ * without racing for the write lock. Provided as TelemetryStoreReadonly
+ * — a distinct service identifier so it can coexist with the writer
+ * TelemetryStore in the same runtime.
  */
-export const TelemetryStoreReadonlyLive = makeTelemetryStoreLayer({ readonly: true, runRetention: false })
+export const TelemetryStoreReadonlyLive = Layer.effect(TelemetryStoreReadonly, makeTelemetryStoreEffect({ readonly: true, runRetention: false })).pipe(Layer.provide(BunFileSystem.layer))
+
+/** Query-worker reader that waits for the sole writer to finish schema bootstrap. */
+export const TelemetryStoreQueryWorkerLive = Layer.effect(
+	TelemetryStoreReadonly,
+	makeTelemetryStoreEffect({ readonly: true, runRetention: false }).pipe(
+		Effect.map((store) => TelemetryStoreReadonly.of(store)),
+		Effect.retry(Schedule.spaced("50 millis")),
+	),
+).pipe(Layer.provide(BunFileSystem.layer))