@aryasaatvik/motel 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/AGENTS.md +180 -0
  2. package/LICENSE +21 -0
  3. package/README.md +137 -0
  4. package/package.json +119 -0
  5. package/skills/motel-debug/SKILL.md +203 -0
  6. package/skills/motel-debug/references/effect.md +38 -0
  7. package/src/App.tsx +414 -0
  8. package/src/StartupGate.test.ts +80 -0
  9. package/src/StartupGate.tsx +263 -0
  10. package/src/cli.ts +205 -0
  11. package/src/config.ts +45 -0
  12. package/src/daemon.test.ts +547 -0
  13. package/src/daemon.ts +486 -0
  14. package/src/domain.test.ts +62 -0
  15. package/src/domain.ts +291 -0
  16. package/src/httpApi.ts +391 -0
  17. package/src/httpListPolicy.test.ts +76 -0
  18. package/src/httpListPolicy.ts +129 -0
  19. package/src/index.tsx +25 -0
  20. package/src/instructions.ts +72 -0
  21. package/src/launchAgent.test.ts +430 -0
  22. package/src/launchAgent.ts +462 -0
  23. package/src/localServer.ts +568 -0
  24. package/src/locator.ts +138 -0
  25. package/src/mcp.ts +433 -0
  26. package/src/motel.test.ts +176 -0
  27. package/src/motel.ts +88 -0
  28. package/src/motelClient.ts +353 -0
  29. package/src/opentui-jsx.d.ts +11 -0
  30. package/src/otlp.test.ts +65 -0
  31. package/src/otlp.ts +162 -0
  32. package/src/otlpProtobuf.ts +35 -0
  33. package/src/queryFilters.ts +39 -0
  34. package/src/registry.ts +115 -0
  35. package/src/runtime.ts +40 -0
  36. package/src/server.ts +11 -0
  37. package/src/services/AsyncIngest.ts +82 -0
  38. package/src/services/LogQueryService.ts +29 -0
  39. package/src/services/TelemetryQuery.ts +62 -0
  40. package/src/services/TelemetryStore.ts +2554 -0
  41. package/src/services/TraceQueryService.ts +43 -0
  42. package/src/services/ingestRpc.ts +39 -0
  43. package/src/services/queryRpc.ts +15 -0
  44. package/src/services/telemetryQueryWorker.ts +32 -0
  45. package/src/services/telemetryWorker.ts +59 -0
  46. package/src/startupBench.ts +19 -0
  47. package/src/startupLifecycle.ts +57 -0
  48. package/src/storybook/aiChatStory.tsx +244 -0
  49. package/src/storybook/fixtures/errorState.ts +44 -0
  50. package/src/storybook/fixtures/imagePaste.ts +34 -0
  51. package/src/storybook/fixtures/index.ts +62 -0
  52. package/src/storybook/fixtures/kitchenSink.ts +148 -0
  53. package/src/storybook/fixtures/rawPrompt.ts +15 -0
  54. package/src/storybook/fixtures/short.ts +27 -0
  55. package/src/storybook/fixtures/toolHeavy.ts +65 -0
  56. package/src/telemetry.test.ts +1053 -0
  57. package/src/ui/AiChatView.tsx +308 -0
  58. package/src/ui/AttrFilterModal.tsx +120 -0
  59. package/src/ui/ServiceLogs.tsx +115 -0
  60. package/src/ui/SpanContentView.tsx +184 -0
  61. package/src/ui/SpanDetail.tsx +215 -0
  62. package/src/ui/SpanDetailFull.tsx +224 -0
  63. package/src/ui/SpanDetailPane.tsx +90 -0
  64. package/src/ui/TraceDetailsPane.tsx +175 -0
  65. package/src/ui/TraceList.tsx +254 -0
  66. package/src/ui/Waterfall.tsx +389 -0
  67. package/src/ui/aiChatModel.test.ts +391 -0
  68. package/src/ui/aiChatModel.ts +773 -0
  69. package/src/ui/aiState.ts +59 -0
  70. package/src/ui/app/TraceListPane.tsx +39 -0
  71. package/src/ui/app/TraceWorkspace.tsx +452 -0
  72. package/src/ui/app/useAppLayout.ts +101 -0
  73. package/src/ui/app/useTraceScreenData.ts +590 -0
  74. package/src/ui/atoms.ts +131 -0
  75. package/src/ui/cachedLoader.test.ts +23 -0
  76. package/src/ui/cachedLoader.ts +60 -0
  77. package/src/ui/filterParser.test.ts +56 -0
  78. package/src/ui/filterParser.ts +45 -0
  79. package/src/ui/format.ts +128 -0
  80. package/src/ui/loaders.ts +101 -0
  81. package/src/ui/persistence.ts +41 -0
  82. package/src/ui/primitives.tsx +198 -0
  83. package/src/ui/state.ts +6 -0
  84. package/src/ui/theme.ts +160 -0
  85. package/src/ui/traceDetailsWidth.repro.test.ts +126 -0
  86. package/src/ui/traceSortNav.repro.seed.ts +62 -0
  87. package/src/ui/traceSortNav.repro.test.ts +231 -0
  88. package/src/ui/useAttrFilterPicker.ts +89 -0
  89. package/src/ui/useKeyboardNav.ts +1023 -0
  90. package/src/ui/waterfallFilter.test.ts +84 -0
  91. package/src/ui/waterfallFilter.ts +59 -0
  92. package/src/ui/waterfallModel.ts +130 -0
  93. package/src/ui/waterfallNav.repro.seed.ts +86 -0
  94. package/src/ui/waterfallNav.repro.test.ts +271 -0
  95. package/src/ui/waterfallNav.test.ts +453 -0
  96. package/src/ui/waterfallNav.ts +75 -0
  97. package/web/dist/assets/index-B01z9BaO.css +2 -0
  98. package/web/dist/assets/index-CYlBaY9O.js +22 -0
  99. package/web/dist/index.html +13 -0
@@ -0,0 +1,2554 @@
1
+ import { Database } from "bun:sqlite"
2
+ import * as BunFileSystem from "@effect/platform-bun/BunFileSystem"
3
+ import { dirname } from "node:path"
4
+ import { Cause, Clock, Effect, FileSystem, Layer, Schedule, Context } from "effect"
5
+ import { config } from "../config.js"
6
+ import type { AiCallDetail, AiCallSummary, FacetItem, LogItem, SpanItem, StatsItem, TraceItem, TraceSummaryItem, TraceSpanEvent, TraceSpanItem } from "../domain.js"
7
+ import { AI_ATTR_MAP, AI_FTS_KEYS, AI_TEXT_SEARCH_KEYS, truncatePreview } from "../domain.js"
8
+ import { attributeMap, nanosToMilliseconds, normalizeOtlpBinaryId, parseAnyValue, spanKindLabel, spanStatusLabel, stringifyValue, type OtlpLogExportRequest, type OtlpTraceExportRequest } from "../otlp.js"
9
+
10
+ const isSqliteLockError = (error: unknown) =>
11
+ error instanceof Error && /(database is locked|database table is locked|SQLITE_BUSY)/i.test(error.message)
12
+
13
+ interface SpanRow {
14
+ readonly trace_id: string
15
+ readonly span_id: string
16
+ readonly parent_span_id: string | null
17
+ readonly service_name: string
18
+ readonly scope_name: string | null
19
+ readonly operation_name: string
20
+ readonly kind: string | null
21
+ readonly start_time_ms: number
22
+ readonly end_time_ms: number
23
+ readonly duration_ms: number
24
+ readonly status: string
25
+ readonly attributes_json: string
26
+ readonly resource_json: string
27
+ readonly events_json: string
28
+ }
29
+
30
+ interface LogRow {
31
+ readonly id: number
32
+ readonly trace_id: string | null
33
+ readonly span_id: string | null
34
+ readonly service_name: string
35
+ readonly scope_name: string | null
36
+ readonly severity_text: string
37
+ readonly timestamp_ms: number
38
+ readonly body: string
39
+ readonly attributes_json: string
40
+ readonly resource_json: string
41
+ }
42
+
43
+ interface LogSearch {
44
+ readonly serviceName?: string | null
45
+ readonly severity?: string | null
46
+ readonly traceId?: string | null
47
+ readonly spanId?: string | null
48
+ readonly body?: string | null
49
+ readonly lookbackMinutes?: number
50
+ readonly limit?: number
51
+ readonly cursorTimestampMs?: number
52
+ readonly cursorId?: string
53
+ readonly attributeFilters?: Readonly<Record<string, string>>
54
+ readonly attributeContainsFilters?: Readonly<Record<string, string>>
55
+ }
56
+
57
+ interface TraceSearch {
58
+ readonly serviceName?: string | null
59
+ readonly operation?: string | null
60
+ readonly status?: "ok" | "error" | null
61
+ readonly minDurationMs?: number | null
62
+ readonly attributeFilters?: Readonly<Record<string, string>>
63
+ /**
64
+ * Full-text match against the AI prompt/response/tool attribute values
65
+ * on any span in the trace (see AI_FTS_KEYS). When set, traces are
66
+ * filtered to those containing at least one span whose indexed LLM
67
+ * content matches. Powered by span_attr_fts (FTS5).
68
+ */
69
+ readonly aiText?: string | null
70
+ readonly lookbackMinutes?: number
71
+ readonly limit?: number
72
+ readonly cursorStartedAtMs?: number
73
+ readonly cursorTraceId?: string
74
+ }
75
+
76
+ interface SpanSearch {
77
+ readonly serviceName?: string | null
78
+ readonly traceId?: string | null
79
+ readonly operation?: string | null
80
+ readonly parentOperation?: string | null
81
+ readonly status?: "ok" | "error" | null
82
+ readonly lookbackMinutes?: number
83
+ readonly limit?: number
84
+ readonly attributeFilters?: Readonly<Record<string, string>>
85
+ readonly attributeContainsFilters?: Readonly<Record<string, string>>
86
+ }
87
+
88
+ interface TraceStatsSearch extends TraceSearch {
89
+ readonly groupBy: string
90
+ readonly agg: "count" | "avg_duration" | "p95_duration" | "error_rate"
91
+ readonly limit?: number
92
+ }
93
+
94
+ interface LogStatsSearch extends LogSearch {
95
+ readonly groupBy: string
96
+ readonly agg: "count"
97
+ readonly lookbackMinutes?: number
98
+ readonly limit?: number
99
+ }
100
+
101
+ // FacetItem and StatsItem imported from domain.ts
102
+
103
+ interface FacetSearch {
104
+ readonly type: "traces" | "logs"
105
+ readonly field: string
106
+ readonly serviceName?: string | null
107
+ readonly key?: string | null
108
+ readonly lookbackMinutes?: number
109
+ readonly limit?: number
110
+ }
111
+
112
+ interface AiCallSearch {
113
+ readonly service?: string | null
114
+ readonly traceId?: string | null
115
+ readonly sessionId?: string | null
116
+ readonly functionId?: string | null
117
+ readonly provider?: string | null
118
+ readonly model?: string | null
119
+ readonly operation?: string | null
120
+ readonly status?: "ok" | "error" | null
121
+ readonly minDurationMs?: number | null
122
+ readonly text?: string | null
123
+ readonly lookbackMinutes?: number
124
+ readonly limit?: number
125
+ }
126
+
127
+ interface AiCallStatsSearch {
128
+ readonly groupBy: "provider" | "model" | "functionId" | "sessionId" | "status"
129
+ readonly agg: "count" | "avg_duration" | "p95_duration" | "total_input_tokens" | "total_output_tokens"
130
+ readonly service?: string | null
131
+ readonly traceId?: string | null
132
+ readonly sessionId?: string | null
133
+ readonly functionId?: string | null
134
+ readonly provider?: string | null
135
+ readonly model?: string | null
136
+ readonly operation?: string | null
137
+ readonly status?: "ok" | "error" | null
138
+ readonly minDurationMs?: number | null
139
+ readonly lookbackMinutes?: number
140
+ readonly limit?: number
141
+ }
142
+
143
+ interface TraceSummaryRow {
144
+ readonly trace_id: string
145
+ readonly service_name: string
146
+ readonly root_operation_name: string
147
+ readonly started_at_ms: number
148
+ readonly ended_at_ms?: number
149
+ readonly active_span_count: number
150
+ readonly duration_ms: number
151
+ readonly span_count: number
152
+ readonly error_count: number
153
+ }
154
+
155
+ type InternalTraceSpanItem = TraceSpanItem & {
156
+ readonly syntheticMissingParent?: boolean
157
+ }
158
+
159
+ const isSpanRunning = (startTimeMs: number, endTimeMs: number) => endTimeMs <= 0 || endTimeMs < startTimeMs
160
+
161
+ const liveDurationMs = (startTimeMs: number, endTimeMs: number, isRunning: boolean) =>
162
+ Math.max(0, (isRunning ? Date.now() : endTimeMs) - startTimeMs)
163
+
164
+ const parseSummaryRow = (row: TraceSummaryRow): TraceSummaryItem => ({
165
+ isRunning: row.active_span_count > 0,
166
+ traceId: row.trace_id,
167
+ serviceName: row.service_name ?? "unknown",
168
+ rootOperationName: row.root_operation_name ?? "unknown",
169
+ startedAt: new Date(row.started_at_ms),
170
+ durationMs: row.active_span_count > 0 ? liveDurationMs(row.started_at_ms, row.ended_at_ms ?? 0, true) : Math.max(0, row.duration_ms),
171
+ spanCount: row.span_count,
172
+ errorCount: row.error_count,
173
+ warnings: [],
174
+ })
175
+
176
+ // Skip attribute facet rows whose value blob is longer than this. Prevents
177
+ // multi-MB text attrs (ai.prompt, ai.prompt.messages, etc.) from dominating
178
+ // picker-open time — SQLite skips reading those pages from disk when the
179
+ // length predicate is evaluated against the page header, taking queries over
180
+ // a 2GB database from ~1.2s down to ~370ms. Keys whose values are ALL fat
181
+ // simply don't appear in the picker, which is the desired behaviour: you'd
182
+ // never want to filter traces by exact-match on a 1MB prompt blob anyway.
183
+ const FACET_VALUE_MAX_LEN = 512
184
+
185
+ const TRACE_SUMMARY_SELECT_SQL = `
186
+ SELECT
187
+ trace_id,
188
+ COALESCE(MIN(CASE WHEN parent_span_id IS NULL THEN service_name END), MIN(service_name)) AS service_name,
189
+ COALESCE(MIN(CASE WHEN parent_span_id IS NULL THEN operation_name END), MIN(operation_name)) AS root_operation_name,
190
+ MIN(start_time_ms) AS started_at_ms,
191
+ MAX(end_time_ms) AS ended_at_ms,
192
+ SUM(CASE WHEN end_time_ms <= 0 OR end_time_ms < start_time_ms THEN 1 ELSE 0 END) AS active_span_count,
193
+ MAX(end_time_ms) - MIN(start_time_ms) AS duration_ms,
194
+ COUNT(*) AS span_count,
195
+ SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) AS error_count
196
+ FROM spans
197
+ `
198
+
199
+ // Memoize small repeated JSON records. Resource attributes are the primary
200
+ // beneficiary because many spans share the same serialized value; compact
201
+ // repeated span attributes also benefit while large unique payloads bypass
202
+ // the cache to keep memory bounded for long-running daemons.
203
+ const RECORD_PARSE_CACHE_MAX_VALUE_LEN = 1024
204
+ const RECORD_PARSE_CACHE_LIMIT = 256
205
+ const recordParseCache = new Map<string, Record<string, string>>()
206
+ const EMPTY_RECORD: Record<string, string> = {}
207
+
208
+ const parseRecord = (value: string): Record<string, string> => {
209
+ if (value === "" || value === "{}") return EMPTY_RECORD
210
+ const cacheable = value.length <= RECORD_PARSE_CACHE_MAX_VALUE_LEN
211
+ if (cacheable) {
212
+ const cached = recordParseCache.get(value)
213
+ if (cached !== undefined) return cached
214
+ }
215
+ let parsed: Record<string, string>
216
+ try {
217
+ const json = JSON.parse(value) as Record<string, unknown>
218
+ parsed = Object.fromEntries(Object.entries(json).map(([key, entry]) => [key, stringifyValue(entry)]))
219
+ } catch {
220
+ parsed = EMPTY_RECORD
221
+ }
222
+ if (cacheable && recordParseCache.size < RECORD_PARSE_CACHE_LIMIT) {
223
+ recordParseCache.set(value, parsed)
224
+ }
225
+ return parsed
226
+ }
227
+
228
+ const parseEvents = (value: string): readonly TraceSpanEvent[] => {
229
+ if (value === "" || value === "[]") return []
230
+ try {
231
+ const parsed = JSON.parse(value) as Array<{ name: string; timestamp: number; attributes: Record<string, string> }>
232
+ if (parsed.length === 0) return []
233
+ return parsed.map((event) => ({
234
+ name: event.name,
235
+ timestamp: new Date(event.timestamp),
236
+ attributes: event.attributes,
237
+ }))
238
+ } catch {
239
+ return []
240
+ }
241
+ }
242
+
243
+ const parseSpanRow = (row: SpanRow): InternalTraceSpanItem => {
244
+ const isRunning = isSpanRunning(row.start_time_ms, row.end_time_ms)
245
+ return {
246
+ spanId: row.span_id,
247
+ parentSpanId: row.parent_span_id,
248
+ serviceName: row.service_name,
249
+ scopeName: row.scope_name,
250
+ kind: row.kind,
251
+ operationName: row.operation_name,
252
+ startTime: new Date(row.start_time_ms),
253
+ isRunning,
254
+ durationMs: liveDurationMs(row.start_time_ms, row.end_time_ms, isRunning),
255
+ status: row.status === "error" ? "error" : "ok",
256
+ depth: 0,
257
+ tags: {
258
+ ...parseRecord(row.resource_json),
259
+ ...parseRecord(row.attributes_json),
260
+ },
261
+ warnings: [],
262
+ events: parseEvents(row.events_json),
263
+ }
264
+ }
265
+
266
+ const parseLogRow = (row: LogRow): LogItem => ({
267
+ id: String(row.id),
268
+ timestamp: new Date(row.timestamp_ms),
269
+ serviceName: row.service_name,
270
+ severityText: row.severity_text,
271
+ body: row.body,
272
+ traceId: row.trace_id,
273
+ spanId: row.span_id,
274
+ scopeName: row.scope_name,
275
+ attributes: {
276
+ ...parseRecord(row.resource_json),
277
+ ...parseRecord(row.attributes_json),
278
+ },
279
+ })
280
+
281
+ const orderTraceSpans = (spans: readonly InternalTraceSpanItem[]) => {
282
+ const childrenByParent = new Map<string | null, InternalTraceSpanItem[]>()
283
+ const spanIds = new Set(spans.map((span) => span.spanId))
284
+
285
+ for (const span of spans) {
286
+ const key = span.parentSpanId && spanIds.has(span.parentSpanId) ? span.parentSpanId : null
287
+ const siblings = childrenByParent.get(key) ?? []
288
+ siblings.push(span)
289
+ childrenByParent.set(key, siblings)
290
+ }
291
+
292
+ for (const siblings of childrenByParent.values()) {
293
+ siblings.sort((left, right) =>
294
+ left.startTime.getTime() - right.startTime.getTime() || Number(Boolean(left.syntheticMissingParent)) - Number(Boolean(right.syntheticMissingParent))
295
+ )
296
+ }
297
+
298
+ const ordered: Array<InternalTraceSpanItem> = []
299
+ const visit = (parent: string | null, depth: number) => {
300
+ for (const child of childrenByParent.get(parent) ?? []) {
301
+ ordered.push({ ...child, depth })
302
+ visit(child.spanId, depth + 1)
303
+ }
304
+ }
305
+
306
+ visit(null, 0)
307
+ return ordered
308
+ }
309
+
310
+ const buildTrace = (traceId: string, spanRows: readonly SpanRow[]): TraceItem => {
311
+ const parsedSpans = spanRows.map(parseSpanRow)
312
+ const spanIds = new Set(parsedSpans.map((span) => span.spanId))
313
+ const missingParentGroups = new Map<string, InternalTraceSpanItem[]>()
314
+
315
+ for (const span of parsedSpans) {
316
+ if (span.parentSpanId !== null && !spanIds.has(span.parentSpanId)) {
317
+ const siblings = missingParentGroups.get(span.parentSpanId) ?? []
318
+ siblings.push(span)
319
+ missingParentGroups.set(span.parentSpanId, siblings)
320
+ }
321
+ }
322
+
323
+ const syntheticParents: InternalTraceSpanItem[] = [...missingParentGroups.entries()].map(([missingParentId, children]) => {
324
+ const firstChild = children[0]!
325
+ const startedAtMs = Math.min(...children.map((child) => child.startTime.getTime()))
326
+ const endedAtMs = Math.max(...children.map((child) => child.startTime.getTime() + child.durationMs))
327
+ return {
328
+ spanId: missingParentId,
329
+ parentSpanId: null,
330
+ serviceName: firstChild.serviceName,
331
+ scopeName: null,
332
+ kind: null,
333
+ operationName: `[missing parent ${missingParentId.slice(0, 8)}]`,
334
+ startTime: new Date(startedAtMs),
335
+ isRunning: children.some((child) => child.isRunning),
336
+ durationMs: Math.max(0, endedAtMs - startedAtMs),
337
+ status: "error",
338
+ depth: 0,
339
+ tags: {},
340
+ warnings: [`missing span ${missingParentId} (${children.length} child${children.length === 1 ? "" : "ren"})`],
341
+ events: [],
342
+ syntheticMissingParent: true,
343
+ }
344
+ })
345
+
346
+ const orderedSpans = orderTraceSpans([...parsedSpans, ...syntheticParents])
347
+ const startedAtMs = Math.min(...orderedSpans.map((span) => span.startTime.getTime()))
348
+ const endedAtMs = Math.max(...orderedSpans.map((span) => span.startTime.getTime() + span.durationMs))
349
+ const isRunning = orderedSpans.some((span) => span.isRunning)
350
+ const rootSpan = orderedSpans.find((span) => !span.syntheticMissingParent && span.parentSpanId === null)
351
+ ?? orderedSpans.find((span) => !span.syntheticMissingParent)
352
+ ?? orderedSpans[0]
353
+ ?? null
354
+ const warnings = syntheticParents.map((span) => span.warnings[0]!).filter((warning) => warning.length > 0)
355
+
356
+ return {
357
+ traceId,
358
+ serviceName: rootSpan?.serviceName ?? "unknown",
359
+ rootOperationName: rootSpan?.operationName ?? "unknown",
360
+ startedAt: new Date(startedAtMs),
361
+ isRunning,
362
+ durationMs: Math.max(0, endedAtMs - startedAtMs),
363
+ spanCount: orderedSpans.length,
364
+ errorCount: orderedSpans.filter((span) => span.status === "error").length,
365
+ warnings,
366
+ spans: orderedSpans.map(({ syntheticMissingParent: _, ...span }) => span),
367
+ }
368
+ }
369
+
370
+ const buildSpanItems = (traceId: string, spanRows: readonly SpanRow[]): readonly SpanItem[] => {
371
+ const trace = buildTrace(traceId, spanRows)
372
+ const spanById = new Map(trace.spans.map((span) => [span.spanId, span]))
373
+ return trace.spans.map((span) => ({
374
+ traceId,
375
+ rootOperationName: trace.rootOperationName,
376
+ parentOperationName: span.parentSpanId ? spanById.get(span.parentSpanId)?.operationName ?? null : null,
377
+ span,
378
+ }))
379
+ }
380
+
381
+ const buildSpanItem = (traceId: string, spanRows: readonly SpanRow[], spanId: string): SpanItem | null =>
382
+ buildSpanItems(traceId, spanRows).find((item) => item.span.spanId === spanId) ?? null
383
+
384
+ const matchesAttributes = (attributes: Readonly<Record<string, string>>, filters: Readonly<Record<string, string>> | undefined) =>
385
+ !filters || Object.entries(filters).every(([key, value]) => attributes[key] === value)
386
+
387
+ const matchesAttributeContains = (attributes: Readonly<Record<string, string>>, filters: Readonly<Record<string, string>> | undefined) =>
388
+ !filters || Object.entries(filters).every(([key, needle]) => {
389
+ const value = attributes[key]
390
+ return value !== undefined && value.toLowerCase().includes(needle.toLowerCase())
391
+ })
392
+
393
+ const percentile = (values: readonly number[], ratio: number) => {
394
+ if (values.length === 0) return 0
395
+ const sorted = [...values].sort((left, right) => left - right)
396
+ const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * ratio) - 1))
397
+ return sorted[index] ?? 0
398
+ }
399
+
400
+ const tokenizeFts = (value: string) => value.match(/[A-Za-z0-9_]+/g)?.filter((token) => token.length > 1) ?? []
401
+
402
+ const toFtsMatchQuery = (value: string) => {
403
+ const tokens = tokenizeFts(value)
404
+ if (tokens.length === 0) return null
405
+ return tokens.map((token) => `${token}*`).join(" AND ")
406
+ }
407
+
408
+ const buildExactAttributeMatchSubquery = (
409
+ tableName: "span_attributes" | "log_attributes",
410
+ idColumns: readonly string[],
411
+ filters: Readonly<Record<string, string>> | undefined,
412
+ ) => {
413
+ const entries = Object.entries(filters ?? {})
414
+ if (entries.length === 0) return null
415
+ const disjunction = entries.map(() => "(key = ? AND value = ?)").join(" OR ")
416
+ return {
417
+ sql: `
418
+ SELECT ${idColumns.join(", ")}
419
+ FROM ${tableName}
420
+ WHERE ${disjunction}
421
+ GROUP BY ${idColumns.join(", ")}
422
+ HAVING COUNT(DISTINCT key) = ${entries.length}
423
+ `,
424
+ params: entries.flatMap(([key, value]) => [key, value]),
425
+ }
426
+ }
427
+
428
+ const buildContainsAttributeMatchSubquery = (
429
+ tableName: "span_attributes" | "log_attributes",
430
+ idColumns: readonly string[],
431
+ filters: Readonly<Record<string, string>> | undefined,
432
+ ) => {
433
+ const entries = Object.entries(filters ?? {})
434
+ if (entries.length === 0) return null
435
+ const disjunction = entries.map(() => "(key = ? AND value LIKE ? COLLATE NOCASE)").join(" OR ")
436
+ return {
437
+ sql: `
438
+ SELECT ${idColumns.join(", ")}
439
+ FROM ${tableName}
440
+ WHERE ${disjunction}
441
+ GROUP BY ${idColumns.join(", ")}
442
+ HAVING COUNT(DISTINCT key) = ${entries.length}
443
+ `,
444
+ params: entries.flatMap(([key, value]) => [key, `%${value}%`]),
445
+ }
446
+ }
447
+
448
+ // Read-only surface of the telemetry store. Pulled out so a readonly
449
+ // SQLite connection (TUI / HTTP query handlers) can be expressed as a
450
+ // distinct service identifier from the writer, without re-declaring
451
+ // every query in a wrapper layer. The writer's value still satisfies
452
+ // this shape — TelemetryStoreLive can provide both identifiers from
453
+ // one underlying object if needed.
454
+ export interface TelemetryStoreReader {
455
+ readonly listServices: Effect.Effect<readonly string[], Error>
456
+ readonly listRecentTraces: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceItem[], Error>
457
+ readonly listTraceSummaries: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceSummaryItem[], Error>
458
+ readonly searchTraces: (input: TraceSearch) => Effect.Effect<readonly TraceItem[], Error>
459
+ readonly searchTraceSummaries: (input: TraceSearch) => Effect.Effect<readonly TraceSummaryItem[], Error>
460
+ readonly traceStats: (input: TraceStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
461
+ readonly getTrace: (traceId: string) => Effect.Effect<TraceItem | null, Error>
462
+ readonly getSpan: (spanId: string) => Effect.Effect<SpanItem | null, Error>
463
+ readonly listTraceSpans: (traceId: string) => Effect.Effect<readonly SpanItem[], Error>
464
+ readonly searchSpans: (input: SpanSearch) => Effect.Effect<readonly SpanItem[], Error>
465
+ readonly searchLogs: (input: LogSearch) => Effect.Effect<readonly LogItem[], Error>
466
+ readonly logStats: (input: LogStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
467
+ readonly listFacets: (input: FacetSearch) => Effect.Effect<readonly FacetItem[], Error>
468
+ readonly listRecentLogs: (serviceName: string) => Effect.Effect<readonly LogItem[], Error>
469
+ readonly listTraceLogs: (traceId: string) => Effect.Effect<readonly LogItem[], Error>
470
+ readonly searchAiCalls: (input: AiCallSearch) => Effect.Effect<readonly AiCallSummary[], Error>
471
+ readonly getAiCall: (spanId: string) => Effect.Effect<AiCallDetail | null, Error>
472
+ readonly aiCallStats: (input: AiCallStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
473
+ }
474
+
475
+ export class TelemetryStoreReadonly extends Context.Service<TelemetryStoreReadonly, TelemetryStoreReader>()("motel/TelemetryStoreReadonly") {}
476
+
477
+ export class TelemetryStore extends Context.Service<
478
+ TelemetryStore,
479
+ TelemetryStoreReader & {
480
+ readonly ingestTraces: (payload: OtlpTraceExportRequest) => Effect.Effect<{ readonly insertedSpans: number }, Error>
481
+ readonly ingestLogs: (payload: OtlpLogExportRequest) => Effect.Effect<{ readonly insertedLogs: number }, Error>
482
+ readonly runRetentionNow: Effect.Effect<void, Error>
483
+ }
484
+ >()("motel/TelemetryStore") {}
485
+
486
+
487
+ /**
488
+ * How this TelemetryStore instance behaves:
489
+ *
490
+ * - `readonly` — opens the SQLite connection read-only and skips every
491
+ * DDL/DML initialisation. Use this from the TUI (and anywhere else
492
+ * that only queries); it avoids the "database is locked" race that
493
+ * happens when a TUI process races a daemon's writer for the schema
494
+ * pragmas on startup. Writes through the service interface become
495
+ * runtime errors — but readers don't call them.
496
+ *
497
+ * - `runRetention` — fork the background cleanup loop (age + size cap
498
+ * eviction, WAL checkpoint). Only one process should own this at a
499
+ * time. The ingest worker owns it; the HTTP thread and TUI skip it.
500
+ */
501
+ export interface TelemetryStoreOptions {
502
+ readonly readonly: boolean
503
+ readonly runRetention: boolean
504
+ }
505
+
506
+ const makeTelemetryStoreEffect = (opts: TelemetryStoreOptions) =>
507
+ Effect.gen(function* () {
508
+ const fileSystem = yield* FileSystem.FileSystem
509
+ yield* fileSystem.makeDirectory(dirname(config.otel.databasePath), { recursive: true })
510
+ const db = yield* Effect.acquireRelease(
511
+ Effect.sync(() => new Database(config.otel.databasePath, {
512
+ create: !opts.readonly,
513
+ readonly: opts.readonly,
514
+ })),
515
+ (db) => Effect.sync(() => {
516
+ if (!opts.readonly) {
517
+ // `PRAGMA optimize` at close persists any stats SQLite gathered
518
+ // during the session, so the next process start gets an accurate
519
+ // query planner on the first query instead of a 3-second cold
520
+ // run. Cheap: it skips work unless stats have drifted.
521
+ try { db.exec(`PRAGMA optimize;`) } catch { /* nothing */ }
522
+ }
523
+ db.close()
524
+ }),
525
+ )
526
+ if (opts.readonly) {
527
+ // Readonly connections skip schema init entirely — the schema
528
+ // already exists (a writer created it) and any `CREATE TABLE IF
529
+ // NOT EXISTS` / `PRAGMA journal_mode = WAL` statement would
530
+ // attempt a write and fight the daemon for the write lock.
531
+ // `query_only = 1` logically blocks any DML the app might
532
+ // accidentally send; still bump cache + mmap since those are
533
+ // safe and keep queries fast.
534
+ db.exec(`
535
+ PRAGMA query_only = 1;
536
+ PRAGMA busy_timeout = 15000;
537
+ PRAGMA cache_size = -65536;
538
+ PRAGMA mmap_size = 268435456;
539
+ `)
540
+ } else {
541
+ db.exec(`
542
+ -- Bump cache above the 2MB default. 64MB fits most hot index pages
543
+ -- (trace_summaries, spans, span_attributes indexes) in RAM even on
544
+ -- multi-GB databases, cutting cold-read latency meaningfully on
545
+ -- picker / search queries that sweep the index.
546
+ PRAGMA cache_size = -65536;
547
+ -- Let SQLite memory-map the first 256MB of the file. This is a
548
+ -- cheap way to avoid read() syscalls on hot pages and lets the OS
549
+ -- page cache serve index lookups directly. Safe on macOS and Linux;
550
+ -- SQLite silently caps at actual file size for smaller DBs.
551
+ PRAGMA mmap_size = 268435456;
552
+ `)
553
+ // auto_vacuum is a header-level setting: it only takes effect on
554
+ // an empty DB, or on the next VACUUM after a change. Setting it
555
+ // here, BEFORE the first CREATE TABLE, is the only path that
556
+ // makes incremental_vacuum work without a full VACUUM. For
557
+ // existing DBs that predate this setting keep their current mode;
558
+ // Motel never performs a surprise full-file VACUUM at startup.
559
+ try { db.exec(`PRAGMA auto_vacuum = INCREMENTAL;`) } catch { /* ignore */ }
560
+ try {
561
+ db.exec(`
562
+ PRAGMA journal_mode = WAL;
563
+ PRAGMA synchronous = NORMAL;
564
+ PRAGMA temp_store = MEMORY;
565
+ -- WAL checkpoint automatically when it grows past ~16MB. Without
566
+ -- this the WAL happily runs into the hundreds of MB and queries
567
+ -- start paying the cost of walking the WAL on every read.
568
+ PRAGMA wal_autocheckpoint = 4000;
569
+ -- Hard floor for the WAL file. Auto-checkpoint controls *when*
570
+ -- pages move out of the WAL; size_limit controls how much the
571
+ -- WAL file is allowed to grow on disk. 128MB is generous enough
572
+ -- to absorb a long write burst without blocking on truncation,
573
+ -- tight enough that a wedged retention loop can't hide a 20GB
574
+ -- WAL the way a default no-limit configuration can.
575
+ PRAGMA journal_size_limit = 134217728;
576
+
577
+ CREATE TABLE IF NOT EXISTS spans (
578
+ trace_id TEXT NOT NULL,
579
+ span_id TEXT NOT NULL,
580
+ parent_span_id TEXT,
581
+ service_name TEXT NOT NULL,
582
+ scope_name TEXT,
583
+ operation_name TEXT NOT NULL,
584
+ kind TEXT,
585
+ start_time_ms INTEGER NOT NULL,
586
+ end_time_ms INTEGER NOT NULL,
587
+ duration_ms REAL NOT NULL,
588
+ status TEXT NOT NULL,
589
+ attributes_json TEXT NOT NULL,
590
+ resource_json TEXT NOT NULL,
591
+ events_json TEXT NOT NULL,
592
+ PRIMARY KEY (trace_id, span_id)
593
+ );
594
+
595
+ CREATE INDEX IF NOT EXISTS idx_spans_service_time ON spans(service_name, start_time_ms DESC);
596
+ CREATE INDEX IF NOT EXISTS idx_spans_trace_time ON spans(trace_id, start_time_ms ASC);
597
+ CREATE INDEX IF NOT EXISTS idx_spans_span_id ON spans(span_id);
598
+ CREATE INDEX IF NOT EXISTS idx_spans_status_time ON spans(status, start_time_ms DESC);
599
+
600
+ CREATE TABLE IF NOT EXISTS logs (
601
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
602
+ trace_id TEXT,
603
+ span_id TEXT,
604
+ service_name TEXT NOT NULL,
605
+ scope_name TEXT,
606
+ severity_text TEXT NOT NULL,
607
+ timestamp_ms INTEGER NOT NULL,
608
+ body TEXT NOT NULL,
609
+ attributes_json TEXT NOT NULL,
610
+ resource_json TEXT NOT NULL
611
+ );
612
+
613
+ CREATE INDEX IF NOT EXISTS idx_logs_service_time ON logs(service_name, timestamp_ms DESC);
614
+ CREATE INDEX IF NOT EXISTS idx_logs_trace_time ON logs(trace_id, timestamp_ms DESC);
615
+ CREATE INDEX IF NOT EXISTS idx_logs_span_time ON logs(span_id, timestamp_ms DESC);
616
+ CREATE INDEX IF NOT EXISTS idx_logs_severity_time ON logs(severity_text, timestamp_ms DESC);
617
+
618
+ CREATE TABLE IF NOT EXISTS trace_summaries (
619
+ trace_id TEXT PRIMARY KEY,
620
+ service_name TEXT NOT NULL,
621
+ root_operation_name TEXT NOT NULL,
622
+ started_at_ms INTEGER NOT NULL,
623
+ ended_at_ms INTEGER NOT NULL,
624
+ active_span_count INTEGER NOT NULL DEFAULT 0,
625
+ duration_ms REAL NOT NULL,
626
+ span_count INTEGER NOT NULL,
627
+ error_count INTEGER NOT NULL
628
+ );
629
+
630
+ CREATE INDEX IF NOT EXISTS idx_trace_summaries_started_at ON trace_summaries(started_at_ms DESC, trace_id DESC);
631
+ CREATE INDEX IF NOT EXISTS idx_trace_summaries_service_started_at ON trace_summaries(service_name, started_at_ms DESC, trace_id DESC);
632
+ CREATE INDEX IF NOT EXISTS idx_trace_summaries_duration ON trace_summaries(duration_ms DESC);
633
+
634
+ CREATE TABLE IF NOT EXISTS span_attributes (
635
+ trace_id TEXT NOT NULL,
636
+ span_id TEXT NOT NULL,
637
+ key TEXT NOT NULL,
638
+ value TEXT NOT NULL,
639
+ PRIMARY KEY (trace_id, span_id, key)
640
+ );
641
+
642
+ CREATE INDEX IF NOT EXISTS idx_span_attributes_key_value ON span_attributes(key, value, trace_id, span_id);
643
+ CREATE INDEX IF NOT EXISTS idx_span_attributes_trace_span ON span_attributes(trace_id, span_id);
644
+
645
+ CREATE TABLE IF NOT EXISTS log_attributes (
646
+ log_id INTEGER NOT NULL,
647
+ key TEXT NOT NULL,
648
+ value TEXT NOT NULL,
649
+ PRIMARY KEY (log_id, key)
650
+ );
651
+
652
+ CREATE INDEX IF NOT EXISTS idx_log_attributes_key_value ON log_attributes(key, value, log_id);
653
+ CREATE INDEX IF NOT EXISTS idx_log_attributes_log_id ON log_attributes(log_id);
654
+
655
+ CREATE TABLE IF NOT EXISTS motel_maintenance (
656
+ key TEXT PRIMARY KEY,
657
+ value TEXT NOT NULL
658
+ );
659
+ `)
660
+ } catch (err) {
661
+ if (!isSqliteLockError(err)) throw err
662
+ console.warn(`motel: writer bootstrap skipped during startup: ${(err as Error).message}`)
663
+ }
664
+ }
665
+
666
+ // Tables detected at runtime. For writer connections these flags are
667
+ // set by the FTS `CREATE VIRTUAL TABLE IF NOT EXISTS` try/catch; for
668
+ // readonly connections we probe `sqlite_master` and set them based on
669
+ // what the writer has already provisioned.
670
+ let hasFts = true
671
+ let hasAttrFts = true
672
+ if (opts.readonly) {
673
+ try {
674
+ const row = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='span_operation_fts'`).get()
675
+ hasFts = row !== null
676
+ } catch { hasFts = false }
677
+ try {
678
+ const row = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='span_attr_fts'`).get()
679
+ const backfill = db.query(`SELECT value FROM motel_maintenance WHERE key = 'span_attr_fts_v1'`).get() as { value: string } | null
680
+ hasAttrFts = row !== null && backfill?.value === "complete"
681
+ } catch { hasAttrFts = false }
682
+ }
683
+
684
+ if (!opts.readonly) {
685
+ try {
686
+ db.exec(`
687
+ CREATE VIRTUAL TABLE IF NOT EXISTS span_operation_fts USING fts5(
688
+ trace_id UNINDEXED,
689
+ span_id UNINDEXED,
690
+ operation_name,
691
+ tokenize='unicode61'
692
+ );
693
+
694
+ CREATE VIRTUAL TABLE IF NOT EXISTS log_body_fts USING fts5(
695
+ log_id UNINDEXED,
696
+ body,
697
+ tokenize='unicode61'
698
+ );
699
+ `)
700
+ } catch {
701
+ hasFts = false
702
+ // FTS is optional; queries will fall back to LIKE if unavailable.
703
+ }
704
+
705
+ // External-content FTS5 over the subset of span_attributes.value rows
706
+ // whose key is in AI_FTS_KEYS (LLM prompts, responses, tool calls,
707
+ // etc.). External content means the inverted index is the only
708
+ // FTS storage — the value text itself continues to live once in
709
+ // span_attributes, not duplicated into the FTS table. On a 2 GB DB
710
+ // with 270 MB of prompt JSON this typically adds ~50-120 MB of
711
+ // index, turning a 500-800ms LIKE scan into a <50ms MATCH.
712
+ //
713
+ // Keys are inlined into the trigger DDL rather than looked up in a
714
+ // side table so the `WHEN` guard stays constant-cost (a subquery
715
+ // would run on every span_attributes insert — ~60/span).
716
+ if (hasFts) {
717
+ try {
718
+ const keyList = AI_FTS_KEYS.map((k) => `'${k.replace(/'/g, "''")}'`).join(", ")
719
+ db.exec(`
720
+ CREATE VIRTUAL TABLE IF NOT EXISTS span_attr_fts USING fts5(
721
+ value,
722
+ content='span_attributes',
723
+ content_rowid='rowid',
724
+ tokenize='unicode61 remove_diacritics 2'
725
+ );
726
+
727
+ -- Mirror inserts into FTS when the key carries LLM content.
728
+ -- NOTE: triggers MUST use fully-qualified name (new.rowid,
729
+ -- new.value) and emit rowid so external-content FTS can
730
+ -- fetch the value back via span_attributes.rowid.
731
+ CREATE TRIGGER IF NOT EXISTS span_attr_fts_ai AFTER INSERT ON span_attributes
732
+ WHEN new.key IN (${keyList})
733
+ BEGIN
734
+ INSERT INTO span_attr_fts(rowid, value) VALUES (new.rowid, new.value);
735
+ END;
736
+
737
+ -- Delete with the same guard so retention & re-ingest stay
738
+ -- in sync. External-content 'delete' command needs the
739
+ -- original value to remove from the inverted index.
740
+ CREATE TRIGGER IF NOT EXISTS span_attr_fts_ad AFTER DELETE ON span_attributes
741
+ WHEN old.key IN (${keyList})
742
+ BEGIN
743
+ INSERT INTO span_attr_fts(span_attr_fts, rowid, value)
744
+ VALUES ('delete', old.rowid, old.value);
745
+ END;
746
+
747
+ -- Handle in-place updates (rare; re-ingest usually goes
748
+ -- DELETE then INSERT but belt-and-braces).
749
+ CREATE TRIGGER IF NOT EXISTS span_attr_fts_au AFTER UPDATE ON span_attributes
750
+ WHEN old.key IN (${keyList}) OR new.key IN (${keyList})
751
+ BEGIN
752
+ INSERT INTO span_attr_fts(span_attr_fts, rowid, value)
753
+ VALUES ('delete', old.rowid, old.value);
754
+ INSERT INTO span_attr_fts(rowid, value)
755
+ SELECT new.rowid, new.value
756
+ WHERE new.key IN (${keyList});
757
+ END;
758
+ `)
759
+ } catch {
760
+ hasAttrFts = false
761
+ }
762
+ }
763
+
764
+ try {
765
+ db.exec(`ALTER TABLE trace_summaries ADD COLUMN active_span_count INTEGER NOT NULL DEFAULT 0`)
766
+ } catch {
767
+ // Existing databases may already have the column.
768
+ }
769
+
770
+ // Prime the query planner. `PRAGMA optimize` is SQLite's modern,
771
+ // lightweight stats refresh: it only re-ANALYZEs indexes whose row
772
+ // counts have drifted significantly since the last run, capped at
773
+ // `analysis_limit` iterations per index so it finishes in a
774
+ // bounded time even on large databases. Without this, queries like
775
+ // the attribute picker facet run with guessed row estimates and
776
+ // pay 3-4s on cold open instead of 400ms.
777
+ try {
778
+ db.exec(`PRAGMA analysis_limit = 1000; PRAGMA optimize;`)
779
+ } catch {
780
+ // ANALYZE / optimize failures are never fatal — queries still work,
781
+ // they just run with default row estimates.
782
+ }
783
+ // Longer busy timeout: the ingest worker holds the write lock for up
784
+ // to a few seconds during big OTLP batches, and the daemon's retention
785
+ // passes can do the same. Apply this AFTER startup maintenance so
786
+ // lock-conflicted bootstrap steps fail fast instead of stalling health
787
+ // for the full 15s timeout.
788
+ try { db.exec(`PRAGMA busy_timeout = 15000;`) } catch { /* ignore */ }
789
+ } // end: if (!opts.readonly) writer init
790
+
791
+ const insertSpan = db.query(`
792
+ INSERT INTO spans (
793
+ trace_id, span_id, parent_span_id, service_name, scope_name, operation_name, kind,
794
+ start_time_ms, end_time_ms, duration_ms, status, attributes_json, resource_json, events_json
795
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
796
+ ON CONFLICT(trace_id, span_id) DO UPDATE SET
797
+ parent_span_id = excluded.parent_span_id,
798
+ service_name = excluded.service_name,
799
+ scope_name = excluded.scope_name,
800
+ operation_name = excluded.operation_name,
801
+ kind = excluded.kind,
802
+ start_time_ms = excluded.start_time_ms,
803
+ end_time_ms = excluded.end_time_ms,
804
+ duration_ms = excluded.duration_ms,
805
+ status = excluded.status,
806
+ attributes_json = excluded.attributes_json,
807
+ resource_json = excluded.resource_json,
808
+ events_json = excluded.events_json
809
+ `)
810
+
811
+ const insertLog = db.query(`
812
+ INSERT INTO logs (
813
+ trace_id, span_id, service_name, scope_name, severity_text, timestamp_ms, body, attributes_json, resource_json
814
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
815
+ `)
816
+
817
+ const upsertTraceSummary = db.query(`
818
+ INSERT OR REPLACE INTO trace_summaries (
819
+ trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
820
+ )
821
+ SELECT trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
822
+ FROM (
823
+ ${TRACE_SUMMARY_SELECT_SQL}
824
+ WHERE trace_id = ?
825
+ GROUP BY trace_id
826
+ )
827
+ `)
828
+
829
+ const reconcileTraceSummaries = Effect.sync(() => {
830
+ const marker = db.query(`SELECT value FROM motel_maintenance WHERE key = 'trace_summary_cursor'`).get() as { value: string } | null
831
+ const cursor = Number(marker?.value ?? 0)
832
+ const rows = db.query(`SELECT rowid, trace_id FROM spans WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`).all(cursor, config.otel.retentionTraceBatch) as Array<{ rowid: number; trace_id: string }>
833
+ if (rows.length === 0) {
834
+ db.query(`INSERT OR REPLACE INTO motel_maintenance(key, value) VALUES ('trace_summary_cursor', '0')`).run()
835
+ return
836
+ }
837
+ const transaction = db.transaction(() => {
838
+ for (const traceId of new Set(rows.map((row) => row.trace_id))) upsertTraceSummary.run(traceId)
839
+ db.query(`INSERT OR REPLACE INTO motel_maintenance(key, value) VALUES ('trace_summary_cursor', ?)`).run(String(rows.at(-1)!.rowid))
840
+ })
841
+ transaction()
842
+ })
843
+
844
+ const deleteSpanAttributes = db.query(`DELETE FROM span_attributes WHERE trace_id = ? AND span_id = ?`)
845
+ const insertSpanAttribute = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES (?, ?, ?, ?)`)
846
+ const spanAttributeInsertManyByCount = new Map<number, ReturnType<Database["query"]>>()
847
+ const insertSpanAttributesMany = (traceId: string, spanId: string, attributes: Readonly<Record<string, string>>) => {
848
+ const entries = Object.entries(attributes)
849
+ if (entries.length === 0) return
850
+ if (entries.length === 1) {
851
+ const [key, value] = entries[0]!
852
+ insertSpanAttribute.run(traceId, spanId, key, value)
853
+ return
854
+ }
855
+ let query = spanAttributeInsertManyByCount.get(entries.length)
856
+ if (!query) {
857
+ query = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES ${entries.map(() => "(?, ?, ?, ?)").join(", ")}`)
858
+ spanAttributeInsertManyByCount.set(entries.length, query)
859
+ }
860
+ query.run(...entries.flatMap(([key, value]) => [traceId, spanId, key, value]))
861
+ }
862
+ const deleteSpanOperationSearch = db.query(`DELETE FROM span_operation_fts WHERE trace_id = ? AND span_id = ?`)
863
+ const insertSpanOperationSearch = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES (?, ?, ?)`)
864
+ const deleteSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
865
+ const insertSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
866
+ const updateSpanOperationSearchMany = (operations: ReadonlyArray<readonly [string, string, string]>) => {
867
+ if (operations.length === 0) return
868
+ if (operations.length === 1) {
869
+ const [traceId, spanId, operationName] = operations[0]!
870
+ deleteSpanOperationSearch.run(traceId, spanId)
871
+ insertSpanOperationSearch.run(traceId, spanId, operationName)
872
+ return
873
+ }
874
+
875
+ let deleteQuery = deleteSpanOperationSearchManyByCount.get(operations.length)
876
+ if (!deleteQuery) {
877
+ deleteQuery = db.query(`DELETE FROM span_operation_fts WHERE ${operations.map(() => "(trace_id = ? AND span_id = ?)").join(" OR ")}`)
878
+ deleteSpanOperationSearchManyByCount.set(operations.length, deleteQuery)
879
+ }
880
+ deleteQuery.run(...operations.flatMap(([traceId, spanId]) => [traceId, spanId]))
881
+
882
+ let insertQuery = insertSpanOperationSearchManyByCount.get(operations.length)
883
+ if (!insertQuery) {
884
+ insertQuery = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES ${operations.map(() => "(?, ?, ?)").join(", ")}`)
885
+ insertSpanOperationSearchManyByCount.set(operations.length, insertQuery)
886
+ }
887
+ insertQuery.run(...operations.flatMap(([traceId, spanId, operationName]) => [traceId, spanId, operationName]))
888
+ }
889
+ const insertLogAttribute = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES (?, ?, ?)`)
890
+ const logAttributeInsertManyByCount = new Map<number, ReturnType<Database["query"]>>()
891
+ const insertLogAttributesMany = (logId: number, attributes: Readonly<Record<string, string>>) => {
892
+ const entries = Object.entries(attributes)
893
+ if (entries.length === 0) return
894
+ if (entries.length === 1) {
895
+ const [key, value] = entries[0]!
896
+ insertLogAttribute.run(logId, key, value)
897
+ return
898
+ }
899
+ let query = logAttributeInsertManyByCount.get(entries.length)
900
+ if (!query) {
901
+ query = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES ${entries.map(() => "(?, ?, ?)").join(", ")}`)
902
+ logAttributeInsertManyByCount.set(entries.length, query)
903
+ }
904
+ query.run(...entries.flatMap(([key, value]) => [logId, key, value]))
905
+ }
906
+ const insertLogBodySearch = db.query(`INSERT INTO log_body_fts (log_id, body) VALUES (?, ?)`)
907
+ const insertLogBodySearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
908
+ const insertLogBodySearchMany = (entries: ReadonlyArray<readonly [string, string]>) => {
909
+ if (entries.length === 0) return
910
+ if (entries.length === 1) {
911
+ const [logId, body] = entries[0]!
912
+ insertLogBodySearch.run(logId, body)
913
+ return
914
+ }
915
+ let query = insertLogBodySearchManyByCount.get(entries.length)
916
+ if (!query) {
917
+ query = db.query(`INSERT INTO log_body_fts (log_id, body) VALUES ${entries.map(() => "(?, ?)").join(", ")}`)
918
+ insertLogBodySearchManyByCount.set(entries.length, query)
919
+ }
920
+ query.run(...entries.flatMap(([logId, body]) => [logId, body]))
921
+ }
922
+
923
+ const maxDbSizeBytes = config.otel.maxDbSizeMb * 1024 * 1024
924
+
925
+ // Freelist-ratio thresholds for the adaptive reclaim loop. Below the
926
+ // LOW threshold there's nothing worth doing; above HIGH we are in the
927
+ // 17GB-DB-with-10GB-freelist failure mode and need to reclaim aggressively
928
+ // even if it costs writer-lock time.
929
+ const FREELIST_LOW_RATIO = 0.05
930
+ const FREELIST_MID_RATIO = 0.20
931
+ const FREELIST_HIGH_RATIO = 0.50
932
+ const VACUUM_PAGES_NORMAL = 2000 // ~8MB/pass
933
+ const VACUUM_PAGES_BUSY = 20000 // ~80MB/pass — used when freelist > 20%
934
+ const VACUUM_PAGES_PANIC = 50000 // ~200MB/pass — only when ratio > 50%
935
+
936
+ const ftsTableNames = ["span_attr_fts", "log_body_fts", "span_operation_fts"] as const
937
+
938
+ const incrementalFtsMerge = (pages: number) => {
939
+ // FTS5 segment merges drop tombstone rows that DELETE leaves behind.
940
+ // Without periodic merges, deleted FTS rows stay on disk indefinitely
941
+ // — a major source of freelist pages on a heavy-deletion workload.
942
+ // `merge=N` is a bounded, online operation: it merges at most N
943
+ // pages of work and returns. Per FTS5 docs, missing tables silently
944
+ // throw; we swallow because not every DB has every FTS table.
945
+ for (const name of ftsTableNames) {
946
+ try { db.query(`INSERT INTO ${name}(${name}) VALUES (?)`).run(`merge=${pages}`) } catch { /* table absent or older schema */ }
947
+ }
948
+ }
949
+
950
+ const reclaimSpace = Effect.fn("motel/TelemetryStore.reclaimSpace")(function* () {
951
+ yield* Effect.sync(() => {
952
+ const pageCount = (db.query(`PRAGMA page_count`).get() as { page_count: number }).page_count
953
+ const freePages = (db.query(`PRAGMA freelist_count`).get() as { freelist_count: number }).freelist_count
954
+ if (pageCount === 0) return
955
+ const ratio = freePages / pageCount
956
+ if (ratio < FREELIST_LOW_RATIO) return
957
+
958
+ // Adaptive vacuum sizing — fixed 2000 pages/min could not keep
959
+ // up with sustained deletions, leaking 10GB of freelist over
960
+ // time. Scale the per-pass work to the size of the backlog so
961
+ // we stay roughly proportional to the deficit.
962
+ const pages =
963
+ ratio >= FREELIST_HIGH_RATIO ? VACUUM_PAGES_PANIC :
964
+ ratio >= FREELIST_MID_RATIO ? VACUUM_PAGES_BUSY :
965
+ VACUUM_PAGES_NORMAL
966
+
967
+ try { db.exec(`PRAGMA incremental_vacuum(${pages});`) } catch { /* ignore */ }
968
+
969
+ // In WAL mode incremental_vacuum only moves pages — the file
970
+ // shrinks on the next checkpoint. PASSIVE silently skips when
971
+ // readers are active (the failure mode the agent's research
972
+ // flagged: checkpoint starvation). Use RESTART normally and
973
+ // TRUNCATE in panic mode to physically shrink the WAL when it
974
+ // has grown.
975
+ const mode = ratio >= FREELIST_HIGH_RATIO ? "TRUNCATE" : "RESTART"
976
+ try { db.exec(`PRAGMA wal_checkpoint(${mode});`) } catch { /* ignore */ }
977
+ })
978
+ })
979
+
980
+ const cleanupExpired = Effect.fn("motel/TelemetryStore.cleanupExpired")(function* () {
981
+ const now = yield* Clock.currentTimeMillis
982
+
983
+ yield* Effect.sync(() => {
984
+ const cutoff = now - config.otel.retentionHours * 60 * 60 * 1000
985
+
986
+ // Evict at TRACE granularity so we never leave a trace half-gutted
987
+ // (previous logic deleted oldest 20% of spans, which happily sliced
988
+ // across traces and corrupted the summary rebuild). Running traces
989
+ // are protected — only `active_span_count = 0` summaries are in
990
+ // scope for eviction.
991
+ const toEvict = new Set<string>()
992
+
993
+ // Time-based: completed traces whose last span ended before cutoff.
994
+ const timeExpired = db.query(
995
+ `SELECT trace_id FROM trace_summaries WHERE active_span_count = 0 AND ended_at_ms > 0 AND ended_at_ms < ? ORDER BY ended_at_ms ASC LIMIT ?`,
996
+ ).all(cutoff, config.otel.retentionTraceBatch) as readonly { trace_id: string }[]
997
+ for (const row of timeExpired) toEvict.add(row.trace_id)
998
+
999
+ // Size-based: if actual data exceeds the target, drop one bounded
1000
+ // batch of the oldest completed traces. `(page_count - freelist_count)`
1001
+ // ignores freed-but-not-vacuumed pages so a large freelist doesn't
1002
+ // trigger a deletion death spiral.
1003
+ const pageCount = (db.query(`PRAGMA page_count`).get() as { page_count: number }).page_count
1004
+ const freePages = (db.query(`PRAGMA freelist_count`).get() as { freelist_count: number }).freelist_count
1005
+ const pageSize = (db.query(`PRAGMA page_size`).get() as { page_size: number }).page_size
1006
+ const dbSize = (pageCount - freePages) * pageSize
1007
+ if (dbSize > maxDbSizeBytes) {
1008
+ const oldest = db.query(
1009
+ `SELECT trace_id FROM trace_summaries WHERE active_span_count = 0 ORDER BY started_at_ms ASC LIMIT ?`,
1010
+ ).all(config.otel.retentionTraceBatch) as readonly { trace_id: string }[]
1011
+ // Set.add dedupes overlap with the time-expired batch above.
1012
+ for (const row of oldest) toEvict.add(row.trace_id)
1013
+ }
1014
+
1015
+ // Logs have their own retention boundary. A correlated log may refer
1016
+ // to a trace that was sampled elsewhere or never reached Motel, so
1017
+ // tying log eviction to trace_summaries lets those rows grow forever.
1018
+ const expiredLogs = db.query(`DELETE FROM logs WHERE id IN (SELECT id FROM logs WHERE timestamp_ms < ? ORDER BY timestamp_ms ASC LIMIT ?)`).run(cutoff, config.otel.retentionLogBatch)
1019
+ let deletedLogs = Number(expiredLogs.changes) > 0
1020
+ if (dbSize > maxDbSizeBytes) {
1021
+ const oversizedLogs = db.query(`DELETE FROM logs WHERE id IN (SELECT id FROM logs ORDER BY timestamp_ms ASC LIMIT ?)`).run(config.otel.retentionLogBatch)
1022
+ deletedLogs = deletedLogs || Number(oversizedLogs.changes) > 0
1023
+ }
1024
+
1025
+ // Batch the trace-id list so the IN placeholders stay under
1026
+ // SQLite's default limit (~999). Each batch wipes every row
1027
+ // reachable from those trace_ids across the cascade tables.
1028
+ const traceIds = Array.from(toEvict)
1029
+ const BATCH_SIZE = 500
1030
+ for (let offset = 0; offset < traceIds.length; offset += BATCH_SIZE) {
1031
+ const batch = traceIds.slice(offset, offset + BATCH_SIZE)
1032
+ const placeholders = batch.map(() => "?").join(",")
1033
+ db.query(`DELETE FROM span_attributes WHERE trace_id IN (${placeholders})`).run(...batch)
1034
+ try {
1035
+ db.query(`DELETE FROM span_operation_fts WHERE trace_id IN (${placeholders})`).run(...batch)
1036
+ } catch {
1037
+ // FTS table may not exist on old DBs.
1038
+ }
1039
+ db.query(`DELETE FROM spans WHERE trace_id IN (${placeholders})`).run(...batch)
1040
+ db.query(`DELETE FROM logs WHERE trace_id IN (${placeholders})`).run(...batch)
1041
+ db.query(`DELETE FROM trace_summaries WHERE trace_id IN (${placeholders})`).run(...batch)
1042
+ }
1043
+
1044
+ // Log-side orphans (log_attributes + FTS) are keyed by log.id,
1045
+ // so prune what no longer has a parent log row.
1046
+ const orphanAttributes = db.query(`DELETE FROM log_attributes WHERE rowid IN (SELECT log_attributes.rowid FROM log_attributes WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = log_attributes.log_id) LIMIT ?)`).run(config.otel.retentionLogBatch)
1047
+ let deletedOrphans = Number(orphanAttributes.changes) > 0
1048
+ try {
1049
+ const orphanFts = db.query(`DELETE FROM log_body_fts WHERE rowid IN (SELECT rowid FROM log_body_fts WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = CAST(log_body_fts.log_id AS INTEGER)) LIMIT ?)`).run(config.otel.retentionLogBatch)
1050
+ deletedOrphans = deletedOrphans || Number(orphanFts.changes) > 0
1051
+ } catch {
1052
+ // FTS table may not exist on old DBs.
1053
+ }
1054
+
1055
+ // Checkpoint after a big delete pass so the freed pages land
1056
+ // in the main DB file and become eligible for incremental
1057
+ // vacuum. Use RESTART (not PASSIVE): PASSIVE silently no-ops
1058
+ // when readers are active, which is the documented mechanism
1059
+ // behind WAL/freelist starvation when ingest is busy.
1060
+ if (toEvict.size === 0 && !deletedLogs && !deletedOrphans) return
1061
+ try { db.exec(`PRAGMA wal_checkpoint(RESTART);`) } catch { /* ignore */ }
1062
+
1063
+ // Incremental FTS5 merge — DELETE on an FTS5-indexed row
1064
+ // leaves a tombstone in the segment tree that only `merge`
1065
+ // reclaims. Skipping this is the second compounding cause
1066
+ // (after fixed-size vacuum) of the slow freelist accretion
1067
+ // that took the DB to 17GB. 100 pages of merge work per
1068
+ // retention tick is bounded and runs in milliseconds.
1069
+ incrementalFtsMerge(100)
1070
+
1071
+ // Actual page reclamation lives in `reclaimSpace`, which
1072
+ // runs on its own faster cadence so the file shrinks even
1073
+ // when no traces are evicted in a given retention tick (e.g.
1074
+ // after a large historical eviction has already happened).
1075
+ })
1076
+ })
1077
+
1078
+ // Retention only runs in the ingest worker so maintenance never blocks
1079
+ // the HTTP event loop and no second writer duplicates cleanup work.
1080
+ if (opts.runRetention) {
1081
+ // Cleanup runs on the telemetry worker, never the HTTP event loop.
1082
+ yield* Effect.forkScoped(Effect.repeat(
1083
+ Effect.andThen(reconcileTraceSummaries, cleanupExpired()).pipe(Effect.catchCause((cause) => Effect.logWarning(`motel: maintenance pass failed: ${Cause.pretty(cause)}`))),
1084
+ Schedule.spaced(`${config.otel.retentionIntervalSeconds} seconds`),
1085
+ ))
1086
+
1087
+ // Page reclamation runs on a separate, faster cadence (10s) and
1088
+ // is independent of the eviction loop. The reason: a single sweep
1089
+ // at 60s intervals can move only ~8MB of pages before the next
1090
+ // burst of inserts grows the freelist again. Decoupling lets us
1091
+ // catch up adaptively (see VACUUM_PAGES_BUSY/PANIC) without
1092
+ // changing the cost of the heavier delete sweep.
1093
+ yield* Effect.forkScoped(Effect.repeat(reclaimSpace(), Schedule.spaced("10 seconds")))
1094
+
1095
+ // Periodically refresh query planner stats. `PRAGMA optimize` is a
1096
+ // no-op when nothing has changed, so this is essentially free on idle
1097
+ // servers and keeps facet/search planner estimates accurate as data
1098
+ // grows. 15 minutes is slower than ingestion rates we care about but
1099
+ // frequent enough that the attribute picker stays snappy.
1100
+ const refreshPlannerStats = Effect.sync(() => {
1101
+ try { db.exec(`PRAGMA optimize;`) } catch { /* ignore */ }
1102
+ })
1103
+ yield* Effect.forkScoped(Effect.repeat(refreshPlannerStats, Schedule.spaced("15 minutes")))
1104
+ }
1105
+
1106
+ // Incrementally rebuild historical AI attributes in bounded batches.
1107
+ // Queries fall back to LIKE until the persistent marker is complete.
1108
+ if (hasAttrFts && !opts.readonly) {
1109
+ const backfillAttrFtsBatch = Effect.sync(() => {
1110
+ try {
1111
+ const keyList = AI_FTS_KEYS.map((k) => `'${k.replace(/'/g, "''")}'`).join(", ")
1112
+ const marker = db.query(`SELECT value FROM motel_maintenance WHERE key = 'span_attr_fts_v1'`).get() as { value: string } | null
1113
+ if (marker?.value === "complete") return false
1114
+ let cursor = 0
1115
+ let maxRowId = 0
1116
+ if (marker) {
1117
+ [cursor, maxRowId] = marker.value.split(":").map(Number)
1118
+ } else {
1119
+ maxRowId = (db.query(`SELECT COALESCE(MAX(rowid), 0) AS value FROM span_attributes`).get() as { value: number }).value
1120
+ db.query(`INSERT INTO span_attr_fts(span_attr_fts) VALUES ('delete-all')`).run()
1121
+ db.query(`INSERT OR REPLACE INTO motel_maintenance(key, value) VALUES ('span_attr_fts_v1', ?)`).run(`0:${maxRowId}`)
1122
+ }
1123
+ const rows = db.query(`SELECT rowid, value FROM span_attributes WHERE key IN (${keyList}) AND rowid > ? AND rowid <= ? ORDER BY rowid ASC LIMIT 500`).all(cursor, maxRowId) as Array<{ rowid: number; value: string }>
1124
+ if (rows.length === 0) {
1125
+ db.query(`UPDATE motel_maintenance SET value = 'complete' WHERE key = 'span_attr_fts_v1'`).run()
1126
+ hasAttrFts = true
1127
+ return false
1128
+ }
1129
+ const insert = db.query(`INSERT INTO span_attr_fts(rowid, value) VALUES (?, ?)`)
1130
+ const transaction = db.transaction(() => {
1131
+ for (const row of rows) insert.run(row.rowid, row.value)
1132
+ db.query(`UPDATE motel_maintenance SET value = ? WHERE key = 'span_attr_fts_v1'`).run(`${rows.at(-1)!.rowid}:${maxRowId}`)
1133
+ })
1134
+ transaction()
1135
+ return true
1136
+ } catch {
1137
+ // Backfill failure is never fatal — new ingests still
1138
+ // populate FTS via the trigger, and queries fall back to
1139
+ // LIKE when FTS lookups return empty.
1140
+ return true
1141
+ }
1142
+ })
1143
+ const backfillAttrFts: Effect.Effect<void> = Effect.suspend(() =>
1144
+ Effect.flatMap(backfillAttrFtsBatch, (pending) =>
1145
+ pending ? Effect.andThen(Effect.sleep("100 millis"), backfillAttrFts) : Effect.void,
1146
+ ),
1147
+ )
1148
+ yield* Effect.forkScoped(backfillAttrFts)
1149
+ }
1150
+
1151
+ const ingestTraces = Effect.fn("motel/TelemetryStore.ingestTraces")(function* (payload: OtlpTraceExportRequest) {
1152
+ return yield* Effect.sync(() => {
1153
+ let insertedSpans = 0
1154
+ const transaction = db.transaction((request: OtlpTraceExportRequest) => {
1155
+ const touchedTraceIds = new Set<string>()
1156
+ const touchedOperations: Array<readonly [string, string, string]> = []
1157
+ for (const resourceSpans of request.resourceSpans ?? []) {
1158
+ const resourceAttributes = attributeMap(resourceSpans.resource?.attributes)
1159
+ const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
1160
+
1161
+ for (const scopeSpans of resourceSpans.scopeSpans ?? []) {
1162
+ const scopeName = scopeSpans.scope?.name ?? null
1163
+
1164
+ for (const span of scopeSpans.spans ?? []) {
1165
+ const traceId = normalizeOtlpBinaryId(span.traceId, 16)
1166
+ const spanId = normalizeOtlpBinaryId(span.spanId, 8)
1167
+ if (!traceId || !spanId) continue
1168
+ const parentSpanId = normalizeOtlpBinaryId(span.parentSpanId, 8)
1169
+ const spanAttributes = attributeMap(span.attributes)
1170
+ const mergedAttributes = { ...resourceAttributes, ...spanAttributes }
1171
+ const startTimeMs = nanosToMilliseconds(span.startTimeUnixNano)
1172
+ const endTimeMs = nanosToMilliseconds(span.endTimeUnixNano)
1173
+ const events = (span.events ?? []).map((event) => ({
1174
+ name: event.name ?? "event",
1175
+ timestamp: nanosToMilliseconds(event.timeUnixNano),
1176
+ attributes: attributeMap(event.attributes),
1177
+ }))
1178
+
1179
+ insertSpan.run(
1180
+ traceId,
1181
+ spanId,
1182
+ parentSpanId,
1183
+ serviceName,
1184
+ scopeName,
1185
+ span.name ?? "unknown",
1186
+ spanKindLabel(span.kind),
1187
+ startTimeMs,
1188
+ endTimeMs,
1189
+ Math.max(0, endTimeMs - startTimeMs),
1190
+ spanStatusLabel(span.status?.code),
1191
+ JSON.stringify(spanAttributes),
1192
+ JSON.stringify(resourceAttributes),
1193
+ JSON.stringify(events),
1194
+ )
1195
+ deleteSpanAttributes.run(traceId, spanId)
1196
+ insertSpanAttributesMany(traceId, spanId, mergedAttributes)
1197
+ touchedOperations.push([traceId, spanId, span.name ?? "unknown"])
1198
+ touchedTraceIds.add(traceId)
1199
+ insertedSpans += 1
1200
+ }
1201
+ }
1202
+ }
1203
+ try {
1204
+ const BATCH_SIZE = 500
1205
+ for (let offset = 0; offset < touchedOperations.length; offset += BATCH_SIZE) {
1206
+ updateSpanOperationSearchMany(touchedOperations.slice(offset, offset + BATCH_SIZE))
1207
+ }
1208
+ } catch {
1209
+ // FTS is optional.
1210
+ }
1211
+ for (const traceId of touchedTraceIds) {
1212
+ upsertTraceSummary.run(traceId)
1213
+ }
1214
+ })
1215
+
1216
+ transaction(payload)
1217
+ return { insertedSpans }
1218
+ })
1219
+ })
1220
+
1221
+ const ingestLogs = Effect.fn("motel/TelemetryStore.ingestLogs")(function* (payload: OtlpLogExportRequest) {
1222
+ return yield* Effect.sync(() => {
1223
+ let insertedLogs = 0
1224
+ const transaction = db.transaction((request: OtlpLogExportRequest) => {
1225
+ const touchedLogBodies: Array<readonly [string, string]> = []
1226
+ for (const resourceLogs of request.resourceLogs ?? []) {
1227
+ const resourceAttributes = attributeMap(resourceLogs.resource?.attributes)
1228
+ const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
1229
+
1230
+ for (const scopeLogs of resourceLogs.scopeLogs ?? []) {
1231
+ const scopeName = scopeLogs.scope?.name ?? null
1232
+
1233
+ for (const record of scopeLogs.logRecords ?? []) {
1234
+ const attributes = attributeMap(record.attributes)
1235
+ const mergedAttributes = { ...resourceAttributes, ...attributes }
1236
+ const timestampMs = nanosToMilliseconds(record.timeUnixNano ?? record.observedTimeUnixNano)
1237
+ const body = stringifyValue(parseAnyValue(record.body))
1238
+ const rawTraceId = attributes.traceId || attributes.trace_id || record.traceId || null
1239
+ const rawSpanId = attributes.spanId || attributes.span_id || record.spanId || null
1240
+ const result = insertLog.run(
1241
+ normalizeOtlpBinaryId(rawTraceId, 16),
1242
+ normalizeOtlpBinaryId(rawSpanId, 8),
1243
+ serviceName,
1244
+ scopeName,
1245
+ record.severityText ?? "INFO",
1246
+ timestampMs,
1247
+ body,
1248
+ JSON.stringify(attributes),
1249
+ JSON.stringify(resourceAttributes),
1250
+ )
1251
+ const logId = Number((result as { lastInsertRowid: number | bigint }).lastInsertRowid)
1252
+ insertLogAttributesMany(logId, mergedAttributes)
1253
+ touchedLogBodies.push([String(logId), body])
1254
+ insertedLogs += 1
1255
+ }
1256
+ }
1257
+ }
1258
+ try {
1259
+ const BATCH_SIZE = 500
1260
+ for (let offset = 0; offset < touchedLogBodies.length; offset += BATCH_SIZE) {
1261
+ insertLogBodySearchMany(touchedLogBodies.slice(offset, offset + BATCH_SIZE))
1262
+ }
1263
+ } catch {
1264
+ // FTS is optional.
1265
+ }
1266
+ })
1267
+
1268
+ transaction(payload)
1269
+ return { insertedLogs }
1270
+ })
1271
+ })
1272
+
1273
+ const listServices = Effect.fn("motel/TelemetryStore.listServices")(function* () {
1274
+ const cutoff = (yield* Clock.currentTimeMillis) - config.otel.traceLookbackMinutes * 60 * 1000
1275
+ const services = yield* Effect.sync(() => {
1276
+ // Discover recent activity from span rows, not trace starts: a
1277
+ // long-running trace can emit a current child after its root ages
1278
+ // outside the lookback window.
1279
+ const rows = db.query(`
1280
+ SELECT service_name FROM spans WHERE start_time_ms >= ?
1281
+ UNION
1282
+ SELECT service_name FROM logs WHERE timestamp_ms >= ?
1283
+ ORDER BY service_name ASC
1284
+ `).all(cutoff, cutoff) as Array<{ service_name: string }>
1285
+ return rows.map((row) => row.service_name)
1286
+ })
1287
+ yield* Effect.annotateCurrentSpan("trace.service_count", services.length)
1288
+ return services
1289
+ })()
1290
+
1291
+ const loadTracesByIds = (traceIds: readonly string[]) => {
1292
+ if (traceIds.length === 0) return [] as readonly TraceItem[]
1293
+ const placeholders = traceIds.map(() => "?").join(", ")
1294
+ const rows = db.query(`
1295
+ SELECT * FROM spans
1296
+ WHERE trace_id IN (${placeholders})
1297
+ ORDER BY start_time_ms ASC
1298
+ `).all(...traceIds) as SpanRow[]
1299
+
1300
+ const grouped = new Map<string, SpanRow[]>()
1301
+ for (const row of rows) {
1302
+ const group = grouped.get(row.trace_id) ?? []
1303
+ group.push(row)
1304
+ grouped.set(row.trace_id, group)
1305
+ }
1306
+
1307
+ return traceIds
1308
+ .map((traceId) => grouped.get(traceId))
1309
+ .filter((group): group is SpanRow[] => group !== undefined)
1310
+ .map((group) => buildTrace(group[0]!.trace_id, group))
1311
+ }
1312
+
1313
+ const listRecentTraces = Effect.fn("motel/TelemetryStore.listRecentTraces")(function* (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number }) {
1314
+ yield* Effect.annotateCurrentSpan("trace.service_name", serviceName ?? "all")
1315
+ const summaries = yield* listTraceSummaries(serviceName, options)
1316
+ const traces = yield* Effect.sync(() => loadTracesByIds(summaries.map((summary) => summary.traceId)))
1317
+ yield* Effect.annotateCurrentSpan("trace.result_count", traces.length)
1318
+ return traces
1319
+ })
1320
+
1321
+ const listTraceSummaries = Effect.fn("motel/TelemetryStore.listTraceSummaries")(function* (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) {
1322
+ yield* Effect.annotateCurrentSpan("trace.service_name", serviceName ?? "all")
1323
+ const cutoff = (yield* Clock.currentTimeMillis) - (options?.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
1324
+ const limit = options?.limit ?? config.otel.traceFetchLimit
1325
+
1326
+ const summaries = yield* Effect.sync(() => {
1327
+ const clauses = ["started_at_ms >= ?"]
1328
+ const params: Array<string | number> = [cutoff]
1329
+
1330
+ if (serviceName) {
1331
+ clauses.push("service_name = ?")
1332
+ params.push(serviceName)
1333
+ }
1334
+
1335
+ if (options?.cursorStartedAtMs != null && options.cursorTraceId) {
1336
+ clauses.push("(started_at_ms < ? OR (started_at_ms = ? AND trace_id < ?))")
1337
+ params.push(options.cursorStartedAtMs, options.cursorStartedAtMs, options.cursorTraceId)
1338
+ }
1339
+
1340
+ return db.query(`
1341
+ SELECT trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
1342
+ FROM trace_summaries
1343
+ WHERE ${clauses.join(" AND ")}
1344
+ ORDER BY started_at_ms DESC, trace_id DESC
1345
+ LIMIT ?
1346
+ `).all(...params, limit) as TraceSummaryRow[]
1347
+ }).pipe(Effect.map((rows) => rows.map(parseSummaryRow)))
1348
+ yield* Effect.annotateCurrentSpan("trace.result_count", summaries.length)
1349
+ return summaries
1350
+ })
1351
+
1352
+ const searchTraceSummaries = Effect.fn("motel/TelemetryStore.searchTraceSummaries")(function* (input: TraceSearch) {
1353
+ const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
1354
+ const limit = input.limit ?? config.otel.traceFetchLimit
1355
+
1356
+ return yield* Effect.sync(() => {
1357
+ const clauses: string[] = ["started_at_ms >= ?"]
1358
+ const params: Array<string | number> = [cutoff]
1359
+
1360
+ if (input.serviceName) {
1361
+ clauses.push("service_name = ?")
1362
+ params.push(input.serviceName)
1363
+ }
1364
+ if (input.status === "error") {
1365
+ clauses.push("error_count > 0")
1366
+ }
1367
+ if (input.status === "ok") {
1368
+ clauses.push("error_count = 0")
1369
+ }
1370
+ if (input.minDurationMs != null) {
1371
+ clauses.push("duration_ms >= ?")
1372
+ params.push(input.minDurationMs)
1373
+ }
1374
+ if (input.cursorStartedAtMs != null && input.cursorTraceId) {
1375
+ clauses.push("(started_at_ms < ? OR (started_at_ms = ? AND trace_id < ?))")
1376
+ params.push(input.cursorStartedAtMs, input.cursorStartedAtMs, input.cursorTraceId)
1377
+ }
1378
+
1379
+ if (input.operation) {
1380
+ const ftsQuery = toFtsMatchQuery(input.operation)
1381
+ if (hasFts && ftsQuery) {
1382
+ clauses.push("trace_id IN (SELECT DISTINCT trace_id FROM span_operation_fts WHERE span_operation_fts MATCH ?)")
1383
+ params.push(ftsQuery)
1384
+ } else {
1385
+ clauses.push("trace_id IN (SELECT DISTINCT trace_id FROM spans WHERE operation_name LIKE ? COLLATE NOCASE)")
1386
+ params.push(`%${input.operation}%`)
1387
+ }
1388
+ }
1389
+
1390
+ const exactAttrMatch = buildExactAttributeMatchSubquery("span_attributes", ["trace_id", "span_id"], input.attributeFilters)
1391
+ if (exactAttrMatch) {
1392
+ clauses.push(`trace_id IN (SELECT DISTINCT trace_id FROM (${exactAttrMatch.sql}))`)
1393
+ params.push(...exactAttrMatch.params)
1394
+ }
1395
+
1396
+ // `:ai <query>` — FTS match against LLM content keys. Joins
1397
+ // span_attr_fts back to span_attributes to collect trace_ids
1398
+ // whose spans carry matching prompt/response content. Falls
1399
+ // through to no-op when the query tokenizes empty (e.g. only
1400
+ // stopwords or operator-chars) so users don't get a silently
1401
+ // empty list.
1402
+ if (input.aiText) {
1403
+ const aiFtsQuery = toFtsMatchQuery(input.aiText)
1404
+ if (hasAttrFts && aiFtsQuery) {
1405
+ clauses.push(`trace_id IN (
1406
+ SELECT DISTINCT sa.trace_id
1407
+ FROM span_attr_fts fts
1408
+ JOIN span_attributes sa ON sa.rowid = fts.rowid
1409
+ WHERE fts.value MATCH ?
1410
+ )`)
1411
+ params.push(aiFtsQuery)
1412
+ }
1413
+ }
1414
+
1415
+ const rows = db.query(`
1416
+ SELECT trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
1417
+ FROM trace_summaries
1418
+ WHERE ${clauses.join(" AND ")}
1419
+ ORDER BY started_at_ms DESC, trace_id DESC
1420
+ LIMIT ?
1421
+ `).all(...params, limit) as TraceSummaryRow[]
1422
+
1423
+ return rows.map(parseSummaryRow)
1424
+ })
1425
+ })
1426
+
1427
+ const getTrace = Effect.fn("motel/TelemetryStore.getTrace")(function* (traceId: string) {
1428
+ yield* Effect.annotateCurrentSpan("trace.trace_id", traceId)
1429
+ return yield* Effect.sync(() => {
1430
+ const rows = db.query(`
1431
+ SELECT * FROM spans WHERE trace_id = ? ORDER BY start_time_ms ASC
1432
+ `).all(traceId) as SpanRow[]
1433
+ return rows.length === 0 ? null : buildTrace(traceId, rows)
1434
+ })
1435
+ })
1436
+
1437
+ const getSpan = Effect.fn("motel/TelemetryStore.getSpan")(function* (spanId: string) {
1438
+ yield* Effect.annotateCurrentSpan("trace.span_id", spanId)
1439
+ return yield* Effect.sync(() => {
1440
+ // Fetch only the target span row (uses idx_spans_span_id)
1441
+ const spanRow = db.query(`SELECT * FROM spans WHERE span_id = ? LIMIT 1`).get(spanId) as SpanRow | null
1442
+ if (!spanRow) return null
1443
+
1444
+ const traceId = spanRow.trace_id
1445
+
1446
+ // Walk the parent chain in one recursive CTE instead of one query
1447
+ // per hop. Root context remains the earliest root in the trace,
1448
+ // matching full trace hydration even when input has multiple roots.
1449
+ let parentOperationName: string | null = null
1450
+ let depth = 0
1451
+ if (spanRow.parent_span_id) {
1452
+ const ancestors = db.query(`
1453
+ WITH RECURSIVE ancestors(span_id, parent_span_id, operation_name, hop) AS (
1454
+ SELECT span_id, parent_span_id, operation_name, 1
1455
+ FROM spans WHERE trace_id = ? AND span_id = ?
1456
+ UNION ALL
1457
+ SELECT s.span_id, s.parent_span_id, s.operation_name, a.hop + 1
1458
+ FROM ancestors a
1459
+ JOIN spans s ON s.trace_id = ? AND s.span_id = a.parent_span_id
1460
+ )
1461
+ SELECT span_id, parent_span_id, operation_name, hop FROM ancestors ORDER BY hop ASC
1462
+ `).all(traceId, spanRow.parent_span_id, traceId) as Array<{ span_id: string; parent_span_id: string | null; operation_name: string; hop: number }>
1463
+
1464
+ parentOperationName = ancestors[0]?.operation_name ?? null
1465
+ depth = ancestors.length
1466
+ }
1467
+
1468
+ const rootRow = db.query(`
1469
+ SELECT operation_name FROM spans
1470
+ WHERE trace_id = ? AND parent_span_id IS NULL
1471
+ ORDER BY start_time_ms ASC LIMIT 1
1472
+ `).get(traceId) as { operation_name: string } | null
1473
+ const rootOperationName = rootRow?.operation_name ?? "unknown"
1474
+
1475
+ const parsed = parseSpanRow(spanRow)
1476
+ return {
1477
+ traceId,
1478
+ rootOperationName,
1479
+ parentOperationName,
1480
+ span: { ...parsed, depth },
1481
+ } satisfies SpanItem
1482
+ })
1483
+ })
1484
+
1485
+ const listTraceSpans = Effect.fn("motel/TelemetryStore.listTraceSpans")(function* (traceId: string) {
1486
+ return yield* Effect.sync(() => {
1487
+ const rows = db.query(`SELECT * FROM spans WHERE trace_id = ? ORDER BY start_time_ms ASC`).all(traceId) as SpanRow[]
1488
+ return rows.length === 0 ? [] as readonly SpanItem[] : buildSpanItems(traceId, rows)
1489
+ })
1490
+ })
1491
+
1492
+ const searchSpans = Effect.fn("motel/TelemetryStore.searchSpans")(function* (input: SpanSearch) {
1493
+ const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
1494
+ const limit = input.limit ?? 100
1495
+ const hasContainsFilters = Object.keys(input.attributeContainsFilters ?? {}).length > 0
1496
+ // Only over-fetch when post-filtering will discard rows. Without
1497
+ // a parentOperation filter the SQL `LIMIT` already returns the
1498
+ // final set, and over-fetching just makes us parse JSON blobs
1499
+ // for rows we'll throw away.
1500
+ const needsPostFilter = !!input.parentOperation
1501
+ const candidateLimit = !needsPostFilter
1502
+ ? limit
1503
+ : hasContainsFilters
1504
+ ? Math.max(limit * 20, 500)
1505
+ : Math.max(limit * 10, 200)
1506
+
1507
+ return yield* Effect.sync(() => {
1508
+ // First pass: fetch only the columns needed to filter and
1509
+ // to drive the parent-context lookup. Parsing the heavy
1510
+ // `*_json` blobs is deferred until after we've sliced down
1511
+ // to the final `limit`.
1512
+ let fromSql = "FROM spans AS s"
1513
+ const joinParams: Array<string | number> = []
1514
+ const clauses: string[] = ["s.start_time_ms >= ?"]
1515
+ const params: Array<string | number> = [cutoff]
1516
+
1517
+ if (input.traceId) {
1518
+ clauses.push("s.trace_id = ?")
1519
+ params.push(input.traceId)
1520
+ }
1521
+ if (input.serviceName) {
1522
+ clauses.push("s.service_name = ?")
1523
+ params.push(input.serviceName)
1524
+ }
1525
+ if (input.operation) {
1526
+ const ftsQuery = toFtsMatchQuery(input.operation)
1527
+ if (hasFts && ftsQuery) {
1528
+ fromSql += ` INNER JOIN (SELECT trace_id, span_id FROM span_operation_fts WHERE span_operation_fts MATCH ?) AS span_operation_match ON span_operation_match.trace_id = s.trace_id AND span_operation_match.span_id = s.span_id`
1529
+ joinParams.push(ftsQuery)
1530
+ } else {
1531
+ clauses.push("s.operation_name LIKE ? COLLATE NOCASE")
1532
+ params.push(`%${input.operation}%`)
1533
+ }
1534
+ }
1535
+ if (input.status) {
1536
+ clauses.push("s.status = ?")
1537
+ params.push(input.status)
1538
+ }
1539
+
1540
+ const exactAttrMatch = buildExactAttributeMatchSubquery("span_attributes", ["trace_id", "span_id"], input.attributeFilters)
1541
+ if (exactAttrMatch) {
1542
+ clauses.push(`EXISTS (SELECT 1 FROM (${exactAttrMatch.sql}) AS span_attr_match WHERE span_attr_match.trace_id = s.trace_id AND span_attr_match.span_id = s.span_id)`)
1543
+ params.push(...exactAttrMatch.params)
1544
+ }
1545
+
1546
+ const containsAttrMatch = buildContainsAttributeMatchSubquery("span_attributes", ["trace_id", "span_id"], input.attributeContainsFilters)
1547
+ if (containsAttrMatch) {
1548
+ clauses.push(`EXISTS (SELECT 1 FROM (${containsAttrMatch.sql}) AS span_attr_contains_match WHERE span_attr_contains_match.trace_id = s.trace_id AND span_attr_contains_match.span_id = s.span_id)`)
1549
+ params.push(...containsAttrMatch.params)
1550
+ }
1551
+
1552
+ const candidateRows = db.query(`
1553
+ SELECT s.trace_id, s.span_id, s.parent_span_id, s.operation_name, s.start_time_ms
1554
+ ${fromSql}
1555
+ WHERE ${clauses.join(" AND ")}
1556
+ ORDER BY s.start_time_ms DESC
1557
+ LIMIT ?
1558
+ `).all(...joinParams, ...params, candidateLimit) as Array<{ trace_id: string; span_id: string; parent_span_id: string | null; operation_name: string; start_time_ms: number }>
1559
+
1560
+ const traceIds = [...new Set(candidateRows.map((row) => row.trace_id))]
1561
+ if (traceIds.length === 0) return [] as readonly SpanItem[]
1562
+
1563
+ const keyOf = (traceId: string, spanId: string) => `${traceId}:${spanId}`
1564
+ const spanContextById = new Map<string, { readonly parentSpanId: string | null; readonly operationName: string }>()
1565
+
1566
+ // Bulk-prefetch parent metadata for every span in every trace
1567
+ // touched by the candidate set. One indexed scan per trace_id
1568
+ // is much cheaper than a per-span lookup loop while computing
1569
+ // depth, and we get the trace-root lookup in the same pass.
1570
+ const placeholders = traceIds.map(() => "?").join(", ")
1571
+ const allSpanRows = db.query(`
1572
+ SELECT trace_id, span_id, parent_span_id, operation_name, start_time_ms
1573
+ FROM spans
1574
+ WHERE trace_id IN (${placeholders})
1575
+ `).all(...traceIds) as Array<{ trace_id: string; span_id: string; parent_span_id: string | null; operation_name: string; start_time_ms: number }>
1576
+
1577
+ const rootOperationByTraceId = new Map<string, { operationName: string; startTimeMs: number }>()
1578
+ for (const row of allSpanRows) {
1579
+ spanContextById.set(keyOf(row.trace_id, row.span_id), {
1580
+ parentSpanId: row.parent_span_id,
1581
+ operationName: row.operation_name,
1582
+ })
1583
+ if (row.parent_span_id === null) {
1584
+ const existing = rootOperationByTraceId.get(row.trace_id)
1585
+ if (!existing || row.start_time_ms < existing.startTimeMs) {
1586
+ rootOperationByTraceId.set(row.trace_id, { operationName: row.operation_name, startTimeMs: row.start_time_ms })
1587
+ }
1588
+ }
1589
+ }
1590
+
1591
+ const getSpanContext = (traceId: string, spanId: string) => spanContextById.get(keyOf(traceId, spanId)) ?? null
1592
+
1593
+ const depthById = new Map<string, number>()
1594
+ const getDepth = (traceId: string, spanId: string, visiting = new Set<string>()): number => {
1595
+ const key = keyOf(traceId, spanId)
1596
+ const cached = depthById.get(key)
1597
+ if (cached !== undefined) return cached
1598
+ if (visiting.has(key)) return 0
1599
+ visiting.add(key)
1600
+ const context = getSpanContext(traceId, spanId)
1601
+ const depth = context?.parentSpanId ? getDepth(traceId, context.parentSpanId, visiting) + 1 : 0
1602
+ depthById.set(key, depth)
1603
+ return depth
1604
+ }
1605
+
1606
+ // Apply parentOperation post-filter on the lite candidate set
1607
+ // (cheap — string compare against cached parent op) and then
1608
+ // slice down to the final result size before parsing any JSON.
1609
+ const parentOperationNeedle = input.parentOperation?.toLowerCase() ?? null
1610
+ const filteredLite: typeof candidateRows = []
1611
+ for (const row of candidateRows) {
1612
+ if (parentOperationNeedle) {
1613
+ const parent = row.parent_span_id ? getSpanContext(row.trace_id, row.parent_span_id) : null
1614
+ if (!parent?.operationName.toLowerCase().includes(parentOperationNeedle)) continue
1615
+ }
1616
+ filteredLite.push(row)
1617
+ if (filteredLite.length >= limit) break
1618
+ }
1619
+
1620
+ if (filteredLite.length === 0) return [] as readonly SpanItem[]
1621
+
1622
+ // Hydrate only the kept rows: one batched fetch of the full
1623
+ // SpanRow (with resource_json / attributes_json / events_json)
1624
+ // using SQLite's row-value `IN` syntax, then parseSpanRow per
1625
+ // kept row. Result order follows `filteredLite` so the caller
1626
+ // sees the same ordering the candidate scan produced.
1627
+ const keptValues = filteredLite.map(() => "(?, ?)").join(", ")
1628
+ const fullRows = db.query(`
1629
+ SELECT * FROM spans WHERE (trace_id, span_id) IN (VALUES ${keptValues})
1630
+ `).all(...filteredLite.flatMap((row) => [row.trace_id, row.span_id])) as SpanRow[]
1631
+ const fullRowByKey = new Map<string, SpanRow>()
1632
+ for (const row of fullRows) {
1633
+ fullRowByKey.set(keyOf(row.trace_id, row.span_id), row)
1634
+ }
1635
+
1636
+ const items: SpanItem[] = []
1637
+ for (const lite of filteredLite) {
1638
+ const row = fullRowByKey.get(keyOf(lite.trace_id, lite.span_id))
1639
+ if (!row) continue
1640
+ const parentContext = row.parent_span_id ? getSpanContext(row.trace_id, row.parent_span_id) : null
1641
+ const parsedSpan = parseSpanRow(row)
1642
+ const span = {
1643
+ ...parsedSpan,
1644
+ depth: getDepth(row.trace_id, row.span_id),
1645
+ warnings: row.parent_span_id && !parentContext
1646
+ ? [`missing span ${row.parent_span_id} (1 child)`]
1647
+ : parsedSpan.warnings,
1648
+ }
1649
+ items.push({
1650
+ traceId: row.trace_id,
1651
+ rootOperationName: rootOperationByTraceId.get(row.trace_id)?.operationName ?? span.operationName,
1652
+ parentOperationName: parentContext?.operationName ?? null,
1653
+ span,
1654
+ })
1655
+ }
1656
+ return items
1657
+ })
1658
+ })
1659
+
1660
+ const searchTraces = Effect.fn("motel/TelemetryStore.searchTraces")(function* (input: TraceSearch) {
1661
+ const summaries = yield* searchTraceSummaries(input)
1662
+ return yield* Effect.sync(() => loadTracesByIds(summaries.map((summary) => summary.traceId)))
1663
+ })
1664
+
1665
+ const searchLogs = Effect.fn("motel/TelemetryStore.searchLogs")(function* (input: LogSearch) {
1666
+ const now = yield* Clock.currentTimeMillis
1667
+ return yield* Effect.sync(() => {
1668
+ const clauses: string[] = []
1669
+ const params: Array<string | number> = []
1670
+
1671
+ if (input.serviceName) {
1672
+ clauses.push(`service_name = ?`)
1673
+ params.push(input.serviceName)
1674
+ }
1675
+ if (input.severity) {
1676
+ clauses.push(`severity_text = ?`)
1677
+ params.push(input.severity.toUpperCase())
1678
+ }
1679
+ if (input.traceId) {
1680
+ clauses.push(`trace_id = ?`)
1681
+ params.push(input.traceId)
1682
+ }
1683
+ if (input.spanId) {
1684
+ clauses.push(`span_id = ?`)
1685
+ params.push(input.spanId)
1686
+ }
1687
+ if (input.body) {
1688
+ const ftsQuery = toFtsMatchQuery(input.body)
1689
+ if (hasFts && ftsQuery) {
1690
+ clauses.push(`id IN (SELECT CAST(log_id AS INTEGER) FROM log_body_fts WHERE log_body_fts MATCH ?)`)
1691
+ params.push(ftsQuery)
1692
+ } else {
1693
+ clauses.push(`body LIKE ? COLLATE NOCASE`)
1694
+ params.push(`%${input.body}%`)
1695
+ }
1696
+ }
1697
+ if (input.lookbackMinutes) {
1698
+ const cutoff = now - input.lookbackMinutes * 60 * 1000
1699
+ clauses.push(`timestamp_ms >= ?`)
1700
+ params.push(cutoff)
1701
+ }
1702
+ if (input.cursorTimestampMs != null && input.cursorId) {
1703
+ clauses.push(`(timestamp_ms < ? OR (timestamp_ms = ? AND id < ?))`)
1704
+ params.push(input.cursorTimestampMs, input.cursorTimestampMs, Number(input.cursorId))
1705
+ }
1706
+
1707
+ const exactAttrMatch = buildExactAttributeMatchSubquery("log_attributes", ["log_id"], input.attributeFilters)
1708
+ if (exactAttrMatch) {
1709
+ clauses.push(`id IN (${exactAttrMatch.sql})`)
1710
+ params.push(...exactAttrMatch.params)
1711
+ }
1712
+
1713
+ const containsAttrMatch = buildContainsAttributeMatchSubquery("log_attributes", ["log_id"], input.attributeContainsFilters)
1714
+ if (containsAttrMatch) {
1715
+ clauses.push(`id IN (${containsAttrMatch.sql})`)
1716
+ params.push(...containsAttrMatch.params)
1717
+ }
1718
+
1719
+ const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""
1720
+ const limit = input.limit ?? config.otel.logFetchLimit
1721
+ const rows = db.query(`
1722
+ SELECT * FROM logs
1723
+ ${where}
1724
+ ORDER BY timestamp_ms DESC, id DESC
1725
+ LIMIT ?
1726
+ `).all(...params, limit) as LogRow[]
1727
+
1728
+ return rows.map(parseLogRow)
1729
+ })
1730
+ })
1731
+
1732
+ const traceStats = Effect.fn("motel/TelemetryStore.traceStats")(function* (input: TraceStatsSearch) {
1733
+ const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
1734
+ const limit = input.limit ?? 20
1735
+ const hasAttrFilters = Object.keys(input.attributeFilters ?? {}).length > 0
1736
+ const isAttrGroupBy = input.groupBy.startsWith("attr.")
1737
+
1738
+ if (isAttrGroupBy || hasAttrFilters || input.operation) {
1739
+ const summaries = yield* searchTraceSummaries({
1740
+ serviceName: input.serviceName,
1741
+ operation: input.operation,
1742
+ status: input.status,
1743
+ minDurationMs: input.minDurationMs,
1744
+ attributeFilters: input.attributeFilters,
1745
+ lookbackMinutes: input.lookbackMinutes,
1746
+ limit: 5000,
1747
+ })
1748
+
1749
+ // For attr.* groupBy, we need to check span attributes — but only the groupBy key
1750
+ let attrLookup: Map<string, string> | null = null
1751
+ if (isAttrGroupBy) {
1752
+ const attrKey = input.groupBy.slice(5)
1753
+ const traceIds = summaries.map((s) => s.traceId)
1754
+ if (traceIds.length > 0) {
1755
+ const placeholders = traceIds.map(() => "?").join(", ")
1756
+ const rows = db.query(`
1757
+ SELECT trace_id, value
1758
+ FROM span_attributes
1759
+ WHERE key = ? AND trace_id IN (${placeholders})
1760
+ GROUP BY trace_id
1761
+ `).all(attrKey, ...traceIds) as Array<{ trace_id: string; value: string }>
1762
+
1763
+ attrLookup = new Map()
1764
+ for (const row of rows) {
1765
+ attrLookup.set(row.trace_id, row.value)
1766
+ }
1767
+ }
1768
+ }
1769
+
1770
+ const groups = new Map<string, { durations: number[]; errorTraces: number }>()
1771
+ for (const summary of summaries) {
1772
+ const group = input.groupBy === "service"
1773
+ ? summary.serviceName
1774
+ : input.groupBy === "operation"
1775
+ ? summary.rootOperationName
1776
+ : input.groupBy === "status"
1777
+ ? summary.errorCount > 0 ? "error" : "ok"
1778
+ : isAttrGroupBy
1779
+ ? attrLookup?.get(summary.traceId) ?? "unknown"
1780
+ : "unknown"
1781
+
1782
+ const bucket = groups.get(group) ?? { durations: [], errorTraces: 0 }
1783
+ bucket.durations.push(summary.durationMs)
1784
+ if (summary.errorCount > 0) bucket.errorTraces++
1785
+ groups.set(group, bucket)
1786
+ }
1787
+
1788
+ const rows = [...groups.entries()].map(([group, bucket]) => {
1789
+ const count = bucket.durations.length
1790
+ const value = input.agg === "count"
1791
+ ? count
1792
+ : input.agg === "avg_duration"
1793
+ ? bucket.durations.reduce((sum, d) => sum + d, 0) / Math.max(1, count)
1794
+ : input.agg === "p95_duration"
1795
+ ? percentile(bucket.durations, 0.95)
1796
+ : bucket.errorTraces / Math.max(1, count)
1797
+ return { group, value, count }
1798
+ })
1799
+
1800
+ return rows.sort((left, right) => right.value - left.value).slice(0, limit)
1801
+ }
1802
+
1803
+ return yield* Effect.sync(() => {
1804
+ const whereClauses: string[] = ["started_at_ms >= ?"]
1805
+ const whereParams: Array<string | number> = [cutoff]
1806
+
1807
+ if (input.serviceName) {
1808
+ whereClauses.push("service_name = ?")
1809
+ whereParams.push(input.serviceName)
1810
+ }
1811
+
1812
+ if (input.status === "error") whereClauses.push("error_count > 0")
1813
+ if (input.status === "ok") whereClauses.push("error_count = 0")
1814
+ if (input.minDurationMs != null) {
1815
+ whereClauses.push("duration_ms >= ?")
1816
+ whereParams.push(input.minDurationMs)
1817
+ }
1818
+
1819
+ const groupExpr = input.groupBy === "service"
1820
+ ? "service_name"
1821
+ : input.groupBy === "operation"
1822
+ ? "root_operation_name"
1823
+ : input.groupBy === "status"
1824
+ ? "CASE WHEN error_count > 0 THEN 'error' ELSE 'ok' END"
1825
+ : "'unknown'"
1826
+
1827
+ const aggExpr = input.agg === "count"
1828
+ ? "COUNT(*)"
1829
+ : input.agg === "avg_duration"
1830
+ ? "AVG(duration_ms)"
1831
+ : "CAST(SUM(CASE WHEN error_count > 0 THEN 1 ELSE 0 END) AS REAL) / COUNT(*)"
1832
+
1833
+ if (input.agg === "p95_duration") {
1834
+ const rows = db.query(`
1835
+ SELECT ${groupExpr} AS grp, duration_ms
1836
+ FROM trace_summaries
1837
+ WHERE ${whereClauses.join(" AND ")}
1838
+ `).all(...whereParams) as Array<{ grp: string; duration_ms: number }>
1839
+
1840
+ const groups = new Map<string, number[]>()
1841
+ for (const row of rows) {
1842
+ const bucket = groups.get(row.grp) ?? []
1843
+ bucket.push(row.duration_ms)
1844
+ groups.set(row.grp, bucket)
1845
+ }
1846
+
1847
+ return [...groups.entries()]
1848
+ .map(([group, durations]) => ({ group, value: percentile(durations, 0.95), count: durations.length }))
1849
+ .sort((left, right) => right.value - left.value)
1850
+ .slice(0, limit)
1851
+ }
1852
+
1853
+ const rows = db.query(`
1854
+ SELECT ${groupExpr} AS grp, ${aggExpr} AS value, COUNT(*) AS count
1855
+ FROM trace_summaries
1856
+ WHERE ${whereClauses.join(" AND ")}
1857
+ GROUP BY grp
1858
+ ORDER BY value DESC
1859
+ LIMIT ?
1860
+ `).all(...whereParams, limit) as Array<{ grp: string; value: number; count: number }>
1861
+
1862
+ return rows.map((row) => ({ group: row.grp, value: row.value, count: row.count }))
1863
+ })
1864
+ })
1865
+
1866
+ const logStats = Effect.fn("motel/TelemetryStore.logStats")(function* (input: LogStatsSearch) {
1867
+ const now = yield* Clock.currentTimeMillis
1868
+ const limit = input.limit ?? 20
1869
+ const hasAttrFilters = Object.keys(input.attributeFilters ?? {}).length > 0
1870
+ const isAttrGroupBy = input.groupBy.startsWith("attr.")
1871
+
1872
+ // For attr.* groupBy or attr filters, fall back to in-memory grouping
1873
+ if (isAttrGroupBy || hasAttrFilters) {
1874
+ const logs = yield* searchLogs({
1875
+ serviceName: input.serviceName,
1876
+ traceId: input.traceId,
1877
+ spanId: input.spanId,
1878
+ body: input.body,
1879
+ lookbackMinutes: input.lookbackMinutes,
1880
+ attributeFilters: input.attributeFilters,
1881
+ limit: 5000,
1882
+ })
1883
+
1884
+ const groups = new Map<string, number>()
1885
+ for (const log of logs) {
1886
+ const group = input.groupBy === "service"
1887
+ ? log.serviceName
1888
+ : input.groupBy === "severity"
1889
+ ? log.severityText
1890
+ : input.groupBy === "scope"
1891
+ ? log.scopeName ?? "unknown"
1892
+ : isAttrGroupBy
1893
+ ? log.attributes[input.groupBy.slice(5)] ?? "unknown"
1894
+ : "unknown"
1895
+ groups.set(group, (groups.get(group) ?? 0) + 1)
1896
+ }
1897
+
1898
+ return [...groups.entries()]
1899
+ .map(([group, count]) => ({ group, value: count, count }))
1900
+ .sort((left, right) => right.value - left.value)
1901
+ .slice(0, limit)
1902
+ }
1903
+
1904
+ // Pure SQL path for standard groupBy fields
1905
+ return yield* Effect.sync(() => {
1906
+ const clauses: string[] = []
1907
+ const params: Array<string | number> = []
1908
+
1909
+ if (input.serviceName) {
1910
+ clauses.push("service_name = ?")
1911
+ params.push(input.serviceName)
1912
+ }
1913
+ if (input.traceId) {
1914
+ clauses.push("trace_id = ?")
1915
+ params.push(input.traceId)
1916
+ }
1917
+ if (input.spanId) {
1918
+ clauses.push("span_id = ?")
1919
+ params.push(input.spanId)
1920
+ }
1921
+ if (input.body) {
1922
+ clauses.push("body LIKE ?")
1923
+ params.push(`%${input.body}%`)
1924
+ }
1925
+ if (input.lookbackMinutes) {
1926
+ clauses.push("timestamp_ms >= ?")
1927
+ params.push(now - input.lookbackMinutes * 60 * 1000)
1928
+ }
1929
+
1930
+ const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""
1931
+
1932
+ const groupExpr = input.groupBy === "service"
1933
+ ? "service_name"
1934
+ : input.groupBy === "severity"
1935
+ ? "severity_text"
1936
+ : input.groupBy === "scope"
1937
+ ? "COALESCE(scope_name, 'unknown')"
1938
+ : "'unknown'"
1939
+
1940
+ const rows = db.query(`
1941
+ SELECT ${groupExpr} AS grp, COUNT(*) AS count
1942
+ FROM logs
1943
+ ${where}
1944
+ GROUP BY grp
1945
+ ORDER BY count DESC
1946
+ LIMIT ?
1947
+ `).all(...params, limit) as Array<{ grp: string; count: number }>
1948
+
1949
+ return rows.map((row) => ({ group: row.grp, value: row.count, count: row.count }))
1950
+ })
1951
+ })
1952
+
1953
+ const listRecentLogs = Effect.fn("motel/TelemetryStore.listRecentLogs")(function* (serviceName: string) {
1954
+ yield* Effect.annotateCurrentSpan("log.service_name", serviceName)
1955
+ const logs = yield* searchLogs({ serviceName, limit: config.otel.logFetchLimit })
1956
+ yield* Effect.annotateCurrentSpan("log.result_count", logs.length)
1957
+ return logs
1958
+ })
1959
+
1960
+ const listFacets = Effect.fn("motel/TelemetryStore.listFacets")(function* (input: FacetSearch) {
1961
+ const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
1962
+ const limit = input.limit ?? 20
1963
+
1964
+ return yield* Effect.sync(() => {
1965
+ if (input.type === "logs") {
1966
+ if (input.field === "service") {
1967
+ const rows = db.query(`
1968
+ SELECT service_name AS value, COUNT(*) AS count
1969
+ FROM logs
1970
+ WHERE timestamp_ms >= ?
1971
+ GROUP BY service_name
1972
+ ORDER BY count DESC, value ASC
1973
+ LIMIT ?
1974
+ `).all(cutoff, limit) as Array<{ value: string; count: number }>
1975
+ return rows
1976
+ }
1977
+ if (input.field === "severity") {
1978
+ const rows = db.query(`
1979
+ SELECT severity_text AS value, COUNT(*) AS count
1980
+ FROM logs
1981
+ WHERE timestamp_ms >= ?
1982
+ ${input.serviceName ? "AND service_name = ?" : ""}
1983
+ GROUP BY severity_text
1984
+ ORDER BY count DESC, value ASC
1985
+ LIMIT ?
1986
+ `).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
1987
+ return rows
1988
+ }
1989
+ if (input.field === "scope") {
1990
+ const rows = db.query(`
1991
+ SELECT COALESCE(scope_name, 'unknown') AS value, COUNT(*) AS count
1992
+ FROM logs
1993
+ WHERE timestamp_ms >= ?
1994
+ ${input.serviceName ? "AND service_name = ?" : ""}
1995
+ GROUP BY COALESCE(scope_name, 'unknown')
1996
+ ORDER BY count DESC, value ASC
1997
+ LIMIT ?
1998
+ `).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
1999
+ return rows
2000
+ }
2001
+ }
2002
+
2003
+ if (input.type === "traces") {
2004
+ if (input.field === "service") {
2005
+ const rows = db.query(`
2006
+ SELECT service_name AS value, COUNT(*) AS count
2007
+ FROM trace_summaries
2008
+ WHERE started_at_ms >= ?
2009
+ GROUP BY service_name
2010
+ ORDER BY count DESC, value ASC
2011
+ LIMIT ?
2012
+ `).all(cutoff, limit) as Array<{ value: string; count: number }>
2013
+ return rows
2014
+ }
2015
+ if (input.field === "operation") {
2016
+ const rows = db.query(`
2017
+ SELECT root_operation_name AS value, COUNT(*) AS count
2018
+ FROM trace_summaries
2019
+ WHERE started_at_ms >= ?
2020
+ ${input.serviceName ? "AND service_name = ?" : ""}
2021
+ GROUP BY root_operation_name
2022
+ ORDER BY count DESC, value ASC
2023
+ LIMIT ?
2024
+ `).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
2025
+ return rows
2026
+ }
2027
+ if (input.field === "status") {
2028
+ const rows = db.query(`
2029
+ SELECT CASE WHEN error_count > 0 THEN 'error' ELSE 'ok' END AS value, COUNT(*) AS count
2030
+ FROM trace_summaries
2031
+ WHERE started_at_ms >= ?
2032
+ ${input.serviceName ? "AND service_name = ?" : ""}
2033
+ GROUP BY value
2034
+ ORDER BY count DESC
2035
+ LIMIT ?
2036
+ `).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
2037
+ return rows
2038
+ }
2039
+ if (input.field === "attribute_keys") {
2040
+ // Count distinct traces each attribute key appears on, optionally
2041
+ // scoped to a service. Keys with many distinct values (e.g. sessionId,
2042
+ // user id, model) rank higher than keys that are constant across every
2043
+ // trace (service.name, telemetry.sdk.*) — the latter can't discriminate
2044
+ // between traces so they're useless as filters.
2045
+ //
2046
+ // Performance note: we skip rows whose value blob is larger than
2047
+ // FACET_VALUE_MAX_LEN. For opencode this hides `ai.prompt`,
2048
+ // `ai.prompt.messages`, and `ai.prompt.tools` — which are 1-6MB text
2049
+ // blobs that you'd never want to filter by exact match anyway. The
2050
+ // WHERE clause lets SQLite skip reading those pages from disk.
2051
+ // COUNT(DISTINCT ...) does its own per-group dedup via a temp B-tree,
2052
+ // so the outer query needs no DISTINCT subquery in front of it. We
2053
+ // pre-filter trace_ids through trace_summaries (an indexed lookup) so
2054
+ // the planner can use a SEMI JOIN against the small in-window set
2055
+ // instead of joining every span_attributes row to trace_summaries.
2056
+ const params: Array<string | number> = []
2057
+ let traceFilter: string
2058
+ if (input.serviceName) {
2059
+ traceFilter = `(SELECT trace_id FROM trace_summaries WHERE started_at_ms >= ? AND service_name = ?)`
2060
+ params.push(cutoff, input.serviceName)
2061
+ } else {
2062
+ traceFilter = `(SELECT trace_id FROM trace_summaries WHERE started_at_ms >= ?)`
2063
+ params.push(cutoff)
2064
+ }
2065
+ params.push(FACET_VALUE_MAX_LEN, limit)
2066
+ const rows = db.query(`
2067
+ SELECT key AS value,
2068
+ COUNT(DISTINCT trace_id) AS count,
2069
+ COUNT(DISTINCT value) AS distinct_values
2070
+ FROM span_attributes
2071
+ WHERE trace_id IN ${traceFilter}
2072
+ AND LENGTH(value) < ?
2073
+ GROUP BY key
2074
+ ORDER BY (CASE WHEN distinct_values = 1 THEN 1 ELSE 0 END) ASC,
2075
+ distinct_values DESC,
2076
+ count DESC,
2077
+ value ASC
2078
+ LIMIT ?
2079
+ `).all(...params) as Array<{ value: string; count: number; distinct_values: number }>
2080
+ return rows.map((row) => ({ value: row.value, count: row.count }))
2081
+ }
2082
+ if (input.field === "attribute_values") {
2083
+ if (!input.key) return [] as FacetItem[]
2084
+ // Skip multi-KB values here too — they blow up GROUP BY on big text.
2085
+ // Matches the attribute_keys pre-filter so the picker stays responsive
2086
+ // if someone hand-crafts a URL that targets a fat key.
2087
+ const params: Array<string | number> = [input.key, FACET_VALUE_MAX_LEN, cutoff]
2088
+ if (input.serviceName) params.push(input.serviceName)
2089
+ params.push(limit)
2090
+ const rows = db.query(`
2091
+ SELECT sa.value AS value, COUNT(DISTINCT sa.trace_id) AS count
2092
+ FROM span_attributes sa
2093
+ JOIN spans s ON s.trace_id = sa.trace_id AND s.span_id = sa.span_id
2094
+ WHERE sa.key = ? AND LENGTH(sa.value) < ?
2095
+ AND s.start_time_ms >= ?
2096
+ ${input.serviceName ? "AND s.service_name = ?" : ""}
2097
+ GROUP BY sa.value
2098
+ ORDER BY count DESC, value ASC
2099
+ LIMIT ?
2100
+ `).all(...params) as Array<{ value: string; count: number }>
2101
+ return rows
2102
+ }
2103
+ }
2104
+
2105
+ return [] as FacetItem[]
2106
+ })
2107
+ })
2108
+
2109
+ const listTraceLogs = Effect.fn("motel/TelemetryStore.listTraceLogs")(function* (traceId: string) {
2110
+ yield* Effect.annotateCurrentSpan("log.trace_id", traceId)
2111
+ const logs = yield* searchLogs({ traceId, limit: config.otel.logFetchLimit })
2112
+ yield* Effect.annotateCurrentSpan("log.result_count", logs.length)
2113
+ return logs
2114
+ })
2115
+
2116
+ // ---------------------------------------------------------------------------
2117
+ // AI Call queries
2118
+ // ---------------------------------------------------------------------------
2119
+
2120
+ /** Extracts ai.streamText -> "streamText", ai.streamText.doStream -> "streamText" */
2121
+ const parseAiOperation = (operationName: string): string => {
2122
+ const parts = operationName.replace(/^ai\./, "").split(".")
2123
+ return parts[0] ?? operationName
2124
+ }
2125
+
2126
+ /** Builds WHERE clauses for AI call search against the spans table (aliased as s) */
2127
+ const buildAiWhereClauses = (input: AiCallSearch | AiCallStatsSearch, cutoff: number) => {
2128
+ const clauses: string[] = [
2129
+ "s.operation_name LIKE 'ai.%'",
2130
+ "s.operation_name NOT LIKE 'ai.%.do%'",
2131
+ "s.start_time_ms >= ?",
2132
+ ]
2133
+ const params: Array<string | number> = [cutoff]
2134
+
2135
+ if (input.service) {
2136
+ clauses.push("s.service_name = ?")
2137
+ params.push(input.service)
2138
+ }
2139
+ if (input.traceId) {
2140
+ clauses.push("s.trace_id = ?")
2141
+ params.push(input.traceId)
2142
+ }
2143
+ if (input.status) {
2144
+ clauses.push("s.status = ?")
2145
+ params.push(input.status)
2146
+ }
2147
+ if (input.minDurationMs != null) {
2148
+ clauses.push("s.duration_ms >= ?")
2149
+ params.push(input.minDurationMs)
2150
+ }
2151
+ if (input.operation) {
2152
+ clauses.push("s.operation_name LIKE ?")
2153
+ params.push(`ai.${input.operation}%`)
2154
+ }
2155
+
2156
+ // Named attribute filters via span_attributes
2157
+ const attrFilters: Array<[string, string]> = []
2158
+ if (input.sessionId) attrFilters.push([AI_ATTR_MAP.sessionId, input.sessionId])
2159
+ if (input.functionId) attrFilters.push([AI_ATTR_MAP.functionId, input.functionId])
2160
+ if (input.provider) attrFilters.push([AI_ATTR_MAP.provider, input.provider])
2161
+ if (input.model) attrFilters.push([AI_ATTR_MAP.model, input.model])
2162
+
2163
+ for (const [key, value] of attrFilters) {
2164
+ clauses.push("EXISTS (SELECT 1 FROM span_attributes WHERE span_attributes.trace_id = s.trace_id AND span_attributes.span_id = s.span_id AND key = ? AND value = ?)")
2165
+ params.push(key, value)
2166
+ }
2167
+
2168
+ // Text search across prompt/response/tool attribute values via
2169
+ // FTS5. Prefers the external-content span_attr_fts index when
2170
+ // available, falls back to case-insensitive LIKE so old DBs
2171
+ // without FTS still work. FTS turns ~500ms full scans of 3 MB
2172
+ // prompt JSON into <50ms MATCH lookups.
2173
+ if ("text" in input && input.text) {
2174
+ const ftsQuery = toFtsMatchQuery(input.text)
2175
+ if (hasAttrFts && ftsQuery) {
2176
+ clauses.push(`EXISTS (
2177
+ SELECT 1 FROM span_attr_fts fts
2178
+ JOIN span_attributes sa ON sa.rowid = fts.rowid
2179
+ WHERE sa.trace_id = s.trace_id
2180
+ AND sa.span_id = s.span_id
2181
+ AND fts.value MATCH ?
2182
+ )`)
2183
+ params.push(ftsQuery)
2184
+ } else {
2185
+ const textKeys = AI_TEXT_SEARCH_KEYS.map(() => "?").join(", ")
2186
+ clauses.push(`EXISTS (SELECT 1 FROM span_attributes WHERE span_attributes.trace_id = s.trace_id AND span_attributes.span_id = s.span_id AND key IN (${textKeys}) AND value LIKE ? COLLATE NOCASE)`)
2187
+ params.push(...AI_TEXT_SEARCH_KEYS, `%${input.text}%`)
2188
+ }
2189
+ }
2190
+
2191
+ return { clauses, params }
2192
+ }
2193
+
2194
+ /** Load attribute values for a set of spans by key */
2195
+ const loadSpanAttrValues = (spans: ReadonlyArray<{ trace_id: string; span_id: string }>, keys: readonly string[]): Map<string, Map<string, string>> => {
2196
+ if (spans.length === 0 || keys.length === 0) return new Map()
2197
+ const spanPlaceholders = spans.map(() => "(?, ?)").join(", ")
2198
+ const keyPlaceholders = keys.map(() => "?").join(", ")
2199
+ const spanParams = spans.flatMap((s) => [s.trace_id, s.span_id])
2200
+
2201
+ const rows = db.query(`
2202
+ SELECT trace_id, span_id, key, value
2203
+ FROM span_attributes
2204
+ WHERE (trace_id, span_id) IN (VALUES ${spanPlaceholders})
2205
+ AND key IN (${keyPlaceholders})
2206
+ `).all(...spanParams, ...keys) as Array<{ trace_id: string; span_id: string; key: string; value: string }>
2207
+
2208
+ const result = new Map<string, Map<string, string>>()
2209
+ for (const row of rows) {
2210
+ const spanKey = `${row.trace_id}:${row.span_id}`
2211
+ let attrs = result.get(spanKey)
2212
+ if (!attrs) {
2213
+ attrs = new Map()
2214
+ result.set(spanKey, attrs)
2215
+ }
2216
+ attrs.set(row.key, row.value)
2217
+ }
2218
+ return result
2219
+ }
2220
+
2221
+ const searchAiCalls = Effect.fn("motel/TelemetryStore.searchAiCalls")(function* (input: AiCallSearch) {
2222
+ const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
2223
+ const limit = input.limit ?? 20
2224
+
2225
+ return yield* Effect.sync(() => {
2226
+ const { clauses, params } = buildAiWhereClauses(input, cutoff)
2227
+
2228
+ const rows = db.query(`
2229
+ SELECT s.trace_id, s.span_id, s.service_name, s.operation_name, s.start_time_ms, s.duration_ms, s.status
2230
+ FROM spans AS s
2231
+ WHERE ${clauses.join(" AND ")}
2232
+ ORDER BY s.start_time_ms DESC
2233
+ LIMIT ?
2234
+ `).all(...params, limit) as Array<{
2235
+ trace_id: string; span_id: string; service_name: string
2236
+ operation_name: string; start_time_ms: number; duration_ms: number; status: string
2237
+ }>
2238
+
2239
+ if (rows.length === 0) return [] as readonly AiCallSummary[]
2240
+
2241
+ // Batch-load the attributes we need for summaries
2242
+ const summaryAttrKeys = [
2243
+ AI_ATTR_MAP.functionId, AI_ATTR_MAP.provider, AI_ATTR_MAP.model,
2244
+ AI_ATTR_MAP.sessionId, AI_ATTR_MAP.userId, AI_ATTR_MAP.finishReason,
2245
+ AI_ATTR_MAP.inputTokens, AI_ATTR_MAP.outputTokens, AI_ATTR_MAP.totalTokens,
2246
+ AI_ATTR_MAP.cachedInputTokens, AI_ATTR_MAP.reasoningTokens,
2247
+ AI_ATTR_MAP.promptMessages, AI_ATTR_MAP.prompt, AI_ATTR_MAP.responseText,
2248
+ ]
2249
+ const attrMap = loadSpanAttrValues(rows, summaryAttrKeys)
2250
+
2251
+ // Count tool call child spans per AI span
2252
+ const spanPlaceholders = rows.map(() => "(?, ?)").join(", ")
2253
+ const spanParams = rows.flatMap((r) => [r.trace_id, r.span_id])
2254
+ const toolCountRows = db.query(`
2255
+ SELECT parent_span_id, COUNT(*) AS cnt
2256
+ FROM spans
2257
+ WHERE (trace_id, parent_span_id) IN (VALUES ${spanPlaceholders})
2258
+ AND operation_name LIKE 'ai.toolCall%'
2259
+ GROUP BY trace_id, parent_span_id
2260
+ `).all(...spanParams) as Array<{ parent_span_id: string; cnt: number }>
2261
+ const toolCounts = new Map(toolCountRows.map((r) => [r.parent_span_id, r.cnt]))
2262
+
2263
+ return rows.map((row): AiCallSummary => {
2264
+ const spanKey = `${row.trace_id}:${row.span_id}`
2265
+ const attrs = attrMap.get(spanKey)
2266
+ const get = (key: string) => attrs?.get(key) ?? null
2267
+ const getNum = (key: string) => {
2268
+ const v = get(key)
2269
+ return v != null ? Number(v) : null
2270
+ }
2271
+
2272
+ const promptContent = get(AI_ATTR_MAP.promptMessages) ?? get(AI_ATTR_MAP.prompt)
2273
+
2274
+ return {
2275
+ traceId: row.trace_id,
2276
+ spanId: row.span_id,
2277
+ operation: parseAiOperation(row.operation_name),
2278
+ service: row.service_name,
2279
+ functionId: get(AI_ATTR_MAP.functionId),
2280
+ provider: get(AI_ATTR_MAP.provider),
2281
+ model: get(AI_ATTR_MAP.model),
2282
+ status: row.status === "error" ? "error" : "ok",
2283
+ startedAt: new Date(row.start_time_ms).toISOString(),
2284
+ durationMs: row.duration_ms,
2285
+ sessionId: get(AI_ATTR_MAP.sessionId),
2286
+ userId: get(AI_ATTR_MAP.userId),
2287
+ promptPreview: truncatePreview(promptContent),
2288
+ responsePreview: truncatePreview(get(AI_ATTR_MAP.responseText)),
2289
+ finishReason: get(AI_ATTR_MAP.finishReason),
2290
+ toolCallCount: toolCounts.get(row.span_id) ?? 0,
2291
+ usage: {
2292
+ inputTokens: getNum(AI_ATTR_MAP.inputTokens),
2293
+ outputTokens: getNum(AI_ATTR_MAP.outputTokens),
2294
+ totalTokens: getNum(AI_ATTR_MAP.totalTokens),
2295
+ cachedInputTokens: getNum(AI_ATTR_MAP.cachedInputTokens),
2296
+ reasoningTokens: getNum(AI_ATTR_MAP.reasoningTokens),
2297
+ },
2298
+ }
2299
+ })
2300
+ })
2301
+ })
2302
+
2303
+ const getAiCall = Effect.fn("motel/TelemetryStore.getAiCall")(function* (spanId: string) {
2304
+ return yield* Effect.sync(() => {
2305
+ const row = db.query(`
2306
+ SELECT * FROM spans WHERE span_id = ? AND operation_name LIKE 'ai.%' LIMIT 1
2307
+ `).get(spanId) as SpanRow | null
2308
+ if (!row) return null
2309
+
2310
+ // Load all attributes for this span
2311
+ const attrRows = db.query(`
2312
+ SELECT key, value FROM span_attributes
2313
+ WHERE trace_id = ? AND span_id = ?
2314
+ `).all(row.trace_id, row.span_id) as Array<{ key: string; value: string }>
2315
+ const attrs = new Map(attrRows.map((r) => [r.key, r.value]))
2316
+ const get = (key: string) => attrs.get(key) ?? null
2317
+ const getNum = (key: string) => {
2318
+ const v = get(key)
2319
+ return v != null ? Number(v) : null
2320
+ }
2321
+
2322
+ // Load tool call child spans
2323
+ const toolCallRows = db.query(`
2324
+ SELECT span_id, operation_name, duration_ms, status, attributes_json
2325
+ FROM spans
2326
+ WHERE trace_id = ? AND parent_span_id = ? AND operation_name LIKE 'ai.toolCall%'
2327
+ ORDER BY start_time_ms ASC
2328
+ `).all(row.trace_id, row.span_id) as SpanRow[]
2329
+
2330
+ const toolCalls = toolCallRows.map((tc) => {
2331
+ const tcAttrs = JSON.parse(tc.attributes_json) as Record<string, string>
2332
+ return {
2333
+ name: tcAttrs["ai.toolCall.name"] ?? tc.operation_name,
2334
+ spanId: tc.span_id,
2335
+ status: tc.status === "error" ? "error" as const : "ok" as const,
2336
+ durationMs: tc.duration_ms,
2337
+ }
2338
+ })
2339
+
2340
+ // Load correlated logs
2341
+ const logRows = db.query(`
2342
+ SELECT * FROM logs WHERE span_id = ? ORDER BY timestamp_ms ASC
2343
+ `).all(row.span_id) as LogRow[]
2344
+ const logs = logRows.map(parseLogRow)
2345
+
2346
+ // Parse prompt - try as JSON first for structured display
2347
+ const promptRaw = get(AI_ATTR_MAP.promptMessages) ?? get(AI_ATTR_MAP.prompt)
2348
+ let promptMessages: unknown = null
2349
+ if (promptRaw) {
2350
+ try { promptMessages = JSON.parse(promptRaw) } catch { promptMessages = promptRaw }
2351
+ }
2352
+
2353
+ // Parse tools
2354
+ const toolsRaw = get(AI_ATTR_MAP.tools)
2355
+ let toolsAvailable: unknown = null
2356
+ if (toolsRaw) {
2357
+ try { toolsAvailable = JSON.parse(toolsRaw) } catch { toolsAvailable = toolsRaw }
2358
+ }
2359
+
2360
+ // Parse provider metadata
2361
+ const providerMetaRaw = get(AI_ATTR_MAP.providerMetadata)
2362
+ let providerMetadata: unknown = null
2363
+ if (providerMetaRaw) {
2364
+ try { providerMetadata = JSON.parse(providerMetaRaw) } catch { providerMetadata = providerMetaRaw }
2365
+ }
2366
+
2367
+ return {
2368
+ traceId: row.trace_id,
2369
+ spanId: row.span_id,
2370
+ operation: parseAiOperation(row.operation_name),
2371
+ service: row.service_name,
2372
+ functionId: get(AI_ATTR_MAP.functionId),
2373
+ provider: get(AI_ATTR_MAP.provider),
2374
+ model: get(AI_ATTR_MAP.model),
2375
+ status: row.status === "error" ? "error" as const : "ok" as const,
2376
+ startedAt: new Date(row.start_time_ms).toISOString(),
2377
+ durationMs: row.duration_ms,
2378
+ sessionId: get(AI_ATTR_MAP.sessionId),
2379
+ userId: get(AI_ATTR_MAP.userId),
2380
+ finishReason: get(AI_ATTR_MAP.finishReason),
2381
+ promptMessages,
2382
+ responseText: get(AI_ATTR_MAP.responseText),
2383
+ toolCalls,
2384
+ toolsAvailable,
2385
+ providerMetadata,
2386
+ usage: {
2387
+ inputTokens: getNum(AI_ATTR_MAP.inputTokens),
2388
+ outputTokens: getNum(AI_ATTR_MAP.outputTokens),
2389
+ totalTokens: getNum(AI_ATTR_MAP.totalTokens),
2390
+ cachedInputTokens: getNum(AI_ATTR_MAP.cachedInputTokens),
2391
+ reasoningTokens: getNum(AI_ATTR_MAP.reasoningTokens),
2392
+ },
2393
+ timing: {
2394
+ msToFirstChunk: getNum(AI_ATTR_MAP.msToFirstChunk),
2395
+ msToFinish: getNum(AI_ATTR_MAP.msToFinish),
2396
+ avgOutputTokensPerSecond: getNum(AI_ATTR_MAP.avgOutputTokensPerSecond),
2397
+ },
2398
+ logs,
2399
+ } satisfies AiCallDetail
2400
+ })
2401
+ })
2402
+
2403
+ const aiCallStats = Effect.fn("motel/TelemetryStore.aiCallStats")(function* (input: AiCallStatsSearch) {
2404
+ const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
2405
+ const limit = input.limit ?? 20
2406
+
2407
+ return yield* Effect.sync(() => {
2408
+ const { clauses, params } = buildAiWhereClauses(input, cutoff)
2409
+
2410
+ // For status groupBy, we can do it purely from the spans table
2411
+ if (input.groupBy === "status") {
2412
+ const rows = db.query(`
2413
+ SELECT s.status AS grp, COUNT(*) AS count, AVG(s.duration_ms) AS avg_dur
2414
+ FROM spans AS s
2415
+ WHERE ${clauses.join(" AND ")}
2416
+ GROUP BY s.status
2417
+ ORDER BY count DESC
2418
+ LIMIT ?
2419
+ `).all(...params, limit) as Array<{ grp: string; count: number; avg_dur: number }>
2420
+
2421
+ if (input.agg === "count") return rows.map((r) => ({ group: r.grp, value: r.count, count: r.count }))
2422
+ if (input.agg === "avg_duration") return rows.map((r) => ({ group: r.grp, value: r.avg_dur, count: r.count }))
2423
+ }
2424
+
2425
+ // For attribute-based groupBy, we need to join span_attributes
2426
+ const groupByAttrKey = input.groupBy === "provider" ? AI_ATTR_MAP.provider
2427
+ : input.groupBy === "model" ? AI_ATTR_MAP.model
2428
+ : input.groupBy === "functionId" ? AI_ATTR_MAP.functionId
2429
+ : input.groupBy === "sessionId" ? AI_ATTR_MAP.sessionId
2430
+ : null
2431
+
2432
+ if (!groupByAttrKey) return []
2433
+
2434
+ // First get the matching spans with their group values
2435
+ const rows = db.query(`
2436
+ SELECT
2437
+ COALESCE(ga.value, 'unknown') AS grp,
2438
+ s.span_id,
2439
+ s.duration_ms,
2440
+ s.status
2441
+ FROM spans AS s
2442
+ LEFT JOIN span_attributes AS ga
2443
+ ON ga.trace_id = s.trace_id AND ga.span_id = s.span_id AND ga.key = ?
2444
+ WHERE ${clauses.join(" AND ")}
2445
+ `).all(groupByAttrKey, ...params) as Array<{ grp: string; span_id: string; duration_ms: number; status: string }>
2446
+
2447
+ // Group and aggregate in JS (need p95 and token aggregation)
2448
+ const groups = new Map<string, { durations: number[]; count: number; spanIds: string[] }>()
2449
+ for (const row of rows) {
2450
+ const bucket = groups.get(row.grp) ?? { durations: [], count: 0, spanIds: [] }
2451
+ bucket.durations.push(row.duration_ms)
2452
+ bucket.count++
2453
+ bucket.spanIds.push(row.span_id)
2454
+ groups.set(row.grp, bucket)
2455
+ }
2456
+
2457
+ // For token aggregations, batch-load from span_attributes
2458
+ if (input.agg === "total_input_tokens" || input.agg === "total_output_tokens") {
2459
+ const tokenKey = input.agg === "total_input_tokens" ? AI_ATTR_MAP.inputTokens : AI_ATTR_MAP.outputTokens
2460
+ const allSpanIds = [...groups.values()].flatMap((b) => b.spanIds)
2461
+ if (allSpanIds.length > 0) {
2462
+ const placeholders = allSpanIds.map(() => "?").join(", ")
2463
+ const tokenRows = db.query(`
2464
+ SELECT span_id, CAST(value AS REAL) AS tokens
2465
+ FROM span_attributes
2466
+ WHERE key = ? AND span_id IN (${placeholders})
2467
+ `).all(tokenKey, ...allSpanIds) as Array<{ span_id: string; tokens: number }>
2468
+
2469
+ const tokenBySpan = new Map(tokenRows.map((r) => [r.span_id, r.tokens]))
2470
+
2471
+ return [...groups.entries()]
2472
+ .map(([group, bucket]) => {
2473
+ const total = bucket.spanIds.reduce((sum, sid) => sum + (tokenBySpan.get(sid) ?? 0), 0)
2474
+ return { group, value: total, count: bucket.count }
2475
+ })
2476
+ .sort((a, b) => b.value - a.value)
2477
+ .slice(0, limit)
2478
+ }
2479
+ }
2480
+
2481
+ return [...groups.entries()]
2482
+ .map(([group, bucket]) => {
2483
+ const value = input.agg === "count"
2484
+ ? bucket.count
2485
+ : input.agg === "avg_duration"
2486
+ ? bucket.durations.reduce((s, d) => s + d, 0) / Math.max(1, bucket.count)
2487
+ : input.agg === "p95_duration"
2488
+ ? percentile(bucket.durations, 0.95)
2489
+ : bucket.count
2490
+ return { group, value, count: bucket.count }
2491
+ })
2492
+ .sort((a, b) => b.value - a.value)
2493
+ .slice(0, limit)
2494
+ })
2495
+ })
2496
+
2497
+ return TelemetryStore.of({
2498
+ ingestTraces,
2499
+ ingestLogs,
2500
+ listServices,
2501
+ listRecentTraces,
2502
+ listTraceSummaries,
2503
+ searchTraces,
2504
+ searchTraceSummaries,
2505
+ traceStats,
2506
+ getTrace,
2507
+ getSpan,
2508
+ listTraceSpans,
2509
+ searchSpans,
2510
+ searchLogs,
2511
+ logStats,
2512
+ listFacets,
2513
+ listRecentLogs,
2514
+ listTraceLogs,
2515
+ searchAiCalls,
2516
+ getAiCall,
2517
+ aiCallStats,
2518
+ runRetentionNow: cleanupExpired(),
2519
+ })
2520
+ })
2521
+
2522
+ /** Compatibility factory for callers constructing a writer/query-capable store layer. */
2523
+ export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) =>
2524
+ Layer.effect(TelemetryStore, makeTelemetryStoreEffect(opts)).pipe(Layer.provide(BunFileSystem.layer))
2525
+
2526
+ /**
2527
+ * Default writer runtime used by tests and direct store consumers.
2528
+ */
2529
+ export const TelemetryStoreLive = makeTelemetryStoreLayer({ readonly: false, runRetention: true })
2530
+
2531
+ /**
2532
+ * The ingest worker's writer. It is the managed daemon's sole owner of
2533
+ * schema migrations, FTS backfill, retention, and page reclamation.
2534
+ */
2535
+ export const TelemetryStoreWorkerLive = TelemetryStoreLive
2536
+
2537
+ /**
2538
+ * Read-only instance for query-only processes (currently the TUI and
2539
+ * HTTP query handlers). Skips every DDL/DML statement at startup so
2540
+ * the connection can be opened while a writer is mid-transaction
2541
+ * without racing for the write lock. Provided as TelemetryStoreReadonly
2542
+ * — a distinct service identifier so it can coexist with the writer
2543
+ * TelemetryStore in the same runtime.
2544
+ */
2545
+ export const TelemetryStoreReadonlyLive = Layer.effect(TelemetryStoreReadonly, makeTelemetryStoreEffect({ readonly: true, runRetention: false })).pipe(Layer.provide(BunFileSystem.layer))
2546
+
2547
+ /** Query-worker reader that waits for the sole writer to finish schema bootstrap. */
2548
+ export const TelemetryStoreQueryWorkerLive = Layer.effect(
2549
+ TelemetryStoreReadonly,
2550
+ makeTelemetryStoreEffect({ readonly: true, runRetention: false }).pipe(
2551
+ Effect.map((store) => TelemetryStoreReadonly.of(store)),
2552
+ Effect.retry(Schedule.spaced("50 millis")),
2553
+ ),
2554
+ ).pipe(Layer.provide(BunFileSystem.layer))