@kitlangton/motel 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +142 -0
- package/LICENSE +21 -0
- package/README.md +199 -0
- package/package.json +92 -0
- package/src/App.tsx +217 -0
- package/src/cli.ts +258 -0
- package/src/config.ts +39 -0
- package/src/daemon.test.ts +59 -0
- package/src/daemon.ts +398 -0
- package/src/domain.ts +233 -0
- package/src/httpApi.ts +384 -0
- package/src/index.tsx +18 -0
- package/src/instructions.ts +72 -0
- package/src/localServer.ts +699 -0
- package/src/locator.ts +138 -0
- package/src/mcp.ts +260 -0
- package/src/motel.ts +86 -0
- package/src/motelClient.ts +201 -0
- package/src/otlp.ts +142 -0
- package/src/queryFilters.ts +39 -0
- package/src/registry.ts +86 -0
- package/src/runtime.ts +38 -0
- package/src/server.ts +10 -0
- package/src/services/LogQueryService.ts +43 -0
- package/src/services/TelemetryStore.ts +1821 -0
- package/src/services/TraceQueryService.ts +71 -0
- package/src/telemetry.test.ts +726 -0
- package/src/ui/ServiceLogs.tsx +112 -0
- package/src/ui/SpanDetail.tsx +134 -0
- package/src/ui/SpanDetailFull.tsx +224 -0
- package/src/ui/SpanDetailPane.tsx +91 -0
- package/src/ui/TraceDetailsPane.tsx +169 -0
- package/src/ui/TraceList.tsx +128 -0
- package/src/ui/Waterfall.tsx +412 -0
- package/src/ui/app/TraceListPane.tsx +34 -0
- package/src/ui/app/TraceWorkspace.tsx +254 -0
- package/src/ui/app/useAppLayout.ts +79 -0
- package/src/ui/app/useTraceScreenData.ts +411 -0
- package/src/ui/format.ts +119 -0
- package/src/ui/primitives.tsx +170 -0
- package/src/ui/state.ts +137 -0
- package/src/ui/theme.ts +153 -0
- package/src/ui/traceDetailsWidth.repro.test.ts +115 -0
- package/src/ui/traceSortNav.repro.seed.ts +62 -0
- package/src/ui/traceSortNav.repro.test.ts +220 -0
- package/src/ui/useKeyboardNav.ts +532 -0
- package/src/ui/waterfallNav.repro.seed.ts +86 -0
- package/src/ui/waterfallNav.repro.test.ts +263 -0
- package/src/ui/waterfallNav.test.ts +422 -0
- package/src/ui/waterfallNav.ts +75 -0
- package/web/dist/assets/index-BEKIiisE.js +27 -0
- package/web/dist/assets/index-DzuHNBGV.css +2 -0
- package/web/dist/index.html +13 -0
|
@@ -0,0 +1,1821 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite"
|
|
2
|
+
import { mkdirSync } from "node:fs"
|
|
3
|
+
import { dirname } from "node:path"
|
|
4
|
+
import { Clock, Effect, Layer, Schedule, Context } from "effect"
|
|
5
|
+
import { config } from "../config.js"
|
|
6
|
+
import type { AiCallDetail, AiCallSummary, FacetItem, LogItem, SpanItem, StatsItem, TraceItem, TraceSummaryItem, TraceSpanEvent, TraceSpanItem } from "../domain.js"
|
|
7
|
+
import { AI_ATTR_MAP, AI_TEXT_SEARCH_KEYS, truncatePreview } from "../domain.js"
|
|
8
|
+
import { attributeMap, nanosToMilliseconds, parseAnyValue, spanKindLabel, spanStatusLabel, stringifyValue, type OtlpLogExportRequest, type OtlpTraceExportRequest } from "../otlp.js"
|
|
9
|
+
|
|
10
|
+
interface SpanRow {
|
|
11
|
+
readonly trace_id: string
|
|
12
|
+
readonly span_id: string
|
|
13
|
+
readonly parent_span_id: string | null
|
|
14
|
+
readonly service_name: string
|
|
15
|
+
readonly scope_name: string | null
|
|
16
|
+
readonly operation_name: string
|
|
17
|
+
readonly kind: string | null
|
|
18
|
+
readonly start_time_ms: number
|
|
19
|
+
readonly end_time_ms: number
|
|
20
|
+
readonly duration_ms: number
|
|
21
|
+
readonly status: string
|
|
22
|
+
readonly attributes_json: string
|
|
23
|
+
readonly resource_json: string
|
|
24
|
+
readonly events_json: string
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
interface LogRow {
|
|
28
|
+
readonly id: number
|
|
29
|
+
readonly trace_id: string | null
|
|
30
|
+
readonly span_id: string | null
|
|
31
|
+
readonly service_name: string
|
|
32
|
+
readonly scope_name: string | null
|
|
33
|
+
readonly severity_text: string
|
|
34
|
+
readonly timestamp_ms: number
|
|
35
|
+
readonly body: string
|
|
36
|
+
readonly attributes_json: string
|
|
37
|
+
readonly resource_json: string
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
interface LogSearch {
|
|
41
|
+
readonly serviceName?: string | null
|
|
42
|
+
readonly severity?: string | null
|
|
43
|
+
readonly traceId?: string | null
|
|
44
|
+
readonly spanId?: string | null
|
|
45
|
+
readonly body?: string | null
|
|
46
|
+
readonly lookbackMinutes?: number
|
|
47
|
+
readonly limit?: number
|
|
48
|
+
readonly cursorTimestampMs?: number
|
|
49
|
+
readonly cursorId?: string
|
|
50
|
+
readonly attributeFilters?: Readonly<Record<string, string>>
|
|
51
|
+
readonly attributeContainsFilters?: Readonly<Record<string, string>>
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
interface TraceSearch {
|
|
55
|
+
readonly serviceName?: string | null
|
|
56
|
+
readonly operation?: string | null
|
|
57
|
+
readonly status?: "ok" | "error" | null
|
|
58
|
+
readonly minDurationMs?: number | null
|
|
59
|
+
readonly attributeFilters?: Readonly<Record<string, string>>
|
|
60
|
+
readonly lookbackMinutes?: number
|
|
61
|
+
readonly limit?: number
|
|
62
|
+
readonly cursorStartedAtMs?: number
|
|
63
|
+
readonly cursorTraceId?: string
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
interface SpanSearch {
|
|
67
|
+
readonly serviceName?: string | null
|
|
68
|
+
readonly traceId?: string | null
|
|
69
|
+
readonly operation?: string | null
|
|
70
|
+
readonly parentOperation?: string | null
|
|
71
|
+
readonly status?: "ok" | "error" | null
|
|
72
|
+
readonly lookbackMinutes?: number
|
|
73
|
+
readonly limit?: number
|
|
74
|
+
readonly attributeFilters?: Readonly<Record<string, string>>
|
|
75
|
+
readonly attributeContainsFilters?: Readonly<Record<string, string>>
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
interface TraceStatsSearch extends TraceSearch {
|
|
79
|
+
readonly groupBy: string
|
|
80
|
+
readonly agg: "count" | "avg_duration" | "p95_duration" | "error_rate"
|
|
81
|
+
readonly limit?: number
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
interface LogStatsSearch extends LogSearch {
|
|
85
|
+
readonly groupBy: string
|
|
86
|
+
readonly agg: "count"
|
|
87
|
+
readonly lookbackMinutes?: number
|
|
88
|
+
readonly limit?: number
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// FacetItem and StatsItem imported from domain.ts
|
|
92
|
+
|
|
93
|
+
interface FacetSearch {
|
|
94
|
+
readonly type: "traces" | "logs"
|
|
95
|
+
readonly field: string
|
|
96
|
+
readonly serviceName?: string | null
|
|
97
|
+
readonly lookbackMinutes?: number
|
|
98
|
+
readonly limit?: number
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
interface AiCallSearch {
|
|
102
|
+
readonly service?: string | null
|
|
103
|
+
readonly traceId?: string | null
|
|
104
|
+
readonly sessionId?: string | null
|
|
105
|
+
readonly functionId?: string | null
|
|
106
|
+
readonly provider?: string | null
|
|
107
|
+
readonly model?: string | null
|
|
108
|
+
readonly operation?: string | null
|
|
109
|
+
readonly status?: "ok" | "error" | null
|
|
110
|
+
readonly minDurationMs?: number | null
|
|
111
|
+
readonly text?: string | null
|
|
112
|
+
readonly lookbackMinutes?: number
|
|
113
|
+
readonly limit?: number
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
interface AiCallStatsSearch {
|
|
117
|
+
readonly groupBy: "provider" | "model" | "functionId" | "sessionId" | "status"
|
|
118
|
+
readonly agg: "count" | "avg_duration" | "p95_duration" | "total_input_tokens" | "total_output_tokens"
|
|
119
|
+
readonly service?: string | null
|
|
120
|
+
readonly traceId?: string | null
|
|
121
|
+
readonly sessionId?: string | null
|
|
122
|
+
readonly functionId?: string | null
|
|
123
|
+
readonly provider?: string | null
|
|
124
|
+
readonly model?: string | null
|
|
125
|
+
readonly operation?: string | null
|
|
126
|
+
readonly status?: "ok" | "error" | null
|
|
127
|
+
readonly minDurationMs?: number | null
|
|
128
|
+
readonly lookbackMinutes?: number
|
|
129
|
+
readonly limit?: number
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
interface TraceSummaryRow {
|
|
133
|
+
readonly trace_id: string
|
|
134
|
+
readonly service_name: string
|
|
135
|
+
readonly root_operation_name: string
|
|
136
|
+
readonly started_at_ms: number
|
|
137
|
+
readonly ended_at_ms?: number
|
|
138
|
+
readonly active_span_count: number
|
|
139
|
+
readonly duration_ms: number
|
|
140
|
+
readonly span_count: number
|
|
141
|
+
readonly error_count: number
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
type InternalTraceSpanItem = TraceSpanItem & {
|
|
145
|
+
readonly syntheticMissingParent?: boolean
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const isSpanRunning = (startTimeMs: number, endTimeMs: number) => endTimeMs <= 0 || endTimeMs < startTimeMs
|
|
149
|
+
|
|
150
|
+
const liveDurationMs = (startTimeMs: number, endTimeMs: number, isRunning: boolean) =>
|
|
151
|
+
Math.max(0, (isRunning ? Date.now() : endTimeMs) - startTimeMs)
|
|
152
|
+
|
|
153
|
+
const parseSummaryRow = (row: TraceSummaryRow): TraceSummaryItem => ({
|
|
154
|
+
isRunning: row.active_span_count > 0,
|
|
155
|
+
traceId: row.trace_id,
|
|
156
|
+
serviceName: row.service_name ?? "unknown",
|
|
157
|
+
rootOperationName: row.root_operation_name ?? "unknown",
|
|
158
|
+
startedAt: new Date(row.started_at_ms),
|
|
159
|
+
durationMs: row.active_span_count > 0 ? liveDurationMs(row.started_at_ms, row.ended_at_ms ?? 0, true) : Math.max(0, row.duration_ms),
|
|
160
|
+
spanCount: row.span_count,
|
|
161
|
+
errorCount: row.error_count,
|
|
162
|
+
warnings: [],
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
const TRACE_SUMMARY_SELECT_SQL = `
|
|
166
|
+
SELECT
|
|
167
|
+
trace_id,
|
|
168
|
+
COALESCE(MIN(CASE WHEN parent_span_id IS NULL THEN service_name END), MIN(service_name)) AS service_name,
|
|
169
|
+
COALESCE(MIN(CASE WHEN parent_span_id IS NULL THEN operation_name END), MIN(operation_name)) AS root_operation_name,
|
|
170
|
+
MIN(start_time_ms) AS started_at_ms,
|
|
171
|
+
MAX(end_time_ms) AS ended_at_ms,
|
|
172
|
+
SUM(CASE WHEN end_time_ms <= 0 OR end_time_ms < start_time_ms THEN 1 ELSE 0 END) AS active_span_count,
|
|
173
|
+
MAX(end_time_ms) - MIN(start_time_ms) AS duration_ms,
|
|
174
|
+
COUNT(*) AS span_count,
|
|
175
|
+
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) AS error_count
|
|
176
|
+
FROM spans
|
|
177
|
+
`
|
|
178
|
+
|
|
179
|
+
const parseRecord = (value: string): Record<string, string> => {
|
|
180
|
+
try {
|
|
181
|
+
const parsed = JSON.parse(value) as Record<string, unknown>
|
|
182
|
+
return Object.fromEntries(Object.entries(parsed).map(([key, entry]) => [key, stringifyValue(entry)]))
|
|
183
|
+
} catch {
|
|
184
|
+
return {}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const parseEvents = (value: string): readonly TraceSpanEvent[] => {
|
|
189
|
+
try {
|
|
190
|
+
const parsed = JSON.parse(value) as Array<{ name: string; timestamp: number; attributes: Record<string, string> }>
|
|
191
|
+
return parsed.map((event) => ({
|
|
192
|
+
name: event.name,
|
|
193
|
+
timestamp: new Date(event.timestamp),
|
|
194
|
+
attributes: event.attributes,
|
|
195
|
+
}))
|
|
196
|
+
} catch {
|
|
197
|
+
return []
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const parseSpanRow = (row: SpanRow): InternalTraceSpanItem => {
|
|
202
|
+
const isRunning = isSpanRunning(row.start_time_ms, row.end_time_ms)
|
|
203
|
+
return {
|
|
204
|
+
spanId: row.span_id,
|
|
205
|
+
parentSpanId: row.parent_span_id,
|
|
206
|
+
serviceName: row.service_name,
|
|
207
|
+
scopeName: row.scope_name,
|
|
208
|
+
kind: row.kind,
|
|
209
|
+
operationName: row.operation_name,
|
|
210
|
+
startTime: new Date(row.start_time_ms),
|
|
211
|
+
isRunning,
|
|
212
|
+
durationMs: liveDurationMs(row.start_time_ms, row.end_time_ms, isRunning),
|
|
213
|
+
status: row.status === "error" ? "error" : "ok",
|
|
214
|
+
depth: 0,
|
|
215
|
+
tags: {
|
|
216
|
+
...parseRecord(row.resource_json),
|
|
217
|
+
...parseRecord(row.attributes_json),
|
|
218
|
+
},
|
|
219
|
+
warnings: [],
|
|
220
|
+
events: parseEvents(row.events_json),
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const parseLogRow = (row: LogRow): LogItem => ({
|
|
225
|
+
id: String(row.id),
|
|
226
|
+
timestamp: new Date(row.timestamp_ms),
|
|
227
|
+
serviceName: row.service_name,
|
|
228
|
+
severityText: row.severity_text,
|
|
229
|
+
body: row.body,
|
|
230
|
+
traceId: row.trace_id,
|
|
231
|
+
spanId: row.span_id,
|
|
232
|
+
scopeName: row.scope_name,
|
|
233
|
+
attributes: {
|
|
234
|
+
...parseRecord(row.resource_json),
|
|
235
|
+
...parseRecord(row.attributes_json),
|
|
236
|
+
},
|
|
237
|
+
})
|
|
238
|
+
|
|
239
|
+
const orderTraceSpans = (spans: readonly InternalTraceSpanItem[]) => {
|
|
240
|
+
const childrenByParent = new Map<string | null, InternalTraceSpanItem[]>()
|
|
241
|
+
const spanIds = new Set(spans.map((span) => span.spanId))
|
|
242
|
+
|
|
243
|
+
for (const span of spans) {
|
|
244
|
+
const key = span.parentSpanId && spanIds.has(span.parentSpanId) ? span.parentSpanId : null
|
|
245
|
+
const siblings = childrenByParent.get(key) ?? []
|
|
246
|
+
siblings.push(span)
|
|
247
|
+
childrenByParent.set(key, siblings)
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
for (const siblings of childrenByParent.values()) {
|
|
251
|
+
siblings.sort((left, right) =>
|
|
252
|
+
left.startTime.getTime() - right.startTime.getTime() || Number(Boolean(left.syntheticMissingParent)) - Number(Boolean(right.syntheticMissingParent))
|
|
253
|
+
)
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const ordered: Array<InternalTraceSpanItem> = []
|
|
257
|
+
const visit = (parent: string | null, depth: number) => {
|
|
258
|
+
for (const child of childrenByParent.get(parent) ?? []) {
|
|
259
|
+
ordered.push({ ...child, depth })
|
|
260
|
+
visit(child.spanId, depth + 1)
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
visit(null, 0)
|
|
265
|
+
return ordered
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
const buildTrace = (traceId: string, spanRows: readonly SpanRow[]): TraceItem => {
|
|
269
|
+
const parsedSpans = spanRows.map(parseSpanRow)
|
|
270
|
+
const spanIds = new Set(parsedSpans.map((span) => span.spanId))
|
|
271
|
+
const missingParentGroups = new Map<string, InternalTraceSpanItem[]>()
|
|
272
|
+
|
|
273
|
+
for (const span of parsedSpans) {
|
|
274
|
+
if (span.parentSpanId !== null && !spanIds.has(span.parentSpanId)) {
|
|
275
|
+
const siblings = missingParentGroups.get(span.parentSpanId) ?? []
|
|
276
|
+
siblings.push(span)
|
|
277
|
+
missingParentGroups.set(span.parentSpanId, siblings)
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const syntheticParents: InternalTraceSpanItem[] = [...missingParentGroups.entries()].map(([missingParentId, children]) => {
|
|
282
|
+
const firstChild = children[0]!
|
|
283
|
+
const startedAtMs = Math.min(...children.map((child) => child.startTime.getTime()))
|
|
284
|
+
const endedAtMs = Math.max(...children.map((child) => child.startTime.getTime() + child.durationMs))
|
|
285
|
+
return {
|
|
286
|
+
spanId: missingParentId,
|
|
287
|
+
parentSpanId: null,
|
|
288
|
+
serviceName: firstChild.serviceName,
|
|
289
|
+
scopeName: null,
|
|
290
|
+
kind: null,
|
|
291
|
+
operationName: `[missing parent ${missingParentId.slice(0, 8)}]`,
|
|
292
|
+
startTime: new Date(startedAtMs),
|
|
293
|
+
isRunning: children.some((child) => child.isRunning),
|
|
294
|
+
durationMs: Math.max(0, endedAtMs - startedAtMs),
|
|
295
|
+
status: "error",
|
|
296
|
+
depth: 0,
|
|
297
|
+
tags: {},
|
|
298
|
+
warnings: [`missing span ${missingParentId} (${children.length} child${children.length === 1 ? "" : "ren"})`],
|
|
299
|
+
events: [],
|
|
300
|
+
syntheticMissingParent: true,
|
|
301
|
+
}
|
|
302
|
+
})
|
|
303
|
+
|
|
304
|
+
const orderedSpans = orderTraceSpans([...parsedSpans, ...syntheticParents])
|
|
305
|
+
const startedAtMs = Math.min(...orderedSpans.map((span) => span.startTime.getTime()))
|
|
306
|
+
const endedAtMs = Math.max(...orderedSpans.map((span) => span.startTime.getTime() + span.durationMs))
|
|
307
|
+
const isRunning = orderedSpans.some((span) => span.isRunning)
|
|
308
|
+
const rootSpan = orderedSpans.find((span) => !span.syntheticMissingParent && span.parentSpanId === null)
|
|
309
|
+
?? orderedSpans.find((span) => !span.syntheticMissingParent)
|
|
310
|
+
?? orderedSpans[0]
|
|
311
|
+
?? null
|
|
312
|
+
const warnings = syntheticParents.map((span) => span.warnings[0]!).filter((warning) => warning.length > 0)
|
|
313
|
+
|
|
314
|
+
return {
|
|
315
|
+
traceId,
|
|
316
|
+
serviceName: rootSpan?.serviceName ?? "unknown",
|
|
317
|
+
rootOperationName: rootSpan?.operationName ?? "unknown",
|
|
318
|
+
startedAt: new Date(startedAtMs),
|
|
319
|
+
isRunning,
|
|
320
|
+
durationMs: Math.max(0, endedAtMs - startedAtMs),
|
|
321
|
+
spanCount: orderedSpans.length,
|
|
322
|
+
errorCount: orderedSpans.filter((span) => span.status === "error").length,
|
|
323
|
+
warnings,
|
|
324
|
+
spans: orderedSpans.map(({ syntheticMissingParent: _, ...span }) => span),
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const buildSpanItems = (traceId: string, spanRows: readonly SpanRow[]): readonly SpanItem[] => {
|
|
329
|
+
const trace = buildTrace(traceId, spanRows)
|
|
330
|
+
const spanById = new Map(trace.spans.map((span) => [span.spanId, span]))
|
|
331
|
+
return trace.spans.map((span) => ({
|
|
332
|
+
traceId,
|
|
333
|
+
rootOperationName: trace.rootOperationName,
|
|
334
|
+
parentOperationName: span.parentSpanId ? spanById.get(span.parentSpanId)?.operationName ?? null : null,
|
|
335
|
+
span,
|
|
336
|
+
}))
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
const buildSpanItem = (traceId: string, spanRows: readonly SpanRow[], spanId: string): SpanItem | null =>
|
|
340
|
+
buildSpanItems(traceId, spanRows).find((item) => item.span.spanId === spanId) ?? null
|
|
341
|
+
|
|
342
|
+
const matchesAttributes = (attributes: Readonly<Record<string, string>>, filters: Readonly<Record<string, string>> | undefined) =>
|
|
343
|
+
!filters || Object.entries(filters).every(([key, value]) => attributes[key] === value)
|
|
344
|
+
|
|
345
|
+
const matchesAttributeContains = (attributes: Readonly<Record<string, string>>, filters: Readonly<Record<string, string>> | undefined) =>
|
|
346
|
+
!filters || Object.entries(filters).every(([key, needle]) => {
|
|
347
|
+
const value = attributes[key]
|
|
348
|
+
return value !== undefined && value.toLowerCase().includes(needle.toLowerCase())
|
|
349
|
+
})
|
|
350
|
+
|
|
351
|
+
const percentile = (values: readonly number[], ratio: number) => {
|
|
352
|
+
if (values.length === 0) return 0
|
|
353
|
+
const sorted = [...values].sort((left, right) => left - right)
|
|
354
|
+
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * ratio) - 1))
|
|
355
|
+
return sorted[index] ?? 0
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const tokenizeFts = (value: string) => value.match(/[A-Za-z0-9_]+/g)?.filter((token) => token.length > 1) ?? []
|
|
359
|
+
|
|
360
|
+
const toFtsMatchQuery = (value: string) => {
|
|
361
|
+
const tokens = tokenizeFts(value)
|
|
362
|
+
if (tokens.length === 0) return null
|
|
363
|
+
return tokens.map((token) => `${token}*`).join(" AND ")
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
const buildExactAttributeMatchSubquery = (
|
|
367
|
+
tableName: "span_attributes" | "log_attributes",
|
|
368
|
+
idColumns: readonly string[],
|
|
369
|
+
filters: Readonly<Record<string, string>> | undefined,
|
|
370
|
+
) => {
|
|
371
|
+
const entries = Object.entries(filters ?? {})
|
|
372
|
+
if (entries.length === 0) return null
|
|
373
|
+
const disjunction = entries.map(() => "(key = ? AND value = ?)").join(" OR ")
|
|
374
|
+
return {
|
|
375
|
+
sql: `
|
|
376
|
+
SELECT ${idColumns.join(", ")}
|
|
377
|
+
FROM ${tableName}
|
|
378
|
+
WHERE ${disjunction}
|
|
379
|
+
GROUP BY ${idColumns.join(", ")}
|
|
380
|
+
HAVING COUNT(DISTINCT key) = ${entries.length}
|
|
381
|
+
`,
|
|
382
|
+
params: entries.flatMap(([key, value]) => [key, value]),
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
const buildContainsAttributeMatchSubquery = (
|
|
387
|
+
tableName: "span_attributes" | "log_attributes",
|
|
388
|
+
idColumns: readonly string[],
|
|
389
|
+
filters: Readonly<Record<string, string>> | undefined,
|
|
390
|
+
) => {
|
|
391
|
+
const entries = Object.entries(filters ?? {})
|
|
392
|
+
if (entries.length === 0) return null
|
|
393
|
+
const disjunction = entries.map(() => "(key = ? AND value LIKE ? COLLATE NOCASE)").join(" OR ")
|
|
394
|
+
return {
|
|
395
|
+
sql: `
|
|
396
|
+
SELECT ${idColumns.join(", ")}
|
|
397
|
+
FROM ${tableName}
|
|
398
|
+
WHERE ${disjunction}
|
|
399
|
+
GROUP BY ${idColumns.join(", ")}
|
|
400
|
+
HAVING COUNT(DISTINCT key) = ${entries.length}
|
|
401
|
+
`,
|
|
402
|
+
params: entries.flatMap(([key, value]) => [key, `%${value}%`]),
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
export class TelemetryStore extends Context.Service<
|
|
407
|
+
TelemetryStore,
|
|
408
|
+
{
|
|
409
|
+
readonly ingestTraces: (payload: OtlpTraceExportRequest) => Effect.Effect<{ readonly insertedSpans: number }, Error>
|
|
410
|
+
readonly ingestLogs: (payload: OtlpLogExportRequest) => Effect.Effect<{ readonly insertedLogs: number }, Error>
|
|
411
|
+
readonly listServices: Effect.Effect<readonly string[], Error>
|
|
412
|
+
readonly listRecentTraces: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceItem[], Error>
|
|
413
|
+
readonly listTraceSummaries: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceSummaryItem[], Error>
|
|
414
|
+
readonly searchTraces: (input: TraceSearch) => Effect.Effect<readonly TraceItem[], Error>
|
|
415
|
+
readonly searchTraceSummaries: (input: TraceSearch) => Effect.Effect<readonly TraceSummaryItem[], Error>
|
|
416
|
+
readonly traceStats: (input: TraceStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
|
|
417
|
+
readonly getTrace: (traceId: string) => Effect.Effect<TraceItem | null, Error>
|
|
418
|
+
readonly getSpan: (spanId: string) => Effect.Effect<SpanItem | null, Error>
|
|
419
|
+
readonly listTraceSpans: (traceId: string) => Effect.Effect<readonly SpanItem[], Error>
|
|
420
|
+
readonly searchSpans: (input: SpanSearch) => Effect.Effect<readonly SpanItem[], Error>
|
|
421
|
+
readonly searchLogs: (input: LogSearch) => Effect.Effect<readonly LogItem[], Error>
|
|
422
|
+
readonly logStats: (input: LogStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
|
|
423
|
+
readonly listFacets: (input: FacetSearch) => Effect.Effect<readonly FacetItem[], Error>
|
|
424
|
+
readonly listRecentLogs: (serviceName: string) => Effect.Effect<readonly LogItem[], Error>
|
|
425
|
+
readonly listTraceLogs: (traceId: string) => Effect.Effect<readonly LogItem[], Error>
|
|
426
|
+
readonly searchAiCalls: (input: AiCallSearch) => Effect.Effect<readonly AiCallSummary[], Error>
|
|
427
|
+
readonly getAiCall: (spanId: string) => Effect.Effect<AiCallDetail | null, Error>
|
|
428
|
+
readonly aiCallStats: (input: AiCallStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
|
|
429
|
+
}
|
|
430
|
+
>()("motel/TelemetryStore") {}
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
export const TelemetryStoreLive = Layer.effect(
|
|
434
|
+
TelemetryStore,
|
|
435
|
+
Effect.gen(function* () {
|
|
436
|
+
mkdirSync(dirname(config.otel.databasePath), { recursive: true })
|
|
437
|
+
const db = yield* Effect.acquireRelease(
|
|
438
|
+
Effect.sync(() => new Database(config.otel.databasePath, { create: true })),
|
|
439
|
+
(db) => Effect.sync(() => db.close()),
|
|
440
|
+
)
|
|
441
|
+
db.exec(`
|
|
442
|
+
PRAGMA journal_mode = WAL;
|
|
443
|
+
PRAGMA synchronous = NORMAL;
|
|
444
|
+
PRAGMA temp_store = MEMORY;
|
|
445
|
+
PRAGMA busy_timeout = 5000;
|
|
446
|
+
|
|
447
|
+
CREATE TABLE IF NOT EXISTS spans (
|
|
448
|
+
trace_id TEXT NOT NULL,
|
|
449
|
+
span_id TEXT NOT NULL,
|
|
450
|
+
parent_span_id TEXT,
|
|
451
|
+
service_name TEXT NOT NULL,
|
|
452
|
+
scope_name TEXT,
|
|
453
|
+
operation_name TEXT NOT NULL,
|
|
454
|
+
kind TEXT,
|
|
455
|
+
start_time_ms INTEGER NOT NULL,
|
|
456
|
+
end_time_ms INTEGER NOT NULL,
|
|
457
|
+
duration_ms REAL NOT NULL,
|
|
458
|
+
status TEXT NOT NULL,
|
|
459
|
+
attributes_json TEXT NOT NULL,
|
|
460
|
+
resource_json TEXT NOT NULL,
|
|
461
|
+
events_json TEXT NOT NULL,
|
|
462
|
+
PRIMARY KEY (trace_id, span_id)
|
|
463
|
+
);
|
|
464
|
+
|
|
465
|
+
CREATE INDEX IF NOT EXISTS idx_spans_service_time ON spans(service_name, start_time_ms DESC);
|
|
466
|
+
CREATE INDEX IF NOT EXISTS idx_spans_trace_time ON spans(trace_id, start_time_ms ASC);
|
|
467
|
+
CREATE INDEX IF NOT EXISTS idx_spans_span_id ON spans(span_id);
|
|
468
|
+
CREATE INDEX IF NOT EXISTS idx_spans_status_time ON spans(status, start_time_ms DESC);
|
|
469
|
+
|
|
470
|
+
CREATE TABLE IF NOT EXISTS logs (
|
|
471
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
472
|
+
trace_id TEXT,
|
|
473
|
+
span_id TEXT,
|
|
474
|
+
service_name TEXT NOT NULL,
|
|
475
|
+
scope_name TEXT,
|
|
476
|
+
severity_text TEXT NOT NULL,
|
|
477
|
+
timestamp_ms INTEGER NOT NULL,
|
|
478
|
+
body TEXT NOT NULL,
|
|
479
|
+
attributes_json TEXT NOT NULL,
|
|
480
|
+
resource_json TEXT NOT NULL
|
|
481
|
+
);
|
|
482
|
+
|
|
483
|
+
CREATE INDEX IF NOT EXISTS idx_logs_service_time ON logs(service_name, timestamp_ms DESC);
|
|
484
|
+
CREATE INDEX IF NOT EXISTS idx_logs_trace_time ON logs(trace_id, timestamp_ms DESC);
|
|
485
|
+
CREATE INDEX IF NOT EXISTS idx_logs_span_time ON logs(span_id, timestamp_ms DESC);
|
|
486
|
+
CREATE INDEX IF NOT EXISTS idx_logs_severity_time ON logs(severity_text, timestamp_ms DESC);
|
|
487
|
+
|
|
488
|
+
CREATE TABLE IF NOT EXISTS trace_summaries (
|
|
489
|
+
trace_id TEXT PRIMARY KEY,
|
|
490
|
+
service_name TEXT NOT NULL,
|
|
491
|
+
root_operation_name TEXT NOT NULL,
|
|
492
|
+
started_at_ms INTEGER NOT NULL,
|
|
493
|
+
ended_at_ms INTEGER NOT NULL,
|
|
494
|
+
active_span_count INTEGER NOT NULL DEFAULT 0,
|
|
495
|
+
duration_ms REAL NOT NULL,
|
|
496
|
+
span_count INTEGER NOT NULL,
|
|
497
|
+
error_count INTEGER NOT NULL
|
|
498
|
+
);
|
|
499
|
+
|
|
500
|
+
CREATE INDEX IF NOT EXISTS idx_trace_summaries_started_at ON trace_summaries(started_at_ms DESC, trace_id DESC);
|
|
501
|
+
CREATE INDEX IF NOT EXISTS idx_trace_summaries_service_started_at ON trace_summaries(service_name, started_at_ms DESC, trace_id DESC);
|
|
502
|
+
CREATE INDEX IF NOT EXISTS idx_trace_summaries_duration ON trace_summaries(duration_ms DESC);
|
|
503
|
+
|
|
504
|
+
CREATE TABLE IF NOT EXISTS span_attributes (
|
|
505
|
+
trace_id TEXT NOT NULL,
|
|
506
|
+
span_id TEXT NOT NULL,
|
|
507
|
+
key TEXT NOT NULL,
|
|
508
|
+
value TEXT NOT NULL,
|
|
509
|
+
PRIMARY KEY (trace_id, span_id, key)
|
|
510
|
+
);
|
|
511
|
+
|
|
512
|
+
CREATE INDEX IF NOT EXISTS idx_span_attributes_key_value ON span_attributes(key, value, trace_id, span_id);
|
|
513
|
+
CREATE INDEX IF NOT EXISTS idx_span_attributes_trace_span ON span_attributes(trace_id, span_id);
|
|
514
|
+
|
|
515
|
+
CREATE TABLE IF NOT EXISTS log_attributes (
|
|
516
|
+
log_id INTEGER NOT NULL,
|
|
517
|
+
key TEXT NOT NULL,
|
|
518
|
+
value TEXT NOT NULL,
|
|
519
|
+
PRIMARY KEY (log_id, key)
|
|
520
|
+
);
|
|
521
|
+
|
|
522
|
+
CREATE INDEX IF NOT EXISTS idx_log_attributes_key_value ON log_attributes(key, value, log_id);
|
|
523
|
+
CREATE INDEX IF NOT EXISTS idx_log_attributes_log_id ON log_attributes(log_id);
|
|
524
|
+
`)
|
|
525
|
+
|
|
526
|
+
let hasFts = true
|
|
527
|
+
try {
|
|
528
|
+
db.exec(`
|
|
529
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS span_operation_fts USING fts5(
|
|
530
|
+
trace_id UNINDEXED,
|
|
531
|
+
span_id UNINDEXED,
|
|
532
|
+
operation_name,
|
|
533
|
+
tokenize='unicode61'
|
|
534
|
+
);
|
|
535
|
+
|
|
536
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS log_body_fts USING fts5(
|
|
537
|
+
log_id UNINDEXED,
|
|
538
|
+
body,
|
|
539
|
+
tokenize='unicode61'
|
|
540
|
+
);
|
|
541
|
+
`)
|
|
542
|
+
} catch {
|
|
543
|
+
hasFts = false
|
|
544
|
+
// FTS is optional; queries will fall back to LIKE if unavailable.
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
try {
|
|
548
|
+
db.exec(`ALTER TABLE trace_summaries ADD COLUMN active_span_count INTEGER NOT NULL DEFAULT 0`)
|
|
549
|
+
} catch {
|
|
550
|
+
// Existing databases may already have the column.
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
const insertSpan = db.query(`
|
|
554
|
+
INSERT INTO spans (
|
|
555
|
+
trace_id, span_id, parent_span_id, service_name, scope_name, operation_name, kind,
|
|
556
|
+
start_time_ms, end_time_ms, duration_ms, status, attributes_json, resource_json, events_json
|
|
557
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
558
|
+
ON CONFLICT(trace_id, span_id) DO UPDATE SET
|
|
559
|
+
parent_span_id = excluded.parent_span_id,
|
|
560
|
+
service_name = excluded.service_name,
|
|
561
|
+
scope_name = excluded.scope_name,
|
|
562
|
+
operation_name = excluded.operation_name,
|
|
563
|
+
kind = excluded.kind,
|
|
564
|
+
start_time_ms = excluded.start_time_ms,
|
|
565
|
+
end_time_ms = excluded.end_time_ms,
|
|
566
|
+
duration_ms = excluded.duration_ms,
|
|
567
|
+
status = excluded.status,
|
|
568
|
+
attributes_json = excluded.attributes_json,
|
|
569
|
+
resource_json = excluded.resource_json,
|
|
570
|
+
events_json = excluded.events_json
|
|
571
|
+
`)
|
|
572
|
+
|
|
573
|
+
const insertLog = db.query(`
|
|
574
|
+
INSERT INTO logs (
|
|
575
|
+
trace_id, span_id, service_name, scope_name, severity_text, timestamp_ms, body, attributes_json, resource_json
|
|
576
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
577
|
+
`)
|
|
578
|
+
|
|
579
|
+
const upsertTraceSummary = db.query(`
|
|
580
|
+
INSERT OR REPLACE INTO trace_summaries (
|
|
581
|
+
trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
|
|
582
|
+
)
|
|
583
|
+
SELECT trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
|
|
584
|
+
FROM (
|
|
585
|
+
${TRACE_SUMMARY_SELECT_SQL}
|
|
586
|
+
WHERE trace_id = ?
|
|
587
|
+
GROUP BY trace_id
|
|
588
|
+
)
|
|
589
|
+
`)
|
|
590
|
+
|
|
591
|
+
const rebuildTraceSummaries = db.query(`
|
|
592
|
+
INSERT INTO trace_summaries (
|
|
593
|
+
trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
|
|
594
|
+
)
|
|
595
|
+
${TRACE_SUMMARY_SELECT_SQL}
|
|
596
|
+
GROUP BY trace_id
|
|
597
|
+
`)
|
|
598
|
+
|
|
599
|
+
db.query(`DELETE FROM trace_summaries`).run()
|
|
600
|
+
rebuildTraceSummaries.run()
|
|
601
|
+
|
|
602
|
+
const deleteSpanAttributes = db.query(`DELETE FROM span_attributes WHERE trace_id = ? AND span_id = ?`)
|
|
603
|
+
const insertSpanAttribute = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES (?, ?, ?, ?)`)
|
|
604
|
+
const deleteSpanOperationSearch = db.query(`DELETE FROM span_operation_fts WHERE trace_id = ? AND span_id = ?`)
|
|
605
|
+
const insertSpanOperationSearch = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES (?, ?, ?)`)
|
|
606
|
+
const insertLogAttribute = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES (?, ?, ?)`)
|
|
607
|
+
const insertLogBodySearch = db.query(`INSERT INTO log_body_fts (log_id, body) VALUES (?, ?)`)
|
|
608
|
+
|
|
609
|
+
const maxDbSizeBytes = config.otel.maxDbSizeMb * 1024 * 1024
|
|
610
|
+
|
|
611
|
+
const cleanupExpired = Effect.fn("motel/TelemetryStore.cleanupExpired")(function* () {
|
|
612
|
+
const now = yield* Clock.currentTimeMillis
|
|
613
|
+
|
|
614
|
+
yield* Effect.sync(() => {
|
|
615
|
+
let deletedData = false
|
|
616
|
+
// Time-based retention
|
|
617
|
+
const cutoff = now - config.otel.retentionHours * 60 * 60 * 1000
|
|
618
|
+
const deletedSpans = db.query(`DELETE FROM spans WHERE start_time_ms < ?`).run(cutoff) as { changes?: number }
|
|
619
|
+
const deletedLogs = db.query(`DELETE FROM logs WHERE timestamp_ms < ?`).run(cutoff) as { changes?: number }
|
|
620
|
+
deletedData = (deletedSpans.changes ?? 0) > 0 || (deletedLogs.changes ?? 0) > 0
|
|
621
|
+
|
|
622
|
+
// Size-based retention: if actual data exceeds max, delete oldest 20% of rows.
|
|
623
|
+
// Use (page_count - freelist_count) to ignore freed-but-not-vacuumed pages;
|
|
624
|
+
// otherwise a large freelist triggers a deletion death spiral.
|
|
625
|
+
const pageCount = (db.query(`PRAGMA page_count`).get() as { page_count: number }).page_count
|
|
626
|
+
const freePages = (db.query(`PRAGMA freelist_count`).get() as { freelist_count: number }).freelist_count
|
|
627
|
+
const pageSize = (db.query(`PRAGMA page_size`).get() as { page_size: number }).page_size
|
|
628
|
+
const dbSize = (pageCount - freePages) * pageSize
|
|
629
|
+
if (dbSize > maxDbSizeBytes) {
|
|
630
|
+
const spanCount = (db.query(`SELECT COUNT(*) AS c FROM spans`).get() as { c: number }).c
|
|
631
|
+
const logCount = (db.query(`SELECT COUNT(*) AS c FROM logs`).get() as { c: number }).c
|
|
632
|
+
const spanCutCount = Math.max(1, Math.floor(spanCount * 0.2))
|
|
633
|
+
const logCutCount = Math.max(1, Math.floor(logCount * 0.2))
|
|
634
|
+
db.query(`DELETE FROM spans WHERE rowid IN (SELECT rowid FROM spans ORDER BY start_time_ms ASC LIMIT ?)`).run(spanCutCount)
|
|
635
|
+
db.query(`DELETE FROM logs WHERE rowid IN (SELECT rowid FROM logs ORDER BY timestamp_ms ASC LIMIT ?)`).run(logCutCount)
|
|
636
|
+
deletedData = true
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
if (deletedData) {
|
|
640
|
+
db.query(`DELETE FROM span_attributes WHERE NOT EXISTS (SELECT 1 FROM spans WHERE spans.trace_id = span_attributes.trace_id AND spans.span_id = span_attributes.span_id)`).run()
|
|
641
|
+
db.query(`DELETE FROM log_attributes WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = log_attributes.log_id)`).run()
|
|
642
|
+
try {
|
|
643
|
+
db.query(`DELETE FROM span_operation_fts WHERE NOT EXISTS (SELECT 1 FROM spans WHERE spans.trace_id = span_operation_fts.trace_id AND spans.span_id = span_operation_fts.span_id)`).run()
|
|
644
|
+
db.query(`DELETE FROM log_body_fts WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = CAST(log_body_fts.log_id AS INTEGER))`).run()
|
|
645
|
+
} catch {
|
|
646
|
+
// FTS tables may not exist.
|
|
647
|
+
}
|
|
648
|
+
db.query(`DELETE FROM trace_summaries`).run()
|
|
649
|
+
rebuildTraceSummaries.run()
|
|
650
|
+
}
|
|
651
|
+
})
|
|
652
|
+
})
|
|
653
|
+
|
|
654
|
+
// Run cleanup every 60 seconds in the background, tied to the layer's scope
|
|
655
|
+
yield* Effect.forkScoped(Effect.repeat(cleanupExpired(), Schedule.spaced("60 seconds")))
|
|
656
|
+
|
|
657
|
+
const ingestTraces = Effect.fn("motel/TelemetryStore.ingestTraces")(function* (payload: OtlpTraceExportRequest) {
|
|
658
|
+
return yield* Effect.sync(() => {
|
|
659
|
+
let insertedSpans = 0
|
|
660
|
+
const transaction = db.transaction((request: OtlpTraceExportRequest) => {
|
|
661
|
+
const touchedTraceIds = new Set<string>()
|
|
662
|
+
for (const resourceSpans of request.resourceSpans ?? []) {
|
|
663
|
+
const resourceAttributes = attributeMap(resourceSpans.resource?.attributes)
|
|
664
|
+
const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
|
|
665
|
+
|
|
666
|
+
for (const scopeSpans of resourceSpans.scopeSpans ?? []) {
|
|
667
|
+
const scopeName = scopeSpans.scope?.name ?? null
|
|
668
|
+
|
|
669
|
+
for (const span of scopeSpans.spans ?? []) {
|
|
670
|
+
const spanAttributes = attributeMap(span.attributes)
|
|
671
|
+
const mergedAttributes = { ...resourceAttributes, ...spanAttributes }
|
|
672
|
+
const startTimeMs = nanosToMilliseconds(span.startTimeUnixNano)
|
|
673
|
+
const endTimeMs = nanosToMilliseconds(span.endTimeUnixNano)
|
|
674
|
+
const events = (span.events ?? []).map((event) => ({
|
|
675
|
+
name: event.name ?? "event",
|
|
676
|
+
timestamp: nanosToMilliseconds(event.timeUnixNano),
|
|
677
|
+
attributes: attributeMap(event.attributes),
|
|
678
|
+
}))
|
|
679
|
+
|
|
680
|
+
insertSpan.run(
|
|
681
|
+
span.traceId,
|
|
682
|
+
span.spanId,
|
|
683
|
+
span.parentSpanId ?? null,
|
|
684
|
+
serviceName,
|
|
685
|
+
scopeName,
|
|
686
|
+
span.name ?? "unknown",
|
|
687
|
+
spanKindLabel(span.kind),
|
|
688
|
+
startTimeMs,
|
|
689
|
+
endTimeMs,
|
|
690
|
+
Math.max(0, endTimeMs - startTimeMs),
|
|
691
|
+
spanStatusLabel(span.status?.code),
|
|
692
|
+
JSON.stringify(spanAttributes),
|
|
693
|
+
JSON.stringify(resourceAttributes),
|
|
694
|
+
JSON.stringify(events),
|
|
695
|
+
)
|
|
696
|
+
deleteSpanAttributes.run(span.traceId, span.spanId)
|
|
697
|
+
for (const [key, value] of Object.entries(mergedAttributes)) {
|
|
698
|
+
insertSpanAttribute.run(span.traceId, span.spanId, key, value)
|
|
699
|
+
}
|
|
700
|
+
try {
|
|
701
|
+
deleteSpanOperationSearch.run(span.traceId, span.spanId)
|
|
702
|
+
insertSpanOperationSearch.run(span.traceId, span.spanId, span.name ?? "unknown")
|
|
703
|
+
} catch {
|
|
704
|
+
// FTS is optional.
|
|
705
|
+
}
|
|
706
|
+
touchedTraceIds.add(span.traceId)
|
|
707
|
+
insertedSpans += 1
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
for (const traceId of touchedTraceIds) {
|
|
712
|
+
upsertTraceSummary.run(traceId)
|
|
713
|
+
}
|
|
714
|
+
})
|
|
715
|
+
|
|
716
|
+
transaction(payload)
|
|
717
|
+
return { insertedSpans }
|
|
718
|
+
})
|
|
719
|
+
})
|
|
720
|
+
|
|
721
|
+
const ingestLogs = Effect.fn("motel/TelemetryStore.ingestLogs")(function* (payload: OtlpLogExportRequest) {
|
|
722
|
+
return yield* Effect.sync(() => {
|
|
723
|
+
let insertedLogs = 0
|
|
724
|
+
const transaction = db.transaction((request: OtlpLogExportRequest) => {
|
|
725
|
+
for (const resourceLogs of request.resourceLogs ?? []) {
|
|
726
|
+
const resourceAttributes = attributeMap(resourceLogs.resource?.attributes)
|
|
727
|
+
const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
|
|
728
|
+
|
|
729
|
+
for (const scopeLogs of resourceLogs.scopeLogs ?? []) {
|
|
730
|
+
const scopeName = scopeLogs.scope?.name ?? null
|
|
731
|
+
|
|
732
|
+
for (const record of scopeLogs.logRecords ?? []) {
|
|
733
|
+
const attributes = attributeMap(record.attributes)
|
|
734
|
+
const mergedAttributes = { ...resourceAttributes, ...attributes }
|
|
735
|
+
const timestampMs = nanosToMilliseconds(record.timeUnixNano ?? record.observedTimeUnixNano)
|
|
736
|
+
const body = stringifyValue(parseAnyValue(record.body))
|
|
737
|
+
const result = insertLog.run(
|
|
738
|
+
attributes.traceId || attributes.trace_id || record.traceId || null,
|
|
739
|
+
attributes.spanId || attributes.span_id || record.spanId || null,
|
|
740
|
+
serviceName,
|
|
741
|
+
scopeName,
|
|
742
|
+
record.severityText ?? "INFO",
|
|
743
|
+
timestampMs,
|
|
744
|
+
body,
|
|
745
|
+
JSON.stringify(attributes),
|
|
746
|
+
JSON.stringify(resourceAttributes),
|
|
747
|
+
)
|
|
748
|
+
const logId = Number((result as { lastInsertRowid: number | bigint }).lastInsertRowid)
|
|
749
|
+
for (const [key, value] of Object.entries(mergedAttributes)) {
|
|
750
|
+
insertLogAttribute.run(logId, key, value)
|
|
751
|
+
}
|
|
752
|
+
try {
|
|
753
|
+
insertLogBodySearch.run(String(logId), body)
|
|
754
|
+
} catch {
|
|
755
|
+
// FTS is optional.
|
|
756
|
+
}
|
|
757
|
+
insertedLogs += 1
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
})
|
|
762
|
+
|
|
763
|
+
transaction(payload)
|
|
764
|
+
return { insertedLogs }
|
|
765
|
+
})
|
|
766
|
+
})
|
|
767
|
+
|
|
768
|
+
const listServices = Effect.fn("motel/TelemetryStore.listServices")(function* () {
|
|
769
|
+
|
|
770
|
+
const cutoff = (yield* Clock.currentTimeMillis) - config.otel.traceLookbackMinutes * 60 * 1000
|
|
771
|
+
return yield* Effect.sync(() => {
|
|
772
|
+
const rows = db.query(`
|
|
773
|
+
SELECT service_name FROM spans WHERE start_time_ms >= ?
|
|
774
|
+
UNION
|
|
775
|
+
SELECT service_name FROM logs WHERE timestamp_ms >= ?
|
|
776
|
+
ORDER BY service_name ASC
|
|
777
|
+
`).all(cutoff, cutoff) as Array<{ service_name: string }>
|
|
778
|
+
return rows.map((row) => row.service_name)
|
|
779
|
+
})
|
|
780
|
+
})()
|
|
781
|
+
|
|
782
|
+
const loadTracesByIds = (traceIds: readonly string[]) => {
|
|
783
|
+
if (traceIds.length === 0) return [] as readonly TraceItem[]
|
|
784
|
+
const placeholders = traceIds.map(() => "?").join(", ")
|
|
785
|
+
const rows = db.query(`
|
|
786
|
+
SELECT * FROM spans
|
|
787
|
+
WHERE trace_id IN (${placeholders})
|
|
788
|
+
ORDER BY start_time_ms ASC
|
|
789
|
+
`).all(...traceIds) as SpanRow[]
|
|
790
|
+
|
|
791
|
+
const grouped = new Map<string, SpanRow[]>()
|
|
792
|
+
for (const row of rows) {
|
|
793
|
+
const group = grouped.get(row.trace_id) ?? []
|
|
794
|
+
group.push(row)
|
|
795
|
+
grouped.set(row.trace_id, group)
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
return traceIds
|
|
799
|
+
.map((traceId) => grouped.get(traceId))
|
|
800
|
+
.filter((group): group is SpanRow[] => group !== undefined)
|
|
801
|
+
.map((group) => buildTrace(group[0]!.trace_id, group))
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
const listRecentTraces = Effect.fn("motel/TelemetryStore.listRecentTraces")(function* (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number }) {
|
|
805
|
+
const summaries = yield* listTraceSummaries(serviceName, options)
|
|
806
|
+
return yield* Effect.sync(() => loadTracesByIds(summaries.map((summary) => summary.traceId)))
|
|
807
|
+
})
|
|
808
|
+
|
|
809
|
+
const listTraceSummaries = Effect.fn("motel/TelemetryStore.listTraceSummaries")(function* (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) {
|
|
810
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (options?.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
811
|
+
const limit = options?.limit ?? config.otel.traceFetchLimit
|
|
812
|
+
|
|
813
|
+
return yield* Effect.sync(() => {
|
|
814
|
+
const clauses = ["started_at_ms >= ?"]
|
|
815
|
+
const params: Array<string | number> = [cutoff]
|
|
816
|
+
|
|
817
|
+
if (serviceName) {
|
|
818
|
+
clauses.push("service_name = ?")
|
|
819
|
+
params.push(serviceName)
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
if (options?.cursorStartedAtMs != null && options.cursorTraceId) {
|
|
823
|
+
clauses.push("(started_at_ms < ? OR (started_at_ms = ? AND trace_id < ?))")
|
|
824
|
+
params.push(options.cursorStartedAtMs, options.cursorStartedAtMs, options.cursorTraceId)
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
return db.query(`
|
|
828
|
+
SELECT trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
|
|
829
|
+
FROM trace_summaries
|
|
830
|
+
WHERE ${clauses.join(" AND ")}
|
|
831
|
+
ORDER BY started_at_ms DESC, trace_id DESC
|
|
832
|
+
LIMIT ?
|
|
833
|
+
`).all(...params, limit) as TraceSummaryRow[]
|
|
834
|
+
}).pipe(Effect.map((rows) => rows.map(parseSummaryRow)))
|
|
835
|
+
})
|
|
836
|
+
|
|
837
|
+
const searchTraceSummaries = Effect.fn("motel/TelemetryStore.searchTraceSummaries")(function* (input: TraceSearch) {
|
|
838
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
839
|
+
const limit = input.limit ?? config.otel.traceFetchLimit
|
|
840
|
+
|
|
841
|
+
return yield* Effect.sync(() => {
|
|
842
|
+
const clauses: string[] = ["started_at_ms >= ?"]
|
|
843
|
+
const params: Array<string | number> = [cutoff]
|
|
844
|
+
|
|
845
|
+
if (input.serviceName) {
|
|
846
|
+
clauses.push("service_name = ?")
|
|
847
|
+
params.push(input.serviceName)
|
|
848
|
+
}
|
|
849
|
+
if (input.status === "error") {
|
|
850
|
+
clauses.push("error_count > 0")
|
|
851
|
+
}
|
|
852
|
+
if (input.status === "ok") {
|
|
853
|
+
clauses.push("error_count = 0")
|
|
854
|
+
}
|
|
855
|
+
if (input.minDurationMs != null) {
|
|
856
|
+
clauses.push("duration_ms >= ?")
|
|
857
|
+
params.push(input.minDurationMs)
|
|
858
|
+
}
|
|
859
|
+
if (input.cursorStartedAtMs != null && input.cursorTraceId) {
|
|
860
|
+
clauses.push("(started_at_ms < ? OR (started_at_ms = ? AND trace_id < ?))")
|
|
861
|
+
params.push(input.cursorStartedAtMs, input.cursorStartedAtMs, input.cursorTraceId)
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
if (input.operation) {
|
|
865
|
+
const ftsQuery = toFtsMatchQuery(input.operation)
|
|
866
|
+
if (hasFts && ftsQuery) {
|
|
867
|
+
clauses.push("trace_id IN (SELECT DISTINCT trace_id FROM span_operation_fts WHERE span_operation_fts MATCH ?)")
|
|
868
|
+
params.push(ftsQuery)
|
|
869
|
+
} else {
|
|
870
|
+
clauses.push("trace_id IN (SELECT DISTINCT trace_id FROM spans WHERE operation_name LIKE ? COLLATE NOCASE)")
|
|
871
|
+
params.push(`%${input.operation}%`)
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
const exactAttrMatch = buildExactAttributeMatchSubquery("span_attributes", ["trace_id", "span_id"], input.attributeFilters)
|
|
876
|
+
if (exactAttrMatch) {
|
|
877
|
+
clauses.push(`trace_id IN (SELECT DISTINCT trace_id FROM (${exactAttrMatch.sql}))`)
|
|
878
|
+
params.push(...exactAttrMatch.params)
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
const rows = db.query(`
|
|
882
|
+
SELECT trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
|
|
883
|
+
FROM trace_summaries
|
|
884
|
+
WHERE ${clauses.join(" AND ")}
|
|
885
|
+
ORDER BY started_at_ms DESC, trace_id DESC
|
|
886
|
+
LIMIT ?
|
|
887
|
+
`).all(...params, limit) as TraceSummaryRow[]
|
|
888
|
+
|
|
889
|
+
return rows.map(parseSummaryRow)
|
|
890
|
+
})
|
|
891
|
+
})
|
|
892
|
+
|
|
893
|
+
const getTrace = Effect.fn("motel/TelemetryStore.getTrace")(function* (traceId: string) {
|
|
894
|
+
return yield* Effect.sync(() => {
|
|
895
|
+
const rows = db.query(`
|
|
896
|
+
SELECT * FROM spans WHERE trace_id = ? ORDER BY start_time_ms ASC
|
|
897
|
+
`).all(traceId) as SpanRow[]
|
|
898
|
+
return rows.length === 0 ? null : buildTrace(traceId, rows)
|
|
899
|
+
})
|
|
900
|
+
})
|
|
901
|
+
|
|
902
|
+
const getSpan = Effect.fn("motel/TelemetryStore.getSpan")(function* (spanId: string) {
|
|
903
|
+
return yield* Effect.sync(() => {
|
|
904
|
+
// Fetch only the target span row (uses idx_spans_span_id)
|
|
905
|
+
const spanRow = db.query(`SELECT * FROM spans WHERE span_id = ? LIMIT 1`).get(spanId) as SpanRow | null
|
|
906
|
+
if (!spanRow) return null
|
|
907
|
+
|
|
908
|
+
const traceId = spanRow.trace_id
|
|
909
|
+
|
|
910
|
+
// Get root operation name (indexed by trace_id)
|
|
911
|
+
const rootRow = db.query(`
|
|
912
|
+
SELECT operation_name FROM spans
|
|
913
|
+
WHERE trace_id = ? AND parent_span_id IS NULL
|
|
914
|
+
ORDER BY start_time_ms ASC LIMIT 1
|
|
915
|
+
`).get(traceId) as { operation_name: string } | null
|
|
916
|
+
const rootOperationName = rootRow?.operation_name ?? "unknown"
|
|
917
|
+
|
|
918
|
+
// Get parent operation name if span has a parent (PK lookup)
|
|
919
|
+
let parentOperationName: string | null = null
|
|
920
|
+
if (spanRow.parent_span_id) {
|
|
921
|
+
const parentRow = db.query(`
|
|
922
|
+
SELECT operation_name FROM spans
|
|
923
|
+
WHERE trace_id = ? AND span_id = ?
|
|
924
|
+
`).get(traceId, spanRow.parent_span_id) as { operation_name: string } | null
|
|
925
|
+
parentOperationName = parentRow?.operation_name ?? null
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
// Compute depth by walking up parent chain (typically 3-5 hops)
|
|
929
|
+
let depth = 0
|
|
930
|
+
let currentParentId = spanRow.parent_span_id
|
|
931
|
+
while (currentParentId) {
|
|
932
|
+
const parentRow = db.query(`
|
|
933
|
+
SELECT parent_span_id FROM spans WHERE trace_id = ? AND span_id = ?
|
|
934
|
+
`).get(traceId, currentParentId) as { parent_span_id: string | null } | null
|
|
935
|
+
if (!parentRow) break
|
|
936
|
+
depth++
|
|
937
|
+
currentParentId = parentRow.parent_span_id
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
const parsed = parseSpanRow(spanRow)
|
|
941
|
+
return {
|
|
942
|
+
traceId,
|
|
943
|
+
rootOperationName,
|
|
944
|
+
parentOperationName,
|
|
945
|
+
span: { ...parsed, depth },
|
|
946
|
+
} satisfies SpanItem
|
|
947
|
+
})
|
|
948
|
+
})
|
|
949
|
+
|
|
950
|
+
const listTraceSpans = Effect.fn("motel/TelemetryStore.listTraceSpans")(function* (traceId: string) {
|
|
951
|
+
return yield* Effect.sync(() => {
|
|
952
|
+
const rows = db.query(`SELECT * FROM spans WHERE trace_id = ? ORDER BY start_time_ms ASC`).all(traceId) as SpanRow[]
|
|
953
|
+
return rows.length === 0 ? [] as readonly SpanItem[] : buildSpanItems(traceId, rows)
|
|
954
|
+
})
|
|
955
|
+
})
|
|
956
|
+
|
|
957
|
+
const searchSpans = Effect.fn("motel/TelemetryStore.searchSpans")(function* (input: SpanSearch) {
|
|
958
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
959
|
+
const limit = input.limit ?? 100
|
|
960
|
+
const hasContainsFilters = Object.keys(input.attributeContainsFilters ?? {}).length > 0
|
|
961
|
+
const candidateLimit = hasContainsFilters ? Math.max(limit * 20, 500) : Math.max(limit * 10, 200)
|
|
962
|
+
|
|
963
|
+
return yield* Effect.sync(() => {
|
|
964
|
+
const clauses: string[] = ["s.start_time_ms >= ?"]
|
|
965
|
+
const params: Array<string | number> = [cutoff]
|
|
966
|
+
|
|
967
|
+
if (input.traceId) {
|
|
968
|
+
clauses.push("s.trace_id = ?")
|
|
969
|
+
params.push(input.traceId)
|
|
970
|
+
}
|
|
971
|
+
if (input.serviceName) {
|
|
972
|
+
clauses.push("s.service_name = ?")
|
|
973
|
+
params.push(input.serviceName)
|
|
974
|
+
}
|
|
975
|
+
if (input.operation) {
|
|
976
|
+
const ftsQuery = toFtsMatchQuery(input.operation)
|
|
977
|
+
if (hasFts && ftsQuery) {
|
|
978
|
+
clauses.push("EXISTS (SELECT 1 FROM span_operation_fts WHERE span_operation_fts.trace_id = s.trace_id AND span_operation_fts.span_id = s.span_id AND span_operation_fts MATCH ?)")
|
|
979
|
+
params.push(ftsQuery)
|
|
980
|
+
} else {
|
|
981
|
+
clauses.push("s.operation_name LIKE ? COLLATE NOCASE")
|
|
982
|
+
params.push(`%${input.operation}%`)
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
if (input.status) {
|
|
986
|
+
clauses.push("s.status = ?")
|
|
987
|
+
params.push(input.status)
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
const exactAttrMatch = buildExactAttributeMatchSubquery("span_attributes", ["trace_id", "span_id"], input.attributeFilters)
|
|
991
|
+
if (exactAttrMatch) {
|
|
992
|
+
clauses.push(`EXISTS (SELECT 1 FROM (${exactAttrMatch.sql}) AS span_attr_match WHERE span_attr_match.trace_id = s.trace_id AND span_attr_match.span_id = s.span_id)`)
|
|
993
|
+
params.push(...exactAttrMatch.params)
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
const containsAttrMatch = buildContainsAttributeMatchSubquery("span_attributes", ["trace_id", "span_id"], input.attributeContainsFilters)
|
|
997
|
+
if (containsAttrMatch) {
|
|
998
|
+
clauses.push(`EXISTS (SELECT 1 FROM (${containsAttrMatch.sql}) AS span_attr_contains_match WHERE span_attr_contains_match.trace_id = s.trace_id AND span_attr_contains_match.span_id = s.span_id)`)
|
|
999
|
+
params.push(...containsAttrMatch.params)
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
const rows = db.query(`
|
|
1003
|
+
SELECT trace_id, span_id
|
|
1004
|
+
FROM spans AS s
|
|
1005
|
+
WHERE ${clauses.join(" AND ")}
|
|
1006
|
+
ORDER BY s.start_time_ms DESC
|
|
1007
|
+
LIMIT ?
|
|
1008
|
+
`).all(...params, candidateLimit) as Array<{ trace_id: string; span_id: string }>
|
|
1009
|
+
|
|
1010
|
+
const traceIds = [...new Set(rows.map((row) => row.trace_id))]
|
|
1011
|
+
if (traceIds.length === 0) return [] as readonly SpanItem[]
|
|
1012
|
+
|
|
1013
|
+
const placeholders = traceIds.map(() => "?").join(", ")
|
|
1014
|
+
const spanRows = db.query(`
|
|
1015
|
+
SELECT * FROM spans
|
|
1016
|
+
WHERE trace_id IN (${placeholders})
|
|
1017
|
+
ORDER BY start_time_ms ASC
|
|
1018
|
+
`).all(...traceIds) as SpanRow[]
|
|
1019
|
+
|
|
1020
|
+
const grouped = new Map<string, SpanRow[]>()
|
|
1021
|
+
for (const row of spanRows) {
|
|
1022
|
+
const group = grouped.get(row.trace_id) ?? []
|
|
1023
|
+
group.push(row)
|
|
1024
|
+
grouped.set(row.trace_id, group)
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
const itemById = new Map<string, SpanItem>()
|
|
1028
|
+
for (const traceId of traceIds) {
|
|
1029
|
+
const traceSpanRows = grouped.get(traceId)
|
|
1030
|
+
if (!traceSpanRows) continue
|
|
1031
|
+
for (const item of buildSpanItems(traceId, traceSpanRows)) {
|
|
1032
|
+
itemById.set(`${item.traceId}:${item.span.spanId}`, item)
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
return rows
|
|
1037
|
+
.map((row) => itemById.get(`${row.trace_id}:${row.span_id}`))
|
|
1038
|
+
.filter((item): item is SpanItem => item !== undefined)
|
|
1039
|
+
.filter((item) => {
|
|
1040
|
+
if (input.parentOperation) {
|
|
1041
|
+
const needle = input.parentOperation.toLowerCase()
|
|
1042
|
+
if (!item.parentOperationName?.toLowerCase().includes(needle)) return false
|
|
1043
|
+
}
|
|
1044
|
+
return true
|
|
1045
|
+
})
|
|
1046
|
+
.slice(0, limit)
|
|
1047
|
+
})
|
|
1048
|
+
})
|
|
1049
|
+
|
|
1050
|
+
const searchTraces = Effect.fn("motel/TelemetryStore.searchTraces")(function* (input: TraceSearch) {
|
|
1051
|
+
const summaries = yield* searchTraceSummaries(input)
|
|
1052
|
+
return yield* Effect.sync(() => loadTracesByIds(summaries.map((summary) => summary.traceId)))
|
|
1053
|
+
})
|
|
1054
|
+
|
|
1055
|
+
const searchLogs = Effect.fn("motel/TelemetryStore.searchLogs")(function* (input: LogSearch) {
|
|
1056
|
+
const now = yield* Clock.currentTimeMillis
|
|
1057
|
+
return yield* Effect.sync(() => {
|
|
1058
|
+
const clauses: string[] = []
|
|
1059
|
+
const params: Array<string | number> = []
|
|
1060
|
+
|
|
1061
|
+
if (input.serviceName) {
|
|
1062
|
+
clauses.push(`service_name = ?`)
|
|
1063
|
+
params.push(input.serviceName)
|
|
1064
|
+
}
|
|
1065
|
+
if (input.severity) {
|
|
1066
|
+
clauses.push(`severity_text = ?`)
|
|
1067
|
+
params.push(input.severity.toUpperCase())
|
|
1068
|
+
}
|
|
1069
|
+
if (input.traceId) {
|
|
1070
|
+
clauses.push(`trace_id = ?`)
|
|
1071
|
+
params.push(input.traceId)
|
|
1072
|
+
}
|
|
1073
|
+
if (input.spanId) {
|
|
1074
|
+
clauses.push(`span_id = ?`)
|
|
1075
|
+
params.push(input.spanId)
|
|
1076
|
+
}
|
|
1077
|
+
if (input.body) {
|
|
1078
|
+
const ftsQuery = toFtsMatchQuery(input.body)
|
|
1079
|
+
if (hasFts && ftsQuery) {
|
|
1080
|
+
clauses.push(`id IN (SELECT CAST(log_id AS INTEGER) FROM log_body_fts WHERE log_body_fts MATCH ?)`)
|
|
1081
|
+
params.push(ftsQuery)
|
|
1082
|
+
} else {
|
|
1083
|
+
clauses.push(`body LIKE ? COLLATE NOCASE`)
|
|
1084
|
+
params.push(`%${input.body}%`)
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
if (input.lookbackMinutes) {
|
|
1088
|
+
const cutoff = now - input.lookbackMinutes * 60 * 1000
|
|
1089
|
+
clauses.push(`timestamp_ms >= ?`)
|
|
1090
|
+
params.push(cutoff)
|
|
1091
|
+
}
|
|
1092
|
+
if (input.cursorTimestampMs != null && input.cursorId) {
|
|
1093
|
+
clauses.push(`(timestamp_ms < ? OR (timestamp_ms = ? AND id < ?))`)
|
|
1094
|
+
params.push(input.cursorTimestampMs, input.cursorTimestampMs, Number(input.cursorId))
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
const exactAttrMatch = buildExactAttributeMatchSubquery("log_attributes", ["log_id"], input.attributeFilters)
|
|
1098
|
+
if (exactAttrMatch) {
|
|
1099
|
+
clauses.push(`id IN (${exactAttrMatch.sql})`)
|
|
1100
|
+
params.push(...exactAttrMatch.params)
|
|
1101
|
+
}
|
|
1102
|
+
|
|
1103
|
+
const containsAttrMatch = buildContainsAttributeMatchSubquery("log_attributes", ["log_id"], input.attributeContainsFilters)
|
|
1104
|
+
if (containsAttrMatch) {
|
|
1105
|
+
clauses.push(`id IN (${containsAttrMatch.sql})`)
|
|
1106
|
+
params.push(...containsAttrMatch.params)
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""
|
|
1110
|
+
const limit = input.limit ?? config.otel.logFetchLimit
|
|
1111
|
+
const rows = db.query(`
|
|
1112
|
+
SELECT * FROM logs
|
|
1113
|
+
${where}
|
|
1114
|
+
ORDER BY timestamp_ms DESC, id DESC
|
|
1115
|
+
LIMIT ?
|
|
1116
|
+
`).all(...params, limit) as LogRow[]
|
|
1117
|
+
|
|
1118
|
+
return rows.map(parseLogRow)
|
|
1119
|
+
})
|
|
1120
|
+
})
|
|
1121
|
+
|
|
1122
|
+
const traceStats = Effect.fn("motel/TelemetryStore.traceStats")(function* (input: TraceStatsSearch) {
|
|
1123
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
1124
|
+
const limit = input.limit ?? 20
|
|
1125
|
+
const hasAttrFilters = Object.keys(input.attributeFilters ?? {}).length > 0
|
|
1126
|
+
const isAttrGroupBy = input.groupBy.startsWith("attr.")
|
|
1127
|
+
|
|
1128
|
+
if (isAttrGroupBy || hasAttrFilters || input.operation) {
|
|
1129
|
+
const summaries = yield* searchTraceSummaries({
|
|
1130
|
+
serviceName: input.serviceName,
|
|
1131
|
+
operation: input.operation,
|
|
1132
|
+
status: input.status,
|
|
1133
|
+
minDurationMs: input.minDurationMs,
|
|
1134
|
+
attributeFilters: input.attributeFilters,
|
|
1135
|
+
lookbackMinutes: input.lookbackMinutes,
|
|
1136
|
+
limit: 5000,
|
|
1137
|
+
})
|
|
1138
|
+
|
|
1139
|
+
// For attr.* groupBy, we need to check span attributes — but only the groupBy key
|
|
1140
|
+
let attrLookup: Map<string, string> | null = null
|
|
1141
|
+
if (isAttrGroupBy) {
|
|
1142
|
+
const attrKey = input.groupBy.slice(5)
|
|
1143
|
+
const traceIds = summaries.map((s) => s.traceId)
|
|
1144
|
+
if (traceIds.length > 0) {
|
|
1145
|
+
const placeholders = traceIds.map(() => "?").join(", ")
|
|
1146
|
+
const rows = db.query(`
|
|
1147
|
+
SELECT trace_id, value
|
|
1148
|
+
FROM span_attributes
|
|
1149
|
+
WHERE key = ? AND trace_id IN (${placeholders})
|
|
1150
|
+
GROUP BY trace_id
|
|
1151
|
+
`).all(attrKey, ...traceIds) as Array<{ trace_id: string; value: string }>
|
|
1152
|
+
|
|
1153
|
+
attrLookup = new Map()
|
|
1154
|
+
for (const row of rows) {
|
|
1155
|
+
attrLookup.set(row.trace_id, row.value)
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
const groups = new Map<string, { durations: number[]; errorTraces: number }>()
|
|
1161
|
+
for (const summary of summaries) {
|
|
1162
|
+
const group = input.groupBy === "service"
|
|
1163
|
+
? summary.serviceName
|
|
1164
|
+
: input.groupBy === "operation"
|
|
1165
|
+
? summary.rootOperationName
|
|
1166
|
+
: input.groupBy === "status"
|
|
1167
|
+
? summary.errorCount > 0 ? "error" : "ok"
|
|
1168
|
+
: isAttrGroupBy
|
|
1169
|
+
? attrLookup?.get(summary.traceId) ?? "unknown"
|
|
1170
|
+
: "unknown"
|
|
1171
|
+
|
|
1172
|
+
const bucket = groups.get(group) ?? { durations: [], errorTraces: 0 }
|
|
1173
|
+
bucket.durations.push(summary.durationMs)
|
|
1174
|
+
if (summary.errorCount > 0) bucket.errorTraces++
|
|
1175
|
+
groups.set(group, bucket)
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
const rows = [...groups.entries()].map(([group, bucket]) => {
|
|
1179
|
+
const count = bucket.durations.length
|
|
1180
|
+
const value = input.agg === "count"
|
|
1181
|
+
? count
|
|
1182
|
+
: input.agg === "avg_duration"
|
|
1183
|
+
? bucket.durations.reduce((sum, d) => sum + d, 0) / Math.max(1, count)
|
|
1184
|
+
: input.agg === "p95_duration"
|
|
1185
|
+
? percentile(bucket.durations, 0.95)
|
|
1186
|
+
: bucket.errorTraces / Math.max(1, count)
|
|
1187
|
+
return { group, value, count }
|
|
1188
|
+
})
|
|
1189
|
+
|
|
1190
|
+
return rows.sort((left, right) => right.value - left.value).slice(0, limit)
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
return yield* Effect.sync(() => {
|
|
1194
|
+
const whereClauses: string[] = ["started_at_ms >= ?"]
|
|
1195
|
+
const whereParams: Array<string | number> = [cutoff]
|
|
1196
|
+
|
|
1197
|
+
if (input.serviceName) {
|
|
1198
|
+
whereClauses.push("service_name = ?")
|
|
1199
|
+
whereParams.push(input.serviceName)
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
if (input.status === "error") whereClauses.push("error_count > 0")
|
|
1203
|
+
if (input.status === "ok") whereClauses.push("error_count = 0")
|
|
1204
|
+
if (input.minDurationMs != null) {
|
|
1205
|
+
whereClauses.push("duration_ms >= ?")
|
|
1206
|
+
whereParams.push(input.minDurationMs)
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
const groupExpr = input.groupBy === "service"
|
|
1210
|
+
? "service_name"
|
|
1211
|
+
: input.groupBy === "operation"
|
|
1212
|
+
? "root_operation_name"
|
|
1213
|
+
: input.groupBy === "status"
|
|
1214
|
+
? "CASE WHEN error_count > 0 THEN 'error' ELSE 'ok' END"
|
|
1215
|
+
: "'unknown'"
|
|
1216
|
+
|
|
1217
|
+
const aggExpr = input.agg === "count"
|
|
1218
|
+
? "COUNT(*)"
|
|
1219
|
+
: input.agg === "avg_duration"
|
|
1220
|
+
? "AVG(duration_ms)"
|
|
1221
|
+
: "CAST(SUM(CASE WHEN error_count > 0 THEN 1 ELSE 0 END) AS REAL) / COUNT(*)"
|
|
1222
|
+
|
|
1223
|
+
if (input.agg === "p95_duration") {
|
|
1224
|
+
const rows = db.query(`
|
|
1225
|
+
SELECT ${groupExpr} AS grp, duration_ms
|
|
1226
|
+
FROM trace_summaries
|
|
1227
|
+
WHERE ${whereClauses.join(" AND ")}
|
|
1228
|
+
`).all(...whereParams) as Array<{ grp: string; duration_ms: number }>
|
|
1229
|
+
|
|
1230
|
+
const groups = new Map<string, number[]>()
|
|
1231
|
+
for (const row of rows) {
|
|
1232
|
+
const bucket = groups.get(row.grp) ?? []
|
|
1233
|
+
bucket.push(row.duration_ms)
|
|
1234
|
+
groups.set(row.grp, bucket)
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
return [...groups.entries()]
|
|
1238
|
+
.map(([group, durations]) => ({ group, value: percentile(durations, 0.95), count: durations.length }))
|
|
1239
|
+
.sort((left, right) => right.value - left.value)
|
|
1240
|
+
.slice(0, limit)
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
const rows = db.query(`
|
|
1244
|
+
SELECT ${groupExpr} AS grp, ${aggExpr} AS value, COUNT(*) AS count
|
|
1245
|
+
FROM trace_summaries
|
|
1246
|
+
WHERE ${whereClauses.join(" AND ")}
|
|
1247
|
+
GROUP BY grp
|
|
1248
|
+
ORDER BY value DESC
|
|
1249
|
+
LIMIT ?
|
|
1250
|
+
`).all(...whereParams, limit) as Array<{ grp: string; value: number; count: number }>
|
|
1251
|
+
|
|
1252
|
+
return rows.map((row) => ({ group: row.grp, value: row.value, count: row.count }))
|
|
1253
|
+
})
|
|
1254
|
+
})
|
|
1255
|
+
|
|
1256
|
+
const logStats = Effect.fn("motel/TelemetryStore.logStats")(function* (input: LogStatsSearch) {
|
|
1257
|
+
const now = yield* Clock.currentTimeMillis
|
|
1258
|
+
const limit = input.limit ?? 20
|
|
1259
|
+
const hasAttrFilters = Object.keys(input.attributeFilters ?? {}).length > 0
|
|
1260
|
+
const isAttrGroupBy = input.groupBy.startsWith("attr.")
|
|
1261
|
+
|
|
1262
|
+
// For attr.* groupBy or attr filters, fall back to in-memory grouping
|
|
1263
|
+
if (isAttrGroupBy || hasAttrFilters) {
|
|
1264
|
+
const logs = yield* searchLogs({
|
|
1265
|
+
serviceName: input.serviceName,
|
|
1266
|
+
traceId: input.traceId,
|
|
1267
|
+
spanId: input.spanId,
|
|
1268
|
+
body: input.body,
|
|
1269
|
+
lookbackMinutes: input.lookbackMinutes,
|
|
1270
|
+
attributeFilters: input.attributeFilters,
|
|
1271
|
+
limit: 5000,
|
|
1272
|
+
})
|
|
1273
|
+
|
|
1274
|
+
const groups = new Map<string, number>()
|
|
1275
|
+
for (const log of logs) {
|
|
1276
|
+
const group = input.groupBy === "service"
|
|
1277
|
+
? log.serviceName
|
|
1278
|
+
: input.groupBy === "severity"
|
|
1279
|
+
? log.severityText
|
|
1280
|
+
: input.groupBy === "scope"
|
|
1281
|
+
? log.scopeName ?? "unknown"
|
|
1282
|
+
: isAttrGroupBy
|
|
1283
|
+
? log.attributes[input.groupBy.slice(5)] ?? "unknown"
|
|
1284
|
+
: "unknown"
|
|
1285
|
+
groups.set(group, (groups.get(group) ?? 0) + 1)
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
return [...groups.entries()]
|
|
1289
|
+
.map(([group, count]) => ({ group, value: count, count }))
|
|
1290
|
+
.sort((left, right) => right.value - left.value)
|
|
1291
|
+
.slice(0, limit)
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
// Pure SQL path for standard groupBy fields
|
|
1295
|
+
return yield* Effect.sync(() => {
|
|
1296
|
+
const clauses: string[] = []
|
|
1297
|
+
const params: Array<string | number> = []
|
|
1298
|
+
|
|
1299
|
+
if (input.serviceName) {
|
|
1300
|
+
clauses.push("service_name = ?")
|
|
1301
|
+
params.push(input.serviceName)
|
|
1302
|
+
}
|
|
1303
|
+
if (input.traceId) {
|
|
1304
|
+
clauses.push("trace_id = ?")
|
|
1305
|
+
params.push(input.traceId)
|
|
1306
|
+
}
|
|
1307
|
+
if (input.spanId) {
|
|
1308
|
+
clauses.push("span_id = ?")
|
|
1309
|
+
params.push(input.spanId)
|
|
1310
|
+
}
|
|
1311
|
+
if (input.body) {
|
|
1312
|
+
clauses.push("body LIKE ?")
|
|
1313
|
+
params.push(`%${input.body}%`)
|
|
1314
|
+
}
|
|
1315
|
+
if (input.lookbackMinutes) {
|
|
1316
|
+
clauses.push("timestamp_ms >= ?")
|
|
1317
|
+
params.push(now - input.lookbackMinutes * 60 * 1000)
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""
|
|
1321
|
+
|
|
1322
|
+
const groupExpr = input.groupBy === "service"
|
|
1323
|
+
? "service_name"
|
|
1324
|
+
: input.groupBy === "severity"
|
|
1325
|
+
? "severity_text"
|
|
1326
|
+
: input.groupBy === "scope"
|
|
1327
|
+
? "COALESCE(scope_name, 'unknown')"
|
|
1328
|
+
: "'unknown'"
|
|
1329
|
+
|
|
1330
|
+
const rows = db.query(`
|
|
1331
|
+
SELECT ${groupExpr} AS grp, COUNT(*) AS count
|
|
1332
|
+
FROM logs
|
|
1333
|
+
${where}
|
|
1334
|
+
GROUP BY grp
|
|
1335
|
+
ORDER BY count DESC
|
|
1336
|
+
LIMIT ?
|
|
1337
|
+
`).all(...params, limit) as Array<{ grp: string; count: number }>
|
|
1338
|
+
|
|
1339
|
+
return rows.map((row) => ({ group: row.grp, value: row.count, count: row.count }))
|
|
1340
|
+
})
|
|
1341
|
+
})
|
|
1342
|
+
|
|
1343
|
+
const listRecentLogs = Effect.fn("motel/TelemetryStore.listRecentLogs")(function* (serviceName: string) {
|
|
1344
|
+
return yield* searchLogs({ serviceName, limit: config.otel.logFetchLimit })
|
|
1345
|
+
})
|
|
1346
|
+
|
|
1347
|
+
const listFacets = Effect.fn("motel/TelemetryStore.listFacets")(function* (input: FacetSearch) {
|
|
1348
|
+
|
|
1349
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
1350
|
+
const limit = input.limit ?? 20
|
|
1351
|
+
|
|
1352
|
+
return yield* Effect.sync(() => {
|
|
1353
|
+
if (input.type === "logs") {
|
|
1354
|
+
if (input.field === "service") {
|
|
1355
|
+
const rows = db.query(`
|
|
1356
|
+
SELECT service_name AS value, COUNT(*) AS count
|
|
1357
|
+
FROM logs
|
|
1358
|
+
WHERE timestamp_ms >= ?
|
|
1359
|
+
GROUP BY service_name
|
|
1360
|
+
ORDER BY count DESC, value ASC
|
|
1361
|
+
LIMIT ?
|
|
1362
|
+
`).all(cutoff, limit) as Array<{ value: string; count: number }>
|
|
1363
|
+
return rows
|
|
1364
|
+
}
|
|
1365
|
+
if (input.field === "severity") {
|
|
1366
|
+
const rows = db.query(`
|
|
1367
|
+
SELECT severity_text AS value, COUNT(*) AS count
|
|
1368
|
+
FROM logs
|
|
1369
|
+
WHERE timestamp_ms >= ?
|
|
1370
|
+
${input.serviceName ? "AND service_name = ?" : ""}
|
|
1371
|
+
GROUP BY severity_text
|
|
1372
|
+
ORDER BY count DESC, value ASC
|
|
1373
|
+
LIMIT ?
|
|
1374
|
+
`).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
|
|
1375
|
+
return rows
|
|
1376
|
+
}
|
|
1377
|
+
if (input.field === "scope") {
|
|
1378
|
+
const rows = db.query(`
|
|
1379
|
+
SELECT COALESCE(scope_name, 'unknown') AS value, COUNT(*) AS count
|
|
1380
|
+
FROM logs
|
|
1381
|
+
WHERE timestamp_ms >= ?
|
|
1382
|
+
${input.serviceName ? "AND service_name = ?" : ""}
|
|
1383
|
+
GROUP BY COALESCE(scope_name, 'unknown')
|
|
1384
|
+
ORDER BY count DESC, value ASC
|
|
1385
|
+
LIMIT ?
|
|
1386
|
+
`).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
|
|
1387
|
+
return rows
|
|
1388
|
+
}
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
if (input.type === "traces") {
|
|
1392
|
+
if (input.field === "service") {
|
|
1393
|
+
const rows = db.query(`
|
|
1394
|
+
SELECT service_name AS value, COUNT(*) AS count
|
|
1395
|
+
FROM trace_summaries
|
|
1396
|
+
WHERE started_at_ms >= ?
|
|
1397
|
+
GROUP BY service_name
|
|
1398
|
+
ORDER BY count DESC, value ASC
|
|
1399
|
+
LIMIT ?
|
|
1400
|
+
`).all(cutoff, limit) as Array<{ value: string; count: number }>
|
|
1401
|
+
return rows
|
|
1402
|
+
}
|
|
1403
|
+
if (input.field === "operation") {
|
|
1404
|
+
const rows = db.query(`
|
|
1405
|
+
SELECT root_operation_name AS value, COUNT(*) AS count
|
|
1406
|
+
FROM trace_summaries
|
|
1407
|
+
WHERE started_at_ms >= ?
|
|
1408
|
+
${input.serviceName ? "AND service_name = ?" : ""}
|
|
1409
|
+
GROUP BY root_operation_name
|
|
1410
|
+
ORDER BY count DESC, value ASC
|
|
1411
|
+
LIMIT ?
|
|
1412
|
+
`).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
|
|
1413
|
+
return rows
|
|
1414
|
+
}
|
|
1415
|
+
if (input.field === "status") {
|
|
1416
|
+
const rows = db.query(`
|
|
1417
|
+
SELECT CASE WHEN error_count > 0 THEN 'error' ELSE 'ok' END AS value, COUNT(*) AS count
|
|
1418
|
+
FROM trace_summaries
|
|
1419
|
+
WHERE started_at_ms >= ?
|
|
1420
|
+
${input.serviceName ? "AND service_name = ?" : ""}
|
|
1421
|
+
GROUP BY value
|
|
1422
|
+
ORDER BY count DESC
|
|
1423
|
+
LIMIT ?
|
|
1424
|
+
`).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
|
|
1425
|
+
return rows
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
return [] as FacetItem[]
|
|
1430
|
+
})
|
|
1431
|
+
})
|
|
1432
|
+
|
|
1433
|
+
const listTraceLogs = Effect.fn("motel/TelemetryStore.listTraceLogs")(function* (traceId: string) {
|
|
1434
|
+
return yield* searchLogs({ traceId, limit: config.otel.logFetchLimit })
|
|
1435
|
+
})
|
|
1436
|
+
|
|
1437
|
+
// ---------------------------------------------------------------------------
|
|
1438
|
+
// AI Call queries
|
|
1439
|
+
// ---------------------------------------------------------------------------
|
|
1440
|
+
|
|
1441
|
+
/** Extracts ai.streamText -> "streamText", ai.streamText.doStream -> "streamText" */
|
|
1442
|
+
const parseAiOperation = (operationName: string): string => {
|
|
1443
|
+
const parts = operationName.replace(/^ai\./, "").split(".")
|
|
1444
|
+
return parts[0] ?? operationName
|
|
1445
|
+
}
|
|
1446
|
+
|
|
1447
|
+
/** Builds WHERE clauses for AI call search against the spans table (aliased as s) */
|
|
1448
|
+
const buildAiWhereClauses = (input: AiCallSearch | AiCallStatsSearch, cutoff: number) => {
|
|
1449
|
+
const clauses: string[] = ["s.operation_name LIKE 'ai.%'", "s.start_time_ms >= ?"]
|
|
1450
|
+
const params: Array<string | number> = [cutoff]
|
|
1451
|
+
|
|
1452
|
+
if (input.service) {
|
|
1453
|
+
clauses.push("s.service_name = ?")
|
|
1454
|
+
params.push(input.service)
|
|
1455
|
+
}
|
|
1456
|
+
if (input.traceId) {
|
|
1457
|
+
clauses.push("s.trace_id = ?")
|
|
1458
|
+
params.push(input.traceId)
|
|
1459
|
+
}
|
|
1460
|
+
if (input.status) {
|
|
1461
|
+
clauses.push("s.status = ?")
|
|
1462
|
+
params.push(input.status)
|
|
1463
|
+
}
|
|
1464
|
+
if (input.minDurationMs != null) {
|
|
1465
|
+
clauses.push("s.duration_ms >= ?")
|
|
1466
|
+
params.push(input.minDurationMs)
|
|
1467
|
+
}
|
|
1468
|
+
if (input.operation) {
|
|
1469
|
+
clauses.push("s.operation_name LIKE ?")
|
|
1470
|
+
params.push(`ai.${input.operation}%`)
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
// Named attribute filters via span_attributes
|
|
1474
|
+
const attrFilters: Array<[string, string]> = []
|
|
1475
|
+
if (input.sessionId) attrFilters.push([AI_ATTR_MAP.sessionId, input.sessionId])
|
|
1476
|
+
if (input.functionId) attrFilters.push([AI_ATTR_MAP.functionId, input.functionId])
|
|
1477
|
+
if (input.provider) attrFilters.push([AI_ATTR_MAP.provider, input.provider])
|
|
1478
|
+
if (input.model) attrFilters.push([AI_ATTR_MAP.model, input.model])
|
|
1479
|
+
|
|
1480
|
+
for (const [key, value] of attrFilters) {
|
|
1481
|
+
clauses.push("EXISTS (SELECT 1 FROM span_attributes WHERE span_attributes.trace_id = s.trace_id AND span_attributes.span_id = s.span_id AND key = ? AND value = ?)")
|
|
1482
|
+
params.push(key, value)
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
// Text search across prompt/response/tool attribute values
|
|
1486
|
+
if ("text" in input && input.text) {
|
|
1487
|
+
const textKeys = AI_TEXT_SEARCH_KEYS.map(() => "?").join(", ")
|
|
1488
|
+
clauses.push(`EXISTS (SELECT 1 FROM span_attributes WHERE span_attributes.trace_id = s.trace_id AND span_attributes.span_id = s.span_id AND key IN (${textKeys}) AND value LIKE ? COLLATE NOCASE)`)
|
|
1489
|
+
params.push(...AI_TEXT_SEARCH_KEYS, `%${input.text}%`)
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1492
|
+
return { clauses, params }
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
/** Load attribute values for a set of spans by key */
|
|
1496
|
+
const loadSpanAttrValues = (spans: ReadonlyArray<{ trace_id: string; span_id: string }>, keys: readonly string[]): Map<string, Map<string, string>> => {
|
|
1497
|
+
if (spans.length === 0 || keys.length === 0) return new Map()
|
|
1498
|
+
const spanPlaceholders = spans.map(() => "(?, ?)").join(", ")
|
|
1499
|
+
const keyPlaceholders = keys.map(() => "?").join(", ")
|
|
1500
|
+
const spanParams = spans.flatMap((s) => [s.trace_id, s.span_id])
|
|
1501
|
+
|
|
1502
|
+
const rows = db.query(`
|
|
1503
|
+
SELECT trace_id, span_id, key, value
|
|
1504
|
+
FROM span_attributes
|
|
1505
|
+
WHERE (trace_id, span_id) IN (VALUES ${spanPlaceholders})
|
|
1506
|
+
AND key IN (${keyPlaceholders})
|
|
1507
|
+
`).all(...spanParams, ...keys) as Array<{ trace_id: string; span_id: string; key: string; value: string }>
|
|
1508
|
+
|
|
1509
|
+
const result = new Map<string, Map<string, string>>()
|
|
1510
|
+
for (const row of rows) {
|
|
1511
|
+
const spanKey = `${row.trace_id}:${row.span_id}`
|
|
1512
|
+
let attrs = result.get(spanKey)
|
|
1513
|
+
if (!attrs) {
|
|
1514
|
+
attrs = new Map()
|
|
1515
|
+
result.set(spanKey, attrs)
|
|
1516
|
+
}
|
|
1517
|
+
attrs.set(row.key, row.value)
|
|
1518
|
+
}
|
|
1519
|
+
return result
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
const searchAiCalls = Effect.fn("motel/TelemetryStore.searchAiCalls")(function* (input: AiCallSearch) {
|
|
1523
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
1524
|
+
const limit = input.limit ?? 20
|
|
1525
|
+
|
|
1526
|
+
return yield* Effect.sync(() => {
|
|
1527
|
+
const { clauses, params } = buildAiWhereClauses(input, cutoff)
|
|
1528
|
+
|
|
1529
|
+
const rows = db.query(`
|
|
1530
|
+
SELECT s.trace_id, s.span_id, s.service_name, s.operation_name, s.start_time_ms, s.duration_ms, s.status
|
|
1531
|
+
FROM spans AS s
|
|
1532
|
+
WHERE ${clauses.join(" AND ")}
|
|
1533
|
+
ORDER BY s.start_time_ms DESC
|
|
1534
|
+
LIMIT ?
|
|
1535
|
+
`).all(...params, limit) as Array<{
|
|
1536
|
+
trace_id: string; span_id: string; service_name: string
|
|
1537
|
+
operation_name: string; start_time_ms: number; duration_ms: number; status: string
|
|
1538
|
+
}>
|
|
1539
|
+
|
|
1540
|
+
if (rows.length === 0) return [] as readonly AiCallSummary[]
|
|
1541
|
+
|
|
1542
|
+
// Batch-load the attributes we need for summaries
|
|
1543
|
+
const summaryAttrKeys = [
|
|
1544
|
+
AI_ATTR_MAP.functionId, AI_ATTR_MAP.provider, AI_ATTR_MAP.model,
|
|
1545
|
+
AI_ATTR_MAP.sessionId, AI_ATTR_MAP.userId, AI_ATTR_MAP.finishReason,
|
|
1546
|
+
AI_ATTR_MAP.inputTokens, AI_ATTR_MAP.outputTokens, AI_ATTR_MAP.totalTokens,
|
|
1547
|
+
AI_ATTR_MAP.cachedInputTokens, AI_ATTR_MAP.reasoningTokens,
|
|
1548
|
+
AI_ATTR_MAP.promptMessages, AI_ATTR_MAP.prompt, AI_ATTR_MAP.responseText,
|
|
1549
|
+
]
|
|
1550
|
+
const attrMap = loadSpanAttrValues(rows, summaryAttrKeys)
|
|
1551
|
+
|
|
1552
|
+
// Count tool call child spans per AI span
|
|
1553
|
+
const spanPlaceholders = rows.map(() => "(?, ?)").join(", ")
|
|
1554
|
+
const spanParams = rows.flatMap((r) => [r.trace_id, r.span_id])
|
|
1555
|
+
const toolCountRows = db.query(`
|
|
1556
|
+
SELECT parent_span_id, COUNT(*) AS cnt
|
|
1557
|
+
FROM spans
|
|
1558
|
+
WHERE (trace_id, parent_span_id) IN (VALUES ${spanPlaceholders})
|
|
1559
|
+
AND operation_name LIKE 'ai.toolCall%'
|
|
1560
|
+
GROUP BY trace_id, parent_span_id
|
|
1561
|
+
`).all(...spanParams) as Array<{ parent_span_id: string; cnt: number }>
|
|
1562
|
+
const toolCounts = new Map(toolCountRows.map((r) => [r.parent_span_id, r.cnt]))
|
|
1563
|
+
|
|
1564
|
+
return rows.map((row): AiCallSummary => {
|
|
1565
|
+
const spanKey = `${row.trace_id}:${row.span_id}`
|
|
1566
|
+
const attrs = attrMap.get(spanKey)
|
|
1567
|
+
const get = (key: string) => attrs?.get(key) ?? null
|
|
1568
|
+
const getNum = (key: string) => {
|
|
1569
|
+
const v = get(key)
|
|
1570
|
+
return v != null ? Number(v) : null
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
const promptContent = get(AI_ATTR_MAP.promptMessages) ?? get(AI_ATTR_MAP.prompt)
|
|
1574
|
+
|
|
1575
|
+
return {
|
|
1576
|
+
traceId: row.trace_id,
|
|
1577
|
+
spanId: row.span_id,
|
|
1578
|
+
operation: parseAiOperation(row.operation_name),
|
|
1579
|
+
service: row.service_name,
|
|
1580
|
+
functionId: get(AI_ATTR_MAP.functionId),
|
|
1581
|
+
provider: get(AI_ATTR_MAP.provider),
|
|
1582
|
+
model: get(AI_ATTR_MAP.model),
|
|
1583
|
+
status: row.status === "error" ? "error" : "ok",
|
|
1584
|
+
startedAt: new Date(row.start_time_ms).toISOString(),
|
|
1585
|
+
durationMs: row.duration_ms,
|
|
1586
|
+
sessionId: get(AI_ATTR_MAP.sessionId),
|
|
1587
|
+
userId: get(AI_ATTR_MAP.userId),
|
|
1588
|
+
promptPreview: truncatePreview(promptContent),
|
|
1589
|
+
responsePreview: truncatePreview(get(AI_ATTR_MAP.responseText)),
|
|
1590
|
+
finishReason: get(AI_ATTR_MAP.finishReason),
|
|
1591
|
+
toolCallCount: toolCounts.get(row.span_id) ?? 0,
|
|
1592
|
+
usage: {
|
|
1593
|
+
inputTokens: getNum(AI_ATTR_MAP.inputTokens),
|
|
1594
|
+
outputTokens: getNum(AI_ATTR_MAP.outputTokens),
|
|
1595
|
+
totalTokens: getNum(AI_ATTR_MAP.totalTokens),
|
|
1596
|
+
cachedInputTokens: getNum(AI_ATTR_MAP.cachedInputTokens),
|
|
1597
|
+
reasoningTokens: getNum(AI_ATTR_MAP.reasoningTokens),
|
|
1598
|
+
},
|
|
1599
|
+
}
|
|
1600
|
+
})
|
|
1601
|
+
})
|
|
1602
|
+
})
|
|
1603
|
+
|
|
1604
|
+
const getAiCall = Effect.fn("motel/TelemetryStore.getAiCall")(function* (spanId: string) {
|
|
1605
|
+
return yield* Effect.sync(() => {
|
|
1606
|
+
const row = db.query(`
|
|
1607
|
+
SELECT * FROM spans WHERE span_id = ? AND operation_name LIKE 'ai.%' LIMIT 1
|
|
1608
|
+
`).get(spanId) as SpanRow | null
|
|
1609
|
+
if (!row) return null
|
|
1610
|
+
|
|
1611
|
+
// Load all attributes for this span
|
|
1612
|
+
const attrRows = db.query(`
|
|
1613
|
+
SELECT key, value FROM span_attributes
|
|
1614
|
+
WHERE trace_id = ? AND span_id = ?
|
|
1615
|
+
`).all(row.trace_id, row.span_id) as Array<{ key: string; value: string }>
|
|
1616
|
+
const attrs = new Map(attrRows.map((r) => [r.key, r.value]))
|
|
1617
|
+
const get = (key: string) => attrs.get(key) ?? null
|
|
1618
|
+
const getNum = (key: string) => {
|
|
1619
|
+
const v = get(key)
|
|
1620
|
+
return v != null ? Number(v) : null
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
// Load tool call child spans
|
|
1624
|
+
const toolCallRows = db.query(`
|
|
1625
|
+
SELECT span_id, operation_name, duration_ms, status, attributes_json
|
|
1626
|
+
FROM spans
|
|
1627
|
+
WHERE trace_id = ? AND parent_span_id = ? AND operation_name LIKE 'ai.toolCall%'
|
|
1628
|
+
ORDER BY start_time_ms ASC
|
|
1629
|
+
`).all(row.trace_id, row.span_id) as SpanRow[]
|
|
1630
|
+
|
|
1631
|
+
const toolCalls = toolCallRows.map((tc) => {
|
|
1632
|
+
const tcAttrs = JSON.parse(tc.attributes_json) as Record<string, string>
|
|
1633
|
+
return {
|
|
1634
|
+
name: tcAttrs["ai.toolCall.name"] ?? tc.operation_name,
|
|
1635
|
+
spanId: tc.span_id,
|
|
1636
|
+
status: tc.status === "error" ? "error" as const : "ok" as const,
|
|
1637
|
+
durationMs: tc.duration_ms,
|
|
1638
|
+
}
|
|
1639
|
+
})
|
|
1640
|
+
|
|
1641
|
+
// Load correlated logs
|
|
1642
|
+
const logRows = db.query(`
|
|
1643
|
+
SELECT * FROM logs WHERE span_id = ? ORDER BY timestamp_ms ASC
|
|
1644
|
+
`).all(row.span_id) as LogRow[]
|
|
1645
|
+
const logs = logRows.map(parseLogRow)
|
|
1646
|
+
|
|
1647
|
+
// Parse prompt - try as JSON first for structured display
|
|
1648
|
+
const promptRaw = get(AI_ATTR_MAP.promptMessages) ?? get(AI_ATTR_MAP.prompt)
|
|
1649
|
+
let promptMessages: unknown = null
|
|
1650
|
+
if (promptRaw) {
|
|
1651
|
+
try { promptMessages = JSON.parse(promptRaw) } catch { promptMessages = promptRaw }
|
|
1652
|
+
}
|
|
1653
|
+
|
|
1654
|
+
// Parse tools
|
|
1655
|
+
const toolsRaw = get(AI_ATTR_MAP.tools)
|
|
1656
|
+
let toolsAvailable: unknown = null
|
|
1657
|
+
if (toolsRaw) {
|
|
1658
|
+
try { toolsAvailable = JSON.parse(toolsRaw) } catch { toolsAvailable = toolsRaw }
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
// Parse provider metadata
|
|
1662
|
+
const providerMetaRaw = get(AI_ATTR_MAP.providerMetadata)
|
|
1663
|
+
let providerMetadata: unknown = null
|
|
1664
|
+
if (providerMetaRaw) {
|
|
1665
|
+
try { providerMetadata = JSON.parse(providerMetaRaw) } catch { providerMetadata = providerMetaRaw }
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
return {
|
|
1669
|
+
traceId: row.trace_id,
|
|
1670
|
+
spanId: row.span_id,
|
|
1671
|
+
operation: parseAiOperation(row.operation_name),
|
|
1672
|
+
service: row.service_name,
|
|
1673
|
+
functionId: get(AI_ATTR_MAP.functionId),
|
|
1674
|
+
provider: get(AI_ATTR_MAP.provider),
|
|
1675
|
+
model: get(AI_ATTR_MAP.model),
|
|
1676
|
+
status: row.status === "error" ? "error" as const : "ok" as const,
|
|
1677
|
+
startedAt: new Date(row.start_time_ms).toISOString(),
|
|
1678
|
+
durationMs: row.duration_ms,
|
|
1679
|
+
sessionId: get(AI_ATTR_MAP.sessionId),
|
|
1680
|
+
userId: get(AI_ATTR_MAP.userId),
|
|
1681
|
+
finishReason: get(AI_ATTR_MAP.finishReason),
|
|
1682
|
+
promptMessages,
|
|
1683
|
+
responseText: get(AI_ATTR_MAP.responseText),
|
|
1684
|
+
toolCalls,
|
|
1685
|
+
toolsAvailable,
|
|
1686
|
+
providerMetadata,
|
|
1687
|
+
usage: {
|
|
1688
|
+
inputTokens: getNum(AI_ATTR_MAP.inputTokens),
|
|
1689
|
+
outputTokens: getNum(AI_ATTR_MAP.outputTokens),
|
|
1690
|
+
totalTokens: getNum(AI_ATTR_MAP.totalTokens),
|
|
1691
|
+
cachedInputTokens: getNum(AI_ATTR_MAP.cachedInputTokens),
|
|
1692
|
+
reasoningTokens: getNum(AI_ATTR_MAP.reasoningTokens),
|
|
1693
|
+
},
|
|
1694
|
+
timing: {
|
|
1695
|
+
msToFirstChunk: getNum(AI_ATTR_MAP.msToFirstChunk),
|
|
1696
|
+
msToFinish: getNum(AI_ATTR_MAP.msToFinish),
|
|
1697
|
+
avgOutputTokensPerSecond: getNum(AI_ATTR_MAP.avgOutputTokensPerSecond),
|
|
1698
|
+
},
|
|
1699
|
+
logs,
|
|
1700
|
+
} satisfies AiCallDetail
|
|
1701
|
+
})
|
|
1702
|
+
})
|
|
1703
|
+
|
|
1704
|
+
const aiCallStats = Effect.fn("motel/TelemetryStore.aiCallStats")(function* (input: AiCallStatsSearch) {
|
|
1705
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
1706
|
+
const limit = input.limit ?? 20
|
|
1707
|
+
|
|
1708
|
+
return yield* Effect.sync(() => {
|
|
1709
|
+
const { clauses, params } = buildAiWhereClauses(input, cutoff)
|
|
1710
|
+
|
|
1711
|
+
// For status groupBy, we can do it purely from the spans table
|
|
1712
|
+
if (input.groupBy === "status") {
|
|
1713
|
+
const rows = db.query(`
|
|
1714
|
+
SELECT s.status AS grp, COUNT(*) AS count, AVG(s.duration_ms) AS avg_dur
|
|
1715
|
+
FROM spans AS s
|
|
1716
|
+
WHERE ${clauses.join(" AND ")}
|
|
1717
|
+
GROUP BY s.status
|
|
1718
|
+
ORDER BY count DESC
|
|
1719
|
+
LIMIT ?
|
|
1720
|
+
`).all(...params, limit) as Array<{ grp: string; count: number; avg_dur: number }>
|
|
1721
|
+
|
|
1722
|
+
if (input.agg === "count") return rows.map((r) => ({ group: r.grp, value: r.count, count: r.count }))
|
|
1723
|
+
if (input.agg === "avg_duration") return rows.map((r) => ({ group: r.grp, value: r.avg_dur, count: r.count }))
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
// For attribute-based groupBy, we need to join span_attributes
|
|
1727
|
+
const groupByAttrKey = input.groupBy === "provider" ? AI_ATTR_MAP.provider
|
|
1728
|
+
: input.groupBy === "model" ? AI_ATTR_MAP.model
|
|
1729
|
+
: input.groupBy === "functionId" ? AI_ATTR_MAP.functionId
|
|
1730
|
+
: input.groupBy === "sessionId" ? AI_ATTR_MAP.sessionId
|
|
1731
|
+
: null
|
|
1732
|
+
|
|
1733
|
+
if (!groupByAttrKey) return []
|
|
1734
|
+
|
|
1735
|
+
// First get the matching spans with their group values
|
|
1736
|
+
const rows = db.query(`
|
|
1737
|
+
SELECT
|
|
1738
|
+
COALESCE(ga.value, 'unknown') AS grp,
|
|
1739
|
+
s.span_id,
|
|
1740
|
+
s.duration_ms,
|
|
1741
|
+
s.status
|
|
1742
|
+
FROM spans AS s
|
|
1743
|
+
LEFT JOIN span_attributes AS ga
|
|
1744
|
+
ON ga.trace_id = s.trace_id AND ga.span_id = s.span_id AND ga.key = ?
|
|
1745
|
+
WHERE ${clauses.join(" AND ")}
|
|
1746
|
+
`).all(groupByAttrKey, ...params) as Array<{ grp: string; span_id: string; duration_ms: number; status: string }>
|
|
1747
|
+
|
|
1748
|
+
// Group and aggregate in JS (need p95 and token aggregation)
|
|
1749
|
+
const groups = new Map<string, { durations: number[]; count: number; spanIds: string[] }>()
|
|
1750
|
+
for (const row of rows) {
|
|
1751
|
+
const bucket = groups.get(row.grp) ?? { durations: [], count: 0, spanIds: [] }
|
|
1752
|
+
bucket.durations.push(row.duration_ms)
|
|
1753
|
+
bucket.count++
|
|
1754
|
+
bucket.spanIds.push(row.span_id)
|
|
1755
|
+
groups.set(row.grp, bucket)
|
|
1756
|
+
}
|
|
1757
|
+
|
|
1758
|
+
// For token aggregations, batch-load from span_attributes
|
|
1759
|
+
if (input.agg === "total_input_tokens" || input.agg === "total_output_tokens") {
|
|
1760
|
+
const tokenKey = input.agg === "total_input_tokens" ? AI_ATTR_MAP.inputTokens : AI_ATTR_MAP.outputTokens
|
|
1761
|
+
const allSpanIds = [...groups.values()].flatMap((b) => b.spanIds)
|
|
1762
|
+
if (allSpanIds.length > 0) {
|
|
1763
|
+
const placeholders = allSpanIds.map(() => "?").join(", ")
|
|
1764
|
+
const tokenRows = db.query(`
|
|
1765
|
+
SELECT span_id, CAST(value AS REAL) AS tokens
|
|
1766
|
+
FROM span_attributes
|
|
1767
|
+
WHERE key = ? AND span_id IN (${placeholders})
|
|
1768
|
+
`).all(tokenKey, ...allSpanIds) as Array<{ span_id: string; tokens: number }>
|
|
1769
|
+
|
|
1770
|
+
const tokenBySpan = new Map(tokenRows.map((r) => [r.span_id, r.tokens]))
|
|
1771
|
+
|
|
1772
|
+
return [...groups.entries()]
|
|
1773
|
+
.map(([group, bucket]) => {
|
|
1774
|
+
const total = bucket.spanIds.reduce((sum, sid) => sum + (tokenBySpan.get(sid) ?? 0), 0)
|
|
1775
|
+
return { group, value: total, count: bucket.count }
|
|
1776
|
+
})
|
|
1777
|
+
.sort((a, b) => b.value - a.value)
|
|
1778
|
+
.slice(0, limit)
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
return [...groups.entries()]
|
|
1783
|
+
.map(([group, bucket]) => {
|
|
1784
|
+
const value = input.agg === "count"
|
|
1785
|
+
? bucket.count
|
|
1786
|
+
: input.agg === "avg_duration"
|
|
1787
|
+
? bucket.durations.reduce((s, d) => s + d, 0) / Math.max(1, bucket.count)
|
|
1788
|
+
: input.agg === "p95_duration"
|
|
1789
|
+
? percentile(bucket.durations, 0.95)
|
|
1790
|
+
: bucket.count
|
|
1791
|
+
return { group, value, count: bucket.count }
|
|
1792
|
+
})
|
|
1793
|
+
.sort((a, b) => b.value - a.value)
|
|
1794
|
+
.slice(0, limit)
|
|
1795
|
+
})
|
|
1796
|
+
})
|
|
1797
|
+
|
|
1798
|
+
return TelemetryStore.of({
|
|
1799
|
+
ingestTraces,
|
|
1800
|
+
ingestLogs,
|
|
1801
|
+
listServices,
|
|
1802
|
+
listRecentTraces,
|
|
1803
|
+
listTraceSummaries,
|
|
1804
|
+
searchTraces,
|
|
1805
|
+
searchTraceSummaries,
|
|
1806
|
+
traceStats,
|
|
1807
|
+
getTrace,
|
|
1808
|
+
getSpan,
|
|
1809
|
+
listTraceSpans,
|
|
1810
|
+
searchSpans,
|
|
1811
|
+
searchLogs,
|
|
1812
|
+
logStats,
|
|
1813
|
+
listFacets,
|
|
1814
|
+
listRecentLogs,
|
|
1815
|
+
listTraceLogs,
|
|
1816
|
+
searchAiCalls,
|
|
1817
|
+
getAiCall,
|
|
1818
|
+
aiCallStats,
|
|
1819
|
+
})
|
|
1820
|
+
}),
|
|
1821
|
+
)
|