@aryasaatvik/motel 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +180 -0
- package/LICENSE +21 -0
- package/README.md +137 -0
- package/package.json +119 -0
- package/skills/motel-debug/SKILL.md +203 -0
- package/skills/motel-debug/references/effect.md +38 -0
- package/src/App.tsx +414 -0
- package/src/StartupGate.test.ts +80 -0
- package/src/StartupGate.tsx +263 -0
- package/src/cli.ts +205 -0
- package/src/config.ts +45 -0
- package/src/daemon.test.ts +547 -0
- package/src/daemon.ts +486 -0
- package/src/domain.test.ts +62 -0
- package/src/domain.ts +291 -0
- package/src/httpApi.ts +391 -0
- package/src/httpListPolicy.test.ts +76 -0
- package/src/httpListPolicy.ts +129 -0
- package/src/index.tsx +25 -0
- package/src/instructions.ts +72 -0
- package/src/launchAgent.test.ts +430 -0
- package/src/launchAgent.ts +462 -0
- package/src/localServer.ts +568 -0
- package/src/locator.ts +138 -0
- package/src/mcp.ts +433 -0
- package/src/motel.test.ts +176 -0
- package/src/motel.ts +88 -0
- package/src/motelClient.ts +353 -0
- package/src/opentui-jsx.d.ts +11 -0
- package/src/otlp.test.ts +65 -0
- package/src/otlp.ts +162 -0
- package/src/otlpProtobuf.ts +35 -0
- package/src/queryFilters.ts +39 -0
- package/src/registry.ts +115 -0
- package/src/runtime.ts +40 -0
- package/src/server.ts +11 -0
- package/src/services/AsyncIngest.ts +82 -0
- package/src/services/LogQueryService.ts +29 -0
- package/src/services/TelemetryQuery.ts +62 -0
- package/src/services/TelemetryStore.ts +2554 -0
- package/src/services/TraceQueryService.ts +43 -0
- package/src/services/ingestRpc.ts +39 -0
- package/src/services/queryRpc.ts +15 -0
- package/src/services/telemetryQueryWorker.ts +32 -0
- package/src/services/telemetryWorker.ts +59 -0
- package/src/startupBench.ts +19 -0
- package/src/startupLifecycle.ts +57 -0
- package/src/storybook/aiChatStory.tsx +244 -0
- package/src/storybook/fixtures/errorState.ts +44 -0
- package/src/storybook/fixtures/imagePaste.ts +34 -0
- package/src/storybook/fixtures/index.ts +62 -0
- package/src/storybook/fixtures/kitchenSink.ts +148 -0
- package/src/storybook/fixtures/rawPrompt.ts +15 -0
- package/src/storybook/fixtures/short.ts +27 -0
- package/src/storybook/fixtures/toolHeavy.ts +65 -0
- package/src/telemetry.test.ts +1053 -0
- package/src/ui/AiChatView.tsx +308 -0
- package/src/ui/AttrFilterModal.tsx +120 -0
- package/src/ui/ServiceLogs.tsx +115 -0
- package/src/ui/SpanContentView.tsx +184 -0
- package/src/ui/SpanDetail.tsx +215 -0
- package/src/ui/SpanDetailFull.tsx +224 -0
- package/src/ui/SpanDetailPane.tsx +90 -0
- package/src/ui/TraceDetailsPane.tsx +175 -0
- package/src/ui/TraceList.tsx +254 -0
- package/src/ui/Waterfall.tsx +389 -0
- package/src/ui/aiChatModel.test.ts +391 -0
- package/src/ui/aiChatModel.ts +773 -0
- package/src/ui/aiState.ts +59 -0
- package/src/ui/app/TraceListPane.tsx +39 -0
- package/src/ui/app/TraceWorkspace.tsx +452 -0
- package/src/ui/app/useAppLayout.ts +101 -0
- package/src/ui/app/useTraceScreenData.ts +590 -0
- package/src/ui/atoms.ts +131 -0
- package/src/ui/cachedLoader.test.ts +23 -0
- package/src/ui/cachedLoader.ts +60 -0
- package/src/ui/filterParser.test.ts +56 -0
- package/src/ui/filterParser.ts +45 -0
- package/src/ui/format.ts +128 -0
- package/src/ui/loaders.ts +101 -0
- package/src/ui/persistence.ts +41 -0
- package/src/ui/primitives.tsx +198 -0
- package/src/ui/state.ts +6 -0
- package/src/ui/theme.ts +160 -0
- package/src/ui/traceDetailsWidth.repro.test.ts +126 -0
- package/src/ui/traceSortNav.repro.seed.ts +62 -0
- package/src/ui/traceSortNav.repro.test.ts +231 -0
- package/src/ui/useAttrFilterPicker.ts +89 -0
- package/src/ui/useKeyboardNav.ts +1023 -0
- package/src/ui/waterfallFilter.test.ts +84 -0
- package/src/ui/waterfallFilter.ts +59 -0
- package/src/ui/waterfallModel.ts +130 -0
- package/src/ui/waterfallNav.repro.seed.ts +86 -0
- package/src/ui/waterfallNav.repro.test.ts +271 -0
- package/src/ui/waterfallNav.test.ts +453 -0
- package/src/ui/waterfallNav.ts +75 -0
- package/web/dist/assets/index-B01z9BaO.css +2 -0
- package/web/dist/assets/index-CYlBaY9O.js +22 -0
- package/web/dist/index.html +13 -0
|
@@ -0,0 +1,2554 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite"
|
|
2
|
+
import * as BunFileSystem from "@effect/platform-bun/BunFileSystem"
|
|
3
|
+
import { dirname } from "node:path"
|
|
4
|
+
import { Cause, Clock, Effect, FileSystem, Layer, Schedule, Context } from "effect"
|
|
5
|
+
import { config } from "../config.js"
|
|
6
|
+
import type { AiCallDetail, AiCallSummary, FacetItem, LogItem, SpanItem, StatsItem, TraceItem, TraceSummaryItem, TraceSpanEvent, TraceSpanItem } from "../domain.js"
|
|
7
|
+
import { AI_ATTR_MAP, AI_FTS_KEYS, AI_TEXT_SEARCH_KEYS, truncatePreview } from "../domain.js"
|
|
8
|
+
import { attributeMap, nanosToMilliseconds, normalizeOtlpBinaryId, parseAnyValue, spanKindLabel, spanStatusLabel, stringifyValue, type OtlpLogExportRequest, type OtlpTraceExportRequest } from "../otlp.js"
|
|
9
|
+
|
|
10
|
+
const isSqliteLockError = (error: unknown) =>
|
|
11
|
+
error instanceof Error && /(database is locked|database table is locked|SQLITE_BUSY)/i.test(error.message)
|
|
12
|
+
|
|
13
|
+
interface SpanRow {
|
|
14
|
+
readonly trace_id: string
|
|
15
|
+
readonly span_id: string
|
|
16
|
+
readonly parent_span_id: string | null
|
|
17
|
+
readonly service_name: string
|
|
18
|
+
readonly scope_name: string | null
|
|
19
|
+
readonly operation_name: string
|
|
20
|
+
readonly kind: string | null
|
|
21
|
+
readonly start_time_ms: number
|
|
22
|
+
readonly end_time_ms: number
|
|
23
|
+
readonly duration_ms: number
|
|
24
|
+
readonly status: string
|
|
25
|
+
readonly attributes_json: string
|
|
26
|
+
readonly resource_json: string
|
|
27
|
+
readonly events_json: string
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
interface LogRow {
|
|
31
|
+
readonly id: number
|
|
32
|
+
readonly trace_id: string | null
|
|
33
|
+
readonly span_id: string | null
|
|
34
|
+
readonly service_name: string
|
|
35
|
+
readonly scope_name: string | null
|
|
36
|
+
readonly severity_text: string
|
|
37
|
+
readonly timestamp_ms: number
|
|
38
|
+
readonly body: string
|
|
39
|
+
readonly attributes_json: string
|
|
40
|
+
readonly resource_json: string
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
interface LogSearch {
|
|
44
|
+
readonly serviceName?: string | null
|
|
45
|
+
readonly severity?: string | null
|
|
46
|
+
readonly traceId?: string | null
|
|
47
|
+
readonly spanId?: string | null
|
|
48
|
+
readonly body?: string | null
|
|
49
|
+
readonly lookbackMinutes?: number
|
|
50
|
+
readonly limit?: number
|
|
51
|
+
readonly cursorTimestampMs?: number
|
|
52
|
+
readonly cursorId?: string
|
|
53
|
+
readonly attributeFilters?: Readonly<Record<string, string>>
|
|
54
|
+
readonly attributeContainsFilters?: Readonly<Record<string, string>>
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
interface TraceSearch {
|
|
58
|
+
readonly serviceName?: string | null
|
|
59
|
+
readonly operation?: string | null
|
|
60
|
+
readonly status?: "ok" | "error" | null
|
|
61
|
+
readonly minDurationMs?: number | null
|
|
62
|
+
readonly attributeFilters?: Readonly<Record<string, string>>
|
|
63
|
+
/**
|
|
64
|
+
* Full-text match against the AI prompt/response/tool attribute values
|
|
65
|
+
* on any span in the trace (see AI_FTS_KEYS). When set, traces are
|
|
66
|
+
* filtered to those containing at least one span whose indexed LLM
|
|
67
|
+
* content matches. Powered by span_attr_fts (FTS5).
|
|
68
|
+
*/
|
|
69
|
+
readonly aiText?: string | null
|
|
70
|
+
readonly lookbackMinutes?: number
|
|
71
|
+
readonly limit?: number
|
|
72
|
+
readonly cursorStartedAtMs?: number
|
|
73
|
+
readonly cursorTraceId?: string
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
interface SpanSearch {
|
|
77
|
+
readonly serviceName?: string | null
|
|
78
|
+
readonly traceId?: string | null
|
|
79
|
+
readonly operation?: string | null
|
|
80
|
+
readonly parentOperation?: string | null
|
|
81
|
+
readonly status?: "ok" | "error" | null
|
|
82
|
+
readonly lookbackMinutes?: number
|
|
83
|
+
readonly limit?: number
|
|
84
|
+
readonly attributeFilters?: Readonly<Record<string, string>>
|
|
85
|
+
readonly attributeContainsFilters?: Readonly<Record<string, string>>
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
interface TraceStatsSearch extends TraceSearch {
|
|
89
|
+
readonly groupBy: string
|
|
90
|
+
readonly agg: "count" | "avg_duration" | "p95_duration" | "error_rate"
|
|
91
|
+
readonly limit?: number
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
interface LogStatsSearch extends LogSearch {
|
|
95
|
+
readonly groupBy: string
|
|
96
|
+
readonly agg: "count"
|
|
97
|
+
readonly lookbackMinutes?: number
|
|
98
|
+
readonly limit?: number
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// FacetItem and StatsItem imported from domain.ts
|
|
102
|
+
|
|
103
|
+
interface FacetSearch {
|
|
104
|
+
readonly type: "traces" | "logs"
|
|
105
|
+
readonly field: string
|
|
106
|
+
readonly serviceName?: string | null
|
|
107
|
+
readonly key?: string | null
|
|
108
|
+
readonly lookbackMinutes?: number
|
|
109
|
+
readonly limit?: number
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
interface AiCallSearch {
|
|
113
|
+
readonly service?: string | null
|
|
114
|
+
readonly traceId?: string | null
|
|
115
|
+
readonly sessionId?: string | null
|
|
116
|
+
readonly functionId?: string | null
|
|
117
|
+
readonly provider?: string | null
|
|
118
|
+
readonly model?: string | null
|
|
119
|
+
readonly operation?: string | null
|
|
120
|
+
readonly status?: "ok" | "error" | null
|
|
121
|
+
readonly minDurationMs?: number | null
|
|
122
|
+
readonly text?: string | null
|
|
123
|
+
readonly lookbackMinutes?: number
|
|
124
|
+
readonly limit?: number
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
interface AiCallStatsSearch {
|
|
128
|
+
readonly groupBy: "provider" | "model" | "functionId" | "sessionId" | "status"
|
|
129
|
+
readonly agg: "count" | "avg_duration" | "p95_duration" | "total_input_tokens" | "total_output_tokens"
|
|
130
|
+
readonly service?: string | null
|
|
131
|
+
readonly traceId?: string | null
|
|
132
|
+
readonly sessionId?: string | null
|
|
133
|
+
readonly functionId?: string | null
|
|
134
|
+
readonly provider?: string | null
|
|
135
|
+
readonly model?: string | null
|
|
136
|
+
readonly operation?: string | null
|
|
137
|
+
readonly status?: "ok" | "error" | null
|
|
138
|
+
readonly minDurationMs?: number | null
|
|
139
|
+
readonly lookbackMinutes?: number
|
|
140
|
+
readonly limit?: number
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
interface TraceSummaryRow {
|
|
144
|
+
readonly trace_id: string
|
|
145
|
+
readonly service_name: string
|
|
146
|
+
readonly root_operation_name: string
|
|
147
|
+
readonly started_at_ms: number
|
|
148
|
+
readonly ended_at_ms?: number
|
|
149
|
+
readonly active_span_count: number
|
|
150
|
+
readonly duration_ms: number
|
|
151
|
+
readonly span_count: number
|
|
152
|
+
readonly error_count: number
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
type InternalTraceSpanItem = TraceSpanItem & {
|
|
156
|
+
readonly syntheticMissingParent?: boolean
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const isSpanRunning = (startTimeMs: number, endTimeMs: number) => endTimeMs <= 0 || endTimeMs < startTimeMs
|
|
160
|
+
|
|
161
|
+
const liveDurationMs = (startTimeMs: number, endTimeMs: number, isRunning: boolean) =>
|
|
162
|
+
Math.max(0, (isRunning ? Date.now() : endTimeMs) - startTimeMs)
|
|
163
|
+
|
|
164
|
+
const parseSummaryRow = (row: TraceSummaryRow): TraceSummaryItem => ({
|
|
165
|
+
isRunning: row.active_span_count > 0,
|
|
166
|
+
traceId: row.trace_id,
|
|
167
|
+
serviceName: row.service_name ?? "unknown",
|
|
168
|
+
rootOperationName: row.root_operation_name ?? "unknown",
|
|
169
|
+
startedAt: new Date(row.started_at_ms),
|
|
170
|
+
durationMs: row.active_span_count > 0 ? liveDurationMs(row.started_at_ms, row.ended_at_ms ?? 0, true) : Math.max(0, row.duration_ms),
|
|
171
|
+
spanCount: row.span_count,
|
|
172
|
+
errorCount: row.error_count,
|
|
173
|
+
warnings: [],
|
|
174
|
+
})
|
|
175
|
+
|
|
176
|
+
// Skip attribute facet rows whose value blob is longer than this. Prevents
|
|
177
|
+
// multi-MB text attrs (ai.prompt, ai.prompt.messages, etc.) from dominating
|
|
178
|
+
// picker-open time — SQLite skips reading those pages from disk when the
|
|
179
|
+
// length predicate is evaluated against the page header, taking queries over
|
|
180
|
+
// a 2GB database from ~1.2s down to ~370ms. Keys whose values are ALL fat
|
|
181
|
+
// simply don't appear in the picker, which is the desired behaviour: you'd
|
|
182
|
+
// never want to filter traces by exact-match on a 1MB prompt blob anyway.
|
|
183
|
+
const FACET_VALUE_MAX_LEN = 512
|
|
184
|
+
|
|
185
|
+
const TRACE_SUMMARY_SELECT_SQL = `
|
|
186
|
+
SELECT
|
|
187
|
+
trace_id,
|
|
188
|
+
COALESCE(MIN(CASE WHEN parent_span_id IS NULL THEN service_name END), MIN(service_name)) AS service_name,
|
|
189
|
+
COALESCE(MIN(CASE WHEN parent_span_id IS NULL THEN operation_name END), MIN(operation_name)) AS root_operation_name,
|
|
190
|
+
MIN(start_time_ms) AS started_at_ms,
|
|
191
|
+
MAX(end_time_ms) AS ended_at_ms,
|
|
192
|
+
SUM(CASE WHEN end_time_ms <= 0 OR end_time_ms < start_time_ms THEN 1 ELSE 0 END) AS active_span_count,
|
|
193
|
+
MAX(end_time_ms) - MIN(start_time_ms) AS duration_ms,
|
|
194
|
+
COUNT(*) AS span_count,
|
|
195
|
+
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) AS error_count
|
|
196
|
+
FROM spans
|
|
197
|
+
`
|
|
198
|
+
|
|
199
|
+
// Memoize small repeated JSON records. Resource attributes are the primary
|
|
200
|
+
// beneficiary because many spans share the same serialized value; compact
|
|
201
|
+
// repeated span attributes also benefit while large unique payloads bypass
|
|
202
|
+
// the cache to keep memory bounded for long-running daemons.
|
|
203
|
+
const RECORD_PARSE_CACHE_MAX_VALUE_LEN = 1024
|
|
204
|
+
const RECORD_PARSE_CACHE_LIMIT = 256
|
|
205
|
+
const recordParseCache = new Map<string, Record<string, string>>()
|
|
206
|
+
const EMPTY_RECORD: Record<string, string> = {}
|
|
207
|
+
|
|
208
|
+
const parseRecord = (value: string): Record<string, string> => {
|
|
209
|
+
if (value === "" || value === "{}") return EMPTY_RECORD
|
|
210
|
+
const cacheable = value.length <= RECORD_PARSE_CACHE_MAX_VALUE_LEN
|
|
211
|
+
if (cacheable) {
|
|
212
|
+
const cached = recordParseCache.get(value)
|
|
213
|
+
if (cached !== undefined) return cached
|
|
214
|
+
}
|
|
215
|
+
let parsed: Record<string, string>
|
|
216
|
+
try {
|
|
217
|
+
const json = JSON.parse(value) as Record<string, unknown>
|
|
218
|
+
parsed = Object.fromEntries(Object.entries(json).map(([key, entry]) => [key, stringifyValue(entry)]))
|
|
219
|
+
} catch {
|
|
220
|
+
parsed = EMPTY_RECORD
|
|
221
|
+
}
|
|
222
|
+
if (cacheable && recordParseCache.size < RECORD_PARSE_CACHE_LIMIT) {
|
|
223
|
+
recordParseCache.set(value, parsed)
|
|
224
|
+
}
|
|
225
|
+
return parsed
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const parseEvents = (value: string): readonly TraceSpanEvent[] => {
|
|
229
|
+
if (value === "" || value === "[]") return []
|
|
230
|
+
try {
|
|
231
|
+
const parsed = JSON.parse(value) as Array<{ name: string; timestamp: number; attributes: Record<string, string> }>
|
|
232
|
+
if (parsed.length === 0) return []
|
|
233
|
+
return parsed.map((event) => ({
|
|
234
|
+
name: event.name,
|
|
235
|
+
timestamp: new Date(event.timestamp),
|
|
236
|
+
attributes: event.attributes,
|
|
237
|
+
}))
|
|
238
|
+
} catch {
|
|
239
|
+
return []
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const parseSpanRow = (row: SpanRow): InternalTraceSpanItem => {
|
|
244
|
+
const isRunning = isSpanRunning(row.start_time_ms, row.end_time_ms)
|
|
245
|
+
return {
|
|
246
|
+
spanId: row.span_id,
|
|
247
|
+
parentSpanId: row.parent_span_id,
|
|
248
|
+
serviceName: row.service_name,
|
|
249
|
+
scopeName: row.scope_name,
|
|
250
|
+
kind: row.kind,
|
|
251
|
+
operationName: row.operation_name,
|
|
252
|
+
startTime: new Date(row.start_time_ms),
|
|
253
|
+
isRunning,
|
|
254
|
+
durationMs: liveDurationMs(row.start_time_ms, row.end_time_ms, isRunning),
|
|
255
|
+
status: row.status === "error" ? "error" : "ok",
|
|
256
|
+
depth: 0,
|
|
257
|
+
tags: {
|
|
258
|
+
...parseRecord(row.resource_json),
|
|
259
|
+
...parseRecord(row.attributes_json),
|
|
260
|
+
},
|
|
261
|
+
warnings: [],
|
|
262
|
+
events: parseEvents(row.events_json),
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const parseLogRow = (row: LogRow): LogItem => ({
|
|
267
|
+
id: String(row.id),
|
|
268
|
+
timestamp: new Date(row.timestamp_ms),
|
|
269
|
+
serviceName: row.service_name,
|
|
270
|
+
severityText: row.severity_text,
|
|
271
|
+
body: row.body,
|
|
272
|
+
traceId: row.trace_id,
|
|
273
|
+
spanId: row.span_id,
|
|
274
|
+
scopeName: row.scope_name,
|
|
275
|
+
attributes: {
|
|
276
|
+
...parseRecord(row.resource_json),
|
|
277
|
+
...parseRecord(row.attributes_json),
|
|
278
|
+
},
|
|
279
|
+
})
|
|
280
|
+
|
|
281
|
+
const orderTraceSpans = (spans: readonly InternalTraceSpanItem[]) => {
|
|
282
|
+
const childrenByParent = new Map<string | null, InternalTraceSpanItem[]>()
|
|
283
|
+
const spanIds = new Set(spans.map((span) => span.spanId))
|
|
284
|
+
|
|
285
|
+
for (const span of spans) {
|
|
286
|
+
const key = span.parentSpanId && spanIds.has(span.parentSpanId) ? span.parentSpanId : null
|
|
287
|
+
const siblings = childrenByParent.get(key) ?? []
|
|
288
|
+
siblings.push(span)
|
|
289
|
+
childrenByParent.set(key, siblings)
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
for (const siblings of childrenByParent.values()) {
|
|
293
|
+
siblings.sort((left, right) =>
|
|
294
|
+
left.startTime.getTime() - right.startTime.getTime() || Number(Boolean(left.syntheticMissingParent)) - Number(Boolean(right.syntheticMissingParent))
|
|
295
|
+
)
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const ordered: Array<InternalTraceSpanItem> = []
|
|
299
|
+
const visit = (parent: string | null, depth: number) => {
|
|
300
|
+
for (const child of childrenByParent.get(parent) ?? []) {
|
|
301
|
+
ordered.push({ ...child, depth })
|
|
302
|
+
visit(child.spanId, depth + 1)
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
visit(null, 0)
|
|
307
|
+
return ordered
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const buildTrace = (traceId: string, spanRows: readonly SpanRow[]): TraceItem => {
|
|
311
|
+
const parsedSpans = spanRows.map(parseSpanRow)
|
|
312
|
+
const spanIds = new Set(parsedSpans.map((span) => span.spanId))
|
|
313
|
+
const missingParentGroups = new Map<string, InternalTraceSpanItem[]>()
|
|
314
|
+
|
|
315
|
+
for (const span of parsedSpans) {
|
|
316
|
+
if (span.parentSpanId !== null && !spanIds.has(span.parentSpanId)) {
|
|
317
|
+
const siblings = missingParentGroups.get(span.parentSpanId) ?? []
|
|
318
|
+
siblings.push(span)
|
|
319
|
+
missingParentGroups.set(span.parentSpanId, siblings)
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
const syntheticParents: InternalTraceSpanItem[] = [...missingParentGroups.entries()].map(([missingParentId, children]) => {
|
|
324
|
+
const firstChild = children[0]!
|
|
325
|
+
const startedAtMs = Math.min(...children.map((child) => child.startTime.getTime()))
|
|
326
|
+
const endedAtMs = Math.max(...children.map((child) => child.startTime.getTime() + child.durationMs))
|
|
327
|
+
return {
|
|
328
|
+
spanId: missingParentId,
|
|
329
|
+
parentSpanId: null,
|
|
330
|
+
serviceName: firstChild.serviceName,
|
|
331
|
+
scopeName: null,
|
|
332
|
+
kind: null,
|
|
333
|
+
operationName: `[missing parent ${missingParentId.slice(0, 8)}]`,
|
|
334
|
+
startTime: new Date(startedAtMs),
|
|
335
|
+
isRunning: children.some((child) => child.isRunning),
|
|
336
|
+
durationMs: Math.max(0, endedAtMs - startedAtMs),
|
|
337
|
+
status: "error",
|
|
338
|
+
depth: 0,
|
|
339
|
+
tags: {},
|
|
340
|
+
warnings: [`missing span ${missingParentId} (${children.length} child${children.length === 1 ? "" : "ren"})`],
|
|
341
|
+
events: [],
|
|
342
|
+
syntheticMissingParent: true,
|
|
343
|
+
}
|
|
344
|
+
})
|
|
345
|
+
|
|
346
|
+
const orderedSpans = orderTraceSpans([...parsedSpans, ...syntheticParents])
|
|
347
|
+
const startedAtMs = Math.min(...orderedSpans.map((span) => span.startTime.getTime()))
|
|
348
|
+
const endedAtMs = Math.max(...orderedSpans.map((span) => span.startTime.getTime() + span.durationMs))
|
|
349
|
+
const isRunning = orderedSpans.some((span) => span.isRunning)
|
|
350
|
+
const rootSpan = orderedSpans.find((span) => !span.syntheticMissingParent && span.parentSpanId === null)
|
|
351
|
+
?? orderedSpans.find((span) => !span.syntheticMissingParent)
|
|
352
|
+
?? orderedSpans[0]
|
|
353
|
+
?? null
|
|
354
|
+
const warnings = syntheticParents.map((span) => span.warnings[0]!).filter((warning) => warning.length > 0)
|
|
355
|
+
|
|
356
|
+
return {
|
|
357
|
+
traceId,
|
|
358
|
+
serviceName: rootSpan?.serviceName ?? "unknown",
|
|
359
|
+
rootOperationName: rootSpan?.operationName ?? "unknown",
|
|
360
|
+
startedAt: new Date(startedAtMs),
|
|
361
|
+
isRunning,
|
|
362
|
+
durationMs: Math.max(0, endedAtMs - startedAtMs),
|
|
363
|
+
spanCount: orderedSpans.length,
|
|
364
|
+
errorCount: orderedSpans.filter((span) => span.status === "error").length,
|
|
365
|
+
warnings,
|
|
366
|
+
spans: orderedSpans.map(({ syntheticMissingParent: _, ...span }) => span),
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
const buildSpanItems = (traceId: string, spanRows: readonly SpanRow[]): readonly SpanItem[] => {
|
|
371
|
+
const trace = buildTrace(traceId, spanRows)
|
|
372
|
+
const spanById = new Map(trace.spans.map((span) => [span.spanId, span]))
|
|
373
|
+
return trace.spans.map((span) => ({
|
|
374
|
+
traceId,
|
|
375
|
+
rootOperationName: trace.rootOperationName,
|
|
376
|
+
parentOperationName: span.parentSpanId ? spanById.get(span.parentSpanId)?.operationName ?? null : null,
|
|
377
|
+
span,
|
|
378
|
+
}))
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
const buildSpanItem = (traceId: string, spanRows: readonly SpanRow[], spanId: string): SpanItem | null =>
|
|
382
|
+
buildSpanItems(traceId, spanRows).find((item) => item.span.spanId === spanId) ?? null
|
|
383
|
+
|
|
384
|
+
const matchesAttributes = (attributes: Readonly<Record<string, string>>, filters: Readonly<Record<string, string>> | undefined) =>
|
|
385
|
+
!filters || Object.entries(filters).every(([key, value]) => attributes[key] === value)
|
|
386
|
+
|
|
387
|
+
const matchesAttributeContains = (attributes: Readonly<Record<string, string>>, filters: Readonly<Record<string, string>> | undefined) =>
|
|
388
|
+
!filters || Object.entries(filters).every(([key, needle]) => {
|
|
389
|
+
const value = attributes[key]
|
|
390
|
+
return value !== undefined && value.toLowerCase().includes(needle.toLowerCase())
|
|
391
|
+
})
|
|
392
|
+
|
|
393
|
+
const percentile = (values: readonly number[], ratio: number) => {
|
|
394
|
+
if (values.length === 0) return 0
|
|
395
|
+
const sorted = [...values].sort((left, right) => left - right)
|
|
396
|
+
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * ratio) - 1))
|
|
397
|
+
return sorted[index] ?? 0
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
const tokenizeFts = (value: string) => value.match(/[A-Za-z0-9_]+/g)?.filter((token) => token.length > 1) ?? []
|
|
401
|
+
|
|
402
|
+
const toFtsMatchQuery = (value: string) => {
|
|
403
|
+
const tokens = tokenizeFts(value)
|
|
404
|
+
if (tokens.length === 0) return null
|
|
405
|
+
return tokens.map((token) => `${token}*`).join(" AND ")
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
const buildExactAttributeMatchSubquery = (
|
|
409
|
+
tableName: "span_attributes" | "log_attributes",
|
|
410
|
+
idColumns: readonly string[],
|
|
411
|
+
filters: Readonly<Record<string, string>> | undefined,
|
|
412
|
+
) => {
|
|
413
|
+
const entries = Object.entries(filters ?? {})
|
|
414
|
+
if (entries.length === 0) return null
|
|
415
|
+
const disjunction = entries.map(() => "(key = ? AND value = ?)").join(" OR ")
|
|
416
|
+
return {
|
|
417
|
+
sql: `
|
|
418
|
+
SELECT ${idColumns.join(", ")}
|
|
419
|
+
FROM ${tableName}
|
|
420
|
+
WHERE ${disjunction}
|
|
421
|
+
GROUP BY ${idColumns.join(", ")}
|
|
422
|
+
HAVING COUNT(DISTINCT key) = ${entries.length}
|
|
423
|
+
`,
|
|
424
|
+
params: entries.flatMap(([key, value]) => [key, value]),
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
const buildContainsAttributeMatchSubquery = (
|
|
429
|
+
tableName: "span_attributes" | "log_attributes",
|
|
430
|
+
idColumns: readonly string[],
|
|
431
|
+
filters: Readonly<Record<string, string>> | undefined,
|
|
432
|
+
) => {
|
|
433
|
+
const entries = Object.entries(filters ?? {})
|
|
434
|
+
if (entries.length === 0) return null
|
|
435
|
+
const disjunction = entries.map(() => "(key = ? AND value LIKE ? COLLATE NOCASE)").join(" OR ")
|
|
436
|
+
return {
|
|
437
|
+
sql: `
|
|
438
|
+
SELECT ${idColumns.join(", ")}
|
|
439
|
+
FROM ${tableName}
|
|
440
|
+
WHERE ${disjunction}
|
|
441
|
+
GROUP BY ${idColumns.join(", ")}
|
|
442
|
+
HAVING COUNT(DISTINCT key) = ${entries.length}
|
|
443
|
+
`,
|
|
444
|
+
params: entries.flatMap(([key, value]) => [key, `%${value}%`]),
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Read-only surface of the telemetry store. Pulled out so a readonly
|
|
449
|
+
// SQLite connection (TUI / HTTP query handlers) can be expressed as a
|
|
450
|
+
// distinct service identifier from the writer, without re-declaring
|
|
451
|
+
// every query in a wrapper layer. The writer's value still satisfies
|
|
452
|
+
// this shape — TelemetryStoreLive can provide both identifiers from
|
|
453
|
+
// one underlying object if needed.
|
|
454
|
+
export interface TelemetryStoreReader {
|
|
455
|
+
readonly listServices: Effect.Effect<readonly string[], Error>
|
|
456
|
+
readonly listRecentTraces: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceItem[], Error>
|
|
457
|
+
readonly listTraceSummaries: (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) => Effect.Effect<readonly TraceSummaryItem[], Error>
|
|
458
|
+
readonly searchTraces: (input: TraceSearch) => Effect.Effect<readonly TraceItem[], Error>
|
|
459
|
+
readonly searchTraceSummaries: (input: TraceSearch) => Effect.Effect<readonly TraceSummaryItem[], Error>
|
|
460
|
+
readonly traceStats: (input: TraceStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
|
|
461
|
+
readonly getTrace: (traceId: string) => Effect.Effect<TraceItem | null, Error>
|
|
462
|
+
readonly getSpan: (spanId: string) => Effect.Effect<SpanItem | null, Error>
|
|
463
|
+
readonly listTraceSpans: (traceId: string) => Effect.Effect<readonly SpanItem[], Error>
|
|
464
|
+
readonly searchSpans: (input: SpanSearch) => Effect.Effect<readonly SpanItem[], Error>
|
|
465
|
+
readonly searchLogs: (input: LogSearch) => Effect.Effect<readonly LogItem[], Error>
|
|
466
|
+
readonly logStats: (input: LogStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
|
|
467
|
+
readonly listFacets: (input: FacetSearch) => Effect.Effect<readonly FacetItem[], Error>
|
|
468
|
+
readonly listRecentLogs: (serviceName: string) => Effect.Effect<readonly LogItem[], Error>
|
|
469
|
+
readonly listTraceLogs: (traceId: string) => Effect.Effect<readonly LogItem[], Error>
|
|
470
|
+
readonly searchAiCalls: (input: AiCallSearch) => Effect.Effect<readonly AiCallSummary[], Error>
|
|
471
|
+
readonly getAiCall: (spanId: string) => Effect.Effect<AiCallDetail | null, Error>
|
|
472
|
+
readonly aiCallStats: (input: AiCallStatsSearch) => Effect.Effect<readonly StatsItem[], Error>
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
export class TelemetryStoreReadonly extends Context.Service<TelemetryStoreReadonly, TelemetryStoreReader>()("motel/TelemetryStoreReadonly") {}
|
|
476
|
+
|
|
477
|
+
export class TelemetryStore extends Context.Service<
|
|
478
|
+
TelemetryStore,
|
|
479
|
+
TelemetryStoreReader & {
|
|
480
|
+
readonly ingestTraces: (payload: OtlpTraceExportRequest) => Effect.Effect<{ readonly insertedSpans: number }, Error>
|
|
481
|
+
readonly ingestLogs: (payload: OtlpLogExportRequest) => Effect.Effect<{ readonly insertedLogs: number }, Error>
|
|
482
|
+
readonly runRetentionNow: Effect.Effect<void, Error>
|
|
483
|
+
}
|
|
484
|
+
>()("motel/TelemetryStore") {}
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
/**
|
|
488
|
+
* How this TelemetryStore instance behaves:
|
|
489
|
+
*
|
|
490
|
+
* - `readonly` — opens the SQLite connection read-only and skips every
|
|
491
|
+
* DDL/DML initialisation. Use this from the TUI (and anywhere else
|
|
492
|
+
* that only queries); it avoids the "database is locked" race that
|
|
493
|
+
* happens when a TUI process races a daemon's writer for the schema
|
|
494
|
+
* pragmas on startup. Writes through the service interface become
|
|
495
|
+
* runtime errors — but readers don't call them.
|
|
496
|
+
*
|
|
497
|
+
* - `runRetention` — fork the background cleanup loop (age + size cap
|
|
498
|
+
* eviction, WAL checkpoint). Only one process should own this at a
|
|
499
|
+
* time. The ingest worker owns it; the HTTP thread and TUI skip it.
|
|
500
|
+
*/
|
|
501
|
+
export interface TelemetryStoreOptions {
|
|
502
|
+
readonly readonly: boolean
|
|
503
|
+
readonly runRetention: boolean
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
const makeTelemetryStoreEffect = (opts: TelemetryStoreOptions) =>
|
|
507
|
+
Effect.gen(function* () {
|
|
508
|
+
const fileSystem = yield* FileSystem.FileSystem
|
|
509
|
+
yield* fileSystem.makeDirectory(dirname(config.otel.databasePath), { recursive: true })
|
|
510
|
+
const db = yield* Effect.acquireRelease(
|
|
511
|
+
Effect.sync(() => new Database(config.otel.databasePath, {
|
|
512
|
+
create: !opts.readonly,
|
|
513
|
+
readonly: opts.readonly,
|
|
514
|
+
})),
|
|
515
|
+
(db) => Effect.sync(() => {
|
|
516
|
+
if (!opts.readonly) {
|
|
517
|
+
// `PRAGMA optimize` at close persists any stats SQLite gathered
|
|
518
|
+
// during the session, so the next process start gets an accurate
|
|
519
|
+
// query planner on the first query instead of a 3-second cold
|
|
520
|
+
// run. Cheap: it skips work unless stats have drifted.
|
|
521
|
+
try { db.exec(`PRAGMA optimize;`) } catch { /* nothing */ }
|
|
522
|
+
}
|
|
523
|
+
db.close()
|
|
524
|
+
}),
|
|
525
|
+
)
|
|
526
|
+
if (opts.readonly) {
|
|
527
|
+
// Readonly connections skip schema init entirely — the schema
|
|
528
|
+
// already exists (a writer created it) and any `CREATE TABLE IF
|
|
529
|
+
// NOT EXISTS` / `PRAGMA journal_mode = WAL` statement would
|
|
530
|
+
// attempt a write and fight the daemon for the write lock.
|
|
531
|
+
// `query_only = 1` logically blocks any DML the app might
|
|
532
|
+
// accidentally send; still bump cache + mmap since those are
|
|
533
|
+
// safe and keep queries fast.
|
|
534
|
+
db.exec(`
|
|
535
|
+
PRAGMA query_only = 1;
|
|
536
|
+
PRAGMA busy_timeout = 15000;
|
|
537
|
+
PRAGMA cache_size = -65536;
|
|
538
|
+
PRAGMA mmap_size = 268435456;
|
|
539
|
+
`)
|
|
540
|
+
} else {
|
|
541
|
+
db.exec(`
|
|
542
|
+
-- Bump cache above the 2MB default. 64MB fits most hot index pages
|
|
543
|
+
-- (trace_summaries, spans, span_attributes indexes) in RAM even on
|
|
544
|
+
-- multi-GB databases, cutting cold-read latency meaningfully on
|
|
545
|
+
-- picker / search queries that sweep the index.
|
|
546
|
+
PRAGMA cache_size = -65536;
|
|
547
|
+
-- Let SQLite memory-map the first 256MB of the file. This is a
|
|
548
|
+
-- cheap way to avoid read() syscalls on hot pages and lets the OS
|
|
549
|
+
-- page cache serve index lookups directly. Safe on macOS and Linux;
|
|
550
|
+
-- SQLite silently caps at actual file size for smaller DBs.
|
|
551
|
+
PRAGMA mmap_size = 268435456;
|
|
552
|
+
`)
|
|
553
|
+
// auto_vacuum is a header-level setting: it only takes effect on
|
|
554
|
+
// an empty DB, or on the next VACUUM after a change. Setting it
|
|
555
|
+
// here, BEFORE the first CREATE TABLE, is the only path that
|
|
556
|
+
// makes incremental_vacuum work without a full VACUUM. For
|
|
557
|
+
// existing DBs that predate this setting keep their current mode;
|
|
558
|
+
// Motel never performs a surprise full-file VACUUM at startup.
|
|
559
|
+
try { db.exec(`PRAGMA auto_vacuum = INCREMENTAL;`) } catch { /* ignore */ }
|
|
560
|
+
try {
|
|
561
|
+
db.exec(`
|
|
562
|
+
PRAGMA journal_mode = WAL;
|
|
563
|
+
PRAGMA synchronous = NORMAL;
|
|
564
|
+
PRAGMA temp_store = MEMORY;
|
|
565
|
+
-- WAL checkpoint automatically when it grows past ~16MB. Without
|
|
566
|
+
-- this the WAL happily runs into the hundreds of MB and queries
|
|
567
|
+
-- start paying the cost of walking the WAL on every read.
|
|
568
|
+
PRAGMA wal_autocheckpoint = 4000;
|
|
569
|
+
-- Hard floor for the WAL file. Auto-checkpoint controls *when*
|
|
570
|
+
-- pages move out of the WAL; size_limit controls how much the
|
|
571
|
+
-- WAL file is allowed to grow on disk. 128MB is generous enough
|
|
572
|
+
-- to absorb a long write burst without blocking on truncation,
|
|
573
|
+
-- tight enough that a wedged retention loop can't hide a 20GB
|
|
574
|
+
-- WAL the way a default no-limit configuration can.
|
|
575
|
+
PRAGMA journal_size_limit = 134217728;
|
|
576
|
+
|
|
577
|
+
CREATE TABLE IF NOT EXISTS spans (
|
|
578
|
+
trace_id TEXT NOT NULL,
|
|
579
|
+
span_id TEXT NOT NULL,
|
|
580
|
+
parent_span_id TEXT,
|
|
581
|
+
service_name TEXT NOT NULL,
|
|
582
|
+
scope_name TEXT,
|
|
583
|
+
operation_name TEXT NOT NULL,
|
|
584
|
+
kind TEXT,
|
|
585
|
+
start_time_ms INTEGER NOT NULL,
|
|
586
|
+
end_time_ms INTEGER NOT NULL,
|
|
587
|
+
duration_ms REAL NOT NULL,
|
|
588
|
+
status TEXT NOT NULL,
|
|
589
|
+
attributes_json TEXT NOT NULL,
|
|
590
|
+
resource_json TEXT NOT NULL,
|
|
591
|
+
events_json TEXT NOT NULL,
|
|
592
|
+
PRIMARY KEY (trace_id, span_id)
|
|
593
|
+
);
|
|
594
|
+
|
|
595
|
+
CREATE INDEX IF NOT EXISTS idx_spans_service_time ON spans(service_name, start_time_ms DESC);
|
|
596
|
+
CREATE INDEX IF NOT EXISTS idx_spans_trace_time ON spans(trace_id, start_time_ms ASC);
|
|
597
|
+
CREATE INDEX IF NOT EXISTS idx_spans_span_id ON spans(span_id);
|
|
598
|
+
CREATE INDEX IF NOT EXISTS idx_spans_status_time ON spans(status, start_time_ms DESC);
|
|
599
|
+
|
|
600
|
+
CREATE TABLE IF NOT EXISTS logs (
|
|
601
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
602
|
+
trace_id TEXT,
|
|
603
|
+
span_id TEXT,
|
|
604
|
+
service_name TEXT NOT NULL,
|
|
605
|
+
scope_name TEXT,
|
|
606
|
+
severity_text TEXT NOT NULL,
|
|
607
|
+
timestamp_ms INTEGER NOT NULL,
|
|
608
|
+
body TEXT NOT NULL,
|
|
609
|
+
attributes_json TEXT NOT NULL,
|
|
610
|
+
resource_json TEXT NOT NULL
|
|
611
|
+
);
|
|
612
|
+
|
|
613
|
+
CREATE INDEX IF NOT EXISTS idx_logs_service_time ON logs(service_name, timestamp_ms DESC);
|
|
614
|
+
CREATE INDEX IF NOT EXISTS idx_logs_trace_time ON logs(trace_id, timestamp_ms DESC);
|
|
615
|
+
CREATE INDEX IF NOT EXISTS idx_logs_span_time ON logs(span_id, timestamp_ms DESC);
|
|
616
|
+
CREATE INDEX IF NOT EXISTS idx_logs_severity_time ON logs(severity_text, timestamp_ms DESC);
|
|
617
|
+
|
|
618
|
+
CREATE TABLE IF NOT EXISTS trace_summaries (
|
|
619
|
+
trace_id TEXT PRIMARY KEY,
|
|
620
|
+
service_name TEXT NOT NULL,
|
|
621
|
+
root_operation_name TEXT NOT NULL,
|
|
622
|
+
started_at_ms INTEGER NOT NULL,
|
|
623
|
+
ended_at_ms INTEGER NOT NULL,
|
|
624
|
+
active_span_count INTEGER NOT NULL DEFAULT 0,
|
|
625
|
+
duration_ms REAL NOT NULL,
|
|
626
|
+
span_count INTEGER NOT NULL,
|
|
627
|
+
error_count INTEGER NOT NULL
|
|
628
|
+
);
|
|
629
|
+
|
|
630
|
+
CREATE INDEX IF NOT EXISTS idx_trace_summaries_started_at ON trace_summaries(started_at_ms DESC, trace_id DESC);
|
|
631
|
+
CREATE INDEX IF NOT EXISTS idx_trace_summaries_service_started_at ON trace_summaries(service_name, started_at_ms DESC, trace_id DESC);
|
|
632
|
+
CREATE INDEX IF NOT EXISTS idx_trace_summaries_duration ON trace_summaries(duration_ms DESC);
|
|
633
|
+
|
|
634
|
+
CREATE TABLE IF NOT EXISTS span_attributes (
|
|
635
|
+
trace_id TEXT NOT NULL,
|
|
636
|
+
span_id TEXT NOT NULL,
|
|
637
|
+
key TEXT NOT NULL,
|
|
638
|
+
value TEXT NOT NULL,
|
|
639
|
+
PRIMARY KEY (trace_id, span_id, key)
|
|
640
|
+
);
|
|
641
|
+
|
|
642
|
+
CREATE INDEX IF NOT EXISTS idx_span_attributes_key_value ON span_attributes(key, value, trace_id, span_id);
|
|
643
|
+
CREATE INDEX IF NOT EXISTS idx_span_attributes_trace_span ON span_attributes(trace_id, span_id);
|
|
644
|
+
|
|
645
|
+
CREATE TABLE IF NOT EXISTS log_attributes (
|
|
646
|
+
log_id INTEGER NOT NULL,
|
|
647
|
+
key TEXT NOT NULL,
|
|
648
|
+
value TEXT NOT NULL,
|
|
649
|
+
PRIMARY KEY (log_id, key)
|
|
650
|
+
);
|
|
651
|
+
|
|
652
|
+
CREATE INDEX IF NOT EXISTS idx_log_attributes_key_value ON log_attributes(key, value, log_id);
|
|
653
|
+
CREATE INDEX IF NOT EXISTS idx_log_attributes_log_id ON log_attributes(log_id);
|
|
654
|
+
|
|
655
|
+
CREATE TABLE IF NOT EXISTS motel_maintenance (
|
|
656
|
+
key TEXT PRIMARY KEY,
|
|
657
|
+
value TEXT NOT NULL
|
|
658
|
+
);
|
|
659
|
+
`)
|
|
660
|
+
} catch (err) {
|
|
661
|
+
if (!isSqliteLockError(err)) throw err
|
|
662
|
+
console.warn(`motel: writer bootstrap skipped during startup: ${(err as Error).message}`)
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
// Tables detected at runtime. For writer connections these flags are
|
|
667
|
+
// set by the FTS `CREATE VIRTUAL TABLE IF NOT EXISTS` try/catch; for
|
|
668
|
+
// readonly connections we probe `sqlite_master` and set them based on
|
|
669
|
+
// what the writer has already provisioned.
|
|
670
|
+
let hasFts = true
|
|
671
|
+
let hasAttrFts = true
|
|
672
|
+
if (opts.readonly) {
|
|
673
|
+
try {
|
|
674
|
+
const row = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='span_operation_fts'`).get()
|
|
675
|
+
hasFts = row !== null
|
|
676
|
+
} catch { hasFts = false }
|
|
677
|
+
try {
|
|
678
|
+
const row = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='span_attr_fts'`).get()
|
|
679
|
+
const backfill = db.query(`SELECT value FROM motel_maintenance WHERE key = 'span_attr_fts_v1'`).get() as { value: string } | null
|
|
680
|
+
hasAttrFts = row !== null && backfill?.value === "complete"
|
|
681
|
+
} catch { hasAttrFts = false }
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
if (!opts.readonly) {
|
|
685
|
+
try {
|
|
686
|
+
db.exec(`
|
|
687
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS span_operation_fts USING fts5(
|
|
688
|
+
trace_id UNINDEXED,
|
|
689
|
+
span_id UNINDEXED,
|
|
690
|
+
operation_name,
|
|
691
|
+
tokenize='unicode61'
|
|
692
|
+
);
|
|
693
|
+
|
|
694
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS log_body_fts USING fts5(
|
|
695
|
+
log_id UNINDEXED,
|
|
696
|
+
body,
|
|
697
|
+
tokenize='unicode61'
|
|
698
|
+
);
|
|
699
|
+
`)
|
|
700
|
+
} catch {
|
|
701
|
+
hasFts = false
|
|
702
|
+
// FTS is optional; queries will fall back to LIKE if unavailable.
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// External-content FTS5 over the subset of span_attributes.value rows
|
|
706
|
+
// whose key is in AI_FTS_KEYS (LLM prompts, responses, tool calls,
|
|
707
|
+
// etc.). External content means the inverted index is the only
|
|
708
|
+
// FTS storage — the value text itself continues to live once in
|
|
709
|
+
// span_attributes, not duplicated into the FTS table. On a 2 GB DB
|
|
710
|
+
// with 270 MB of prompt JSON this typically adds ~50-120 MB of
|
|
711
|
+
// index, turning a 500-800ms LIKE scan into a <50ms MATCH.
|
|
712
|
+
//
|
|
713
|
+
// Keys are inlined into the trigger DDL rather than looked up in a
|
|
714
|
+
// side table so the `WHEN` guard stays constant-cost (a subquery
|
|
715
|
+
// would run on every span_attributes insert — ~60/span).
|
|
716
|
+
if (hasFts) {
|
|
717
|
+
try {
|
|
718
|
+
const keyList = AI_FTS_KEYS.map((k) => `'${k.replace(/'/g, "''")}'`).join(", ")
|
|
719
|
+
db.exec(`
|
|
720
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS span_attr_fts USING fts5(
|
|
721
|
+
value,
|
|
722
|
+
content='span_attributes',
|
|
723
|
+
content_rowid='rowid',
|
|
724
|
+
tokenize='unicode61 remove_diacritics 2'
|
|
725
|
+
);
|
|
726
|
+
|
|
727
|
+
-- Mirror inserts into FTS when the key carries LLM content.
|
|
728
|
+
-- NOTE: triggers MUST use fully-qualified name (new.rowid,
|
|
729
|
+
-- new.value) and emit rowid so external-content FTS can
|
|
730
|
+
-- fetch the value back via span_attributes.rowid.
|
|
731
|
+
CREATE TRIGGER IF NOT EXISTS span_attr_fts_ai AFTER INSERT ON span_attributes
|
|
732
|
+
WHEN new.key IN (${keyList})
|
|
733
|
+
BEGIN
|
|
734
|
+
INSERT INTO span_attr_fts(rowid, value) VALUES (new.rowid, new.value);
|
|
735
|
+
END;
|
|
736
|
+
|
|
737
|
+
-- Delete with the same guard so retention & re-ingest stay
|
|
738
|
+
-- in sync. External-content 'delete' command needs the
|
|
739
|
+
-- original value to remove from the inverted index.
|
|
740
|
+
CREATE TRIGGER IF NOT EXISTS span_attr_fts_ad AFTER DELETE ON span_attributes
|
|
741
|
+
WHEN old.key IN (${keyList})
|
|
742
|
+
BEGIN
|
|
743
|
+
INSERT INTO span_attr_fts(span_attr_fts, rowid, value)
|
|
744
|
+
VALUES ('delete', old.rowid, old.value);
|
|
745
|
+
END;
|
|
746
|
+
|
|
747
|
+
-- Handle in-place updates (rare; re-ingest usually goes
|
|
748
|
+
-- DELETE then INSERT but belt-and-braces).
|
|
749
|
+
CREATE TRIGGER IF NOT EXISTS span_attr_fts_au AFTER UPDATE ON span_attributes
|
|
750
|
+
WHEN old.key IN (${keyList}) OR new.key IN (${keyList})
|
|
751
|
+
BEGIN
|
|
752
|
+
INSERT INTO span_attr_fts(span_attr_fts, rowid, value)
|
|
753
|
+
VALUES ('delete', old.rowid, old.value);
|
|
754
|
+
INSERT INTO span_attr_fts(rowid, value)
|
|
755
|
+
SELECT new.rowid, new.value
|
|
756
|
+
WHERE new.key IN (${keyList});
|
|
757
|
+
END;
|
|
758
|
+
`)
|
|
759
|
+
} catch {
|
|
760
|
+
hasAttrFts = false
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
try {
|
|
765
|
+
db.exec(`ALTER TABLE trace_summaries ADD COLUMN active_span_count INTEGER NOT NULL DEFAULT 0`)
|
|
766
|
+
} catch {
|
|
767
|
+
// Existing databases may already have the column.
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// Prime the query planner. `PRAGMA optimize` is SQLite's modern,
|
|
771
|
+
// lightweight stats refresh: it only re-ANALYZEs indexes whose row
|
|
772
|
+
// counts have drifted significantly since the last run, capped at
|
|
773
|
+
// `analysis_limit` iterations per index so it finishes in a
|
|
774
|
+
// bounded time even on large databases. Without this, queries like
|
|
775
|
+
// the attribute picker facet run with guessed row estimates and
|
|
776
|
+
// pay 3-4s on cold open instead of 400ms.
|
|
777
|
+
try {
|
|
778
|
+
db.exec(`PRAGMA analysis_limit = 1000; PRAGMA optimize;`)
|
|
779
|
+
} catch {
|
|
780
|
+
// ANALYZE / optimize failures are never fatal — queries still work,
|
|
781
|
+
// they just run with default row estimates.
|
|
782
|
+
}
|
|
783
|
+
// Longer busy timeout: the ingest worker holds the write lock for up
|
|
784
|
+
// to a few seconds during big OTLP batches, and the daemon's retention
|
|
785
|
+
// passes can do the same. Apply this AFTER startup maintenance so
|
|
786
|
+
// lock-conflicted bootstrap steps fail fast instead of stalling health
|
|
787
|
+
// for the full 15s timeout.
|
|
788
|
+
try { db.exec(`PRAGMA busy_timeout = 15000;`) } catch { /* ignore */ }
|
|
789
|
+
} // end: if (!opts.readonly) writer init
|
|
790
|
+
|
|
791
|
+
const insertSpan = db.query(`
|
|
792
|
+
INSERT INTO spans (
|
|
793
|
+
trace_id, span_id, parent_span_id, service_name, scope_name, operation_name, kind,
|
|
794
|
+
start_time_ms, end_time_ms, duration_ms, status, attributes_json, resource_json, events_json
|
|
795
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
796
|
+
ON CONFLICT(trace_id, span_id) DO UPDATE SET
|
|
797
|
+
parent_span_id = excluded.parent_span_id,
|
|
798
|
+
service_name = excluded.service_name,
|
|
799
|
+
scope_name = excluded.scope_name,
|
|
800
|
+
operation_name = excluded.operation_name,
|
|
801
|
+
kind = excluded.kind,
|
|
802
|
+
start_time_ms = excluded.start_time_ms,
|
|
803
|
+
end_time_ms = excluded.end_time_ms,
|
|
804
|
+
duration_ms = excluded.duration_ms,
|
|
805
|
+
status = excluded.status,
|
|
806
|
+
attributes_json = excluded.attributes_json,
|
|
807
|
+
resource_json = excluded.resource_json,
|
|
808
|
+
events_json = excluded.events_json
|
|
809
|
+
`)
|
|
810
|
+
|
|
811
|
+
const insertLog = db.query(`
|
|
812
|
+
INSERT INTO logs (
|
|
813
|
+
trace_id, span_id, service_name, scope_name, severity_text, timestamp_ms, body, attributes_json, resource_json
|
|
814
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
815
|
+
`)
|
|
816
|
+
|
|
817
|
+
const upsertTraceSummary = db.query(`
|
|
818
|
+
INSERT OR REPLACE INTO trace_summaries (
|
|
819
|
+
trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
|
|
820
|
+
)
|
|
821
|
+
SELECT trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
|
|
822
|
+
FROM (
|
|
823
|
+
${TRACE_SUMMARY_SELECT_SQL}
|
|
824
|
+
WHERE trace_id = ?
|
|
825
|
+
GROUP BY trace_id
|
|
826
|
+
)
|
|
827
|
+
`)
|
|
828
|
+
|
|
829
|
+
const reconcileTraceSummaries = Effect.sync(() => {
|
|
830
|
+
const marker = db.query(`SELECT value FROM motel_maintenance WHERE key = 'trace_summary_cursor'`).get() as { value: string } | null
|
|
831
|
+
const cursor = Number(marker?.value ?? 0)
|
|
832
|
+
const rows = db.query(`SELECT rowid, trace_id FROM spans WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`).all(cursor, config.otel.retentionTraceBatch) as Array<{ rowid: number; trace_id: string }>
|
|
833
|
+
if (rows.length === 0) {
|
|
834
|
+
db.query(`INSERT OR REPLACE INTO motel_maintenance(key, value) VALUES ('trace_summary_cursor', '0')`).run()
|
|
835
|
+
return
|
|
836
|
+
}
|
|
837
|
+
const transaction = db.transaction(() => {
|
|
838
|
+
for (const traceId of new Set(rows.map((row) => row.trace_id))) upsertTraceSummary.run(traceId)
|
|
839
|
+
db.query(`INSERT OR REPLACE INTO motel_maintenance(key, value) VALUES ('trace_summary_cursor', ?)`).run(String(rows.at(-1)!.rowid))
|
|
840
|
+
})
|
|
841
|
+
transaction()
|
|
842
|
+
})
|
|
843
|
+
|
|
844
|
+
const deleteSpanAttributes = db.query(`DELETE FROM span_attributes WHERE trace_id = ? AND span_id = ?`)
|
|
845
|
+
const insertSpanAttribute = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES (?, ?, ?, ?)`)
|
|
846
|
+
const spanAttributeInsertManyByCount = new Map<number, ReturnType<Database["query"]>>()
|
|
847
|
+
const insertSpanAttributesMany = (traceId: string, spanId: string, attributes: Readonly<Record<string, string>>) => {
|
|
848
|
+
const entries = Object.entries(attributes)
|
|
849
|
+
if (entries.length === 0) return
|
|
850
|
+
if (entries.length === 1) {
|
|
851
|
+
const [key, value] = entries[0]!
|
|
852
|
+
insertSpanAttribute.run(traceId, spanId, key, value)
|
|
853
|
+
return
|
|
854
|
+
}
|
|
855
|
+
let query = spanAttributeInsertManyByCount.get(entries.length)
|
|
856
|
+
if (!query) {
|
|
857
|
+
query = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES ${entries.map(() => "(?, ?, ?, ?)").join(", ")}`)
|
|
858
|
+
spanAttributeInsertManyByCount.set(entries.length, query)
|
|
859
|
+
}
|
|
860
|
+
query.run(...entries.flatMap(([key, value]) => [traceId, spanId, key, value]))
|
|
861
|
+
}
|
|
862
|
+
const deleteSpanOperationSearch = db.query(`DELETE FROM span_operation_fts WHERE trace_id = ? AND span_id = ?`)
|
|
863
|
+
const insertSpanOperationSearch = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES (?, ?, ?)`)
|
|
864
|
+
const deleteSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
|
|
865
|
+
const insertSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
|
|
866
|
+
const updateSpanOperationSearchMany = (operations: ReadonlyArray<readonly [string, string, string]>) => {
|
|
867
|
+
if (operations.length === 0) return
|
|
868
|
+
if (operations.length === 1) {
|
|
869
|
+
const [traceId, spanId, operationName] = operations[0]!
|
|
870
|
+
deleteSpanOperationSearch.run(traceId, spanId)
|
|
871
|
+
insertSpanOperationSearch.run(traceId, spanId, operationName)
|
|
872
|
+
return
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
let deleteQuery = deleteSpanOperationSearchManyByCount.get(operations.length)
|
|
876
|
+
if (!deleteQuery) {
|
|
877
|
+
deleteQuery = db.query(`DELETE FROM span_operation_fts WHERE ${operations.map(() => "(trace_id = ? AND span_id = ?)").join(" OR ")}`)
|
|
878
|
+
deleteSpanOperationSearchManyByCount.set(operations.length, deleteQuery)
|
|
879
|
+
}
|
|
880
|
+
deleteQuery.run(...operations.flatMap(([traceId, spanId]) => [traceId, spanId]))
|
|
881
|
+
|
|
882
|
+
let insertQuery = insertSpanOperationSearchManyByCount.get(operations.length)
|
|
883
|
+
if (!insertQuery) {
|
|
884
|
+
insertQuery = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES ${operations.map(() => "(?, ?, ?)").join(", ")}`)
|
|
885
|
+
insertSpanOperationSearchManyByCount.set(operations.length, insertQuery)
|
|
886
|
+
}
|
|
887
|
+
insertQuery.run(...operations.flatMap(([traceId, spanId, operationName]) => [traceId, spanId, operationName]))
|
|
888
|
+
}
|
|
889
|
+
const insertLogAttribute = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES (?, ?, ?)`)
|
|
890
|
+
const logAttributeInsertManyByCount = new Map<number, ReturnType<Database["query"]>>()
|
|
891
|
+
const insertLogAttributesMany = (logId: number, attributes: Readonly<Record<string, string>>) => {
|
|
892
|
+
const entries = Object.entries(attributes)
|
|
893
|
+
if (entries.length === 0) return
|
|
894
|
+
if (entries.length === 1) {
|
|
895
|
+
const [key, value] = entries[0]!
|
|
896
|
+
insertLogAttribute.run(logId, key, value)
|
|
897
|
+
return
|
|
898
|
+
}
|
|
899
|
+
let query = logAttributeInsertManyByCount.get(entries.length)
|
|
900
|
+
if (!query) {
|
|
901
|
+
query = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES ${entries.map(() => "(?, ?, ?)").join(", ")}`)
|
|
902
|
+
logAttributeInsertManyByCount.set(entries.length, query)
|
|
903
|
+
}
|
|
904
|
+
query.run(...entries.flatMap(([key, value]) => [logId, key, value]))
|
|
905
|
+
}
|
|
906
|
+
const insertLogBodySearch = db.query(`INSERT INTO log_body_fts (log_id, body) VALUES (?, ?)`)
|
|
907
|
+
const insertLogBodySearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
|
|
908
|
+
const insertLogBodySearchMany = (entries: ReadonlyArray<readonly [string, string]>) => {
|
|
909
|
+
if (entries.length === 0) return
|
|
910
|
+
if (entries.length === 1) {
|
|
911
|
+
const [logId, body] = entries[0]!
|
|
912
|
+
insertLogBodySearch.run(logId, body)
|
|
913
|
+
return
|
|
914
|
+
}
|
|
915
|
+
let query = insertLogBodySearchManyByCount.get(entries.length)
|
|
916
|
+
if (!query) {
|
|
917
|
+
query = db.query(`INSERT INTO log_body_fts (log_id, body) VALUES ${entries.map(() => "(?, ?)").join(", ")}`)
|
|
918
|
+
insertLogBodySearchManyByCount.set(entries.length, query)
|
|
919
|
+
}
|
|
920
|
+
query.run(...entries.flatMap(([logId, body]) => [logId, body]))
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
const maxDbSizeBytes = config.otel.maxDbSizeMb * 1024 * 1024
|
|
924
|
+
|
|
925
|
+
// Freelist-ratio thresholds for the adaptive reclaim loop. Below the
|
|
926
|
+
// LOW threshold there's nothing worth doing; above HIGH we are in the
|
|
927
|
+
// 17GB-DB-with-10GB-freelist failure mode and need to reclaim aggressively
|
|
928
|
+
// even if it costs writer-lock time.
|
|
929
|
+
const FREELIST_LOW_RATIO = 0.05
|
|
930
|
+
const FREELIST_MID_RATIO = 0.20
|
|
931
|
+
const FREELIST_HIGH_RATIO = 0.50
|
|
932
|
+
const VACUUM_PAGES_NORMAL = 2000 // ~8MB/pass
|
|
933
|
+
const VACUUM_PAGES_BUSY = 20000 // ~80MB/pass — used when freelist > 20%
|
|
934
|
+
const VACUUM_PAGES_PANIC = 50000 // ~200MB/pass — only when ratio > 50%
|
|
935
|
+
|
|
936
|
+
const ftsTableNames = ["span_attr_fts", "log_body_fts", "span_operation_fts"] as const
|
|
937
|
+
|
|
938
|
+
const incrementalFtsMerge = (pages: number) => {
|
|
939
|
+
// FTS5 segment merges drop tombstone rows that DELETE leaves behind.
|
|
940
|
+
// Without periodic merges, deleted FTS rows stay on disk indefinitely
|
|
941
|
+
// — a major source of freelist pages on a heavy-deletion workload.
|
|
942
|
+
// `merge=N` is a bounded, online operation: it merges at most N
|
|
943
|
+
// pages of work and returns. Per FTS5 docs, missing tables silently
|
|
944
|
+
// throw; we swallow because not every DB has every FTS table.
|
|
945
|
+
for (const name of ftsTableNames) {
|
|
946
|
+
try { db.query(`INSERT INTO ${name}(${name}) VALUES (?)`).run(`merge=${pages}`) } catch { /* table absent or older schema */ }
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
const reclaimSpace = Effect.fn("motel/TelemetryStore.reclaimSpace")(function* () {
|
|
951
|
+
yield* Effect.sync(() => {
|
|
952
|
+
const pageCount = (db.query(`PRAGMA page_count`).get() as { page_count: number }).page_count
|
|
953
|
+
const freePages = (db.query(`PRAGMA freelist_count`).get() as { freelist_count: number }).freelist_count
|
|
954
|
+
if (pageCount === 0) return
|
|
955
|
+
const ratio = freePages / pageCount
|
|
956
|
+
if (ratio < FREELIST_LOW_RATIO) return
|
|
957
|
+
|
|
958
|
+
// Adaptive vacuum sizing — fixed 2000 pages/min could not keep
|
|
959
|
+
// up with sustained deletions, leaking 10GB of freelist over
|
|
960
|
+
// time. Scale the per-pass work to the size of the backlog so
|
|
961
|
+
// we stay roughly proportional to the deficit.
|
|
962
|
+
const pages =
|
|
963
|
+
ratio >= FREELIST_HIGH_RATIO ? VACUUM_PAGES_PANIC :
|
|
964
|
+
ratio >= FREELIST_MID_RATIO ? VACUUM_PAGES_BUSY :
|
|
965
|
+
VACUUM_PAGES_NORMAL
|
|
966
|
+
|
|
967
|
+
try { db.exec(`PRAGMA incremental_vacuum(${pages});`) } catch { /* ignore */ }
|
|
968
|
+
|
|
969
|
+
// In WAL mode incremental_vacuum only moves pages — the file
|
|
970
|
+
// shrinks on the next checkpoint. PASSIVE silently skips when
|
|
971
|
+
// readers are active (the failure mode the agent's research
|
|
972
|
+
// flagged: checkpoint starvation). Use RESTART normally and
|
|
973
|
+
// TRUNCATE in panic mode to physically shrink the WAL when it
|
|
974
|
+
// has grown.
|
|
975
|
+
const mode = ratio >= FREELIST_HIGH_RATIO ? "TRUNCATE" : "RESTART"
|
|
976
|
+
try { db.exec(`PRAGMA wal_checkpoint(${mode});`) } catch { /* ignore */ }
|
|
977
|
+
})
|
|
978
|
+
})
|
|
979
|
+
|
|
980
|
+
const cleanupExpired = Effect.fn("motel/TelemetryStore.cleanupExpired")(function* () {
|
|
981
|
+
const now = yield* Clock.currentTimeMillis
|
|
982
|
+
|
|
983
|
+
yield* Effect.sync(() => {
|
|
984
|
+
const cutoff = now - config.otel.retentionHours * 60 * 60 * 1000
|
|
985
|
+
|
|
986
|
+
// Evict at TRACE granularity so we never leave a trace half-gutted
|
|
987
|
+
// (previous logic deleted oldest 20% of spans, which happily sliced
|
|
988
|
+
// across traces and corrupted the summary rebuild). Running traces
|
|
989
|
+
// are protected — only `active_span_count = 0` summaries are in
|
|
990
|
+
// scope for eviction.
|
|
991
|
+
const toEvict = new Set<string>()
|
|
992
|
+
|
|
993
|
+
// Time-based: completed traces whose last span ended before cutoff.
|
|
994
|
+
const timeExpired = db.query(
|
|
995
|
+
`SELECT trace_id FROM trace_summaries WHERE active_span_count = 0 AND ended_at_ms > 0 AND ended_at_ms < ? ORDER BY ended_at_ms ASC LIMIT ?`,
|
|
996
|
+
).all(cutoff, config.otel.retentionTraceBatch) as readonly { trace_id: string }[]
|
|
997
|
+
for (const row of timeExpired) toEvict.add(row.trace_id)
|
|
998
|
+
|
|
999
|
+
// Size-based: if actual data exceeds the target, drop one bounded
|
|
1000
|
+
// batch of the oldest completed traces. `(page_count - freelist_count)`
|
|
1001
|
+
// ignores freed-but-not-vacuumed pages so a large freelist doesn't
|
|
1002
|
+
// trigger a deletion death spiral.
|
|
1003
|
+
const pageCount = (db.query(`PRAGMA page_count`).get() as { page_count: number }).page_count
|
|
1004
|
+
const freePages = (db.query(`PRAGMA freelist_count`).get() as { freelist_count: number }).freelist_count
|
|
1005
|
+
const pageSize = (db.query(`PRAGMA page_size`).get() as { page_size: number }).page_size
|
|
1006
|
+
const dbSize = (pageCount - freePages) * pageSize
|
|
1007
|
+
if (dbSize > maxDbSizeBytes) {
|
|
1008
|
+
const oldest = db.query(
|
|
1009
|
+
`SELECT trace_id FROM trace_summaries WHERE active_span_count = 0 ORDER BY started_at_ms ASC LIMIT ?`,
|
|
1010
|
+
).all(config.otel.retentionTraceBatch) as readonly { trace_id: string }[]
|
|
1011
|
+
// Set.add dedupes overlap with the time-expired batch above.
|
|
1012
|
+
for (const row of oldest) toEvict.add(row.trace_id)
|
|
1013
|
+
}
|
|
1014
|
+
|
|
1015
|
+
// Logs have their own retention boundary. A correlated log may refer
|
|
1016
|
+
// to a trace that was sampled elsewhere or never reached Motel, so
|
|
1017
|
+
// tying log eviction to trace_summaries lets those rows grow forever.
|
|
1018
|
+
const expiredLogs = db.query(`DELETE FROM logs WHERE id IN (SELECT id FROM logs WHERE timestamp_ms < ? ORDER BY timestamp_ms ASC LIMIT ?)`).run(cutoff, config.otel.retentionLogBatch)
|
|
1019
|
+
let deletedLogs = Number(expiredLogs.changes) > 0
|
|
1020
|
+
if (dbSize > maxDbSizeBytes) {
|
|
1021
|
+
const oversizedLogs = db.query(`DELETE FROM logs WHERE id IN (SELECT id FROM logs ORDER BY timestamp_ms ASC LIMIT ?)`).run(config.otel.retentionLogBatch)
|
|
1022
|
+
deletedLogs = deletedLogs || Number(oversizedLogs.changes) > 0
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
// Batch the trace-id list so the IN placeholders stay under
|
|
1026
|
+
// SQLite's default limit (~999). Each batch wipes every row
|
|
1027
|
+
// reachable from those trace_ids across the cascade tables.
|
|
1028
|
+
const traceIds = Array.from(toEvict)
|
|
1029
|
+
const BATCH_SIZE = 500
|
|
1030
|
+
for (let offset = 0; offset < traceIds.length; offset += BATCH_SIZE) {
|
|
1031
|
+
const batch = traceIds.slice(offset, offset + BATCH_SIZE)
|
|
1032
|
+
const placeholders = batch.map(() => "?").join(",")
|
|
1033
|
+
db.query(`DELETE FROM span_attributes WHERE trace_id IN (${placeholders})`).run(...batch)
|
|
1034
|
+
try {
|
|
1035
|
+
db.query(`DELETE FROM span_operation_fts WHERE trace_id IN (${placeholders})`).run(...batch)
|
|
1036
|
+
} catch {
|
|
1037
|
+
// FTS table may not exist on old DBs.
|
|
1038
|
+
}
|
|
1039
|
+
db.query(`DELETE FROM spans WHERE trace_id IN (${placeholders})`).run(...batch)
|
|
1040
|
+
db.query(`DELETE FROM logs WHERE trace_id IN (${placeholders})`).run(...batch)
|
|
1041
|
+
db.query(`DELETE FROM trace_summaries WHERE trace_id IN (${placeholders})`).run(...batch)
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
// Log-side orphans (log_attributes + FTS) are keyed by log.id,
|
|
1045
|
+
// so prune what no longer has a parent log row.
|
|
1046
|
+
const orphanAttributes = db.query(`DELETE FROM log_attributes WHERE rowid IN (SELECT log_attributes.rowid FROM log_attributes WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = log_attributes.log_id) LIMIT ?)`).run(config.otel.retentionLogBatch)
|
|
1047
|
+
let deletedOrphans = Number(orphanAttributes.changes) > 0
|
|
1048
|
+
try {
|
|
1049
|
+
const orphanFts = db.query(`DELETE FROM log_body_fts WHERE rowid IN (SELECT rowid FROM log_body_fts WHERE NOT EXISTS (SELECT 1 FROM logs WHERE logs.id = CAST(log_body_fts.log_id AS INTEGER)) LIMIT ?)`).run(config.otel.retentionLogBatch)
|
|
1050
|
+
deletedOrphans = deletedOrphans || Number(orphanFts.changes) > 0
|
|
1051
|
+
} catch {
|
|
1052
|
+
// FTS table may not exist on old DBs.
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
// Checkpoint after a big delete pass so the freed pages land
|
|
1056
|
+
// in the main DB file and become eligible for incremental
|
|
1057
|
+
// vacuum. Use RESTART (not PASSIVE): PASSIVE silently no-ops
|
|
1058
|
+
// when readers are active, which is the documented mechanism
|
|
1059
|
+
// behind WAL/freelist starvation when ingest is busy.
|
|
1060
|
+
if (toEvict.size === 0 && !deletedLogs && !deletedOrphans) return
|
|
1061
|
+
try { db.exec(`PRAGMA wal_checkpoint(RESTART);`) } catch { /* ignore */ }
|
|
1062
|
+
|
|
1063
|
+
// Incremental FTS5 merge — DELETE on an FTS5-indexed row
|
|
1064
|
+
// leaves a tombstone in the segment tree that only `merge`
|
|
1065
|
+
// reclaims. Skipping this is the second compounding cause
|
|
1066
|
+
// (after fixed-size vacuum) of the slow freelist accretion
|
|
1067
|
+
// that took the DB to 17GB. 100 pages of merge work per
|
|
1068
|
+
// retention tick is bounded and runs in milliseconds.
|
|
1069
|
+
incrementalFtsMerge(100)
|
|
1070
|
+
|
|
1071
|
+
// Actual page reclamation lives in `reclaimSpace`, which
|
|
1072
|
+
// runs on its own faster cadence so the file shrinks even
|
|
1073
|
+
// when no traces are evicted in a given retention tick (e.g.
|
|
1074
|
+
// after a large historical eviction has already happened).
|
|
1075
|
+
})
|
|
1076
|
+
})
|
|
1077
|
+
|
|
1078
|
+
// Retention only runs in the ingest worker so maintenance never blocks
|
|
1079
|
+
// the HTTP event loop and no second writer duplicates cleanup work.
|
|
1080
|
+
if (opts.runRetention) {
|
|
1081
|
+
// Cleanup runs on the telemetry worker, never the HTTP event loop.
|
|
1082
|
+
yield* Effect.forkScoped(Effect.repeat(
|
|
1083
|
+
Effect.andThen(reconcileTraceSummaries, cleanupExpired()).pipe(Effect.catchCause((cause) => Effect.logWarning(`motel: maintenance pass failed: ${Cause.pretty(cause)}`))),
|
|
1084
|
+
Schedule.spaced(`${config.otel.retentionIntervalSeconds} seconds`),
|
|
1085
|
+
))
|
|
1086
|
+
|
|
1087
|
+
// Page reclamation runs on a separate, faster cadence (10s) and
|
|
1088
|
+
// is independent of the eviction loop. The reason: a single sweep
|
|
1089
|
+
// at 60s intervals can move only ~8MB of pages before the next
|
|
1090
|
+
// burst of inserts grows the freelist again. Decoupling lets us
|
|
1091
|
+
// catch up adaptively (see VACUUM_PAGES_BUSY/PANIC) without
|
|
1092
|
+
// changing the cost of the heavier delete sweep.
|
|
1093
|
+
yield* Effect.forkScoped(Effect.repeat(reclaimSpace(), Schedule.spaced("10 seconds")))
|
|
1094
|
+
|
|
1095
|
+
// Periodically refresh query planner stats. `PRAGMA optimize` is a
|
|
1096
|
+
// no-op when nothing has changed, so this is essentially free on idle
|
|
1097
|
+
// servers and keeps facet/search planner estimates accurate as data
|
|
1098
|
+
// grows. 15 minutes is slower than ingestion rates we care about but
|
|
1099
|
+
// frequent enough that the attribute picker stays snappy.
|
|
1100
|
+
const refreshPlannerStats = Effect.sync(() => {
|
|
1101
|
+
try { db.exec(`PRAGMA optimize;`) } catch { /* ignore */ }
|
|
1102
|
+
})
|
|
1103
|
+
yield* Effect.forkScoped(Effect.repeat(refreshPlannerStats, Schedule.spaced("15 minutes")))
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
// Incrementally rebuild historical AI attributes in bounded batches.
|
|
1107
|
+
// Queries fall back to LIKE until the persistent marker is complete.
|
|
1108
|
+
if (hasAttrFts && !opts.readonly) {
|
|
1109
|
+
const backfillAttrFtsBatch = Effect.sync(() => {
|
|
1110
|
+
try {
|
|
1111
|
+
const keyList = AI_FTS_KEYS.map((k) => `'${k.replace(/'/g, "''")}'`).join(", ")
|
|
1112
|
+
const marker = db.query(`SELECT value FROM motel_maintenance WHERE key = 'span_attr_fts_v1'`).get() as { value: string } | null
|
|
1113
|
+
if (marker?.value === "complete") return false
|
|
1114
|
+
let cursor = 0
|
|
1115
|
+
let maxRowId = 0
|
|
1116
|
+
if (marker) {
|
|
1117
|
+
[cursor, maxRowId] = marker.value.split(":").map(Number)
|
|
1118
|
+
} else {
|
|
1119
|
+
maxRowId = (db.query(`SELECT COALESCE(MAX(rowid), 0) AS value FROM span_attributes`).get() as { value: number }).value
|
|
1120
|
+
db.query(`INSERT INTO span_attr_fts(span_attr_fts) VALUES ('delete-all')`).run()
|
|
1121
|
+
db.query(`INSERT OR REPLACE INTO motel_maintenance(key, value) VALUES ('span_attr_fts_v1', ?)`).run(`0:${maxRowId}`)
|
|
1122
|
+
}
|
|
1123
|
+
const rows = db.query(`SELECT rowid, value FROM span_attributes WHERE key IN (${keyList}) AND rowid > ? AND rowid <= ? ORDER BY rowid ASC LIMIT 500`).all(cursor, maxRowId) as Array<{ rowid: number; value: string }>
|
|
1124
|
+
if (rows.length === 0) {
|
|
1125
|
+
db.query(`UPDATE motel_maintenance SET value = 'complete' WHERE key = 'span_attr_fts_v1'`).run()
|
|
1126
|
+
hasAttrFts = true
|
|
1127
|
+
return false
|
|
1128
|
+
}
|
|
1129
|
+
const insert = db.query(`INSERT INTO span_attr_fts(rowid, value) VALUES (?, ?)`)
|
|
1130
|
+
const transaction = db.transaction(() => {
|
|
1131
|
+
for (const row of rows) insert.run(row.rowid, row.value)
|
|
1132
|
+
db.query(`UPDATE motel_maintenance SET value = ? WHERE key = 'span_attr_fts_v1'`).run(`${rows.at(-1)!.rowid}:${maxRowId}`)
|
|
1133
|
+
})
|
|
1134
|
+
transaction()
|
|
1135
|
+
return true
|
|
1136
|
+
} catch {
|
|
1137
|
+
// Backfill failure is never fatal — new ingests still
|
|
1138
|
+
// populate FTS via the trigger, and queries fall back to
|
|
1139
|
+
// LIKE when FTS lookups return empty.
|
|
1140
|
+
return true
|
|
1141
|
+
}
|
|
1142
|
+
})
|
|
1143
|
+
const backfillAttrFts: Effect.Effect<void> = Effect.suspend(() =>
|
|
1144
|
+
Effect.flatMap(backfillAttrFtsBatch, (pending) =>
|
|
1145
|
+
pending ? Effect.andThen(Effect.sleep("100 millis"), backfillAttrFts) : Effect.void,
|
|
1146
|
+
),
|
|
1147
|
+
)
|
|
1148
|
+
yield* Effect.forkScoped(backfillAttrFts)
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
const ingestTraces = Effect.fn("motel/TelemetryStore.ingestTraces")(function* (payload: OtlpTraceExportRequest) {
|
|
1152
|
+
return yield* Effect.sync(() => {
|
|
1153
|
+
let insertedSpans = 0
|
|
1154
|
+
const transaction = db.transaction((request: OtlpTraceExportRequest) => {
|
|
1155
|
+
const touchedTraceIds = new Set<string>()
|
|
1156
|
+
const touchedOperations: Array<readonly [string, string, string]> = []
|
|
1157
|
+
for (const resourceSpans of request.resourceSpans ?? []) {
|
|
1158
|
+
const resourceAttributes = attributeMap(resourceSpans.resource?.attributes)
|
|
1159
|
+
const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
|
|
1160
|
+
|
|
1161
|
+
for (const scopeSpans of resourceSpans.scopeSpans ?? []) {
|
|
1162
|
+
const scopeName = scopeSpans.scope?.name ?? null
|
|
1163
|
+
|
|
1164
|
+
for (const span of scopeSpans.spans ?? []) {
|
|
1165
|
+
const traceId = normalizeOtlpBinaryId(span.traceId, 16)
|
|
1166
|
+
const spanId = normalizeOtlpBinaryId(span.spanId, 8)
|
|
1167
|
+
if (!traceId || !spanId) continue
|
|
1168
|
+
const parentSpanId = normalizeOtlpBinaryId(span.parentSpanId, 8)
|
|
1169
|
+
const spanAttributes = attributeMap(span.attributes)
|
|
1170
|
+
const mergedAttributes = { ...resourceAttributes, ...spanAttributes }
|
|
1171
|
+
const startTimeMs = nanosToMilliseconds(span.startTimeUnixNano)
|
|
1172
|
+
const endTimeMs = nanosToMilliseconds(span.endTimeUnixNano)
|
|
1173
|
+
const events = (span.events ?? []).map((event) => ({
|
|
1174
|
+
name: event.name ?? "event",
|
|
1175
|
+
timestamp: nanosToMilliseconds(event.timeUnixNano),
|
|
1176
|
+
attributes: attributeMap(event.attributes),
|
|
1177
|
+
}))
|
|
1178
|
+
|
|
1179
|
+
insertSpan.run(
|
|
1180
|
+
traceId,
|
|
1181
|
+
spanId,
|
|
1182
|
+
parentSpanId,
|
|
1183
|
+
serviceName,
|
|
1184
|
+
scopeName,
|
|
1185
|
+
span.name ?? "unknown",
|
|
1186
|
+
spanKindLabel(span.kind),
|
|
1187
|
+
startTimeMs,
|
|
1188
|
+
endTimeMs,
|
|
1189
|
+
Math.max(0, endTimeMs - startTimeMs),
|
|
1190
|
+
spanStatusLabel(span.status?.code),
|
|
1191
|
+
JSON.stringify(spanAttributes),
|
|
1192
|
+
JSON.stringify(resourceAttributes),
|
|
1193
|
+
JSON.stringify(events),
|
|
1194
|
+
)
|
|
1195
|
+
deleteSpanAttributes.run(traceId, spanId)
|
|
1196
|
+
insertSpanAttributesMany(traceId, spanId, mergedAttributes)
|
|
1197
|
+
touchedOperations.push([traceId, spanId, span.name ?? "unknown"])
|
|
1198
|
+
touchedTraceIds.add(traceId)
|
|
1199
|
+
insertedSpans += 1
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
try {
|
|
1204
|
+
const BATCH_SIZE = 500
|
|
1205
|
+
for (let offset = 0; offset < touchedOperations.length; offset += BATCH_SIZE) {
|
|
1206
|
+
updateSpanOperationSearchMany(touchedOperations.slice(offset, offset + BATCH_SIZE))
|
|
1207
|
+
}
|
|
1208
|
+
} catch {
|
|
1209
|
+
// FTS is optional.
|
|
1210
|
+
}
|
|
1211
|
+
for (const traceId of touchedTraceIds) {
|
|
1212
|
+
upsertTraceSummary.run(traceId)
|
|
1213
|
+
}
|
|
1214
|
+
})
|
|
1215
|
+
|
|
1216
|
+
transaction(payload)
|
|
1217
|
+
return { insertedSpans }
|
|
1218
|
+
})
|
|
1219
|
+
})
|
|
1220
|
+
|
|
1221
|
+
const ingestLogs = Effect.fn("motel/TelemetryStore.ingestLogs")(function* (payload: OtlpLogExportRequest) {
|
|
1222
|
+
return yield* Effect.sync(() => {
|
|
1223
|
+
let insertedLogs = 0
|
|
1224
|
+
const transaction = db.transaction((request: OtlpLogExportRequest) => {
|
|
1225
|
+
const touchedLogBodies: Array<readonly [string, string]> = []
|
|
1226
|
+
for (const resourceLogs of request.resourceLogs ?? []) {
|
|
1227
|
+
const resourceAttributes = attributeMap(resourceLogs.resource?.attributes)
|
|
1228
|
+
const serviceName = resourceAttributes["service.name"] || resourceAttributes["service_name"] || "unknown"
|
|
1229
|
+
|
|
1230
|
+
for (const scopeLogs of resourceLogs.scopeLogs ?? []) {
|
|
1231
|
+
const scopeName = scopeLogs.scope?.name ?? null
|
|
1232
|
+
|
|
1233
|
+
for (const record of scopeLogs.logRecords ?? []) {
|
|
1234
|
+
const attributes = attributeMap(record.attributes)
|
|
1235
|
+
const mergedAttributes = { ...resourceAttributes, ...attributes }
|
|
1236
|
+
const timestampMs = nanosToMilliseconds(record.timeUnixNano ?? record.observedTimeUnixNano)
|
|
1237
|
+
const body = stringifyValue(parseAnyValue(record.body))
|
|
1238
|
+
const rawTraceId = attributes.traceId || attributes.trace_id || record.traceId || null
|
|
1239
|
+
const rawSpanId = attributes.spanId || attributes.span_id || record.spanId || null
|
|
1240
|
+
const result = insertLog.run(
|
|
1241
|
+
normalizeOtlpBinaryId(rawTraceId, 16),
|
|
1242
|
+
normalizeOtlpBinaryId(rawSpanId, 8),
|
|
1243
|
+
serviceName,
|
|
1244
|
+
scopeName,
|
|
1245
|
+
record.severityText ?? "INFO",
|
|
1246
|
+
timestampMs,
|
|
1247
|
+
body,
|
|
1248
|
+
JSON.stringify(attributes),
|
|
1249
|
+
JSON.stringify(resourceAttributes),
|
|
1250
|
+
)
|
|
1251
|
+
const logId = Number((result as { lastInsertRowid: number | bigint }).lastInsertRowid)
|
|
1252
|
+
insertLogAttributesMany(logId, mergedAttributes)
|
|
1253
|
+
touchedLogBodies.push([String(logId), body])
|
|
1254
|
+
insertedLogs += 1
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
try {
|
|
1259
|
+
const BATCH_SIZE = 500
|
|
1260
|
+
for (let offset = 0; offset < touchedLogBodies.length; offset += BATCH_SIZE) {
|
|
1261
|
+
insertLogBodySearchMany(touchedLogBodies.slice(offset, offset + BATCH_SIZE))
|
|
1262
|
+
}
|
|
1263
|
+
} catch {
|
|
1264
|
+
// FTS is optional.
|
|
1265
|
+
}
|
|
1266
|
+
})
|
|
1267
|
+
|
|
1268
|
+
transaction(payload)
|
|
1269
|
+
return { insertedLogs }
|
|
1270
|
+
})
|
|
1271
|
+
})
|
|
1272
|
+
|
|
1273
|
+
const listServices = Effect.fn("motel/TelemetryStore.listServices")(function* () {
|
|
1274
|
+
const cutoff = (yield* Clock.currentTimeMillis) - config.otel.traceLookbackMinutes * 60 * 1000
|
|
1275
|
+
const services = yield* Effect.sync(() => {
|
|
1276
|
+
// Discover recent activity from span rows, not trace starts: a
|
|
1277
|
+
// long-running trace can emit a current child after its root ages
|
|
1278
|
+
// outside the lookback window.
|
|
1279
|
+
const rows = db.query(`
|
|
1280
|
+
SELECT service_name FROM spans WHERE start_time_ms >= ?
|
|
1281
|
+
UNION
|
|
1282
|
+
SELECT service_name FROM logs WHERE timestamp_ms >= ?
|
|
1283
|
+
ORDER BY service_name ASC
|
|
1284
|
+
`).all(cutoff, cutoff) as Array<{ service_name: string }>
|
|
1285
|
+
return rows.map((row) => row.service_name)
|
|
1286
|
+
})
|
|
1287
|
+
yield* Effect.annotateCurrentSpan("trace.service_count", services.length)
|
|
1288
|
+
return services
|
|
1289
|
+
})()
|
|
1290
|
+
|
|
1291
|
+
const loadTracesByIds = (traceIds: readonly string[]) => {
|
|
1292
|
+
if (traceIds.length === 0) return [] as readonly TraceItem[]
|
|
1293
|
+
const placeholders = traceIds.map(() => "?").join(", ")
|
|
1294
|
+
const rows = db.query(`
|
|
1295
|
+
SELECT * FROM spans
|
|
1296
|
+
WHERE trace_id IN (${placeholders})
|
|
1297
|
+
ORDER BY start_time_ms ASC
|
|
1298
|
+
`).all(...traceIds) as SpanRow[]
|
|
1299
|
+
|
|
1300
|
+
const grouped = new Map<string, SpanRow[]>()
|
|
1301
|
+
for (const row of rows) {
|
|
1302
|
+
const group = grouped.get(row.trace_id) ?? []
|
|
1303
|
+
group.push(row)
|
|
1304
|
+
grouped.set(row.trace_id, group)
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
return traceIds
|
|
1308
|
+
.map((traceId) => grouped.get(traceId))
|
|
1309
|
+
.filter((group): group is SpanRow[] => group !== undefined)
|
|
1310
|
+
.map((group) => buildTrace(group[0]!.trace_id, group))
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
const listRecentTraces = Effect.fn("motel/TelemetryStore.listRecentTraces")(function* (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number }) {
|
|
1314
|
+
yield* Effect.annotateCurrentSpan("trace.service_name", serviceName ?? "all")
|
|
1315
|
+
const summaries = yield* listTraceSummaries(serviceName, options)
|
|
1316
|
+
const traces = yield* Effect.sync(() => loadTracesByIds(summaries.map((summary) => summary.traceId)))
|
|
1317
|
+
yield* Effect.annotateCurrentSpan("trace.result_count", traces.length)
|
|
1318
|
+
return traces
|
|
1319
|
+
})
|
|
1320
|
+
|
|
1321
|
+
const listTraceSummaries = Effect.fn("motel/TelemetryStore.listTraceSummaries")(function* (serviceName: string | null, options?: { readonly lookbackMinutes?: number; readonly limit?: number; readonly cursorStartedAtMs?: number; readonly cursorTraceId?: string }) {
|
|
1322
|
+
yield* Effect.annotateCurrentSpan("trace.service_name", serviceName ?? "all")
|
|
1323
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (options?.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
1324
|
+
const limit = options?.limit ?? config.otel.traceFetchLimit
|
|
1325
|
+
|
|
1326
|
+
const summaries = yield* Effect.sync(() => {
|
|
1327
|
+
const clauses = ["started_at_ms >= ?"]
|
|
1328
|
+
const params: Array<string | number> = [cutoff]
|
|
1329
|
+
|
|
1330
|
+
if (serviceName) {
|
|
1331
|
+
clauses.push("service_name = ?")
|
|
1332
|
+
params.push(serviceName)
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
if (options?.cursorStartedAtMs != null && options.cursorTraceId) {
|
|
1336
|
+
clauses.push("(started_at_ms < ? OR (started_at_ms = ? AND trace_id < ?))")
|
|
1337
|
+
params.push(options.cursorStartedAtMs, options.cursorStartedAtMs, options.cursorTraceId)
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
return db.query(`
|
|
1341
|
+
SELECT trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
|
|
1342
|
+
FROM trace_summaries
|
|
1343
|
+
WHERE ${clauses.join(" AND ")}
|
|
1344
|
+
ORDER BY started_at_ms DESC, trace_id DESC
|
|
1345
|
+
LIMIT ?
|
|
1346
|
+
`).all(...params, limit) as TraceSummaryRow[]
|
|
1347
|
+
}).pipe(Effect.map((rows) => rows.map(parseSummaryRow)))
|
|
1348
|
+
yield* Effect.annotateCurrentSpan("trace.result_count", summaries.length)
|
|
1349
|
+
return summaries
|
|
1350
|
+
})
|
|
1351
|
+
|
|
1352
|
+
const searchTraceSummaries = Effect.fn("motel/TelemetryStore.searchTraceSummaries")(function* (input: TraceSearch) {
|
|
1353
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
1354
|
+
const limit = input.limit ?? config.otel.traceFetchLimit
|
|
1355
|
+
|
|
1356
|
+
return yield* Effect.sync(() => {
|
|
1357
|
+
const clauses: string[] = ["started_at_ms >= ?"]
|
|
1358
|
+
const params: Array<string | number> = [cutoff]
|
|
1359
|
+
|
|
1360
|
+
if (input.serviceName) {
|
|
1361
|
+
clauses.push("service_name = ?")
|
|
1362
|
+
params.push(input.serviceName)
|
|
1363
|
+
}
|
|
1364
|
+
if (input.status === "error") {
|
|
1365
|
+
clauses.push("error_count > 0")
|
|
1366
|
+
}
|
|
1367
|
+
if (input.status === "ok") {
|
|
1368
|
+
clauses.push("error_count = 0")
|
|
1369
|
+
}
|
|
1370
|
+
if (input.minDurationMs != null) {
|
|
1371
|
+
clauses.push("duration_ms >= ?")
|
|
1372
|
+
params.push(input.minDurationMs)
|
|
1373
|
+
}
|
|
1374
|
+
if (input.cursorStartedAtMs != null && input.cursorTraceId) {
|
|
1375
|
+
clauses.push("(started_at_ms < ? OR (started_at_ms = ? AND trace_id < ?))")
|
|
1376
|
+
params.push(input.cursorStartedAtMs, input.cursorStartedAtMs, input.cursorTraceId)
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
if (input.operation) {
|
|
1380
|
+
const ftsQuery = toFtsMatchQuery(input.operation)
|
|
1381
|
+
if (hasFts && ftsQuery) {
|
|
1382
|
+
clauses.push("trace_id IN (SELECT DISTINCT trace_id FROM span_operation_fts WHERE span_operation_fts MATCH ?)")
|
|
1383
|
+
params.push(ftsQuery)
|
|
1384
|
+
} else {
|
|
1385
|
+
clauses.push("trace_id IN (SELECT DISTINCT trace_id FROM spans WHERE operation_name LIKE ? COLLATE NOCASE)")
|
|
1386
|
+
params.push(`%${input.operation}%`)
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
const exactAttrMatch = buildExactAttributeMatchSubquery("span_attributes", ["trace_id", "span_id"], input.attributeFilters)
|
|
1391
|
+
if (exactAttrMatch) {
|
|
1392
|
+
clauses.push(`trace_id IN (SELECT DISTINCT trace_id FROM (${exactAttrMatch.sql}))`)
|
|
1393
|
+
params.push(...exactAttrMatch.params)
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
// `:ai <query>` — FTS match against LLM content keys. Joins
|
|
1397
|
+
// span_attr_fts back to span_attributes to collect trace_ids
|
|
1398
|
+
// whose spans carry matching prompt/response content. Falls
|
|
1399
|
+
// through to no-op when the query tokenizes empty (e.g. only
|
|
1400
|
+
// stopwords or operator-chars) so users don't get a silently
|
|
1401
|
+
// empty list.
|
|
1402
|
+
if (input.aiText) {
|
|
1403
|
+
const aiFtsQuery = toFtsMatchQuery(input.aiText)
|
|
1404
|
+
if (hasAttrFts && aiFtsQuery) {
|
|
1405
|
+
clauses.push(`trace_id IN (
|
|
1406
|
+
SELECT DISTINCT sa.trace_id
|
|
1407
|
+
FROM span_attr_fts fts
|
|
1408
|
+
JOIN span_attributes sa ON sa.rowid = fts.rowid
|
|
1409
|
+
WHERE fts.value MATCH ?
|
|
1410
|
+
)`)
|
|
1411
|
+
params.push(aiFtsQuery)
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
const rows = db.query(`
|
|
1416
|
+
SELECT trace_id, service_name, root_operation_name, started_at_ms, ended_at_ms, active_span_count, duration_ms, span_count, error_count
|
|
1417
|
+
FROM trace_summaries
|
|
1418
|
+
WHERE ${clauses.join(" AND ")}
|
|
1419
|
+
ORDER BY started_at_ms DESC, trace_id DESC
|
|
1420
|
+
LIMIT ?
|
|
1421
|
+
`).all(...params, limit) as TraceSummaryRow[]
|
|
1422
|
+
|
|
1423
|
+
return rows.map(parseSummaryRow)
|
|
1424
|
+
})
|
|
1425
|
+
})
|
|
1426
|
+
|
|
1427
|
+
const getTrace = Effect.fn("motel/TelemetryStore.getTrace")(function* (traceId: string) {
|
|
1428
|
+
yield* Effect.annotateCurrentSpan("trace.trace_id", traceId)
|
|
1429
|
+
return yield* Effect.sync(() => {
|
|
1430
|
+
const rows = db.query(`
|
|
1431
|
+
SELECT * FROM spans WHERE trace_id = ? ORDER BY start_time_ms ASC
|
|
1432
|
+
`).all(traceId) as SpanRow[]
|
|
1433
|
+
return rows.length === 0 ? null : buildTrace(traceId, rows)
|
|
1434
|
+
})
|
|
1435
|
+
})
|
|
1436
|
+
|
|
1437
|
+
const getSpan = Effect.fn("motel/TelemetryStore.getSpan")(function* (spanId: string) {
|
|
1438
|
+
yield* Effect.annotateCurrentSpan("trace.span_id", spanId)
|
|
1439
|
+
return yield* Effect.sync(() => {
|
|
1440
|
+
// Fetch only the target span row (uses idx_spans_span_id)
|
|
1441
|
+
const spanRow = db.query(`SELECT * FROM spans WHERE span_id = ? LIMIT 1`).get(spanId) as SpanRow | null
|
|
1442
|
+
if (!spanRow) return null
|
|
1443
|
+
|
|
1444
|
+
const traceId = spanRow.trace_id
|
|
1445
|
+
|
|
1446
|
+
// Walk the parent chain in one recursive CTE instead of one query
|
|
1447
|
+
// per hop. Root context remains the earliest root in the trace,
|
|
1448
|
+
// matching full trace hydration even when input has multiple roots.
|
|
1449
|
+
let parentOperationName: string | null = null
|
|
1450
|
+
let depth = 0
|
|
1451
|
+
if (spanRow.parent_span_id) {
|
|
1452
|
+
const ancestors = db.query(`
|
|
1453
|
+
WITH RECURSIVE ancestors(span_id, parent_span_id, operation_name, hop) AS (
|
|
1454
|
+
SELECT span_id, parent_span_id, operation_name, 1
|
|
1455
|
+
FROM spans WHERE trace_id = ? AND span_id = ?
|
|
1456
|
+
UNION ALL
|
|
1457
|
+
SELECT s.span_id, s.parent_span_id, s.operation_name, a.hop + 1
|
|
1458
|
+
FROM ancestors a
|
|
1459
|
+
JOIN spans s ON s.trace_id = ? AND s.span_id = a.parent_span_id
|
|
1460
|
+
)
|
|
1461
|
+
SELECT span_id, parent_span_id, operation_name, hop FROM ancestors ORDER BY hop ASC
|
|
1462
|
+
`).all(traceId, spanRow.parent_span_id, traceId) as Array<{ span_id: string; parent_span_id: string | null; operation_name: string; hop: number }>
|
|
1463
|
+
|
|
1464
|
+
parentOperationName = ancestors[0]?.operation_name ?? null
|
|
1465
|
+
depth = ancestors.length
|
|
1466
|
+
}
|
|
1467
|
+
|
|
1468
|
+
const rootRow = db.query(`
|
|
1469
|
+
SELECT operation_name FROM spans
|
|
1470
|
+
WHERE trace_id = ? AND parent_span_id IS NULL
|
|
1471
|
+
ORDER BY start_time_ms ASC LIMIT 1
|
|
1472
|
+
`).get(traceId) as { operation_name: string } | null
|
|
1473
|
+
const rootOperationName = rootRow?.operation_name ?? "unknown"
|
|
1474
|
+
|
|
1475
|
+
const parsed = parseSpanRow(spanRow)
|
|
1476
|
+
return {
|
|
1477
|
+
traceId,
|
|
1478
|
+
rootOperationName,
|
|
1479
|
+
parentOperationName,
|
|
1480
|
+
span: { ...parsed, depth },
|
|
1481
|
+
} satisfies SpanItem
|
|
1482
|
+
})
|
|
1483
|
+
})
|
|
1484
|
+
|
|
1485
|
+
const listTraceSpans = Effect.fn("motel/TelemetryStore.listTraceSpans")(function* (traceId: string) {
|
|
1486
|
+
return yield* Effect.sync(() => {
|
|
1487
|
+
const rows = db.query(`SELECT * FROM spans WHERE trace_id = ? ORDER BY start_time_ms ASC`).all(traceId) as SpanRow[]
|
|
1488
|
+
return rows.length === 0 ? [] as readonly SpanItem[] : buildSpanItems(traceId, rows)
|
|
1489
|
+
})
|
|
1490
|
+
})
|
|
1491
|
+
|
|
1492
|
+
const searchSpans = Effect.fn("motel/TelemetryStore.searchSpans")(function* (input: SpanSearch) {
|
|
1493
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
1494
|
+
const limit = input.limit ?? 100
|
|
1495
|
+
const hasContainsFilters = Object.keys(input.attributeContainsFilters ?? {}).length > 0
|
|
1496
|
+
// Only over-fetch when post-filtering will discard rows. Without
|
|
1497
|
+
// a parentOperation filter the SQL `LIMIT` already returns the
|
|
1498
|
+
// final set, and over-fetching just makes us parse JSON blobs
|
|
1499
|
+
// for rows we'll throw away.
|
|
1500
|
+
const needsPostFilter = !!input.parentOperation
|
|
1501
|
+
const candidateLimit = !needsPostFilter
|
|
1502
|
+
? limit
|
|
1503
|
+
: hasContainsFilters
|
|
1504
|
+
? Math.max(limit * 20, 500)
|
|
1505
|
+
: Math.max(limit * 10, 200)
|
|
1506
|
+
|
|
1507
|
+
return yield* Effect.sync(() => {
|
|
1508
|
+
// First pass: fetch only the columns needed to filter and
|
|
1509
|
+
// to drive the parent-context lookup. Parsing the heavy
|
|
1510
|
+
// `*_json` blobs is deferred until after we've sliced down
|
|
1511
|
+
// to the final `limit`.
|
|
1512
|
+
let fromSql = "FROM spans AS s"
|
|
1513
|
+
const joinParams: Array<string | number> = []
|
|
1514
|
+
const clauses: string[] = ["s.start_time_ms >= ?"]
|
|
1515
|
+
const params: Array<string | number> = [cutoff]
|
|
1516
|
+
|
|
1517
|
+
if (input.traceId) {
|
|
1518
|
+
clauses.push("s.trace_id = ?")
|
|
1519
|
+
params.push(input.traceId)
|
|
1520
|
+
}
|
|
1521
|
+
if (input.serviceName) {
|
|
1522
|
+
clauses.push("s.service_name = ?")
|
|
1523
|
+
params.push(input.serviceName)
|
|
1524
|
+
}
|
|
1525
|
+
if (input.operation) {
|
|
1526
|
+
const ftsQuery = toFtsMatchQuery(input.operation)
|
|
1527
|
+
if (hasFts && ftsQuery) {
|
|
1528
|
+
fromSql += ` INNER JOIN (SELECT trace_id, span_id FROM span_operation_fts WHERE span_operation_fts MATCH ?) AS span_operation_match ON span_operation_match.trace_id = s.trace_id AND span_operation_match.span_id = s.span_id`
|
|
1529
|
+
joinParams.push(ftsQuery)
|
|
1530
|
+
} else {
|
|
1531
|
+
clauses.push("s.operation_name LIKE ? COLLATE NOCASE")
|
|
1532
|
+
params.push(`%${input.operation}%`)
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
if (input.status) {
|
|
1536
|
+
clauses.push("s.status = ?")
|
|
1537
|
+
params.push(input.status)
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
const exactAttrMatch = buildExactAttributeMatchSubquery("span_attributes", ["trace_id", "span_id"], input.attributeFilters)
|
|
1541
|
+
if (exactAttrMatch) {
|
|
1542
|
+
clauses.push(`EXISTS (SELECT 1 FROM (${exactAttrMatch.sql}) AS span_attr_match WHERE span_attr_match.trace_id = s.trace_id AND span_attr_match.span_id = s.span_id)`)
|
|
1543
|
+
params.push(...exactAttrMatch.params)
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
const containsAttrMatch = buildContainsAttributeMatchSubquery("span_attributes", ["trace_id", "span_id"], input.attributeContainsFilters)
|
|
1547
|
+
if (containsAttrMatch) {
|
|
1548
|
+
clauses.push(`EXISTS (SELECT 1 FROM (${containsAttrMatch.sql}) AS span_attr_contains_match WHERE span_attr_contains_match.trace_id = s.trace_id AND span_attr_contains_match.span_id = s.span_id)`)
|
|
1549
|
+
params.push(...containsAttrMatch.params)
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
const candidateRows = db.query(`
|
|
1553
|
+
SELECT s.trace_id, s.span_id, s.parent_span_id, s.operation_name, s.start_time_ms
|
|
1554
|
+
${fromSql}
|
|
1555
|
+
WHERE ${clauses.join(" AND ")}
|
|
1556
|
+
ORDER BY s.start_time_ms DESC
|
|
1557
|
+
LIMIT ?
|
|
1558
|
+
`).all(...joinParams, ...params, candidateLimit) as Array<{ trace_id: string; span_id: string; parent_span_id: string | null; operation_name: string; start_time_ms: number }>
|
|
1559
|
+
|
|
1560
|
+
const traceIds = [...new Set(candidateRows.map((row) => row.trace_id))]
|
|
1561
|
+
if (traceIds.length === 0) return [] as readonly SpanItem[]
|
|
1562
|
+
|
|
1563
|
+
const keyOf = (traceId: string, spanId: string) => `${traceId}:${spanId}`
|
|
1564
|
+
const spanContextById = new Map<string, { readonly parentSpanId: string | null; readonly operationName: string }>()
|
|
1565
|
+
|
|
1566
|
+
// Bulk-prefetch parent metadata for every span in every trace
|
|
1567
|
+
// touched by the candidate set. One indexed scan per trace_id
|
|
1568
|
+
// is much cheaper than a per-span lookup loop while computing
|
|
1569
|
+
// depth, and we get the trace-root lookup in the same pass.
|
|
1570
|
+
const placeholders = traceIds.map(() => "?").join(", ")
|
|
1571
|
+
const allSpanRows = db.query(`
|
|
1572
|
+
SELECT trace_id, span_id, parent_span_id, operation_name, start_time_ms
|
|
1573
|
+
FROM spans
|
|
1574
|
+
WHERE trace_id IN (${placeholders})
|
|
1575
|
+
`).all(...traceIds) as Array<{ trace_id: string; span_id: string; parent_span_id: string | null; operation_name: string; start_time_ms: number }>
|
|
1576
|
+
|
|
1577
|
+
const rootOperationByTraceId = new Map<string, { operationName: string; startTimeMs: number }>()
|
|
1578
|
+
for (const row of allSpanRows) {
|
|
1579
|
+
spanContextById.set(keyOf(row.trace_id, row.span_id), {
|
|
1580
|
+
parentSpanId: row.parent_span_id,
|
|
1581
|
+
operationName: row.operation_name,
|
|
1582
|
+
})
|
|
1583
|
+
if (row.parent_span_id === null) {
|
|
1584
|
+
const existing = rootOperationByTraceId.get(row.trace_id)
|
|
1585
|
+
if (!existing || row.start_time_ms < existing.startTimeMs) {
|
|
1586
|
+
rootOperationByTraceId.set(row.trace_id, { operationName: row.operation_name, startTimeMs: row.start_time_ms })
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
const getSpanContext = (traceId: string, spanId: string) => spanContextById.get(keyOf(traceId, spanId)) ?? null
|
|
1592
|
+
|
|
1593
|
+
const depthById = new Map<string, number>()
|
|
1594
|
+
const getDepth = (traceId: string, spanId: string, visiting = new Set<string>()): number => {
|
|
1595
|
+
const key = keyOf(traceId, spanId)
|
|
1596
|
+
const cached = depthById.get(key)
|
|
1597
|
+
if (cached !== undefined) return cached
|
|
1598
|
+
if (visiting.has(key)) return 0
|
|
1599
|
+
visiting.add(key)
|
|
1600
|
+
const context = getSpanContext(traceId, spanId)
|
|
1601
|
+
const depth = context?.parentSpanId ? getDepth(traceId, context.parentSpanId, visiting) + 1 : 0
|
|
1602
|
+
depthById.set(key, depth)
|
|
1603
|
+
return depth
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
// Apply parentOperation post-filter on the lite candidate set
|
|
1607
|
+
// (cheap — string compare against cached parent op) and then
|
|
1608
|
+
// slice down to the final result size before parsing any JSON.
|
|
1609
|
+
const parentOperationNeedle = input.parentOperation?.toLowerCase() ?? null
|
|
1610
|
+
const filteredLite: typeof candidateRows = []
|
|
1611
|
+
for (const row of candidateRows) {
|
|
1612
|
+
if (parentOperationNeedle) {
|
|
1613
|
+
const parent = row.parent_span_id ? getSpanContext(row.trace_id, row.parent_span_id) : null
|
|
1614
|
+
if (!parent?.operationName.toLowerCase().includes(parentOperationNeedle)) continue
|
|
1615
|
+
}
|
|
1616
|
+
filteredLite.push(row)
|
|
1617
|
+
if (filteredLite.length >= limit) break
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
if (filteredLite.length === 0) return [] as readonly SpanItem[]
|
|
1621
|
+
|
|
1622
|
+
// Hydrate only the kept rows: one batched fetch of the full
|
|
1623
|
+
// SpanRow (with resource_json / attributes_json / events_json)
|
|
1624
|
+
// using SQLite's row-value `IN` syntax, then parseSpanRow per
|
|
1625
|
+
// kept row. Result order follows `filteredLite` so the caller
|
|
1626
|
+
// sees the same ordering the candidate scan produced.
|
|
1627
|
+
const keptValues = filteredLite.map(() => "(?, ?)").join(", ")
|
|
1628
|
+
const fullRows = db.query(`
|
|
1629
|
+
SELECT * FROM spans WHERE (trace_id, span_id) IN (VALUES ${keptValues})
|
|
1630
|
+
`).all(...filteredLite.flatMap((row) => [row.trace_id, row.span_id])) as SpanRow[]
|
|
1631
|
+
const fullRowByKey = new Map<string, SpanRow>()
|
|
1632
|
+
for (const row of fullRows) {
|
|
1633
|
+
fullRowByKey.set(keyOf(row.trace_id, row.span_id), row)
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
const items: SpanItem[] = []
|
|
1637
|
+
for (const lite of filteredLite) {
|
|
1638
|
+
const row = fullRowByKey.get(keyOf(lite.trace_id, lite.span_id))
|
|
1639
|
+
if (!row) continue
|
|
1640
|
+
const parentContext = row.parent_span_id ? getSpanContext(row.trace_id, row.parent_span_id) : null
|
|
1641
|
+
const parsedSpan = parseSpanRow(row)
|
|
1642
|
+
const span = {
|
|
1643
|
+
...parsedSpan,
|
|
1644
|
+
depth: getDepth(row.trace_id, row.span_id),
|
|
1645
|
+
warnings: row.parent_span_id && !parentContext
|
|
1646
|
+
? [`missing span ${row.parent_span_id} (1 child)`]
|
|
1647
|
+
: parsedSpan.warnings,
|
|
1648
|
+
}
|
|
1649
|
+
items.push({
|
|
1650
|
+
traceId: row.trace_id,
|
|
1651
|
+
rootOperationName: rootOperationByTraceId.get(row.trace_id)?.operationName ?? span.operationName,
|
|
1652
|
+
parentOperationName: parentContext?.operationName ?? null,
|
|
1653
|
+
span,
|
|
1654
|
+
})
|
|
1655
|
+
}
|
|
1656
|
+
return items
|
|
1657
|
+
})
|
|
1658
|
+
})
|
|
1659
|
+
|
|
1660
|
+
const searchTraces = Effect.fn("motel/TelemetryStore.searchTraces")(function* (input: TraceSearch) {
|
|
1661
|
+
const summaries = yield* searchTraceSummaries(input)
|
|
1662
|
+
return yield* Effect.sync(() => loadTracesByIds(summaries.map((summary) => summary.traceId)))
|
|
1663
|
+
})
|
|
1664
|
+
|
|
1665
|
+
const searchLogs = Effect.fn("motel/TelemetryStore.searchLogs")(function* (input: LogSearch) {
|
|
1666
|
+
const now = yield* Clock.currentTimeMillis
|
|
1667
|
+
return yield* Effect.sync(() => {
|
|
1668
|
+
const clauses: string[] = []
|
|
1669
|
+
const params: Array<string | number> = []
|
|
1670
|
+
|
|
1671
|
+
if (input.serviceName) {
|
|
1672
|
+
clauses.push(`service_name = ?`)
|
|
1673
|
+
params.push(input.serviceName)
|
|
1674
|
+
}
|
|
1675
|
+
if (input.severity) {
|
|
1676
|
+
clauses.push(`severity_text = ?`)
|
|
1677
|
+
params.push(input.severity.toUpperCase())
|
|
1678
|
+
}
|
|
1679
|
+
if (input.traceId) {
|
|
1680
|
+
clauses.push(`trace_id = ?`)
|
|
1681
|
+
params.push(input.traceId)
|
|
1682
|
+
}
|
|
1683
|
+
if (input.spanId) {
|
|
1684
|
+
clauses.push(`span_id = ?`)
|
|
1685
|
+
params.push(input.spanId)
|
|
1686
|
+
}
|
|
1687
|
+
if (input.body) {
|
|
1688
|
+
const ftsQuery = toFtsMatchQuery(input.body)
|
|
1689
|
+
if (hasFts && ftsQuery) {
|
|
1690
|
+
clauses.push(`id IN (SELECT CAST(log_id AS INTEGER) FROM log_body_fts WHERE log_body_fts MATCH ?)`)
|
|
1691
|
+
params.push(ftsQuery)
|
|
1692
|
+
} else {
|
|
1693
|
+
clauses.push(`body LIKE ? COLLATE NOCASE`)
|
|
1694
|
+
params.push(`%${input.body}%`)
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
if (input.lookbackMinutes) {
|
|
1698
|
+
const cutoff = now - input.lookbackMinutes * 60 * 1000
|
|
1699
|
+
clauses.push(`timestamp_ms >= ?`)
|
|
1700
|
+
params.push(cutoff)
|
|
1701
|
+
}
|
|
1702
|
+
if (input.cursorTimestampMs != null && input.cursorId) {
|
|
1703
|
+
clauses.push(`(timestamp_ms < ? OR (timestamp_ms = ? AND id < ?))`)
|
|
1704
|
+
params.push(input.cursorTimestampMs, input.cursorTimestampMs, Number(input.cursorId))
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
const exactAttrMatch = buildExactAttributeMatchSubquery("log_attributes", ["log_id"], input.attributeFilters)
|
|
1708
|
+
if (exactAttrMatch) {
|
|
1709
|
+
clauses.push(`id IN (${exactAttrMatch.sql})`)
|
|
1710
|
+
params.push(...exactAttrMatch.params)
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
const containsAttrMatch = buildContainsAttributeMatchSubquery("log_attributes", ["log_id"], input.attributeContainsFilters)
|
|
1714
|
+
if (containsAttrMatch) {
|
|
1715
|
+
clauses.push(`id IN (${containsAttrMatch.sql})`)
|
|
1716
|
+
params.push(...containsAttrMatch.params)
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""
|
|
1720
|
+
const limit = input.limit ?? config.otel.logFetchLimit
|
|
1721
|
+
const rows = db.query(`
|
|
1722
|
+
SELECT * FROM logs
|
|
1723
|
+
${where}
|
|
1724
|
+
ORDER BY timestamp_ms DESC, id DESC
|
|
1725
|
+
LIMIT ?
|
|
1726
|
+
`).all(...params, limit) as LogRow[]
|
|
1727
|
+
|
|
1728
|
+
return rows.map(parseLogRow)
|
|
1729
|
+
})
|
|
1730
|
+
})
|
|
1731
|
+
|
|
1732
|
+
const traceStats = Effect.fn("motel/TelemetryStore.traceStats")(function* (input: TraceStatsSearch) {
|
|
1733
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
1734
|
+
const limit = input.limit ?? 20
|
|
1735
|
+
const hasAttrFilters = Object.keys(input.attributeFilters ?? {}).length > 0
|
|
1736
|
+
const isAttrGroupBy = input.groupBy.startsWith("attr.")
|
|
1737
|
+
|
|
1738
|
+
if (isAttrGroupBy || hasAttrFilters || input.operation) {
|
|
1739
|
+
const summaries = yield* searchTraceSummaries({
|
|
1740
|
+
serviceName: input.serviceName,
|
|
1741
|
+
operation: input.operation,
|
|
1742
|
+
status: input.status,
|
|
1743
|
+
minDurationMs: input.minDurationMs,
|
|
1744
|
+
attributeFilters: input.attributeFilters,
|
|
1745
|
+
lookbackMinutes: input.lookbackMinutes,
|
|
1746
|
+
limit: 5000,
|
|
1747
|
+
})
|
|
1748
|
+
|
|
1749
|
+
// For attr.* groupBy, we need to check span attributes — but only the groupBy key
|
|
1750
|
+
let attrLookup: Map<string, string> | null = null
|
|
1751
|
+
if (isAttrGroupBy) {
|
|
1752
|
+
const attrKey = input.groupBy.slice(5)
|
|
1753
|
+
const traceIds = summaries.map((s) => s.traceId)
|
|
1754
|
+
if (traceIds.length > 0) {
|
|
1755
|
+
const placeholders = traceIds.map(() => "?").join(", ")
|
|
1756
|
+
const rows = db.query(`
|
|
1757
|
+
SELECT trace_id, value
|
|
1758
|
+
FROM span_attributes
|
|
1759
|
+
WHERE key = ? AND trace_id IN (${placeholders})
|
|
1760
|
+
GROUP BY trace_id
|
|
1761
|
+
`).all(attrKey, ...traceIds) as Array<{ trace_id: string; value: string }>
|
|
1762
|
+
|
|
1763
|
+
attrLookup = new Map()
|
|
1764
|
+
for (const row of rows) {
|
|
1765
|
+
attrLookup.set(row.trace_id, row.value)
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
const groups = new Map<string, { durations: number[]; errorTraces: number }>()
|
|
1771
|
+
for (const summary of summaries) {
|
|
1772
|
+
const group = input.groupBy === "service"
|
|
1773
|
+
? summary.serviceName
|
|
1774
|
+
: input.groupBy === "operation"
|
|
1775
|
+
? summary.rootOperationName
|
|
1776
|
+
: input.groupBy === "status"
|
|
1777
|
+
? summary.errorCount > 0 ? "error" : "ok"
|
|
1778
|
+
: isAttrGroupBy
|
|
1779
|
+
? attrLookup?.get(summary.traceId) ?? "unknown"
|
|
1780
|
+
: "unknown"
|
|
1781
|
+
|
|
1782
|
+
const bucket = groups.get(group) ?? { durations: [], errorTraces: 0 }
|
|
1783
|
+
bucket.durations.push(summary.durationMs)
|
|
1784
|
+
if (summary.errorCount > 0) bucket.errorTraces++
|
|
1785
|
+
groups.set(group, bucket)
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
const rows = [...groups.entries()].map(([group, bucket]) => {
|
|
1789
|
+
const count = bucket.durations.length
|
|
1790
|
+
const value = input.agg === "count"
|
|
1791
|
+
? count
|
|
1792
|
+
: input.agg === "avg_duration"
|
|
1793
|
+
? bucket.durations.reduce((sum, d) => sum + d, 0) / Math.max(1, count)
|
|
1794
|
+
: input.agg === "p95_duration"
|
|
1795
|
+
? percentile(bucket.durations, 0.95)
|
|
1796
|
+
: bucket.errorTraces / Math.max(1, count)
|
|
1797
|
+
return { group, value, count }
|
|
1798
|
+
})
|
|
1799
|
+
|
|
1800
|
+
return rows.sort((left, right) => right.value - left.value).slice(0, limit)
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
return yield* Effect.sync(() => {
|
|
1804
|
+
const whereClauses: string[] = ["started_at_ms >= ?"]
|
|
1805
|
+
const whereParams: Array<string | number> = [cutoff]
|
|
1806
|
+
|
|
1807
|
+
if (input.serviceName) {
|
|
1808
|
+
whereClauses.push("service_name = ?")
|
|
1809
|
+
whereParams.push(input.serviceName)
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
if (input.status === "error") whereClauses.push("error_count > 0")
|
|
1813
|
+
if (input.status === "ok") whereClauses.push("error_count = 0")
|
|
1814
|
+
if (input.minDurationMs != null) {
|
|
1815
|
+
whereClauses.push("duration_ms >= ?")
|
|
1816
|
+
whereParams.push(input.minDurationMs)
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
const groupExpr = input.groupBy === "service"
|
|
1820
|
+
? "service_name"
|
|
1821
|
+
: input.groupBy === "operation"
|
|
1822
|
+
? "root_operation_name"
|
|
1823
|
+
: input.groupBy === "status"
|
|
1824
|
+
? "CASE WHEN error_count > 0 THEN 'error' ELSE 'ok' END"
|
|
1825
|
+
: "'unknown'"
|
|
1826
|
+
|
|
1827
|
+
const aggExpr = input.agg === "count"
|
|
1828
|
+
? "COUNT(*)"
|
|
1829
|
+
: input.agg === "avg_duration"
|
|
1830
|
+
? "AVG(duration_ms)"
|
|
1831
|
+
: "CAST(SUM(CASE WHEN error_count > 0 THEN 1 ELSE 0 END) AS REAL) / COUNT(*)"
|
|
1832
|
+
|
|
1833
|
+
if (input.agg === "p95_duration") {
|
|
1834
|
+
const rows = db.query(`
|
|
1835
|
+
SELECT ${groupExpr} AS grp, duration_ms
|
|
1836
|
+
FROM trace_summaries
|
|
1837
|
+
WHERE ${whereClauses.join(" AND ")}
|
|
1838
|
+
`).all(...whereParams) as Array<{ grp: string; duration_ms: number }>
|
|
1839
|
+
|
|
1840
|
+
const groups = new Map<string, number[]>()
|
|
1841
|
+
for (const row of rows) {
|
|
1842
|
+
const bucket = groups.get(row.grp) ?? []
|
|
1843
|
+
bucket.push(row.duration_ms)
|
|
1844
|
+
groups.set(row.grp, bucket)
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
return [...groups.entries()]
|
|
1848
|
+
.map(([group, durations]) => ({ group, value: percentile(durations, 0.95), count: durations.length }))
|
|
1849
|
+
.sort((left, right) => right.value - left.value)
|
|
1850
|
+
.slice(0, limit)
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1853
|
+
const rows = db.query(`
|
|
1854
|
+
SELECT ${groupExpr} AS grp, ${aggExpr} AS value, COUNT(*) AS count
|
|
1855
|
+
FROM trace_summaries
|
|
1856
|
+
WHERE ${whereClauses.join(" AND ")}
|
|
1857
|
+
GROUP BY grp
|
|
1858
|
+
ORDER BY value DESC
|
|
1859
|
+
LIMIT ?
|
|
1860
|
+
`).all(...whereParams, limit) as Array<{ grp: string; value: number; count: number }>
|
|
1861
|
+
|
|
1862
|
+
return rows.map((row) => ({ group: row.grp, value: row.value, count: row.count }))
|
|
1863
|
+
})
|
|
1864
|
+
})
|
|
1865
|
+
|
|
1866
|
+
const logStats = Effect.fn("motel/TelemetryStore.logStats")(function* (input: LogStatsSearch) {
|
|
1867
|
+
const now = yield* Clock.currentTimeMillis
|
|
1868
|
+
const limit = input.limit ?? 20
|
|
1869
|
+
const hasAttrFilters = Object.keys(input.attributeFilters ?? {}).length > 0
|
|
1870
|
+
const isAttrGroupBy = input.groupBy.startsWith("attr.")
|
|
1871
|
+
|
|
1872
|
+
// For attr.* groupBy or attr filters, fall back to in-memory grouping
|
|
1873
|
+
if (isAttrGroupBy || hasAttrFilters) {
|
|
1874
|
+
const logs = yield* searchLogs({
|
|
1875
|
+
serviceName: input.serviceName,
|
|
1876
|
+
traceId: input.traceId,
|
|
1877
|
+
spanId: input.spanId,
|
|
1878
|
+
body: input.body,
|
|
1879
|
+
lookbackMinutes: input.lookbackMinutes,
|
|
1880
|
+
attributeFilters: input.attributeFilters,
|
|
1881
|
+
limit: 5000,
|
|
1882
|
+
})
|
|
1883
|
+
|
|
1884
|
+
const groups = new Map<string, number>()
|
|
1885
|
+
for (const log of logs) {
|
|
1886
|
+
const group = input.groupBy === "service"
|
|
1887
|
+
? log.serviceName
|
|
1888
|
+
: input.groupBy === "severity"
|
|
1889
|
+
? log.severityText
|
|
1890
|
+
: input.groupBy === "scope"
|
|
1891
|
+
? log.scopeName ?? "unknown"
|
|
1892
|
+
: isAttrGroupBy
|
|
1893
|
+
? log.attributes[input.groupBy.slice(5)] ?? "unknown"
|
|
1894
|
+
: "unknown"
|
|
1895
|
+
groups.set(group, (groups.get(group) ?? 0) + 1)
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1898
|
+
return [...groups.entries()]
|
|
1899
|
+
.map(([group, count]) => ({ group, value: count, count }))
|
|
1900
|
+
.sort((left, right) => right.value - left.value)
|
|
1901
|
+
.slice(0, limit)
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
// Pure SQL path for standard groupBy fields
|
|
1905
|
+
return yield* Effect.sync(() => {
|
|
1906
|
+
const clauses: string[] = []
|
|
1907
|
+
const params: Array<string | number> = []
|
|
1908
|
+
|
|
1909
|
+
if (input.serviceName) {
|
|
1910
|
+
clauses.push("service_name = ?")
|
|
1911
|
+
params.push(input.serviceName)
|
|
1912
|
+
}
|
|
1913
|
+
if (input.traceId) {
|
|
1914
|
+
clauses.push("trace_id = ?")
|
|
1915
|
+
params.push(input.traceId)
|
|
1916
|
+
}
|
|
1917
|
+
if (input.spanId) {
|
|
1918
|
+
clauses.push("span_id = ?")
|
|
1919
|
+
params.push(input.spanId)
|
|
1920
|
+
}
|
|
1921
|
+
if (input.body) {
|
|
1922
|
+
clauses.push("body LIKE ?")
|
|
1923
|
+
params.push(`%${input.body}%`)
|
|
1924
|
+
}
|
|
1925
|
+
if (input.lookbackMinutes) {
|
|
1926
|
+
clauses.push("timestamp_ms >= ?")
|
|
1927
|
+
params.push(now - input.lookbackMinutes * 60 * 1000)
|
|
1928
|
+
}
|
|
1929
|
+
|
|
1930
|
+
const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""
|
|
1931
|
+
|
|
1932
|
+
const groupExpr = input.groupBy === "service"
|
|
1933
|
+
? "service_name"
|
|
1934
|
+
: input.groupBy === "severity"
|
|
1935
|
+
? "severity_text"
|
|
1936
|
+
: input.groupBy === "scope"
|
|
1937
|
+
? "COALESCE(scope_name, 'unknown')"
|
|
1938
|
+
: "'unknown'"
|
|
1939
|
+
|
|
1940
|
+
const rows = db.query(`
|
|
1941
|
+
SELECT ${groupExpr} AS grp, COUNT(*) AS count
|
|
1942
|
+
FROM logs
|
|
1943
|
+
${where}
|
|
1944
|
+
GROUP BY grp
|
|
1945
|
+
ORDER BY count DESC
|
|
1946
|
+
LIMIT ?
|
|
1947
|
+
`).all(...params, limit) as Array<{ grp: string; count: number }>
|
|
1948
|
+
|
|
1949
|
+
return rows.map((row) => ({ group: row.grp, value: row.count, count: row.count }))
|
|
1950
|
+
})
|
|
1951
|
+
})
|
|
1952
|
+
|
|
1953
|
+
const listRecentLogs = Effect.fn("motel/TelemetryStore.listRecentLogs")(function* (serviceName: string) {
|
|
1954
|
+
yield* Effect.annotateCurrentSpan("log.service_name", serviceName)
|
|
1955
|
+
const logs = yield* searchLogs({ serviceName, limit: config.otel.logFetchLimit })
|
|
1956
|
+
yield* Effect.annotateCurrentSpan("log.result_count", logs.length)
|
|
1957
|
+
return logs
|
|
1958
|
+
})
|
|
1959
|
+
|
|
1960
|
+
const listFacets = Effect.fn("motel/TelemetryStore.listFacets")(function* (input: FacetSearch) {
|
|
1961
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
1962
|
+
const limit = input.limit ?? 20
|
|
1963
|
+
|
|
1964
|
+
return yield* Effect.sync(() => {
|
|
1965
|
+
if (input.type === "logs") {
|
|
1966
|
+
if (input.field === "service") {
|
|
1967
|
+
const rows = db.query(`
|
|
1968
|
+
SELECT service_name AS value, COUNT(*) AS count
|
|
1969
|
+
FROM logs
|
|
1970
|
+
WHERE timestamp_ms >= ?
|
|
1971
|
+
GROUP BY service_name
|
|
1972
|
+
ORDER BY count DESC, value ASC
|
|
1973
|
+
LIMIT ?
|
|
1974
|
+
`).all(cutoff, limit) as Array<{ value: string; count: number }>
|
|
1975
|
+
return rows
|
|
1976
|
+
}
|
|
1977
|
+
if (input.field === "severity") {
|
|
1978
|
+
const rows = db.query(`
|
|
1979
|
+
SELECT severity_text AS value, COUNT(*) AS count
|
|
1980
|
+
FROM logs
|
|
1981
|
+
WHERE timestamp_ms >= ?
|
|
1982
|
+
${input.serviceName ? "AND service_name = ?" : ""}
|
|
1983
|
+
GROUP BY severity_text
|
|
1984
|
+
ORDER BY count DESC, value ASC
|
|
1985
|
+
LIMIT ?
|
|
1986
|
+
`).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
|
|
1987
|
+
return rows
|
|
1988
|
+
}
|
|
1989
|
+
if (input.field === "scope") {
|
|
1990
|
+
const rows = db.query(`
|
|
1991
|
+
SELECT COALESCE(scope_name, 'unknown') AS value, COUNT(*) AS count
|
|
1992
|
+
FROM logs
|
|
1993
|
+
WHERE timestamp_ms >= ?
|
|
1994
|
+
${input.serviceName ? "AND service_name = ?" : ""}
|
|
1995
|
+
GROUP BY COALESCE(scope_name, 'unknown')
|
|
1996
|
+
ORDER BY count DESC, value ASC
|
|
1997
|
+
LIMIT ?
|
|
1998
|
+
`).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
|
|
1999
|
+
return rows
|
|
2000
|
+
}
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
if (input.type === "traces") {
|
|
2004
|
+
if (input.field === "service") {
|
|
2005
|
+
const rows = db.query(`
|
|
2006
|
+
SELECT service_name AS value, COUNT(*) AS count
|
|
2007
|
+
FROM trace_summaries
|
|
2008
|
+
WHERE started_at_ms >= ?
|
|
2009
|
+
GROUP BY service_name
|
|
2010
|
+
ORDER BY count DESC, value ASC
|
|
2011
|
+
LIMIT ?
|
|
2012
|
+
`).all(cutoff, limit) as Array<{ value: string; count: number }>
|
|
2013
|
+
return rows
|
|
2014
|
+
}
|
|
2015
|
+
if (input.field === "operation") {
|
|
2016
|
+
const rows = db.query(`
|
|
2017
|
+
SELECT root_operation_name AS value, COUNT(*) AS count
|
|
2018
|
+
FROM trace_summaries
|
|
2019
|
+
WHERE started_at_ms >= ?
|
|
2020
|
+
${input.serviceName ? "AND service_name = ?" : ""}
|
|
2021
|
+
GROUP BY root_operation_name
|
|
2022
|
+
ORDER BY count DESC, value ASC
|
|
2023
|
+
LIMIT ?
|
|
2024
|
+
`).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
|
|
2025
|
+
return rows
|
|
2026
|
+
}
|
|
2027
|
+
if (input.field === "status") {
|
|
2028
|
+
const rows = db.query(`
|
|
2029
|
+
SELECT CASE WHEN error_count > 0 THEN 'error' ELSE 'ok' END AS value, COUNT(*) AS count
|
|
2030
|
+
FROM trace_summaries
|
|
2031
|
+
WHERE started_at_ms >= ?
|
|
2032
|
+
${input.serviceName ? "AND service_name = ?" : ""}
|
|
2033
|
+
GROUP BY value
|
|
2034
|
+
ORDER BY count DESC
|
|
2035
|
+
LIMIT ?
|
|
2036
|
+
`).all(...(input.serviceName ? [cutoff, input.serviceName, limit] : [cutoff, limit])) as Array<{ value: string; count: number }>
|
|
2037
|
+
return rows
|
|
2038
|
+
}
|
|
2039
|
+
if (input.field === "attribute_keys") {
|
|
2040
|
+
// Count distinct traces each attribute key appears on, optionally
|
|
2041
|
+
// scoped to a service. Keys with many distinct values (e.g. sessionId,
|
|
2042
|
+
// user id, model) rank higher than keys that are constant across every
|
|
2043
|
+
// trace (service.name, telemetry.sdk.*) — the latter can't discriminate
|
|
2044
|
+
// between traces so they're useless as filters.
|
|
2045
|
+
//
|
|
2046
|
+
// Performance note: we skip rows whose value blob is larger than
|
|
2047
|
+
// FACET_VALUE_MAX_LEN. For opencode this hides `ai.prompt`,
|
|
2048
|
+
// `ai.prompt.messages`, and `ai.prompt.tools` — which are 1-6MB text
|
|
2049
|
+
// blobs that you'd never want to filter by exact match anyway. The
|
|
2050
|
+
// WHERE clause lets SQLite skip reading those pages from disk.
|
|
2051
|
+
// COUNT(DISTINCT ...) does its own per-group dedup via a temp B-tree,
|
|
2052
|
+
// so the outer query needs no DISTINCT subquery in front of it. We
|
|
2053
|
+
// pre-filter trace_ids through trace_summaries (an indexed lookup) so
|
|
2054
|
+
// the planner can use a SEMI JOIN against the small in-window set
|
|
2055
|
+
// instead of joining every span_attributes row to trace_summaries.
|
|
2056
|
+
const params: Array<string | number> = []
|
|
2057
|
+
let traceFilter: string
|
|
2058
|
+
if (input.serviceName) {
|
|
2059
|
+
traceFilter = `(SELECT trace_id FROM trace_summaries WHERE started_at_ms >= ? AND service_name = ?)`
|
|
2060
|
+
params.push(cutoff, input.serviceName)
|
|
2061
|
+
} else {
|
|
2062
|
+
traceFilter = `(SELECT trace_id FROM trace_summaries WHERE started_at_ms >= ?)`
|
|
2063
|
+
params.push(cutoff)
|
|
2064
|
+
}
|
|
2065
|
+
params.push(FACET_VALUE_MAX_LEN, limit)
|
|
2066
|
+
const rows = db.query(`
|
|
2067
|
+
SELECT key AS value,
|
|
2068
|
+
COUNT(DISTINCT trace_id) AS count,
|
|
2069
|
+
COUNT(DISTINCT value) AS distinct_values
|
|
2070
|
+
FROM span_attributes
|
|
2071
|
+
WHERE trace_id IN ${traceFilter}
|
|
2072
|
+
AND LENGTH(value) < ?
|
|
2073
|
+
GROUP BY key
|
|
2074
|
+
ORDER BY (CASE WHEN distinct_values = 1 THEN 1 ELSE 0 END) ASC,
|
|
2075
|
+
distinct_values DESC,
|
|
2076
|
+
count DESC,
|
|
2077
|
+
value ASC
|
|
2078
|
+
LIMIT ?
|
|
2079
|
+
`).all(...params) as Array<{ value: string; count: number; distinct_values: number }>
|
|
2080
|
+
return rows.map((row) => ({ value: row.value, count: row.count }))
|
|
2081
|
+
}
|
|
2082
|
+
if (input.field === "attribute_values") {
|
|
2083
|
+
if (!input.key) return [] as FacetItem[]
|
|
2084
|
+
// Skip multi-KB values here too — they blow up GROUP BY on big text.
|
|
2085
|
+
// Matches the attribute_keys pre-filter so the picker stays responsive
|
|
2086
|
+
// if someone hand-crafts a URL that targets a fat key.
|
|
2087
|
+
const params: Array<string | number> = [input.key, FACET_VALUE_MAX_LEN, cutoff]
|
|
2088
|
+
if (input.serviceName) params.push(input.serviceName)
|
|
2089
|
+
params.push(limit)
|
|
2090
|
+
const rows = db.query(`
|
|
2091
|
+
SELECT sa.value AS value, COUNT(DISTINCT sa.trace_id) AS count
|
|
2092
|
+
FROM span_attributes sa
|
|
2093
|
+
JOIN spans s ON s.trace_id = sa.trace_id AND s.span_id = sa.span_id
|
|
2094
|
+
WHERE sa.key = ? AND LENGTH(sa.value) < ?
|
|
2095
|
+
AND s.start_time_ms >= ?
|
|
2096
|
+
${input.serviceName ? "AND s.service_name = ?" : ""}
|
|
2097
|
+
GROUP BY sa.value
|
|
2098
|
+
ORDER BY count DESC, value ASC
|
|
2099
|
+
LIMIT ?
|
|
2100
|
+
`).all(...params) as Array<{ value: string; count: number }>
|
|
2101
|
+
return rows
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
2104
|
+
|
|
2105
|
+
return [] as FacetItem[]
|
|
2106
|
+
})
|
|
2107
|
+
})
|
|
2108
|
+
|
|
2109
|
+
const listTraceLogs = Effect.fn("motel/TelemetryStore.listTraceLogs")(function* (traceId: string) {
|
|
2110
|
+
yield* Effect.annotateCurrentSpan("log.trace_id", traceId)
|
|
2111
|
+
const logs = yield* searchLogs({ traceId, limit: config.otel.logFetchLimit })
|
|
2112
|
+
yield* Effect.annotateCurrentSpan("log.result_count", logs.length)
|
|
2113
|
+
return logs
|
|
2114
|
+
})
|
|
2115
|
+
|
|
2116
|
+
// ---------------------------------------------------------------------------
|
|
2117
|
+
// AI Call queries
|
|
2118
|
+
// ---------------------------------------------------------------------------
|
|
2119
|
+
|
|
2120
|
+
/** Extracts ai.streamText -> "streamText", ai.streamText.doStream -> "streamText" */
|
|
2121
|
+
const parseAiOperation = (operationName: string): string => {
|
|
2122
|
+
const parts = operationName.replace(/^ai\./, "").split(".")
|
|
2123
|
+
return parts[0] ?? operationName
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
/** Builds WHERE clauses for AI call search against the spans table (aliased as s) */
|
|
2127
|
+
const buildAiWhereClauses = (input: AiCallSearch | AiCallStatsSearch, cutoff: number) => {
|
|
2128
|
+
const clauses: string[] = [
|
|
2129
|
+
"s.operation_name LIKE 'ai.%'",
|
|
2130
|
+
"s.operation_name NOT LIKE 'ai.%.do%'",
|
|
2131
|
+
"s.start_time_ms >= ?",
|
|
2132
|
+
]
|
|
2133
|
+
const params: Array<string | number> = [cutoff]
|
|
2134
|
+
|
|
2135
|
+
if (input.service) {
|
|
2136
|
+
clauses.push("s.service_name = ?")
|
|
2137
|
+
params.push(input.service)
|
|
2138
|
+
}
|
|
2139
|
+
if (input.traceId) {
|
|
2140
|
+
clauses.push("s.trace_id = ?")
|
|
2141
|
+
params.push(input.traceId)
|
|
2142
|
+
}
|
|
2143
|
+
if (input.status) {
|
|
2144
|
+
clauses.push("s.status = ?")
|
|
2145
|
+
params.push(input.status)
|
|
2146
|
+
}
|
|
2147
|
+
if (input.minDurationMs != null) {
|
|
2148
|
+
clauses.push("s.duration_ms >= ?")
|
|
2149
|
+
params.push(input.minDurationMs)
|
|
2150
|
+
}
|
|
2151
|
+
if (input.operation) {
|
|
2152
|
+
clauses.push("s.operation_name LIKE ?")
|
|
2153
|
+
params.push(`ai.${input.operation}%`)
|
|
2154
|
+
}
|
|
2155
|
+
|
|
2156
|
+
// Named attribute filters via span_attributes
|
|
2157
|
+
const attrFilters: Array<[string, string]> = []
|
|
2158
|
+
if (input.sessionId) attrFilters.push([AI_ATTR_MAP.sessionId, input.sessionId])
|
|
2159
|
+
if (input.functionId) attrFilters.push([AI_ATTR_MAP.functionId, input.functionId])
|
|
2160
|
+
if (input.provider) attrFilters.push([AI_ATTR_MAP.provider, input.provider])
|
|
2161
|
+
if (input.model) attrFilters.push([AI_ATTR_MAP.model, input.model])
|
|
2162
|
+
|
|
2163
|
+
for (const [key, value] of attrFilters) {
|
|
2164
|
+
clauses.push("EXISTS (SELECT 1 FROM span_attributes WHERE span_attributes.trace_id = s.trace_id AND span_attributes.span_id = s.span_id AND key = ? AND value = ?)")
|
|
2165
|
+
params.push(key, value)
|
|
2166
|
+
}
|
|
2167
|
+
|
|
2168
|
+
// Text search across prompt/response/tool attribute values via
|
|
2169
|
+
// FTS5. Prefers the external-content span_attr_fts index when
|
|
2170
|
+
// available, falls back to case-insensitive LIKE so old DBs
|
|
2171
|
+
// without FTS still work. FTS turns ~500ms full scans of 3 MB
|
|
2172
|
+
// prompt JSON into <50ms MATCH lookups.
|
|
2173
|
+
if ("text" in input && input.text) {
|
|
2174
|
+
const ftsQuery = toFtsMatchQuery(input.text)
|
|
2175
|
+
if (hasAttrFts && ftsQuery) {
|
|
2176
|
+
clauses.push(`EXISTS (
|
|
2177
|
+
SELECT 1 FROM span_attr_fts fts
|
|
2178
|
+
JOIN span_attributes sa ON sa.rowid = fts.rowid
|
|
2179
|
+
WHERE sa.trace_id = s.trace_id
|
|
2180
|
+
AND sa.span_id = s.span_id
|
|
2181
|
+
AND fts.value MATCH ?
|
|
2182
|
+
)`)
|
|
2183
|
+
params.push(ftsQuery)
|
|
2184
|
+
} else {
|
|
2185
|
+
const textKeys = AI_TEXT_SEARCH_KEYS.map(() => "?").join(", ")
|
|
2186
|
+
clauses.push(`EXISTS (SELECT 1 FROM span_attributes WHERE span_attributes.trace_id = s.trace_id AND span_attributes.span_id = s.span_id AND key IN (${textKeys}) AND value LIKE ? COLLATE NOCASE)`)
|
|
2187
|
+
params.push(...AI_TEXT_SEARCH_KEYS, `%${input.text}%`)
|
|
2188
|
+
}
|
|
2189
|
+
}
|
|
2190
|
+
|
|
2191
|
+
return { clauses, params }
|
|
2192
|
+
}
|
|
2193
|
+
|
|
2194
|
+
/** Load attribute values for a set of spans by key */
|
|
2195
|
+
const loadSpanAttrValues = (spans: ReadonlyArray<{ trace_id: string; span_id: string }>, keys: readonly string[]): Map<string, Map<string, string>> => {
|
|
2196
|
+
if (spans.length === 0 || keys.length === 0) return new Map()
|
|
2197
|
+
const spanPlaceholders = spans.map(() => "(?, ?)").join(", ")
|
|
2198
|
+
const keyPlaceholders = keys.map(() => "?").join(", ")
|
|
2199
|
+
const spanParams = spans.flatMap((s) => [s.trace_id, s.span_id])
|
|
2200
|
+
|
|
2201
|
+
const rows = db.query(`
|
|
2202
|
+
SELECT trace_id, span_id, key, value
|
|
2203
|
+
FROM span_attributes
|
|
2204
|
+
WHERE (trace_id, span_id) IN (VALUES ${spanPlaceholders})
|
|
2205
|
+
AND key IN (${keyPlaceholders})
|
|
2206
|
+
`).all(...spanParams, ...keys) as Array<{ trace_id: string; span_id: string; key: string; value: string }>
|
|
2207
|
+
|
|
2208
|
+
const result = new Map<string, Map<string, string>>()
|
|
2209
|
+
for (const row of rows) {
|
|
2210
|
+
const spanKey = `${row.trace_id}:${row.span_id}`
|
|
2211
|
+
let attrs = result.get(spanKey)
|
|
2212
|
+
if (!attrs) {
|
|
2213
|
+
attrs = new Map()
|
|
2214
|
+
result.set(spanKey, attrs)
|
|
2215
|
+
}
|
|
2216
|
+
attrs.set(row.key, row.value)
|
|
2217
|
+
}
|
|
2218
|
+
return result
|
|
2219
|
+
}
|
|
2220
|
+
|
|
2221
|
+
const searchAiCalls = Effect.fn("motel/TelemetryStore.searchAiCalls")(function* (input: AiCallSearch) {
|
|
2222
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
2223
|
+
const limit = input.limit ?? 20
|
|
2224
|
+
|
|
2225
|
+
return yield* Effect.sync(() => {
|
|
2226
|
+
const { clauses, params } = buildAiWhereClauses(input, cutoff)
|
|
2227
|
+
|
|
2228
|
+
const rows = db.query(`
|
|
2229
|
+
SELECT s.trace_id, s.span_id, s.service_name, s.operation_name, s.start_time_ms, s.duration_ms, s.status
|
|
2230
|
+
FROM spans AS s
|
|
2231
|
+
WHERE ${clauses.join(" AND ")}
|
|
2232
|
+
ORDER BY s.start_time_ms DESC
|
|
2233
|
+
LIMIT ?
|
|
2234
|
+
`).all(...params, limit) as Array<{
|
|
2235
|
+
trace_id: string; span_id: string; service_name: string
|
|
2236
|
+
operation_name: string; start_time_ms: number; duration_ms: number; status: string
|
|
2237
|
+
}>
|
|
2238
|
+
|
|
2239
|
+
if (rows.length === 0) return [] as readonly AiCallSummary[]
|
|
2240
|
+
|
|
2241
|
+
// Batch-load the attributes we need for summaries
|
|
2242
|
+
const summaryAttrKeys = [
|
|
2243
|
+
AI_ATTR_MAP.functionId, AI_ATTR_MAP.provider, AI_ATTR_MAP.model,
|
|
2244
|
+
AI_ATTR_MAP.sessionId, AI_ATTR_MAP.userId, AI_ATTR_MAP.finishReason,
|
|
2245
|
+
AI_ATTR_MAP.inputTokens, AI_ATTR_MAP.outputTokens, AI_ATTR_MAP.totalTokens,
|
|
2246
|
+
AI_ATTR_MAP.cachedInputTokens, AI_ATTR_MAP.reasoningTokens,
|
|
2247
|
+
AI_ATTR_MAP.promptMessages, AI_ATTR_MAP.prompt, AI_ATTR_MAP.responseText,
|
|
2248
|
+
]
|
|
2249
|
+
const attrMap = loadSpanAttrValues(rows, summaryAttrKeys)
|
|
2250
|
+
|
|
2251
|
+
// Count tool call child spans per AI span
|
|
2252
|
+
const spanPlaceholders = rows.map(() => "(?, ?)").join(", ")
|
|
2253
|
+
const spanParams = rows.flatMap((r) => [r.trace_id, r.span_id])
|
|
2254
|
+
const toolCountRows = db.query(`
|
|
2255
|
+
SELECT parent_span_id, COUNT(*) AS cnt
|
|
2256
|
+
FROM spans
|
|
2257
|
+
WHERE (trace_id, parent_span_id) IN (VALUES ${spanPlaceholders})
|
|
2258
|
+
AND operation_name LIKE 'ai.toolCall%'
|
|
2259
|
+
GROUP BY trace_id, parent_span_id
|
|
2260
|
+
`).all(...spanParams) as Array<{ parent_span_id: string; cnt: number }>
|
|
2261
|
+
const toolCounts = new Map(toolCountRows.map((r) => [r.parent_span_id, r.cnt]))
|
|
2262
|
+
|
|
2263
|
+
return rows.map((row): AiCallSummary => {
|
|
2264
|
+
const spanKey = `${row.trace_id}:${row.span_id}`
|
|
2265
|
+
const attrs = attrMap.get(spanKey)
|
|
2266
|
+
const get = (key: string) => attrs?.get(key) ?? null
|
|
2267
|
+
const getNum = (key: string) => {
|
|
2268
|
+
const v = get(key)
|
|
2269
|
+
return v != null ? Number(v) : null
|
|
2270
|
+
}
|
|
2271
|
+
|
|
2272
|
+
const promptContent = get(AI_ATTR_MAP.promptMessages) ?? get(AI_ATTR_MAP.prompt)
|
|
2273
|
+
|
|
2274
|
+
return {
|
|
2275
|
+
traceId: row.trace_id,
|
|
2276
|
+
spanId: row.span_id,
|
|
2277
|
+
operation: parseAiOperation(row.operation_name),
|
|
2278
|
+
service: row.service_name,
|
|
2279
|
+
functionId: get(AI_ATTR_MAP.functionId),
|
|
2280
|
+
provider: get(AI_ATTR_MAP.provider),
|
|
2281
|
+
model: get(AI_ATTR_MAP.model),
|
|
2282
|
+
status: row.status === "error" ? "error" : "ok",
|
|
2283
|
+
startedAt: new Date(row.start_time_ms).toISOString(),
|
|
2284
|
+
durationMs: row.duration_ms,
|
|
2285
|
+
sessionId: get(AI_ATTR_MAP.sessionId),
|
|
2286
|
+
userId: get(AI_ATTR_MAP.userId),
|
|
2287
|
+
promptPreview: truncatePreview(promptContent),
|
|
2288
|
+
responsePreview: truncatePreview(get(AI_ATTR_MAP.responseText)),
|
|
2289
|
+
finishReason: get(AI_ATTR_MAP.finishReason),
|
|
2290
|
+
toolCallCount: toolCounts.get(row.span_id) ?? 0,
|
|
2291
|
+
usage: {
|
|
2292
|
+
inputTokens: getNum(AI_ATTR_MAP.inputTokens),
|
|
2293
|
+
outputTokens: getNum(AI_ATTR_MAP.outputTokens),
|
|
2294
|
+
totalTokens: getNum(AI_ATTR_MAP.totalTokens),
|
|
2295
|
+
cachedInputTokens: getNum(AI_ATTR_MAP.cachedInputTokens),
|
|
2296
|
+
reasoningTokens: getNum(AI_ATTR_MAP.reasoningTokens),
|
|
2297
|
+
},
|
|
2298
|
+
}
|
|
2299
|
+
})
|
|
2300
|
+
})
|
|
2301
|
+
})
|
|
2302
|
+
|
|
2303
|
+
const getAiCall = Effect.fn("motel/TelemetryStore.getAiCall")(function* (spanId: string) {
|
|
2304
|
+
return yield* Effect.sync(() => {
|
|
2305
|
+
const row = db.query(`
|
|
2306
|
+
SELECT * FROM spans WHERE span_id = ? AND operation_name LIKE 'ai.%' LIMIT 1
|
|
2307
|
+
`).get(spanId) as SpanRow | null
|
|
2308
|
+
if (!row) return null
|
|
2309
|
+
|
|
2310
|
+
// Load all attributes for this span
|
|
2311
|
+
const attrRows = db.query(`
|
|
2312
|
+
SELECT key, value FROM span_attributes
|
|
2313
|
+
WHERE trace_id = ? AND span_id = ?
|
|
2314
|
+
`).all(row.trace_id, row.span_id) as Array<{ key: string; value: string }>
|
|
2315
|
+
const attrs = new Map(attrRows.map((r) => [r.key, r.value]))
|
|
2316
|
+
const get = (key: string) => attrs.get(key) ?? null
|
|
2317
|
+
const getNum = (key: string) => {
|
|
2318
|
+
const v = get(key)
|
|
2319
|
+
return v != null ? Number(v) : null
|
|
2320
|
+
}
|
|
2321
|
+
|
|
2322
|
+
// Load tool call child spans
|
|
2323
|
+
const toolCallRows = db.query(`
|
|
2324
|
+
SELECT span_id, operation_name, duration_ms, status, attributes_json
|
|
2325
|
+
FROM spans
|
|
2326
|
+
WHERE trace_id = ? AND parent_span_id = ? AND operation_name LIKE 'ai.toolCall%'
|
|
2327
|
+
ORDER BY start_time_ms ASC
|
|
2328
|
+
`).all(row.trace_id, row.span_id) as SpanRow[]
|
|
2329
|
+
|
|
2330
|
+
const toolCalls = toolCallRows.map((tc) => {
|
|
2331
|
+
const tcAttrs = JSON.parse(tc.attributes_json) as Record<string, string>
|
|
2332
|
+
return {
|
|
2333
|
+
name: tcAttrs["ai.toolCall.name"] ?? tc.operation_name,
|
|
2334
|
+
spanId: tc.span_id,
|
|
2335
|
+
status: tc.status === "error" ? "error" as const : "ok" as const,
|
|
2336
|
+
durationMs: tc.duration_ms,
|
|
2337
|
+
}
|
|
2338
|
+
})
|
|
2339
|
+
|
|
2340
|
+
// Load correlated logs
|
|
2341
|
+
const logRows = db.query(`
|
|
2342
|
+
SELECT * FROM logs WHERE span_id = ? ORDER BY timestamp_ms ASC
|
|
2343
|
+
`).all(row.span_id) as LogRow[]
|
|
2344
|
+
const logs = logRows.map(parseLogRow)
|
|
2345
|
+
|
|
2346
|
+
// Parse prompt - try as JSON first for structured display
|
|
2347
|
+
const promptRaw = get(AI_ATTR_MAP.promptMessages) ?? get(AI_ATTR_MAP.prompt)
|
|
2348
|
+
let promptMessages: unknown = null
|
|
2349
|
+
if (promptRaw) {
|
|
2350
|
+
try { promptMessages = JSON.parse(promptRaw) } catch { promptMessages = promptRaw }
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
// Parse tools
|
|
2354
|
+
const toolsRaw = get(AI_ATTR_MAP.tools)
|
|
2355
|
+
let toolsAvailable: unknown = null
|
|
2356
|
+
if (toolsRaw) {
|
|
2357
|
+
try { toolsAvailable = JSON.parse(toolsRaw) } catch { toolsAvailable = toolsRaw }
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2360
|
+
// Parse provider metadata
|
|
2361
|
+
const providerMetaRaw = get(AI_ATTR_MAP.providerMetadata)
|
|
2362
|
+
let providerMetadata: unknown = null
|
|
2363
|
+
if (providerMetaRaw) {
|
|
2364
|
+
try { providerMetadata = JSON.parse(providerMetaRaw) } catch { providerMetadata = providerMetaRaw }
|
|
2365
|
+
}
|
|
2366
|
+
|
|
2367
|
+
return {
|
|
2368
|
+
traceId: row.trace_id,
|
|
2369
|
+
spanId: row.span_id,
|
|
2370
|
+
operation: parseAiOperation(row.operation_name),
|
|
2371
|
+
service: row.service_name,
|
|
2372
|
+
functionId: get(AI_ATTR_MAP.functionId),
|
|
2373
|
+
provider: get(AI_ATTR_MAP.provider),
|
|
2374
|
+
model: get(AI_ATTR_MAP.model),
|
|
2375
|
+
status: row.status === "error" ? "error" as const : "ok" as const,
|
|
2376
|
+
startedAt: new Date(row.start_time_ms).toISOString(),
|
|
2377
|
+
durationMs: row.duration_ms,
|
|
2378
|
+
sessionId: get(AI_ATTR_MAP.sessionId),
|
|
2379
|
+
userId: get(AI_ATTR_MAP.userId),
|
|
2380
|
+
finishReason: get(AI_ATTR_MAP.finishReason),
|
|
2381
|
+
promptMessages,
|
|
2382
|
+
responseText: get(AI_ATTR_MAP.responseText),
|
|
2383
|
+
toolCalls,
|
|
2384
|
+
toolsAvailable,
|
|
2385
|
+
providerMetadata,
|
|
2386
|
+
usage: {
|
|
2387
|
+
inputTokens: getNum(AI_ATTR_MAP.inputTokens),
|
|
2388
|
+
outputTokens: getNum(AI_ATTR_MAP.outputTokens),
|
|
2389
|
+
totalTokens: getNum(AI_ATTR_MAP.totalTokens),
|
|
2390
|
+
cachedInputTokens: getNum(AI_ATTR_MAP.cachedInputTokens),
|
|
2391
|
+
reasoningTokens: getNum(AI_ATTR_MAP.reasoningTokens),
|
|
2392
|
+
},
|
|
2393
|
+
timing: {
|
|
2394
|
+
msToFirstChunk: getNum(AI_ATTR_MAP.msToFirstChunk),
|
|
2395
|
+
msToFinish: getNum(AI_ATTR_MAP.msToFinish),
|
|
2396
|
+
avgOutputTokensPerSecond: getNum(AI_ATTR_MAP.avgOutputTokensPerSecond),
|
|
2397
|
+
},
|
|
2398
|
+
logs,
|
|
2399
|
+
} satisfies AiCallDetail
|
|
2400
|
+
})
|
|
2401
|
+
})
|
|
2402
|
+
|
|
2403
|
+
const aiCallStats = Effect.fn("motel/TelemetryStore.aiCallStats")(function* (input: AiCallStatsSearch) {
|
|
2404
|
+
const cutoff = (yield* Clock.currentTimeMillis) - (input.lookbackMinutes ?? config.otel.traceLookbackMinutes) * 60 * 1000
|
|
2405
|
+
const limit = input.limit ?? 20
|
|
2406
|
+
|
|
2407
|
+
return yield* Effect.sync(() => {
|
|
2408
|
+
const { clauses, params } = buildAiWhereClauses(input, cutoff)
|
|
2409
|
+
|
|
2410
|
+
// For status groupBy, we can do it purely from the spans table
|
|
2411
|
+
if (input.groupBy === "status") {
|
|
2412
|
+
const rows = db.query(`
|
|
2413
|
+
SELECT s.status AS grp, COUNT(*) AS count, AVG(s.duration_ms) AS avg_dur
|
|
2414
|
+
FROM spans AS s
|
|
2415
|
+
WHERE ${clauses.join(" AND ")}
|
|
2416
|
+
GROUP BY s.status
|
|
2417
|
+
ORDER BY count DESC
|
|
2418
|
+
LIMIT ?
|
|
2419
|
+
`).all(...params, limit) as Array<{ grp: string; count: number; avg_dur: number }>
|
|
2420
|
+
|
|
2421
|
+
if (input.agg === "count") return rows.map((r) => ({ group: r.grp, value: r.count, count: r.count }))
|
|
2422
|
+
if (input.agg === "avg_duration") return rows.map((r) => ({ group: r.grp, value: r.avg_dur, count: r.count }))
|
|
2423
|
+
}
|
|
2424
|
+
|
|
2425
|
+
// For attribute-based groupBy, we need to join span_attributes
|
|
2426
|
+
const groupByAttrKey = input.groupBy === "provider" ? AI_ATTR_MAP.provider
|
|
2427
|
+
: input.groupBy === "model" ? AI_ATTR_MAP.model
|
|
2428
|
+
: input.groupBy === "functionId" ? AI_ATTR_MAP.functionId
|
|
2429
|
+
: input.groupBy === "sessionId" ? AI_ATTR_MAP.sessionId
|
|
2430
|
+
: null
|
|
2431
|
+
|
|
2432
|
+
if (!groupByAttrKey) return []
|
|
2433
|
+
|
|
2434
|
+
// First get the matching spans with their group values
|
|
2435
|
+
const rows = db.query(`
|
|
2436
|
+
SELECT
|
|
2437
|
+
COALESCE(ga.value, 'unknown') AS grp,
|
|
2438
|
+
s.span_id,
|
|
2439
|
+
s.duration_ms,
|
|
2440
|
+
s.status
|
|
2441
|
+
FROM spans AS s
|
|
2442
|
+
LEFT JOIN span_attributes AS ga
|
|
2443
|
+
ON ga.trace_id = s.trace_id AND ga.span_id = s.span_id AND ga.key = ?
|
|
2444
|
+
WHERE ${clauses.join(" AND ")}
|
|
2445
|
+
`).all(groupByAttrKey, ...params) as Array<{ grp: string; span_id: string; duration_ms: number; status: string }>
|
|
2446
|
+
|
|
2447
|
+
// Group and aggregate in JS (need p95 and token aggregation)
|
|
2448
|
+
const groups = new Map<string, { durations: number[]; count: number; spanIds: string[] }>()
|
|
2449
|
+
for (const row of rows) {
|
|
2450
|
+
const bucket = groups.get(row.grp) ?? { durations: [], count: 0, spanIds: [] }
|
|
2451
|
+
bucket.durations.push(row.duration_ms)
|
|
2452
|
+
bucket.count++
|
|
2453
|
+
bucket.spanIds.push(row.span_id)
|
|
2454
|
+
groups.set(row.grp, bucket)
|
|
2455
|
+
}
|
|
2456
|
+
|
|
2457
|
+
// For token aggregations, batch-load from span_attributes
|
|
2458
|
+
if (input.agg === "total_input_tokens" || input.agg === "total_output_tokens") {
|
|
2459
|
+
const tokenKey = input.agg === "total_input_tokens" ? AI_ATTR_MAP.inputTokens : AI_ATTR_MAP.outputTokens
|
|
2460
|
+
const allSpanIds = [...groups.values()].flatMap((b) => b.spanIds)
|
|
2461
|
+
if (allSpanIds.length > 0) {
|
|
2462
|
+
const placeholders = allSpanIds.map(() => "?").join(", ")
|
|
2463
|
+
const tokenRows = db.query(`
|
|
2464
|
+
SELECT span_id, CAST(value AS REAL) AS tokens
|
|
2465
|
+
FROM span_attributes
|
|
2466
|
+
WHERE key = ? AND span_id IN (${placeholders})
|
|
2467
|
+
`).all(tokenKey, ...allSpanIds) as Array<{ span_id: string; tokens: number }>
|
|
2468
|
+
|
|
2469
|
+
const tokenBySpan = new Map(tokenRows.map((r) => [r.span_id, r.tokens]))
|
|
2470
|
+
|
|
2471
|
+
return [...groups.entries()]
|
|
2472
|
+
.map(([group, bucket]) => {
|
|
2473
|
+
const total = bucket.spanIds.reduce((sum, sid) => sum + (tokenBySpan.get(sid) ?? 0), 0)
|
|
2474
|
+
return { group, value: total, count: bucket.count }
|
|
2475
|
+
})
|
|
2476
|
+
.sort((a, b) => b.value - a.value)
|
|
2477
|
+
.slice(0, limit)
|
|
2478
|
+
}
|
|
2479
|
+
}
|
|
2480
|
+
|
|
2481
|
+
return [...groups.entries()]
|
|
2482
|
+
.map(([group, bucket]) => {
|
|
2483
|
+
const value = input.agg === "count"
|
|
2484
|
+
? bucket.count
|
|
2485
|
+
: input.agg === "avg_duration"
|
|
2486
|
+
? bucket.durations.reduce((s, d) => s + d, 0) / Math.max(1, bucket.count)
|
|
2487
|
+
: input.agg === "p95_duration"
|
|
2488
|
+
? percentile(bucket.durations, 0.95)
|
|
2489
|
+
: bucket.count
|
|
2490
|
+
return { group, value, count: bucket.count }
|
|
2491
|
+
})
|
|
2492
|
+
.sort((a, b) => b.value - a.value)
|
|
2493
|
+
.slice(0, limit)
|
|
2494
|
+
})
|
|
2495
|
+
})
|
|
2496
|
+
|
|
2497
|
+
return TelemetryStore.of({
|
|
2498
|
+
ingestTraces,
|
|
2499
|
+
ingestLogs,
|
|
2500
|
+
listServices,
|
|
2501
|
+
listRecentTraces,
|
|
2502
|
+
listTraceSummaries,
|
|
2503
|
+
searchTraces,
|
|
2504
|
+
searchTraceSummaries,
|
|
2505
|
+
traceStats,
|
|
2506
|
+
getTrace,
|
|
2507
|
+
getSpan,
|
|
2508
|
+
listTraceSpans,
|
|
2509
|
+
searchSpans,
|
|
2510
|
+
searchLogs,
|
|
2511
|
+
logStats,
|
|
2512
|
+
listFacets,
|
|
2513
|
+
listRecentLogs,
|
|
2514
|
+
listTraceLogs,
|
|
2515
|
+
searchAiCalls,
|
|
2516
|
+
getAiCall,
|
|
2517
|
+
aiCallStats,
|
|
2518
|
+
runRetentionNow: cleanupExpired(),
|
|
2519
|
+
})
|
|
2520
|
+
})
|
|
2521
|
+
|
|
2522
|
+
/** Compatibility factory for callers constructing a writer/query-capable store layer. */
|
|
2523
|
+
export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) =>
|
|
2524
|
+
Layer.effect(TelemetryStore, makeTelemetryStoreEffect(opts)).pipe(Layer.provide(BunFileSystem.layer))
|
|
2525
|
+
|
|
2526
|
+
/**
|
|
2527
|
+
* Default writer runtime used by tests and direct store consumers.
|
|
2528
|
+
*/
|
|
2529
|
+
export const TelemetryStoreLive = makeTelemetryStoreLayer({ readonly: false, runRetention: true })
|
|
2530
|
+
|
|
2531
|
+
/**
|
|
2532
|
+
* The ingest worker's writer. It is the managed daemon's sole owner of
|
|
2533
|
+
* schema migrations, FTS backfill, retention, and page reclamation.
|
|
2534
|
+
*/
|
|
2535
|
+
export const TelemetryStoreWorkerLive = TelemetryStoreLive
|
|
2536
|
+
|
|
2537
|
+
/**
|
|
2538
|
+
* Read-only instance for query-only processes (currently the TUI and
|
|
2539
|
+
* HTTP query handlers). Skips every DDL/DML statement at startup so
|
|
2540
|
+
* the connection can be opened while a writer is mid-transaction
|
|
2541
|
+
* without racing for the write lock. Provided as TelemetryStoreReadonly
|
|
2542
|
+
* — a distinct service identifier so it can coexist with the writer
|
|
2543
|
+
* TelemetryStore in the same runtime.
|
|
2544
|
+
*/
|
|
2545
|
+
export const TelemetryStoreReadonlyLive = Layer.effect(TelemetryStoreReadonly, makeTelemetryStoreEffect({ readonly: true, runRetention: false })).pipe(Layer.provide(BunFileSystem.layer))
|
|
2546
|
+
|
|
2547
|
+
/** Query-worker reader that waits for the sole writer to finish schema bootstrap. */
|
|
2548
|
+
export const TelemetryStoreQueryWorkerLive = Layer.effect(
|
|
2549
|
+
TelemetryStoreReadonly,
|
|
2550
|
+
makeTelemetryStoreEffect({ readonly: true, runRetention: false }).pipe(
|
|
2551
|
+
Effect.map((store) => TelemetryStoreReadonly.of(store)),
|
|
2552
|
+
Effect.retry(Schedule.spaced("50 millis")),
|
|
2553
|
+
),
|
|
2554
|
+
).pipe(Layer.provide(BunFileSystem.layer))
|