@aryasaatvik/motel 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +20 -17
- package/README.md +53 -4
- package/package.json +3 -2
- package/src/services/TelemetryStore.ts +23 -10
- package/src/services/retention.test.ts +150 -1
- package/src/services/retention.ts +44 -1
- package/src/telemetry.test.ts +224 -0
package/AGENTS.md
CHANGED
|
@@ -130,27 +130,30 @@ The repo is wired up with `@effect/language-service` as a `tsconfig.json` `plugi
|
|
|
130
130
|
- Health: `http://127.0.0.1:27686/api/health`
|
|
131
131
|
|
|
132
132
|
## Env Vars
|
|
133
|
-
- `
|
|
134
|
-
- `
|
|
133
|
+
Integer settings treat blank, non-numeric, and non-positive values as unset. `src/config.ts` resolves the base URL as `MOTEL_OTEL_BASE_URL`, then `MOTEL_OTEL_QUERY_URL`, then `MOTEL_OTEL_COLLECTOR_URL`, then `http://127.0.0.1:27686`. Host and port follow that URL unless set (`src/config.ts` uses port 80 when the URL has no port). The managed daemon in `src/daemon.ts` stops at `MOTEL_OTEL_QUERY_URL` and uses port 27686 when its URL has no port, then writes that listen address into the server's base URL, query URL, host, port, and exporter URLs. The server's database file is `MOTEL_OTEL_DB_PATH`, or `telemetry.sqlite` in the runtime directory, independent of the listen address.
|
|
134
|
+
- `MOTEL_OTEL_ENABLED`: defaults to `false`. Blank, `0`, `false`, `no`, and `off` stay off; any other value makes the TUI and CLI export motel's own traces and logs.
|
|
135
|
+
- `MOTEL_OTEL_SERVICE_NAME`: defaults to `motel-otel-tui`. A nonblank value is the exporter service name, the CLI default service, and the initial TUI service. Blank keeps the remembered TUI service.
|
|
135
136
|
- `MOTEL_OTEL_BASE_URL`: defaults to `http://127.0.0.1:27686`
|
|
136
|
-
- `
|
|
137
|
-
- `
|
|
138
|
-
- `
|
|
139
|
-
- `
|
|
140
|
-
- `
|
|
141
|
-
- `
|
|
142
|
-
- `MOTEL_RUNTIME_DIR`:
|
|
143
|
-
- `
|
|
144
|
-
- `
|
|
145
|
-
- `
|
|
137
|
+
- `MOTEL_OTEL_QUERY_URL`: base URL fallback when `MOTEL_OTEL_BASE_URL` is unset. The configured query URL is the resolved base URL.
|
|
138
|
+
- `MOTEL_OTEL_COLLECTOR_URL`: base URL fallback when both URLs above are unset. The managed daemon leaves it unused.
|
|
139
|
+
- `MOTEL_OTEL_HOST`: defaults to the base URL hostname (`127.0.0.1`)
|
|
140
|
+
- `MOTEL_OTEL_PORT`: defaults to the base URL port (`27686`; `80` from `src/config.ts` when the URL has no port)
|
|
141
|
+
- `MOTEL_OTEL_EXPORTER_URL`: defaults to the base URL plus `v1/traces`
|
|
142
|
+
- `MOTEL_OTEL_LOGS_EXPORTER_URL`: defaults to the base URL plus `v1/logs`
|
|
143
|
+
- `MOTEL_RUNTIME_DIR`: defaults to `${XDG_STATE_HOME:-~/.local/state}/motel` (default database, daemon log, lock, and instance registry; primarily overridden for isolated tests and custom managed instances)
|
|
144
|
+
- `MOTEL_OTEL_DB_PATH`: defaults to `$MOTEL_RUNTIME_DIR/telemetry.sqlite` (one shared DB per machine)
|
|
145
|
+
- `MOTEL_URL`: when set, the MCP client uses this base URL instead of the instance registry
|
|
146
|
+
- `MOTEL_OTEL_TRACE_LOOKBACK_MINUTES`: defaults to `1440` (24h) for service discovery and store queries that omit a lookback. HTTP list and search routes send their own `lookback` (default 60 minutes, maximum 1440).
|
|
147
|
+
- `MOTEL_OTEL_TRACE_LIMIT`: defaults to `100` for the TUI and CLI. HTTP list routes send their own `limit`.
|
|
148
|
+
- `MOTEL_OTEL_LOG_LIMIT`: defaults to `80` for TUI service logs, TUI trace logs, and CLI log commands. HTTP log routes send their own `limit`.
|
|
146
149
|
- `MOTEL_OTEL_RETENTION_HOURS`: defaults to `168` (7d)
|
|
147
|
-
- `MOTEL_OTEL_MAX_DB_SIZE_MB`: defaults to `1024` (
|
|
148
|
-
- `MOTEL_OTEL_RETENTION_TRACE_BATCH`: defaults to `100` completed traces per cleanup pass
|
|
149
|
-
- `MOTEL_OTEL_RETENTION_LOG_BATCH`: defaults to `5000` logs per cleanup pass
|
|
150
|
+
- `MOTEL_OTEL_MAX_DB_SIZE_MB`: defaults to `1024` (active-data target, not a hard file cap)
|
|
151
|
+
- `MOTEL_OTEL_RETENTION_TRACE_BATCH`: defaults to `100` completed traces marked per cleanup pass
|
|
152
|
+
- `MOTEL_OTEL_RETENTION_LOG_BATCH`: defaults to `5000` logs marked per cleanup pass
|
|
150
153
|
- `MOTEL_OTEL_QUERY_CAPACITY`: defaults to `8` outstanding queries per runtime (maximum `64`)
|
|
151
154
|
- `MOTEL_OTEL_QUERY_DEADLINE_MS`: defaults to `5000` including queue and startup time (maximum `30000`)
|
|
152
|
-
- `MOTEL_OTEL_RETENTION_ROW_BATCH`: defaults to `1000`
|
|
153
|
-
- `MOTEL_OTEL_RETENTION_INTERVAL_SECONDS`: defaults to `10`
|
|
155
|
+
- `MOTEL_OTEL_RETENTION_ROW_BATCH`: defaults to `1000` rows deleted per cleanup pass, split across traces and logs. Bounds row count, not the size of one field.
|
|
156
|
+
- `MOTEL_OTEL_RETENTION_INTERVAL_SECONDS`: defaults to `10` after an idle pass. Cleanup that is still making progress waits 100 ms.
|
|
154
157
|
|
|
155
158
|
## TUI Keys
|
|
156
159
|
- `?`: toggle shortcut help
|
package/README.md
CHANGED
|
@@ -42,7 +42,7 @@ npm install -g @aryasaatvik/motel
|
|
|
42
42
|
Don't have Bun?
|
|
43
43
|
|
|
44
44
|
```bash
|
|
45
|
-
curl -fsSL https://bun.sh/install | bash
|
|
45
|
+
curl -fsSL https://bun.sh/install | bash -s "bun-v1.4.4"
|
|
46
46
|
```
|
|
47
47
|
|
|
48
48
|
`motel` starts the local OTLP ingest server on
|
|
@@ -50,7 +50,7 @@ curl -fsSL https://bun.sh/install | bash
|
|
|
50
50
|
the keyboard cheat sheet, or `c` to copy paste-ready setup instructions
|
|
51
51
|
for any Effect/OTEL app you want to trace.
|
|
52
52
|
|
|
53
|
-
Requirements: [Bun](https://bun.sh/) matching `package.json` `engines.bun` (`>=1.
|
|
53
|
+
Requirements: [Bun](https://bun.sh/) matching `package.json` `engines.bun` (`>=1.4.4`).
|
|
54
54
|
|
|
55
55
|
### macOS service from the maintained checkout
|
|
56
56
|
|
|
@@ -90,12 +90,61 @@ The store retains seven days of telemetry by default and targets a 1 GB
|
|
|
90
90
|
active-data ceiling using bounded background batches. Recent data is preserved while
|
|
91
91
|
the oldest completed traces and logs are removed first. Configure the policy
|
|
92
92
|
with `MOTEL_OTEL_RETENTION_HOURS`, `MOTEL_OTEL_MAX_DB_SIZE_MB`,
|
|
93
|
-
`MOTEL_OTEL_RETENTION_TRACE_BATCH`, `
|
|
94
|
-
`
|
|
93
|
+
`MOTEL_OTEL_RETENTION_TRACE_BATCH`, `MOTEL_OTEL_RETENTION_ROW_BATCH`,
|
|
94
|
+
`MOTEL_OTEL_RETENTION_LOG_BATCH`, and
|
|
95
|
+
`MOTEL_OTEL_RETENTION_INTERVAL_SECONDS`.
|
|
96
|
+
Existing databases created without
|
|
95
97
|
incremental auto-vacuum are never silently rewritten at startup; deleted pages
|
|
96
98
|
are reused, but shrinking such a historical file requires an explicit offline
|
|
97
99
|
SQLite `VACUUM` chosen by the user.
|
|
98
100
|
|
|
101
|
+
## Configuration
|
|
102
|
+
|
|
103
|
+
Processes that load `src/config.ts` resolve one base URL, in order:
|
|
104
|
+
`MOTEL_OTEL_BASE_URL`, `MOTEL_OTEL_QUERY_URL`, `MOTEL_OTEL_COLLECTOR_URL`,
|
|
105
|
+
then `http://127.0.0.1:27686`. The query URL is that base URL. Host is
|
|
106
|
+
`MOTEL_OTEL_HOST`, or the base URL hostname. Port is `MOTEL_OTEL_PORT` when
|
|
107
|
+
it is a positive integer, otherwise the base URL port, otherwise 80. Trace
|
|
108
|
+
and log exporter URLs default to that base with `v1/traces` and `v1/logs`.
|
|
109
|
+
|
|
110
|
+
`motel start`, and the daemon the TUI launches, build the listen address
|
|
111
|
+
from `MOTEL_OTEL_BASE_URL`, then `MOTEL_OTEL_QUERY_URL`, then
|
|
112
|
+
`http://127.0.0.1:27686`. A missing port becomes 27686. The spawned server
|
|
113
|
+
receives that listen address as its base URL, query URL, host, port, and
|
|
114
|
+
exporter URLs (the listen origin plus `/v1/traces` and `/v1/logs`). The
|
|
115
|
+
database file is resolved separately: `MOTEL_OTEL_DB_PATH`, or
|
|
116
|
+
`telemetry.sqlite` in the runtime directory.
|
|
117
|
+
|
|
118
|
+
Integer settings treat blank, non-numeric, and non-positive values as unset.
|
|
119
|
+
`MOTEL_OTEL_QUERY_CAPACITY` is capped at 64 and
|
|
120
|
+
`MOTEL_OTEL_QUERY_DEADLINE_MS` at 30000.
|
|
121
|
+
|
|
122
|
+
| Variable | Default | What it does |
|
|
123
|
+
| --- | --- | --- |
|
|
124
|
+
| `MOTEL_OTEL_BASE_URL` | `http://127.0.0.1:27686` | Resolved base URL and query URL. First choice in the fallback above. |
|
|
125
|
+
| `MOTEL_OTEL_QUERY_URL` | unset | Base URL when `MOTEL_OTEL_BASE_URL` is unset. The query URL equals the resolved base URL. |
|
|
126
|
+
| `MOTEL_OTEL_COLLECTOR_URL` | unset | Base URL only when both URLs above are unset. The managed daemon leaves it unused. |
|
|
127
|
+
| `MOTEL_OTEL_HOST` | base URL hostname (`127.0.0.1`) | Hostname the server binds. |
|
|
128
|
+
| `MOTEL_OTEL_PORT` | base URL port (`27686`) | Port the server binds. `src/config.ts` uses 80 when the URL has no port; the managed daemon uses 27686. |
|
|
129
|
+
| `MOTEL_OTEL_SERVICE_NAME` | `motel-otel-tui` | Name on motel's own exporter, and the CLI default service. A nonblank value is also the TUI's initial service. Blank keeps the remembered TUI service. |
|
|
130
|
+
| `MOTEL_OTEL_ENABLED` | `false` | Export motel's own traces and logs from the TUI and CLI. Blank, `0`, `false`, `no`, and `off` stay off; any other value turns it on. |
|
|
131
|
+
| `MOTEL_OTEL_EXPORTER_URL` | base URL + `/v1/traces` | Where that self-export posts spans. |
|
|
132
|
+
| `MOTEL_OTEL_LOGS_EXPORTER_URL` | base URL + `/v1/logs` | Where that self-export posts logs. |
|
|
133
|
+
| `MOTEL_RUNTIME_DIR` | `${XDG_STATE_HOME:-~/.local/state}/motel` | State directory: default database, daemon log, lock, and instance registry. |
|
|
134
|
+
| `MOTEL_OTEL_DB_PATH` | `$MOTEL_RUNTIME_DIR/telemetry.sqlite` | SQLite file. One shared database per machine unless this is set. |
|
|
135
|
+
| `MOTEL_URL` | unset | Base URL for the MCP client. When set, discovery uses it instead of the instance registry. |
|
|
136
|
+
| `MOTEL_OTEL_TRACE_LOOKBACK_MINUTES` | `1440` | Age window, in minutes, for service discovery and for store queries that omit a lookback. HTTP list and search routes send their own `lookback` (default 60 minutes, maximum 1440). |
|
|
137
|
+
| `MOTEL_OTEL_TRACE_LIMIT` | `100` | Trace count for the TUI and for CLI trace commands that omit a limit. HTTP list routes send their own `limit`. |
|
|
138
|
+
| `MOTEL_OTEL_LOG_LIMIT` | `80` | Log count for TUI service logs, TUI trace logs, and CLI log commands. HTTP log routes send their own `limit`. |
|
|
139
|
+
| `MOTEL_OTEL_QUERY_CAPACITY` | `8` (maximum 64) | Outstanding readonly queries per runtime. Overload is HTTP 503 `QUERY_OVERLOADED`. |
|
|
140
|
+
| `MOTEL_OTEL_QUERY_DEADLINE_MS` | `5000` (maximum 30000) | Queue plus startup deadline for one readonly query. Expiry is HTTP 504 `QUERY_DEADLINE`. |
|
|
141
|
+
| `MOTEL_OTEL_RETENTION_HOURS` | `168` | Age cutoff for completed traces and for logs. |
|
|
142
|
+
| `MOTEL_OTEL_MAX_DB_SIZE_MB` | `1024` | Active-data target in MB: `(page_count - freelist_count) * page_size`. Not a hard file-size cap. |
|
|
143
|
+
| `MOTEL_OTEL_RETENTION_TRACE_BATCH` | `100` | Completed traces marked per cleanup pass. |
|
|
144
|
+
| `MOTEL_OTEL_RETENTION_ROW_BATCH` | `1000` | Rows deleted per cleanup pass, split across traces and logs. Bounds row count, not the size of one field. |
|
|
145
|
+
| `MOTEL_OTEL_RETENTION_LOG_BATCH` | `5000` | Logs marked per cleanup pass. |
|
|
146
|
+
| `MOTEL_OTEL_RETENTION_INTERVAL_SECONDS` | `10` | Delay after an idle cleanup pass. Cleanup that is still making progress waits 100 ms. |
|
|
147
|
+
|
|
99
148
|
## How agents connect
|
|
100
149
|
|
|
101
150
|
Agents with the `motel-debug` skill installed will automatically use
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aryasaatvik/motel",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.3",
|
|
4
4
|
"description": "A local OpenTelemetry ingest + TUI viewer for development, backed by SQLite.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -26,8 +26,9 @@
|
|
|
26
26
|
"workspaces": [
|
|
27
27
|
"web"
|
|
28
28
|
],
|
|
29
|
+
"packageManager": "bun@1.4.4",
|
|
29
30
|
"engines": {
|
|
30
|
-
"bun": ">=1.
|
|
31
|
+
"bun": ">=1.4.4"
|
|
31
32
|
},
|
|
32
33
|
"publishConfig": {
|
|
33
34
|
"access": "public",
|
|
@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite"
|
|
|
2
2
|
import * as BunFileSystem from "@effect/platform-bun/BunFileSystem"
|
|
3
3
|
import { dirname } from "node:path"
|
|
4
4
|
import { Cause, Clock, Effect, FileSystem, Layer, Schedule, Context } from "effect"
|
|
5
|
-
import { installRetentionSchema, checkpointPassive, mergeFts, retainBatch, repairSearchRows } from "./retention.js"
|
|
5
|
+
import { installRetentionSchema, checkpointPassive, mergeFts, retainBatch, repairSearchRows, SPAN_SEARCH_MAPPING_READY_KEY } from "./retention.js"
|
|
6
6
|
import { WriterDiagnostics } from "../ingestReadiness.ts"
|
|
7
7
|
import { config } from "../config.js"
|
|
8
8
|
import type { AiCallDetail, AiCallSummary, FacetItem, LogItem, SpanItem, StatsItem, TraceItem, TraceSummaryItem, TraceSpanEvent, TraceSpanItem } from "../domain.js"
|
|
@@ -872,14 +872,21 @@ const makeTelemetryStoreEffect = (opts: TelemetryStoreOptions) =>
|
|
|
872
872
|
query.run(...entries.flatMap(([key, value]) => [traceId, spanId, key, value]))
|
|
873
873
|
}
|
|
874
874
|
const deleteSpanOperationSearch = db.query(`DELETE FROM span_operation_fts WHERE trace_id = ? AND span_id = ?`)
|
|
875
|
+
const spanSearchMappingReady = hasRetention ? db.query("SELECT 1 FROM motel_maintenance WHERE key = ? AND value = 'complete'") : null
|
|
876
|
+
const deleteMappedSpanOperationSearch = hasRetention ? db.query(`DELETE FROM span_operation_fts WHERE rowid IN (
|
|
877
|
+
SELECT id FROM span_search_rows WHERE trace_id = ? AND span_id = ?
|
|
878
|
+
)`) : null
|
|
875
879
|
const insertSpanOperationSearch = db.query(`INSERT INTO span_operation_fts (trace_id, span_id, operation_name) VALUES (?, ?, ?)`)
|
|
876
880
|
const deleteSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
|
|
877
881
|
const insertSpanOperationSearchManyByCount = new Map<number, ReturnType<Database["query"]>>()
|
|
878
882
|
const updateSpanOperationSearchMany = (operations: ReadonlyArray<readonly [string, string, string]>) => {
|
|
879
883
|
if (operations.length === 0) return
|
|
884
|
+
const mappedDelete = spanSearchMappingReady !== null && deleteMappedSpanOperationSearch !== null
|
|
885
|
+
&& spanSearchMappingReady.get(SPAN_SEARCH_MAPPING_READY_KEY) !== null ? deleteMappedSpanOperationSearch : null
|
|
880
886
|
if (operations.length === 1) {
|
|
881
887
|
const [traceId, spanId, operationName] = operations[0]!
|
|
882
|
-
|
|
888
|
+
if (mappedDelete !== null) mappedDelete.run(traceId, spanId)
|
|
889
|
+
else deleteSpanOperationSearch.run(traceId, spanId)
|
|
883
890
|
db.query("DELETE FROM span_search_rows WHERE trace_id = ? AND span_id = ?").run(traceId, spanId)
|
|
884
891
|
const result = insertSpanOperationSearch.run(traceId, spanId, operationName)
|
|
885
892
|
mapSpanSearch?.run(Number(result.lastInsertRowid), traceId, spanId)
|
|
@@ -891,7 +898,9 @@ const makeTelemetryStoreEffect = (opts: TelemetryStoreOptions) =>
|
|
|
891
898
|
deleteQuery = db.query(`DELETE FROM span_operation_fts WHERE ${operations.map(() => "(trace_id = ? AND span_id = ?)").join(" OR ")}`)
|
|
892
899
|
deleteSpanOperationSearchManyByCount.set(operations.length, deleteQuery)
|
|
893
900
|
}
|
|
894
|
-
|
|
901
|
+
if (mappedDelete !== null) {
|
|
902
|
+
for (const [traceId, spanId] of operations) mappedDelete.run(traceId, spanId)
|
|
903
|
+
} else deleteQuery.run(...operations.flatMap(([traceId, spanId]) => [traceId, spanId]))
|
|
895
904
|
for (const [traceId, spanId] of operations) db.query("DELETE FROM span_search_rows WHERE trace_id = ? AND span_id = ?").run(traceId, spanId)
|
|
896
905
|
|
|
897
906
|
let insertQuery = insertSpanOperationSearchManyByCount.get(operations.length)
|
|
@@ -904,6 +913,13 @@ const makeTelemetryStoreEffect = (opts: TelemetryStoreOptions) =>
|
|
|
904
913
|
const firstId = Number(result.lastInsertRowid) - operations.length + 1
|
|
905
914
|
for (const [index, [traceId, spanId]] of operations.entries()) mapSpanSearch?.run(firstId + index, traceId, spanId)
|
|
906
915
|
}
|
|
916
|
+
// An optional search-index failure must leave every FTS row mapped.
|
|
917
|
+
const updateSpanOperationSearch = db.transaction((operations: Parameters<typeof updateSpanOperationSearchMany>[0]) => {
|
|
918
|
+
const BATCH_SIZE = 500
|
|
919
|
+
for (let offset = 0; offset < operations.length; offset += BATCH_SIZE) {
|
|
920
|
+
updateSpanOperationSearchMany(operations.slice(offset, offset + BATCH_SIZE))
|
|
921
|
+
}
|
|
922
|
+
})
|
|
907
923
|
const insertLogAttribute = db.query(`INSERT INTO log_attributes (log_id, key, value) VALUES (?, ?, ?)`)
|
|
908
924
|
const logAttributeInsertManyByCount = new Map<number, ReturnType<Database["query"]>>()
|
|
909
925
|
const insertLogAttributesMany = (logId: number, attributes: Readonly<Record<string, string>>) => {
|
|
@@ -979,8 +995,8 @@ const makeTelemetryStoreEffect = (opts: TelemetryStoreOptions) =>
|
|
|
979
995
|
logs: config.otel.retentionLogBatch,
|
|
980
996
|
rows: config.otel.retentionRowBatch,
|
|
981
997
|
})
|
|
982
|
-
|
|
983
|
-
|
|
998
|
+
const spanMappingPending = repairSearchRows(db, Math.min(100, config.otel.retentionRowBatch))
|
|
999
|
+
maintenancePending = (result.pending && result.rows > 0) || spanMappingPending
|
|
984
1000
|
mergeFts(db, 100)
|
|
985
1001
|
checkpoint()
|
|
986
1002
|
})
|
|
@@ -1114,12 +1130,9 @@ const makeTelemetryStoreEffect = (opts: TelemetryStoreOptions) =>
|
|
|
1114
1130
|
}
|
|
1115
1131
|
}
|
|
1116
1132
|
try {
|
|
1117
|
-
|
|
1118
|
-
for (let offset = 0; offset < touchedOperations.length; offset += BATCH_SIZE) {
|
|
1119
|
-
updateSpanOperationSearchMany(touchedOperations.slice(offset, offset + BATCH_SIZE))
|
|
1120
|
-
}
|
|
1133
|
+
updateSpanOperationSearch(touchedOperations)
|
|
1121
1134
|
} catch {
|
|
1122
|
-
// FTS is optional.
|
|
1135
|
+
// FTS is optional; its nested transaction restores the index and mapping together.
|
|
1123
1136
|
}
|
|
1124
1137
|
for (const traceId of touchedTraceIds) {
|
|
1125
1138
|
upsertTraceSummary.run(traceId)
|
|
@@ -3,7 +3,7 @@ import { expect, test } from "bun:test"
|
|
|
3
3
|
import { mkdtempSync, rmSync } from "node:fs"
|
|
4
4
|
import { tmpdir } from "node:os"
|
|
5
5
|
import { join } from "node:path"
|
|
6
|
-
import { checkpointPassive, installRetentionSchema, mergeFts, repairSearchRows, retainBatch } from "./retention.ts"
|
|
6
|
+
import { checkpointPassive, installRetentionSchema, mergeFts, repairSearchRows, retainBatch, SPAN_SEARCH_MAPPING_READY_KEY } from "./retention.ts"
|
|
7
7
|
|
|
8
8
|
const schema = (db: Database) => {
|
|
9
9
|
db.exec(`PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 15000;
|
|
@@ -168,3 +168,152 @@ test("stale upgraded summaries cannot evict active spans under age or size reten
|
|
|
168
168
|
} finally { db.close() }
|
|
169
169
|
}
|
|
170
170
|
})
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
test("operation mapping readiness waits for complete legacy repair and stays idle after reopen", () => {
|
|
174
|
+
const root = mkdtempSync(join(tmpdir(), "motel-mapping-ready-"))
|
|
175
|
+
const path = join(root, "test.sqlite")
|
|
176
|
+
let db = new Database(path)
|
|
177
|
+
try {
|
|
178
|
+
schema(db)
|
|
179
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "complete" })
|
|
180
|
+
db.query("DELETE FROM motel_maintenance WHERE key = ?").run(SPAN_SEARCH_MAPPING_READY_KEY)
|
|
181
|
+
db.exec("INSERT INTO spans(trace_id,span_id) VALUES ('a','s'), ('b','s'); INSERT INTO span_operation_fts(trace_id,span_id,operation_name) VALUES ('a','s','alpha'), ('b','s','beta'), ('orphan','s','orphan')")
|
|
182
|
+
// All three unmapped historical rows are below the pre-upgrade durable cursor.
|
|
183
|
+
db.exec("INSERT OR REPLACE INTO motel_maintenance VALUES ('retention_repair_span_operation_fts', '999')")
|
|
184
|
+
db.exec("CREATE TRIGGER refuse_mapping_marker BEFORE INSERT ON motel_maintenance WHEN NEW.key = 'span_search_mapping_complete' BEGIN SELECT RAISE(ABORT, 'marker failure'); END")
|
|
185
|
+
expect(() => installRetentionSchema(db)).toThrow("marker failure")
|
|
186
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = 'retention_repair_span_operation_fts'").get()).toEqual({ value: "999" })
|
|
187
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toBeNull()
|
|
188
|
+
db.exec("DROP TRIGGER refuse_mapping_marker")
|
|
189
|
+
installRetentionSchema(db)
|
|
190
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = 'retention_repair_span_operation_fts'").get()).toBeNull()
|
|
191
|
+
expect(repairSearchRows(db, 1)).toBe(true)
|
|
192
|
+
expect(count(db, "span_search_rows")).toBe(1)
|
|
193
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "repairing:3" })
|
|
194
|
+
db.close()
|
|
195
|
+
db = new Database(path)
|
|
196
|
+
installRetentionSchema(db)
|
|
197
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = 'retention_repair_span_operation_fts'").get()).toEqual({ value: "1" })
|
|
198
|
+
expect(repairSearchRows(db, 1)).toBe(true)
|
|
199
|
+
expect(count(db, "span_search_rows")).toBe(2)
|
|
200
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "repairing:3" })
|
|
201
|
+
expect(repairSearchRows(db, 1)).toBe(false)
|
|
202
|
+
expect(count(db, "span_operation_fts")).toBe(2)
|
|
203
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "complete" })
|
|
204
|
+
expect(repairSearchRows(db, 1)).toBe(false)
|
|
205
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "complete" })
|
|
206
|
+
db.close()
|
|
207
|
+
db = new Database(path)
|
|
208
|
+
installRetentionSchema(db)
|
|
209
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = 'retention_repair_span_operation_fts'").get()).toEqual({ value: "3" })
|
|
210
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "complete" })
|
|
211
|
+
const before = db.query("SELECT total_changes() AS n").get()
|
|
212
|
+
expect(repairSearchRows(db, 1)).toBe(false)
|
|
213
|
+
expect(db.query("SELECT total_changes() AS n").get()).toEqual(before)
|
|
214
|
+
} finally { db.close(); rmSync(root, { recursive: true, force: true }) }
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
test("a final partial span repair returns idle despite remaining log repair", () => {
|
|
219
|
+
const db = new Database(":memory:")
|
|
220
|
+
try {
|
|
221
|
+
schema(db)
|
|
222
|
+
db.query("DELETE FROM motel_maintenance WHERE key = ?").run(SPAN_SEARCH_MAPPING_READY_KEY)
|
|
223
|
+
db.exec("INSERT INTO spans(trace_id, span_id) VALUES ('a', 's'), ('b', 's'); INSERT INTO span_operation_fts(trace_id, span_id, operation_name) VALUES ('a', 's', 'alpha'), ('b', 's', 'beta')")
|
|
224
|
+
for (let id = 1; id <= 7; id++) {
|
|
225
|
+
db.query("INSERT INTO logs VALUES (?, NULL, 1000, 'INFO')").run(id)
|
|
226
|
+
db.query("INSERT INTO log_attributes VALUES (?, 'key', 'value')").run(id)
|
|
227
|
+
db.query("INSERT INTO log_body_fts(log_id, body) VALUES (?, 'searchable')").run(String(id))
|
|
228
|
+
}
|
|
229
|
+
installRetentionSchema(db)
|
|
230
|
+
expect(repairSearchRows(db, 3)).toBe(false)
|
|
231
|
+
expect(count(db, "span_search_rows")).toBe(2)
|
|
232
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "complete" })
|
|
233
|
+
expect(count(db, "log_body_fts")).toBe(7)
|
|
234
|
+
expect(count(db, "log_search_rows")).toBe(3)
|
|
235
|
+
} finally { db.close() }
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
test("an unavailable optional span index does not request fast repair retries", () => {
|
|
239
|
+
const db = new Database(":memory:")
|
|
240
|
+
try {
|
|
241
|
+
schema(db)
|
|
242
|
+
db.query("UPDATE motel_maintenance SET value = 'repairing:3' WHERE key = ?").run(SPAN_SEARCH_MAPPING_READY_KEY)
|
|
243
|
+
db.exec("DROP TABLE span_operation_fts")
|
|
244
|
+
expect(repairSearchRows(db, 3)).toBe(false)
|
|
245
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "repairing:3" })
|
|
246
|
+
} finally { db.close() }
|
|
247
|
+
})
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
test("legacy span mapping finishes its fixed range despite mapped arrivals and reopen", () => {
|
|
251
|
+
const root = mkdtempSync(join(tmpdir(), "motel-mapping-target-"))
|
|
252
|
+
const path = join(root, "test.sqlite")
|
|
253
|
+
let db = new Database(path)
|
|
254
|
+
const state = () => db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)
|
|
255
|
+
const append = (id: number, mapped: boolean) => {
|
|
256
|
+
db.query("INSERT INTO spans(trace_id, span_id) VALUES (?, 's')").run(String(id))
|
|
257
|
+
db.query("INSERT INTO span_operation_fts(rowid, trace_id, span_id, operation_name) VALUES (?, ?, 's', 'operation')").run(id, String(id))
|
|
258
|
+
if (mapped) db.query("INSERT INTO span_search_rows VALUES (?, ?, 's')").run(id, String(id))
|
|
259
|
+
}
|
|
260
|
+
try {
|
|
261
|
+
schema(db)
|
|
262
|
+
db.query("DELETE FROM motel_maintenance WHERE key = ?").run(SPAN_SEARCH_MAPPING_READY_KEY)
|
|
263
|
+
for (let id = 1; id <= 4; id++) append(id, false)
|
|
264
|
+
db.exec("INSERT OR REPLACE INTO motel_maintenance VALUES ('retention_repair_span_operation_fts', '999')")
|
|
265
|
+
installRetentionSchema(db)
|
|
266
|
+
const captured = state()
|
|
267
|
+
expect(repairSearchRows(db, 2)).toBe(true)
|
|
268
|
+
for (let id = 5; id <= 8; id++) append(id, true)
|
|
269
|
+
db.close()
|
|
270
|
+
db = new Database(path)
|
|
271
|
+
installRetentionSchema(db)
|
|
272
|
+
expect(state()).toEqual(captured)
|
|
273
|
+
// Complete the exact full-sized final batch without scanning the mapped arrivals.
|
|
274
|
+
expect(repairSearchRows(db, 2)).toBe(false)
|
|
275
|
+
expect(state()).toEqual({ value: "complete" })
|
|
276
|
+
expect(captured).toEqual({ value: "repairing:4" })
|
|
277
|
+
expect(count(db, "span_search_rows")).toBe(8)
|
|
278
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = 'retention_repair_span_operation_fts'").get()).toEqual({ value: "4" })
|
|
279
|
+
// Ordinary incremental repair still visits later rows, without requesting fast cadence.
|
|
280
|
+
expect(repairSearchRows(db, 2)).toBe(false)
|
|
281
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = 'retention_repair_span_operation_fts'").get()).toEqual({ value: "6" })
|
|
282
|
+
expect(repairSearchRows(db, 2)).toBe(false)
|
|
283
|
+
const before = db.query("SELECT total_changes() AS n").get()
|
|
284
|
+
expect(repairSearchRows(db, 2)).toBe(false)
|
|
285
|
+
expect(db.query("SELECT total_changes() AS n").get()).toEqual(before)
|
|
286
|
+
} finally { db.close(); rmSync(root, { recursive: true, force: true }) }
|
|
287
|
+
})
|
|
288
|
+
|
|
289
|
+
test("fixed-range repair rolls back failures and completes after its target row is removed", () => {
|
|
290
|
+
const db = new Database(":memory:")
|
|
291
|
+
try {
|
|
292
|
+
schema(db)
|
|
293
|
+
db.query("DELETE FROM motel_maintenance WHERE key = ?").run(SPAN_SEARCH_MAPPING_READY_KEY)
|
|
294
|
+
db.exec("INSERT INTO spans(trace_id,span_id) VALUES ('a','s'),('b','s'); INSERT INTO span_operation_fts(trace_id,span_id,operation_name) VALUES ('a','s','alpha'),('b','s','beta')")
|
|
295
|
+
installRetentionSchema(db)
|
|
296
|
+
expect(repairSearchRows(db, 1)).toBe(true)
|
|
297
|
+
db.exec("CREATE TRIGGER refuse_repair BEFORE INSERT ON span_search_rows WHEN NEW.trace_id = 'b' BEGIN SELECT RAISE(ABORT, 'repair failure'); END")
|
|
298
|
+
expect(() => repairSearchRows(db, 1)).toThrow("repair failure")
|
|
299
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "repairing:2" })
|
|
300
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = 'retention_repair_span_operation_fts'").get()).toEqual({ value: "1" })
|
|
301
|
+
db.exec("DROP TRIGGER refuse_repair; DELETE FROM span_operation_fts WHERE rowid = 2")
|
|
302
|
+
expect(repairSearchRows(db, 1)).toBe(false)
|
|
303
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY)).toEqual({ value: "complete" })
|
|
304
|
+
} finally { db.close() }
|
|
305
|
+
})
|
|
306
|
+
|
|
307
|
+
test("malformed persistent span migration state fails without resetting its cursor", () => {
|
|
308
|
+
for (const state of ["repairing", "repairing:0", "repairing:1.5", "repairing:9007199254740992", "unknown"]) {
|
|
309
|
+
const db = new Database(":memory:")
|
|
310
|
+
try {
|
|
311
|
+
schema(db)
|
|
312
|
+
db.query("UPDATE motel_maintenance SET value = ? WHERE key = ?").run(state, SPAN_SEARCH_MAPPING_READY_KEY)
|
|
313
|
+
db.exec("INSERT OR REPLACE INTO motel_maintenance VALUES ('retention_repair_span_operation_fts', '7')")
|
|
314
|
+
expect(() => installRetentionSchema(db)).toThrow("Invalid span search mapping migration state")
|
|
315
|
+
expect(() => repairSearchRows(db, 1)).toThrow("Invalid span search mapping migration state")
|
|
316
|
+
expect(db.query("SELECT value FROM motel_maintenance WHERE key = 'retention_repair_span_operation_fts'").get()).toEqual({ value: "7" })
|
|
317
|
+
} finally { db.close() }
|
|
318
|
+
}
|
|
319
|
+
})
|
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite"
|
|
2
2
|
|
|
3
|
+
export const SPAN_SEARCH_MAPPING_READY_KEY = "span_search_mapping_complete"
|
|
4
|
+
|
|
5
|
+
const spanMappingTarget = (state: string): number | null => {
|
|
6
|
+
if (state === "complete") return null
|
|
7
|
+
const match = /^repairing:([1-9][0-9]*)$/.exec(state)
|
|
8
|
+
const target = match === null ? NaN : Number(match[1])
|
|
9
|
+
if (!Number.isSafeInteger(target)) throw new Error("Invalid span search mapping migration state")
|
|
10
|
+
return target
|
|
11
|
+
}
|
|
12
|
+
|
|
3
13
|
/** Additive schema: unfinished eviction is hidden from every public query. */
|
|
4
14
|
export const installRetentionSchema = (db: Database) => {
|
|
5
15
|
db.exec(`
|
|
@@ -22,6 +32,21 @@ export const installRetentionSchema = (db: Database) => {
|
|
|
22
32
|
WHERE NOT EXISTS (SELECT 1 FROM retention_traces d WHERE d.trace_id = logs.trace_id)
|
|
23
33
|
AND NOT EXISTS (SELECT 1 FROM retention_logs d WHERE d.id = logs.id);
|
|
24
34
|
`)
|
|
35
|
+
if (db.query("SELECT 1 FROM sqlite_master WHERE name = 'span_operation_fts' AND type = 'table'").get()) {
|
|
36
|
+
db.transaction(() => {
|
|
37
|
+
const marker = db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY) as { value: string } | null
|
|
38
|
+
if (marker !== null) {
|
|
39
|
+
spanMappingTarget(marker.value)
|
|
40
|
+
return
|
|
41
|
+
}
|
|
42
|
+
// Earlier optional index failures can strand unmapped rows behind the old cursor.
|
|
43
|
+
db.query("DELETE FROM motel_maintenance WHERE key = 'retention_repair_span_operation_fts'").run()
|
|
44
|
+
const last = db.query("SELECT rowid AS target FROM span_operation_fts ORDER BY rowid DESC LIMIT 1").get() as { target: number } | null
|
|
45
|
+
const state = last === null ? "complete" : `repairing:${last.target}`
|
|
46
|
+
spanMappingTarget(state)
|
|
47
|
+
db.query("INSERT INTO motel_maintenance(key, value) VALUES (?, ?)").run(SPAN_SEARCH_MAPPING_READY_KEY, state)
|
|
48
|
+
}).immediate()
|
|
49
|
+
}
|
|
25
50
|
}
|
|
26
51
|
|
|
27
52
|
export type Checkpoint = { busy: number; log: number; checkpointed: number; deferred: boolean }
|
|
@@ -133,13 +158,23 @@ export const retainBatch = (db: Database, options: RetentionOptions) => db.trans
|
|
|
133
158
|
|
|
134
159
|
/** Bounded repair scans only rowids beyond its durable high-water mark; idle passes do not write. */
|
|
135
160
|
export const repairSearchRows = (db: Database, limit: number) => db.transaction(() => {
|
|
161
|
+
let spanMappingPending = false
|
|
136
162
|
for (const table of ["log_attributes", "log_body_fts", "span_operation_fts"] as const) {
|
|
137
163
|
if (!db.query("SELECT 1 FROM sqlite_master WHERE name = ?").get(table)) continue
|
|
138
164
|
const key = `retention_repair_${table}`
|
|
139
165
|
const marker = (db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(key) as { value: string } | null)?.value
|
|
140
166
|
const cursor = Number(marker ?? 0)
|
|
141
167
|
const columns = table === "span_operation_fts" ? "trace_id, span_id" : "log_id"
|
|
142
|
-
|
|
168
|
+
let target: number | null = null
|
|
169
|
+
if (table === "span_operation_fts") {
|
|
170
|
+
const state = db.query("SELECT value FROM motel_maintenance WHERE key = ?").get(SPAN_SEARCH_MAPPING_READY_KEY) as { value: string } | null
|
|
171
|
+
if (state === null) throw new Error("Missing span search mapping migration state")
|
|
172
|
+
target = spanMappingTarget(state.value)
|
|
173
|
+
}
|
|
174
|
+
const rows = (target === null
|
|
175
|
+
? db.query(`SELECT rowid AS repair_id, ${columns} FROM ${table} WHERE rowid > ? ORDER BY rowid LIMIT ?`).all(cursor, limit)
|
|
176
|
+
: db.query(`SELECT rowid AS repair_id, ${columns} FROM ${table} WHERE rowid > ? AND rowid <= ? ORDER BY rowid LIMIT ?`).all(cursor, target, limit)
|
|
177
|
+
) as { repair_id: number; log_id?: number | string; trace_id?: string; span_id?: string }[]
|
|
143
178
|
for (const row of rows) {
|
|
144
179
|
const exists = table === "span_operation_fts"
|
|
145
180
|
? db.query("SELECT 1 FROM spans WHERE trace_id = ? AND span_id = ?").get(row.trace_id!, row.span_id!)
|
|
@@ -154,5 +189,13 @@ export const repairSearchRows = (db: Database, limit: number) => db.transaction(
|
|
|
154
189
|
}
|
|
155
190
|
}
|
|
156
191
|
if (rows.length > 0) db.query("INSERT OR REPLACE INTO motel_maintenance VALUES (?, ?)").run(key, String(rows.at(-1)!.repair_id))
|
|
192
|
+
if (target !== null) {
|
|
193
|
+
const nextCursor = rows.at(-1)?.repair_id ?? cursor
|
|
194
|
+
spanMappingPending = db.query("SELECT 1 FROM span_operation_fts WHERE rowid > ? AND rowid <= ? LIMIT 1").get(nextCursor, target) !== null
|
|
195
|
+
if (!spanMappingPending) {
|
|
196
|
+
db.query("UPDATE motel_maintenance SET value = 'complete' WHERE key = ?").run(SPAN_SEARCH_MAPPING_READY_KEY)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
157
199
|
}
|
|
200
|
+
return spanMappingPending
|
|
158
201
|
})()
|
package/src/telemetry.test.ts
CHANGED
|
@@ -407,6 +407,194 @@ describe("motel telemetry store", () => {
|
|
|
407
407
|
}
|
|
408
408
|
})
|
|
409
409
|
|
|
410
|
+
|
|
411
|
+
it("keeps operation search and mappings atomic across updates, legacy repair and ingest rollback", async () => {
|
|
412
|
+
const isolatedPath = join(tempDir, "operation-savepoint.sqlite")
|
|
413
|
+
const script = `
|
|
414
|
+
const assert = (await import('node:assert/strict')).default
|
|
415
|
+
const { Database } = await import('bun:sqlite')
|
|
416
|
+
const { Effect, ManagedRuntime } = await import('effect')
|
|
417
|
+
const { TelemetryStore, makeTelemetryStoreLayer } = await import('./src/services/TelemetryStore.ts')
|
|
418
|
+
const { repairSearchRows, installRetentionSchema, SPAN_SEARCH_MAPPING_READY_KEY } = await import('./src/services/retention.ts')
|
|
419
|
+
const runtime = ManagedRuntime.make(makeTelemetryStoreLayer({ readonly: false, runRetention: false }))
|
|
420
|
+
let db
|
|
421
|
+
const now = BigInt(Date.now()) * 1000000n
|
|
422
|
+
const span = (traceId, spanId, name) => ({ traceId, spanId, name, startTimeUnixNano: String(now), endTimeUnixNano: String(now + 1000000n) })
|
|
423
|
+
const ingest = (spans) => runtime.runPromise(Effect.flatMap(TelemetryStore, (store) => store.ingestTraces({ resourceSpans: [{ resource: { attributes: [{ key: 'service.name', value: { stringValue: 'fts-regression' } }] }, scopeSpans: [{ spans }] }] })))
|
|
424
|
+
const names = (traceId) => db.query('SELECT operation_name FROM span_operation_fts WHERE trace_id = ? ORDER BY operation_name').all(traceId).map((row) => row.operation_name)
|
|
425
|
+
const ready = () => db.query('SELECT value FROM motel_maintenance WHERE key = ?').get(SPAN_SEARCH_MAPPING_READY_KEY)?.value
|
|
426
|
+
const mappingCounts = () => db.query('SELECT (SELECT count(*) FROM span_operation_fts) AS fts, (SELECT count(*) FROM span_search_rows) AS mapping, (SELECT count(*) FROM span_operation_fts f LEFT JOIN span_search_rows m ON m.id = f.rowid WHERE m.id IS NULL) AS unmapped').get()
|
|
427
|
+
try {
|
|
428
|
+
await ingest([span('normal', 'root', 'oldoperation')])
|
|
429
|
+
db = new Database(process.env.MOTEL_OTEL_DB_PATH)
|
|
430
|
+
assert.equal(ready(), 'complete')
|
|
431
|
+
await ingest([span('normal', 'root', 'newoperation')])
|
|
432
|
+
assert.deepEqual(names('normal'), ['newoperation'])
|
|
433
|
+
assert.equal((await runtime.runPromise(Effect.flatMap(TelemetryStore, (store) => store.searchSpans({ operation: 'oldoperation' })))).length, 0)
|
|
434
|
+
assert.equal((await runtime.runPromise(Effect.flatMap(TelemetryStore, (store) => store.searchSpans({ operation: 'newoperation' })))).length, 1)
|
|
435
|
+
|
|
436
|
+
await ingest(Array.from({ length: 32 }, (_, i) => span('mixed-' + i, 'root', 'mixedold')))
|
|
437
|
+
await ingest(Array.from({ length: 64 }, (_, i) => span('mixed-' + i, 'root', 'mixednew')))
|
|
438
|
+
assert.equal(db.query("SELECT count(*) AS n FROM span_operation_fts WHERE operation_name = 'mixedold'").get().n, 0)
|
|
439
|
+
assert.equal(db.query("SELECT count(*) AS n FROM span_operation_fts WHERE operation_name = 'mixednew'").get().n, 64)
|
|
440
|
+
|
|
441
|
+
await ingest([span('duplicate', 'root', 'first'), span('duplicate', 'root', 'second')])
|
|
442
|
+
assert.deepEqual(names('duplicate'), ['first', 'second'])
|
|
443
|
+
const acrossBoundary = [span('duplicate', 'root', 'early')]
|
|
444
|
+
for (let i = 1; i < 499; i++) acrossBoundary.push(span('filler-' + i, 'root', 'filler'))
|
|
445
|
+
acrossBoundary.push(span('duplicate', 'root', 'middle'), span('duplicate', 'root', 'lateone'), span('duplicate', 'root', 'latetwo'))
|
|
446
|
+
await ingest(acrossBoundary)
|
|
447
|
+
assert.deepEqual(names('duplicate'), ['lateone', 'latetwo'])
|
|
448
|
+
assert.equal(db.query("SELECT operation_name FROM spans WHERE trace_id = 'duplicate' AND span_id = 'root'").get().operation_name, 'latetwo')
|
|
449
|
+
|
|
450
|
+
// Simulate an upgraded database with unmapped historical FTS rows.
|
|
451
|
+
db.query("DELETE FROM motel_maintenance WHERE key IN (?, 'retention_repair_span_operation_fts')").run(SPAN_SEARCH_MAPPING_READY_KEY)
|
|
452
|
+
db.exec("DELETE FROM span_search_rows WHERE trace_id = 'normal'")
|
|
453
|
+
db.query("INSERT OR REPLACE INTO motel_maintenance VALUES ('retention_repair_span_operation_fts', '999999')").run()
|
|
454
|
+
installRetentionSchema(db)
|
|
455
|
+
await ingest([span('normal', 'root', 'legacyupdated')])
|
|
456
|
+
assert.deepEqual(names('normal'), ['legacyupdated'])
|
|
457
|
+
assert.match(ready(), /^repairing:[1-9][0-9]*$/)
|
|
458
|
+
for (let pass = 0; ready() !== 'complete' && pass < 20; pass++) repairSearchRows(db, 100)
|
|
459
|
+
assert.equal(ready(), 'complete')
|
|
460
|
+
|
|
461
|
+
// Fail mapping after FTS insertion; every chunk of the optional unit must roll back.
|
|
462
|
+
const before = names('normal')
|
|
463
|
+
const beforeCounts = mappingCounts()
|
|
464
|
+
db.exec("CREATE TRIGGER reject_mapping BEFORE INSERT ON span_search_rows WHEN NEW.trace_id = 'normal' BEGIN SELECT RAISE(ABORT, 'mapping failure'); END")
|
|
465
|
+
const failedChunks = Array.from({ length: 500 }, (_, i) => span('failed-chunk-' + i, 'root', 'mustrollback'))
|
|
466
|
+
failedChunks.push(span('normal', 'root', 'committedspan'))
|
|
467
|
+
await ingest(failedChunks)
|
|
468
|
+
assert.deepEqual(names('normal'), before)
|
|
469
|
+
assert.deepEqual(mappingCounts(), beforeCounts)
|
|
470
|
+
assert.equal(db.query("SELECT count(*) AS n FROM span_operation_fts WHERE operation_name = 'mustrollback'").get().n, 0)
|
|
471
|
+
assert.equal(db.query("SELECT operation_name FROM spans WHERE trace_id = 'normal' AND span_id = 'root'").get().operation_name, 'committedspan')
|
|
472
|
+
assert.equal(ready(), 'complete')
|
|
473
|
+
db.exec('DROP TRIGGER reject_mapping')
|
|
474
|
+
await ingest([span('normal', 'root', 'recovered')])
|
|
475
|
+
assert.deepEqual(names('normal'), ['recovered'])
|
|
476
|
+
|
|
477
|
+
const stableNames = names('normal')
|
|
478
|
+
const stableCounts = mappingCounts()
|
|
479
|
+
db.exec("CREATE TRIGGER reject_summary BEFORE INSERT ON trace_summaries WHEN NEW.trace_id = 'normal' BEGIN SELECT RAISE(ABORT, 'summary failure'); END")
|
|
480
|
+
await assert.rejects(ingest([span('normal', 'root', 'outerrollback')]))
|
|
481
|
+
assert.deepEqual(names('normal'), stableNames)
|
|
482
|
+
assert.deepEqual(mappingCounts(), stableCounts)
|
|
483
|
+
assert.equal(db.query("SELECT operation_name FROM spans WHERE trace_id = 'normal' AND span_id = 'root'").get().operation_name, 'recovered')
|
|
484
|
+
assert.equal(ready(), 'complete')
|
|
485
|
+
db.exec('DROP TRIGGER reject_summary')
|
|
486
|
+
} finally {
|
|
487
|
+
db?.close()
|
|
488
|
+
await runtime.dispose()
|
|
489
|
+
}
|
|
490
|
+
`
|
|
491
|
+
const child = Bun.spawn({
|
|
492
|
+
cmd: [process.execPath, "--no-env-file", "-e", script], cwd: process.cwd(),
|
|
493
|
+
env: { ...process.env, MOTEL_OTEL_DB_PATH: isolatedPath, MOTEL_OTEL_ENABLED: "false" },
|
|
494
|
+
stdout: "ignore", stderr: "pipe",
|
|
495
|
+
})
|
|
496
|
+
const stderr = await new Response(child.stderr).text()
|
|
497
|
+
expect(await child.exited, stderr).toBe(0)
|
|
498
|
+
})
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
it("repairs legacy span mappings at the pending cadence and idles after completion", async () => {
|
|
502
|
+
const isolatedPath = join(tempDir, "mapping-repair-pacing.sqlite")
|
|
503
|
+
const script = `
|
|
504
|
+
const assert = (await import('node:assert/strict')).default
|
|
505
|
+
const { Database } = await import('bun:sqlite')
|
|
506
|
+
const { Clock, Deferred, Effect, Layer, ManagedRuntime } = await import('effect')
|
|
507
|
+
const { TestClock } = await import('effect/testing')
|
|
508
|
+
const { TelemetryStore, makeTelemetryStoreLayer } = await import('./src/services/TelemetryStore.ts')
|
|
509
|
+
const { WriterDiagnostics } = await import('./src/ingestReadiness.ts')
|
|
510
|
+
const { SPAN_SEARCH_MAPPING_READY_KEY } = await import('./src/services/retention.ts')
|
|
511
|
+
const now = Date.now()
|
|
512
|
+
const nanos = BigInt(now) * 1000000n
|
|
513
|
+
const seed = ManagedRuntime.make(makeTelemetryStoreLayer({ readonly: false, runRetention: false }))
|
|
514
|
+
try {
|
|
515
|
+
await seed.runPromise(Effect.flatMap(TelemetryStore, (store) => store.ingestTraces({
|
|
516
|
+
resourceSpans: [{
|
|
517
|
+
resource: { attributes: [{ key: 'service.name', value: { stringValue: 'mapping-pacing' } }] },
|
|
518
|
+
scopeSpans: [{ spans: Array.from({ length: 250 }, (_, id) => ({
|
|
519
|
+
traceId: 'pacing-' + id, spanId: 'root', name: 'legacyoperation',
|
|
520
|
+
startTimeUnixNano: String(nanos), endTimeUnixNano: String(nanos + 1000000n),
|
|
521
|
+
})) }],
|
|
522
|
+
}],
|
|
523
|
+
})))
|
|
524
|
+
} finally { await seed.dispose() }
|
|
525
|
+
const db = new Database(process.env.MOTEL_OTEL_DB_PATH)
|
|
526
|
+
try {
|
|
527
|
+
db.exec('DELETE FROM span_search_rows')
|
|
528
|
+
db.query('DELETE FROM motel_maintenance WHERE key = ?').run(SPAN_SEARCH_MAPPING_READY_KEY)
|
|
529
|
+
db.exec("INSERT OR REPLACE INTO motel_maintenance VALUES ('retention_repair_span_operation_fts', '999999')")
|
|
530
|
+
await Effect.runPromise(Effect.scoped(Effect.gen(function* () {
|
|
531
|
+
const clock = yield* TestClock.make()
|
|
532
|
+
yield* clock.setTime(now + 2)
|
|
533
|
+
const firstPass = Deferred.makeUnsafe()
|
|
534
|
+
const passes = []
|
|
535
|
+
const publish = (event) => {
|
|
536
|
+
if (event._tag !== 'maintenance' || event.value.operation !== 'retention' || event.value.outcome === 'running') return
|
|
537
|
+
passes.push({
|
|
538
|
+
outcome: event.value.outcome,
|
|
539
|
+
mapped: db.query('SELECT count(*) AS n FROM span_search_rows').get().n,
|
|
540
|
+
state: db.query('SELECT value FROM motel_maintenance WHERE key = ?').get(SPAN_SEARCH_MAPPING_READY_KEY)?.value,
|
|
541
|
+
})
|
|
542
|
+
Deferred.doneUnsafe(firstPass, Effect.void)
|
|
543
|
+
}
|
|
544
|
+
const runtime = ManagedRuntime.make(makeTelemetryStoreLayer({ readonly: false, runRetention: true }).pipe(
|
|
545
|
+
Layer.provide(Layer.succeed(Clock.Clock, clock)),
|
|
546
|
+
Layer.provide(Layer.succeed(WriterDiagnostics, publish)),
|
|
547
|
+
))
|
|
548
|
+
yield* Effect.addFinalizer(() => runtime.disposeEffect)
|
|
549
|
+
yield* Effect.promise(() => runtime.runPromise(Effect.asVoid(TelemetryStore)))
|
|
550
|
+
yield* Deferred.await(firstPass).pipe(Effect.timeout('5 seconds'))
|
|
551
|
+
assert.deepEqual(passes, [{ outcome: 'completed', mapped: 100, state: 'repairing:250' }])
|
|
552
|
+
yield* clock.adjust('100 millis')
|
|
553
|
+
assert.deepEqual(passes, [
|
|
554
|
+
{ outcome: 'completed', mapped: 100, state: 'repairing:250' },
|
|
555
|
+
{ outcome: 'completed', mapped: 200, state: 'repairing:250' },
|
|
556
|
+
])
|
|
557
|
+
yield* clock.adjust('100 millis')
|
|
558
|
+
assert.deepEqual(passes, [
|
|
559
|
+
{ outcome: 'completed', mapped: 100, state: 'repairing:250' },
|
|
560
|
+
{ outcome: 'completed', mapped: 200, state: 'repairing:250' },
|
|
561
|
+
{ outcome: 'completed', mapped: 250, state: 'complete' },
|
|
562
|
+
])
|
|
563
|
+
yield* clock.adjust('100 millis')
|
|
564
|
+
assert.equal(passes.length, 3)
|
|
565
|
+
yield* clock.adjust('9900 millis')
|
|
566
|
+
assert.deepEqual(passes.at(-1), { outcome: 'completed', mapped: 250, state: 'complete' })
|
|
567
|
+
assert.equal(passes.length, 4)
|
|
568
|
+
yield* runtime.disposeEffect
|
|
569
|
+
yield* clock.adjust('30 seconds')
|
|
570
|
+
assert.equal(passes.length, 4)
|
|
571
|
+
})))
|
|
572
|
+
} finally { db.close() }
|
|
573
|
+
`
|
|
574
|
+
const child = Bun.spawn({
|
|
575
|
+
cmd: [process.execPath, "--no-env-file", "-e", script], cwd: process.cwd(),
|
|
576
|
+
env: {
|
|
577
|
+
...process.env,
|
|
578
|
+
MOTEL_OTEL_DB_PATH: isolatedPath, MOTEL_OTEL_ENABLED: "false",
|
|
579
|
+
MOTEL_OTEL_RETENTION_ROW_BATCH: "1000", MOTEL_OTEL_RETENTION_INTERVAL_SECONDS: "10",
|
|
580
|
+
MOTEL_OTEL_RETENTION_TRACE_BATCH: "100", MOTEL_OTEL_RETENTION_HOURS: "168",
|
|
581
|
+
MOTEL_OTEL_MAX_DB_SIZE_MB: "1024",
|
|
582
|
+
},
|
|
583
|
+
stdout: "ignore", stderr: "pipe",
|
|
584
|
+
})
|
|
585
|
+
let timedOut = false
|
|
586
|
+
const deadline = setTimeout(() => { timedOut = true; child.kill() }, 15_000)
|
|
587
|
+
try {
|
|
588
|
+
const [exitCode, stderr] = await Promise.all([child.exited, new Response(child.stderr).text()])
|
|
589
|
+
expect(timedOut, stderr).toBe(false)
|
|
590
|
+
expect(exitCode, stderr).toBe(0)
|
|
591
|
+
} finally {
|
|
592
|
+
clearTimeout(deadline)
|
|
593
|
+
if (child.exitCode === null) child.kill()
|
|
594
|
+
await child.exited
|
|
595
|
+
}
|
|
596
|
+
}, 20_000)
|
|
597
|
+
|
|
410
598
|
it("incrementally backfills historical AI FTS content", async () => {
|
|
411
599
|
const historicalPath = join(tempDir, "historical-fts.sqlite")
|
|
412
600
|
const seedScript = `
|
|
@@ -468,6 +656,42 @@ describe("motel telemetry store", () => {
|
|
|
468
656
|
expect(result?.span.depth).toBe(1)
|
|
469
657
|
})
|
|
470
658
|
|
|
659
|
+
|
|
660
|
+
it("opens and searches a read-only pre-retention database without creating mappings", async () => {
|
|
661
|
+
const legacyPath = join(tempDir, "readonly-pre-retention.sqlite")
|
|
662
|
+
const script = `
|
|
663
|
+
const assert = (await import('node:assert/strict')).default
|
|
664
|
+
const { Database } = await import('bun:sqlite')
|
|
665
|
+
const { Effect, ManagedRuntime } = await import('effect')
|
|
666
|
+
const { TelemetryStore, makeTelemetryStoreLayer, TelemetryStoreReadonly, TelemetryStoreReadonlyLive } = await import('./src/services/TelemetryStore.ts')
|
|
667
|
+
const writer = ManagedRuntime.make(makeTelemetryStoreLayer({ readonly: false, runRetention: false }))
|
|
668
|
+
try {
|
|
669
|
+
const now = BigInt(Date.now()) * 1000000n
|
|
670
|
+
await writer.runPromise(Effect.flatMap(TelemetryStore, (store) => store.ingestTraces({ resourceSpans: [{ resource: { attributes: [{ key: 'service.name', value: { stringValue: 'legacy-readonly' } }] }, scopeSpans: [{ spans: [{ traceId: 'legacy-read', spanId: 'root', name: 'legacyoperation', startTimeUnixNano: String(now), endTimeUnixNano: String(now + 1000000n) }] }] }] })))
|
|
671
|
+
} finally { await writer.dispose() }
|
|
672
|
+
const legacy = new Database(process.env.MOTEL_OTEL_DB_PATH)
|
|
673
|
+
legacy.exec('DROP VIEW retained_spans; DROP VIEW retained_logs; DROP VIEW retained_trace_summaries; DROP TABLE span_search_rows; DROP TABLE log_search_rows; DROP TABLE retention_traces; DROP TABLE retention_logs;')
|
|
674
|
+
const version = legacy.query('PRAGMA schema_version').get().schema_version
|
|
675
|
+
legacy.close()
|
|
676
|
+
const reader = ManagedRuntime.make(TelemetryStoreReadonlyLive)
|
|
677
|
+
try {
|
|
678
|
+
const trace = await reader.runPromise(Effect.flatMap(TelemetryStoreReadonly, (store) => store.getTrace('legacy-read')))
|
|
679
|
+
assert.equal(trace?.spans.length, 1)
|
|
680
|
+
const found = await reader.runPromise(Effect.flatMap(TelemetryStoreReadonly, (store) => store.searchSpans({ operation: 'legacyoperation' })))
|
|
681
|
+
assert.equal(found.length, 1)
|
|
682
|
+
assert.equal(found[0].traceId, 'legacy-read')
|
|
683
|
+
} finally { await reader.dispose() }
|
|
684
|
+
const verify = new Database(process.env.MOTEL_OTEL_DB_PATH, { readonly: true })
|
|
685
|
+
try {
|
|
686
|
+
assert.equal(verify.query('PRAGMA schema_version').get().schema_version, version)
|
|
687
|
+
assert.equal(verify.query("SELECT count(*) AS n FROM sqlite_master WHERE name IN ('span_search_rows', 'log_search_rows', 'retained_spans')").get().n, 0)
|
|
688
|
+
} finally { verify.close() }
|
|
689
|
+
`
|
|
690
|
+
const child = Bun.spawn({ cmd: [process.execPath, "--no-env-file", "-e", script], cwd: process.cwd(), env: { ...process.env, MOTEL_OTEL_DB_PATH: legacyPath }, stdout: "ignore", stderr: "pipe" })
|
|
691
|
+
const [exitCode, stderr] = await Promise.all([child.exited, new Response(child.stderr).text()])
|
|
692
|
+
expect(exitCode, stderr).toBe(0)
|
|
693
|
+
})
|
|
694
|
+
|
|
471
695
|
it("returns trace details through the query worker when spans share empty event arrays", async () => {
|
|
472
696
|
const workerDbPath = join(tempDir, "query-worker-shared-events.sqlite")
|
|
473
697
|
const script = `
|