@kitlangton/motel 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -0
- package/package.json +5 -3
- package/src/App.tsx +233 -59
- package/src/daemon.test.ts +144 -7
- package/src/daemon.ts +113 -8
- package/src/domain.test.ts +62 -0
- package/src/domain.ts +16 -0
- package/src/localServer.ts +111 -121
- package/src/mcp.ts +172 -0
- package/src/motelClient.ts +166 -14
- package/src/registry.ts +26 -23
- package/src/runtime.ts +8 -2
- package/src/server.ts +10 -9
- package/src/services/AsyncIngest.ts +52 -0
- package/src/services/TelemetryStore.ts +151 -26
- package/src/services/TraceQueryService.ts +3 -1
- package/src/services/ingestRpc.ts +41 -0
- package/src/services/telemetryWorker.ts +62 -0
- package/src/storybook/aiChatStory.tsx +243 -0
- package/src/storybook/fixtures/errorState.ts +44 -0
- package/src/storybook/fixtures/imagePaste.ts +34 -0
- package/src/storybook/fixtures/index.ts +62 -0
- package/src/storybook/fixtures/kitchenSink.ts +148 -0
- package/src/storybook/fixtures/rawPrompt.ts +15 -0
- package/src/storybook/fixtures/short.ts +27 -0
- package/src/storybook/fixtures/toolHeavy.ts +65 -0
- package/src/telemetry.test.ts +28 -0
- package/src/ui/AiChatView.tsx +292 -0
- package/src/ui/SpanContentView.tsx +181 -0
- package/src/ui/SpanDetail.tsx +98 -17
- package/src/ui/TraceDetailsPane.tsx +2 -1
- package/src/ui/Waterfall.tsx +38 -138
- package/src/ui/aiChatModel.test.ts +347 -0
- package/src/ui/aiChatModel.ts +736 -0
- package/src/ui/aiState.ts +71 -0
- package/src/ui/app/TraceWorkspace.tsx +291 -120
- package/src/ui/app/useAppLayout.ts +14 -11
- package/src/ui/app/useTraceScreenData.ts +173 -39
- package/src/ui/atoms.ts +131 -0
- package/src/ui/loaders.ts +120 -0
- package/src/ui/persistence.ts +41 -0
- package/src/ui/primitives.tsx +27 -13
- package/src/ui/state.ts +4 -199
- package/src/ui/useAttrFilterPicker.ts +63 -23
- package/src/ui/useKeyboardNav.ts +552 -364
- package/src/ui/waterfallModel.ts +130 -0
- package/src/ui/waterfallNav.test.ts +17 -1
- package/src/ui/waterfallNav.ts +1 -1
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Main-thread client for the telemetry worker's ingest RPCs.
|
|
3
|
+
*
|
|
4
|
+
* The HTTP handlers for POST /v1/traces and POST /v1/logs call into
|
|
5
|
+
* this service instead of `TelemetryStore.ingestTraces/Logs`. Each
|
|
6
|
+
* method sends a typed message to the worker, awaits the reply, and
|
|
7
|
+
* returns the worker's result as an Effect. While the worker is
|
|
8
|
+
* serialising a big batch into SQLite, the main thread's event loop
|
|
9
|
+
* is FREE to answer /api/* queries — that's the whole point of the
|
|
10
|
+
* offload. Without this, /api/health and friends queued behind long
|
|
11
|
+
* ingests and reported p95 latencies of 3-5 seconds; after, they
|
|
12
|
+
* stay responsive regardless of ingest load.
|
|
13
|
+
*
|
|
14
|
+
* The worker is spawned as a scope'd resource inside the layer. The
|
|
15
|
+
* protocol pool is sized at 1 because SQLite only supports a single
|
|
16
|
+
* writer at a time anyway — running N concurrent workers would just
|
|
17
|
+
* queue them on SQLite's lock. When the outer scope closes (server
|
|
18
|
+
* shutdown), `BunWorker.layer`'s finalizer sends a close message and
|
|
19
|
+
* terminates the worker if it doesn't exit gracefully in 5s.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import * as BunWorker from "@effect/platform-bun/BunWorker"
|
|
23
|
+
import { Context, Layer } from "effect"
|
|
24
|
+
import * as RpcClient from "effect/unstable/rpc/RpcClient"
|
|
25
|
+
import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"
|
|
26
|
+
import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"
|
|
27
|
+
import { IngestRpcs } from "./ingestRpc.ts"
|
|
28
|
+
|
|
29
|
+
// RpcClient.make always surfaces RpcClientError in addition to the
|
|
30
|
+
// group's declared errors (transport failures, worker crashes, etc.),
|
|
31
|
+
// so the service shape has to mirror that. Without the explicit error
|
|
32
|
+
// type param, TS treats the declared and observed client types as
|
|
33
|
+
// unrelated structural mismatches.
|
|
34
|
+
export class AsyncIngest extends Context.Service<
|
|
35
|
+
AsyncIngest,
|
|
36
|
+
RpcClient.FromGroup<typeof IngestRpcs, RpcClientError>
|
|
37
|
+
>()("@motel/AsyncIngest") {}
|
|
38
|
+
|
|
39
|
+
// Protocol: RpcClient.layerProtocolWorker manages a worker pool and
|
|
40
|
+
// speaks msgpack over structured-clone messages. `size: 1` matches
|
|
41
|
+
// SQLite's single-writer constraint.
|
|
42
|
+
const WorkerProtocol = RpcClient.layerProtocolWorker({ size: 1 }).pipe(
|
|
43
|
+
Layer.provide(RpcSerialization.layerMsgPack),
|
|
44
|
+
Layer.provide(
|
|
45
|
+
BunWorker.layer(() => new Worker(new URL("./telemetryWorker.ts", import.meta.url))),
|
|
46
|
+
),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
export const AsyncIngestLive = Layer.effect(
|
|
50
|
+
AsyncIngest,
|
|
51
|
+
RpcClient.make(IngestRpcs),
|
|
52
|
+
).pipe(Layer.provide(WorkerProtocol))
|
|
@@ -447,26 +447,74 @@ export class TelemetryStore extends Context.Service<
|
|
|
447
447
|
>()("motel/TelemetryStore") {}
|
|
448
448
|
|
|
449
449
|
|
|
450
|
-
|
|
450
|
+
/**
|
|
451
|
+
* How this TelemetryStore instance behaves:
|
|
452
|
+
*
|
|
453
|
+
* - `readonly` — opens the SQLite connection read-only and skips every
|
|
454
|
+
* DDL/DML initialisation. Use this from the TUI (and anywhere else
|
|
455
|
+
* that only queries); it avoids the "database is locked" race that
|
|
456
|
+
* happens when a TUI process races a daemon's writer for the schema
|
|
457
|
+
* pragmas on startup. Writes through the service interface become
|
|
458
|
+
* runtime errors — but readers don't call them.
|
|
459
|
+
*
|
|
460
|
+
* - `runRetention` — fork the background cleanup loop (age + size cap
|
|
461
|
+
* eviction, WAL checkpoint). Only one process should own this at a
|
|
462
|
+
* time. Currently the main daemon (localServer) does; the ingest
|
|
463
|
+
* worker and the TUI skip it.
|
|
464
|
+
*/
|
|
465
|
+
export interface TelemetryStoreOptions {
|
|
466
|
+
readonly readonly: boolean
|
|
467
|
+
readonly runRetention: boolean
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
export const makeTelemetryStoreLayer = (opts: TelemetryStoreOptions) => Layer.effect(
|
|
451
471
|
TelemetryStore,
|
|
452
472
|
Effect.gen(function* () {
|
|
453
473
|
mkdirSync(dirname(config.otel.databasePath), { recursive: true })
|
|
454
474
|
const db = yield* Effect.acquireRelease(
|
|
455
|
-
Effect.sync(() => new Database(config.otel.databasePath, {
|
|
475
|
+
Effect.sync(() => new Database(config.otel.databasePath, {
|
|
476
|
+
create: !opts.readonly,
|
|
477
|
+
readonly: opts.readonly,
|
|
478
|
+
})),
|
|
456
479
|
(db) => Effect.sync(() => {
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
480
|
+
if (!opts.readonly) {
|
|
481
|
+
// `PRAGMA optimize` at close persists any stats SQLite gathered
|
|
482
|
+
// during the session, so the next process start gets an accurate
|
|
483
|
+
// query planner on the first query instead of a 3-second cold
|
|
484
|
+
// run. Cheap: it skips work unless stats have drifted.
|
|
485
|
+
try { db.exec(`PRAGMA optimize;`) } catch { /* nothing */ }
|
|
486
|
+
}
|
|
462
487
|
db.close()
|
|
463
488
|
}),
|
|
464
489
|
)
|
|
490
|
+
if (opts.readonly) {
|
|
491
|
+
// Readonly connections skip schema init entirely — the schema
|
|
492
|
+
// already exists (a writer created it) and any `CREATE TABLE IF
|
|
493
|
+
// NOT EXISTS` / `PRAGMA journal_mode = WAL` statement would
|
|
494
|
+
// attempt a write and fight the daemon for the write lock.
|
|
495
|
+
// `query_only = 1` logically blocks any DML the app might
|
|
496
|
+
// accidentally send; still bump cache + mmap since those are
|
|
497
|
+
// safe and keep queries fast.
|
|
498
|
+
db.exec(`
|
|
499
|
+
PRAGMA query_only = 1;
|
|
500
|
+
PRAGMA busy_timeout = 15000;
|
|
501
|
+
PRAGMA cache_size = -65536;
|
|
502
|
+
PRAGMA mmap_size = 268435456;
|
|
503
|
+
`)
|
|
504
|
+
} else {
|
|
465
505
|
db.exec(`
|
|
466
506
|
PRAGMA journal_mode = WAL;
|
|
467
507
|
PRAGMA synchronous = NORMAL;
|
|
468
508
|
PRAGMA temp_store = MEMORY;
|
|
469
|
-
|
|
509
|
+
-- Longer busy timeout: the ingest worker holds the write lock
|
|
510
|
+
-- for up to a few seconds during big OTLP batches, and the main
|
|
511
|
+
-- daemon's retention passes can do the same. 15s gives either
|
|
512
|
+
-- side enough slack to serialise instead of erroring.
|
|
513
|
+
PRAGMA busy_timeout = 15000;
|
|
514
|
+
-- WAL checkpoint automatically when it grows past ~16MB. Without
|
|
515
|
+
-- this the WAL happily runs into the hundreds of MB and queries
|
|
516
|
+
-- start paying the cost of walking the WAL on every read.
|
|
517
|
+
PRAGMA wal_autocheckpoint = 4000;
|
|
470
518
|
-- Bump cache above the 2MB default. 64MB fits most hot index pages
|
|
471
519
|
-- (trace_summaries, spans, span_attributes indexes) in RAM even on
|
|
472
520
|
-- multi-GB databases, cutting cold-read latency meaningfully on
|
|
@@ -556,8 +604,26 @@ export const TelemetryStoreLive = Layer.effect(
|
|
|
556
604
|
CREATE INDEX IF NOT EXISTS idx_log_attributes_key_value ON log_attributes(key, value, log_id);
|
|
557
605
|
CREATE INDEX IF NOT EXISTS idx_log_attributes_log_id ON log_attributes(log_id);
|
|
558
606
|
`)
|
|
607
|
+
}
|
|
559
608
|
|
|
609
|
+
// Tables detected at runtime. For writer connections these flags are
|
|
610
|
+
// set by the FTS `CREATE VIRTUAL TABLE IF NOT EXISTS` try/catch; for
|
|
611
|
+
// readonly connections we probe `sqlite_master` and set them based on
|
|
612
|
+
// what the writer has already provisioned.
|
|
560
613
|
let hasFts = true
|
|
614
|
+
let hasAttrFts = true
|
|
615
|
+
if (opts.readonly) {
|
|
616
|
+
try {
|
|
617
|
+
const row = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='span_operation_fts'`).get()
|
|
618
|
+
hasFts = row !== null
|
|
619
|
+
} catch { hasFts = false }
|
|
620
|
+
try {
|
|
621
|
+
const row = db.query(`SELECT name FROM sqlite_master WHERE type='table' AND name='span_attr_fts'`).get()
|
|
622
|
+
hasAttrFts = row !== null
|
|
623
|
+
} catch { hasAttrFts = false }
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
if (!opts.readonly) {
|
|
561
627
|
try {
|
|
562
628
|
db.exec(`
|
|
563
629
|
CREATE VIRTUAL TABLE IF NOT EXISTS span_operation_fts USING fts5(
|
|
@@ -589,7 +655,6 @@ export const TelemetryStoreLive = Layer.effect(
|
|
|
589
655
|
// Keys are inlined into the trigger DDL rather than looked up in a
|
|
590
656
|
// side table so the `WHEN` guard stays constant-cost (a subquery
|
|
591
657
|
// would run on every span_attributes insert — ~60/span).
|
|
592
|
-
let hasAttrFts = hasFts
|
|
593
658
|
if (hasFts) {
|
|
594
659
|
try {
|
|
595
660
|
const keyList = AI_FTS_KEYS.map((k) => `'${k.replace(/'/g, "''")}'`).join(", ")
|
|
@@ -661,6 +726,7 @@ export const TelemetryStoreLive = Layer.effect(
|
|
|
661
726
|
// ANALYZE / optimize failures are never fatal — queries still work,
|
|
662
727
|
// they just run with default row estimates.
|
|
663
728
|
}
|
|
729
|
+
} // end: if (!opts.readonly) writer init
|
|
664
730
|
|
|
665
731
|
const insertSpan = db.query(`
|
|
666
732
|
INSERT INTO spans (
|
|
@@ -708,8 +774,14 @@ export const TelemetryStoreLive = Layer.effect(
|
|
|
708
774
|
GROUP BY trace_id
|
|
709
775
|
`)
|
|
710
776
|
|
|
711
|
-
|
|
712
|
-
|
|
777
|
+
// One-time full rebuild of the trace_summaries table at open so
|
|
778
|
+
// any drift from interrupted ingests gets reconciled. Writer-only
|
|
779
|
+
// because the DELETE + INSERT would fail on a readonly connection
|
|
780
|
+
// (and would fight the daemon's writer for the lock anyway).
|
|
781
|
+
if (!opts.readonly) {
|
|
782
|
+
db.query(`DELETE FROM trace_summaries`).run()
|
|
783
|
+
rebuildTraceSummaries.run()
|
|
784
|
+
}
|
|
713
785
|
|
|
714
786
|
const deleteSpanAttributes = db.query(`DELETE FROM span_attributes WHERE trace_id = ? AND span_id = ?`)
|
|
715
787
|
const insertSpanAttribute = db.query(`INSERT INTO span_attributes (trace_id, span_id, key, value) VALUES (?, ?, ?, ?)`)
|
|
@@ -792,28 +864,55 @@ export const TelemetryStoreLive = Layer.effect(
|
|
|
792
864
|
} catch {
|
|
793
865
|
// FTS table may not exist on old DBs.
|
|
794
866
|
}
|
|
867
|
+
|
|
868
|
+
// Truncate the WAL after a big delete pass. Without this the
|
|
869
|
+
// WAL keeps growing (observed: 640MB) because wal_autocheckpoint
|
|
870
|
+
// only triggers when WAL pages exceed the threshold during
|
|
871
|
+
// writes — a retention pass that evicts millions of rows can
|
|
872
|
+
// blow far past that before the auto-checkpoint fires. Using
|
|
873
|
+
// PASSIVE so active readers aren't interrupted; if the WAL
|
|
874
|
+
// can't be fully reclaimed right now, we'll try again next
|
|
875
|
+
// cycle.
|
|
876
|
+
try { db.exec(`PRAGMA wal_checkpoint(PASSIVE);`) } catch { /* ignore */ }
|
|
877
|
+
|
|
878
|
+
// Incremental vacuum reclaims some of the freed pages back
|
|
879
|
+
// to the OS so the file size actually shrinks over time
|
|
880
|
+
// instead of just growing the freelist. Bounded to 2000
|
|
881
|
+
// pages per pass (≈8MB) to avoid a long-running transaction.
|
|
882
|
+
try { db.exec(`PRAGMA incremental_vacuum(2000);`) } catch { /* ignore */ }
|
|
795
883
|
})
|
|
796
884
|
})
|
|
797
885
|
|
|
798
|
-
//
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
886
|
+
// Retention only runs in processes that opt in (currently the main
|
|
887
|
+
// daemon). The ingest worker and TUI skip it to avoid two writers
|
|
888
|
+
// competing for the write lock with overlapping DELETE passes.
|
|
889
|
+
if (opts.runRetention) {
|
|
890
|
+
// Enable incremental vacuum so retention can reclaim freed
|
|
891
|
+
// pages over time instead of needing a stop-the-world VACUUM.
|
|
892
|
+
// Idempotent: repeat calls after the first are no-ops.
|
|
893
|
+
try { db.exec(`PRAGMA auto_vacuum = INCREMENTAL;`) } catch { /* ignore */ }
|
|
894
|
+
|
|
895
|
+
// Run cleanup every 60 seconds in the background, tied to the layer's scope
|
|
896
|
+
yield* Effect.forkScoped(Effect.repeat(cleanupExpired(), Schedule.spaced("60 seconds")))
|
|
897
|
+
|
|
898
|
+
// Periodically refresh query planner stats. `PRAGMA optimize` is a
|
|
899
|
+
// no-op when nothing has changed, so this is essentially free on idle
|
|
900
|
+
// servers and keeps facet/search planner estimates accurate as data
|
|
901
|
+
// grows. 15 minutes is slower than ingestion rates we care about but
|
|
902
|
+
// frequent enough that the attribute picker stays snappy.
|
|
903
|
+
const refreshPlannerStats = Effect.sync(() => {
|
|
904
|
+
try { db.exec(`PRAGMA optimize;`) } catch { /* ignore */ }
|
|
905
|
+
})
|
|
906
|
+
yield* Effect.forkScoped(Effect.repeat(refreshPlannerStats, Schedule.spaced("15 minutes")))
|
|
907
|
+
}
|
|
810
908
|
|
|
811
909
|
// One-time backfill for existing DBs: if span_attr_fts is empty but
|
|
812
910
|
// span_attributes has rows with AI_FTS_KEYS, populate the index.
|
|
813
911
|
// Runs forked so server startup isn't blocked; queries hitting the
|
|
814
912
|
// FTS will just return empty until the fill lands. On a 2 GB DB with
|
|
815
|
-
// ~400 matching rows this takes ~3-8 seconds.
|
|
816
|
-
|
|
913
|
+
// ~400 matching rows this takes ~3-8 seconds. Writer-only because
|
|
914
|
+
// it does INSERT INTO ... — readonly connections would error.
|
|
915
|
+
if (hasAttrFts && !opts.readonly) {
|
|
817
916
|
const backfillAttrFts = Effect.sync(() => {
|
|
818
917
|
try {
|
|
819
918
|
const ftsCount = (db.query(`SELECT COUNT(*) AS c FROM span_attr_fts`).get() as { c: number }).c
|
|
@@ -1705,7 +1804,11 @@ export const TelemetryStoreLive = Layer.effect(
|
|
|
1705
1804
|
|
|
1706
1805
|
/** Builds WHERE clauses for AI call search against the spans table (aliased as s) */
|
|
1707
1806
|
const buildAiWhereClauses = (input: AiCallSearch | AiCallStatsSearch, cutoff: number) => {
|
|
1708
|
-
const clauses: string[] = [
|
|
1807
|
+
const clauses: string[] = [
|
|
1808
|
+
"s.operation_name LIKE 'ai.%'",
|
|
1809
|
+
"s.operation_name NOT LIKE 'ai.%.do%'",
|
|
1810
|
+
"s.start_time_ms >= ?",
|
|
1811
|
+
]
|
|
1709
1812
|
const params: Array<string | number> = [cutoff]
|
|
1710
1813
|
|
|
1711
1814
|
if (input.service) {
|
|
@@ -2094,3 +2197,25 @@ export const TelemetryStoreLive = Layer.effect(
|
|
|
2094
2197
|
})
|
|
2095
2198
|
}),
|
|
2096
2199
|
)
|
|
2200
|
+
|
|
2201
|
+
/**
|
|
2202
|
+
* Default writer instance: the main daemon uses this. Owns schema
|
|
2203
|
+
* migrations, FTS backfill, and the retention loop.
|
|
2204
|
+
*/
|
|
2205
|
+
export const TelemetryStoreLive = makeTelemetryStoreLayer({ readonly: false, runRetention: true })
|
|
2206
|
+
|
|
2207
|
+
/**
|
|
2208
|
+
* Writer instance that SKIPS retention. The ingest worker uses this
|
|
2209
|
+
* so the daemon and the worker aren't both running DELETE passes at
|
|
2210
|
+
* the same time (they'd just serialise behind the write lock and
|
|
2211
|
+
* duplicate work).
|
|
2212
|
+
*/
|
|
2213
|
+
export const TelemetryStoreWorkerLive = makeTelemetryStoreLayer({ readonly: false, runRetention: false })
|
|
2214
|
+
|
|
2215
|
+
/**
|
|
2216
|
+
* Read-only instance for query-only processes (currently the TUI).
|
|
2217
|
+
* Skips every DDL/DML statement at startup so the connection can be
|
|
2218
|
+
* opened while a writer is mid-transaction without racing for the
|
|
2219
|
+
* write lock. Writes through the service interface will throw.
|
|
2220
|
+
*/
|
|
2221
|
+
export const TelemetryStoreReadonlyLive = makeTelemetryStoreLayer({ readonly: true, runRetention: false })
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Effect, Layer, Context } from "effect"
|
|
2
|
-
import type { SpanItem, TraceItem, TraceSummaryItem } from "../domain.js"
|
|
2
|
+
import type { AiCallDetail, SpanItem, TraceItem, TraceSummaryItem } from "../domain.js"
|
|
3
3
|
import { TelemetryStore } from "./TelemetryStore.js"
|
|
4
4
|
|
|
5
5
|
export class TraceQueryService extends Context.Service<
|
|
@@ -14,6 +14,7 @@ export class TraceQueryService extends Context.Service<
|
|
|
14
14
|
readonly traceStats: (input: { readonly groupBy: string; readonly agg: "count" | "avg_duration" | "p95_duration" | "error_rate"; readonly serviceName?: string | null; readonly operation?: string | null; readonly status?: "ok" | "error" | null; readonly minDurationMs?: number | null; readonly lookbackMinutes?: number; readonly limit?: number; readonly attributeFilters?: Readonly<Record<string, string>> }) => Effect.Effect<readonly { readonly group: string; readonly value: number; readonly count: number }[], Error>
|
|
15
15
|
readonly getTrace: (traceId: string) => Effect.Effect<TraceItem | null, Error>
|
|
16
16
|
readonly getSpan: (spanId: string) => Effect.Effect<SpanItem | null, Error>
|
|
17
|
+
readonly getAiCall: (spanId: string) => Effect.Effect<AiCallDetail | null, Error>
|
|
17
18
|
readonly listTraceSpans: (traceId: string) => Effect.Effect<readonly SpanItem[], Error>
|
|
18
19
|
readonly searchSpans: (input: { readonly serviceName?: string | null; readonly operation?: string | null; readonly parentOperation?: string | null; readonly status?: "ok" | "error" | null; readonly lookbackMinutes?: number; readonly limit?: number; readonly attributeFilters?: Readonly<Record<string, string>> }) => Effect.Effect<readonly SpanItem[], Error>
|
|
19
20
|
}
|
|
@@ -68,6 +69,7 @@ export const TraceQueryServiceLive = Layer.effect(
|
|
|
68
69
|
traceStats: store.traceStats,
|
|
69
70
|
getTrace,
|
|
70
71
|
getSpan,
|
|
72
|
+
getAiCall: store.getAiCall,
|
|
71
73
|
listTraceSpans: store.listTraceSpans,
|
|
72
74
|
searchSpans: store.searchSpans,
|
|
73
75
|
})
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RPC contract for OTLP ingest. Lives in its own file so both the
|
|
3
|
+
* main thread (client) and the telemetry worker (server) can import
|
|
4
|
+
* the schema without pulling in each other's runtime code.
|
|
5
|
+
*
|
|
6
|
+
* Only ingestTraces and ingestLogs run through RPC — those are the
|
|
7
|
+
* methods whose SQLite writes used to block the main event loop for
|
|
8
|
+
* seconds at a time. Every other TelemetryStore method stays on the
|
|
9
|
+
* main thread with its own direct DB connection; SQLite's WAL mode
|
|
10
|
+
* lets the reader (main) and writer (worker) hold independent
|
|
11
|
+
* connections to the same file concurrently without contention.
|
|
12
|
+
*
|
|
13
|
+
* Payloads are typed as Schema.Unknown because OTLP's protobuf-JSON
|
|
14
|
+
* shape is enormous and nested — the store validates structurally
|
|
15
|
+
* during the actual insert loop, and serialising a schema through
|
|
16
|
+
* the worker boundary would add overhead that beats the purpose of
|
|
17
|
+
* the offload. If a payload is malformed we surface it as an
|
|
18
|
+
* IngestError rather than a RpcSchemaError, which keeps the failure
|
|
19
|
+
* mode consistent with the old direct-call behaviour.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { Schema } from "effect"
|
|
23
|
+
import * as Rpc from "effect/unstable/rpc/Rpc"
|
|
24
|
+
import * as RpcGroup from "effect/unstable/rpc/RpcGroup"
|
|
25
|
+
|
|
26
|
+
export class IngestError extends Schema.TaggedErrorClass<IngestError>()("IngestError", {
|
|
27
|
+
message: Schema.String,
|
|
28
|
+
}) {}
|
|
29
|
+
|
|
30
|
+
export const IngestRpcs = RpcGroup.make(
|
|
31
|
+
Rpc.make("ingestTraces", {
|
|
32
|
+
payload: { payload: Schema.Unknown },
|
|
33
|
+
success: Schema.Struct({ insertedSpans: Schema.Number }),
|
|
34
|
+
error: IngestError,
|
|
35
|
+
}),
|
|
36
|
+
Rpc.make("ingestLogs", {
|
|
37
|
+
payload: { payload: Schema.Unknown },
|
|
38
|
+
success: Schema.Struct({ insertedLogs: Schema.Number }),
|
|
39
|
+
error: IngestError,
|
|
40
|
+
}),
|
|
41
|
+
)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker-thread entry point for OTLP ingest.
|
|
3
|
+
*
|
|
4
|
+
* Spawned by the main process via `new Worker(new URL("./telemetryWorker.ts", import.meta.url))`.
|
|
5
|
+
* This file runs inside a Bun Worker, so anything it imports is
|
|
6
|
+
* evaluated in a FRESH module graph on the worker side. In particular
|
|
7
|
+
* `TelemetryStoreWorkerLive` opens its own SQLite connection here — the main
|
|
8
|
+
* thread's store connection is unrelated. SQLite's WAL journal mode
|
|
9
|
+
* lets both connections coexist against the same `.sqlite` file: the
|
|
10
|
+
* worker writes, the main thread reads, and neither blocks the other.
|
|
11
|
+
*
|
|
12
|
+
* The worker only exposes `ingestTraces` / `ingestLogs` (see
|
|
13
|
+
* ingestRpc.ts). Query methods stay on the main thread because they're
|
|
14
|
+
* already fast (1-14ms) and round-tripping them through structured-
|
|
15
|
+
* clone would add more overhead than it saves. This is a deliberately
|
|
16
|
+
* narrow interface — the payoff is that main-thread HTTP queries
|
|
17
|
+
* never queue behind a heavy OTLP batch again.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { BunRuntime } from "@effect/platform-bun"
|
|
21
|
+
import * as BunWorkerRunner from "@effect/platform-bun/BunWorkerRunner"
|
|
22
|
+
import { Effect, Layer } from "effect"
|
|
23
|
+
import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"
|
|
24
|
+
import * as RpcServer from "effect/unstable/rpc/RpcServer"
|
|
25
|
+
import type { OtlpLogExportRequest, OtlpTraceExportRequest } from "../otlp.ts"
|
|
26
|
+
import { IngestError, IngestRpcs } from "./ingestRpc.ts"
|
|
27
|
+
import { TelemetryStore, TelemetryStoreWorkerLive } from "./TelemetryStore.ts"
|
|
28
|
+
|
|
29
|
+
// Wire the two RPC methods to the existing TelemetryStore service.
|
|
30
|
+
// The store's ingest methods already carry their own Effect.fn spans,
|
|
31
|
+
// so the worker-side traces show up correctly attributed — the RPC
|
|
32
|
+
// framework also auto-spans each incoming request with method +
|
|
33
|
+
// payload-size attributes, giving us visibility into how ingest is
|
|
34
|
+
// splitting its time across the queue / wire / SQL stages.
|
|
35
|
+
const IngestHandlers = IngestRpcs.toLayer(
|
|
36
|
+
Effect.gen(function*() {
|
|
37
|
+
const store = yield* TelemetryStore
|
|
38
|
+
return {
|
|
39
|
+
ingestTraces: ({ payload }) =>
|
|
40
|
+
store.ingestTraces(payload as OtlpTraceExportRequest).pipe(
|
|
41
|
+
Effect.mapError((cause) => new IngestError({ message: String(cause) })),
|
|
42
|
+
),
|
|
43
|
+
ingestLogs: ({ payload }) =>
|
|
44
|
+
store.ingestLogs(payload as OtlpLogExportRequest).pipe(
|
|
45
|
+
Effect.mapError((cause) => new IngestError({ message: String(cause) })),
|
|
46
|
+
),
|
|
47
|
+
}
|
|
48
|
+
}),
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
const WorkerLive = RpcServer.layer(IngestRpcs).pipe(
|
|
52
|
+
Layer.provide(IngestHandlers),
|
|
53
|
+
Layer.provide(TelemetryStoreWorkerLive),
|
|
54
|
+
Layer.provide(RpcServer.layerProtocolWorkerRunner),
|
|
55
|
+
Layer.provide(RpcSerialization.layerMsgPack),
|
|
56
|
+
Layer.provide(BunWorkerRunner.layer),
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
// BunRuntime.runMain installs signal handlers so the scope closes
|
|
60
|
+
// cleanly on termination; the BunHttpServer layer pattern from the
|
|
61
|
+
// main server carries over here.
|
|
62
|
+
Layer.launch(WorkerLive).pipe(BunRuntime.runMain)
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
// Storybook-style preview for AiChatView. Renders the component
|
|
2
|
+
// against a menu of fixtures so we can iterate on the rendering
|
|
3
|
+
// without needing real LLM traffic captured.
|
|
4
|
+
//
|
|
5
|
+
// Run it: `bun run story:chat`
|
|
6
|
+
// Keys:
|
|
7
|
+
// 1..N switch fixture
|
|
8
|
+
// j / k scroll down / up (1 line)
|
|
9
|
+
// ctrl-d/u half-page
|
|
10
|
+
// gg / G jump to top / bottom
|
|
11
|
+
// r force re-render with a fresh date (to sanity-check headers)
|
|
12
|
+
// q / ^c quit
|
|
13
|
+
|
|
14
|
+
import { RegistryProvider } from "@effect/atom-react"
|
|
15
|
+
import { RGBA, TextAttributes, createCliRenderer } from "@opentui/core"
|
|
16
|
+
import { createRoot, useKeyboard, useRenderer, useTerminalDimensions } from "@opentui/react"
|
|
17
|
+
import { useEffect, useMemo, useRef, useState } from "react"
|
|
18
|
+
import { buildChunks, type Chunk } from "../ui/aiChatModel.ts"
|
|
19
|
+
import { AiChatView } from "../ui/AiChatView.tsx"
|
|
20
|
+
import { Divider, TextLine } from "../ui/primitives.tsx"
|
|
21
|
+
import type { AiCallDetailState } from "../ui/state.ts"
|
|
22
|
+
import { applyTheme, colors, SEPARATOR } from "../ui/theme.ts"
|
|
23
|
+
import type { ChatFixture } from "./fixtures/index.ts"
|
|
24
|
+
import { errorFixture } from "./fixtures/errorState.ts"
|
|
25
|
+
import { imagePasteFixture } from "./fixtures/imagePaste.ts"
|
|
26
|
+
import { kitchenSinkFixture } from "./fixtures/kitchenSink.ts"
|
|
27
|
+
import { rawPromptFixture } from "./fixtures/rawPrompt.ts"
|
|
28
|
+
import { shortFixture } from "./fixtures/short.ts"
|
|
29
|
+
import { toolHeavyFixture } from "./fixtures/toolHeavy.ts"
|
|
30
|
+
|
|
31
|
+
// Kitchen-sink first so launching the story lands on something that
|
|
32
|
+
// shows every rendering branch at once. The other fixtures exercise
|
|
33
|
+
// one case at a time for regression testing.
|
|
34
|
+
const FIXTURES: readonly ChatFixture[] = [
|
|
35
|
+
kitchenSinkFixture,
|
|
36
|
+
shortFixture,
|
|
37
|
+
toolHeavyFixture,
|
|
38
|
+
imagePasteFixture,
|
|
39
|
+
rawPromptFixture,
|
|
40
|
+
errorFixture,
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
const HEADER_ROWS = 2
|
|
44
|
+
const FOOTER_ROWS = 1
|
|
45
|
+
|
|
46
|
+
const StoryApp = () => {
|
|
47
|
+
applyTheme("motel-default")
|
|
48
|
+
const renderer = useRenderer()
|
|
49
|
+
const { width, height } = useTerminalDimensions()
|
|
50
|
+
const w = width ?? 120
|
|
51
|
+
const h = height ?? 32
|
|
52
|
+
const [fixtureIdx, setFixtureIdx] = useState(0)
|
|
53
|
+
const [selectedChunkId, setSelectedChunkId] = useState<string | null>(null)
|
|
54
|
+
const [detailChunkId, setDetailChunkId] = useState<string | null>(null)
|
|
55
|
+
const [detailScrollOffset, setDetailScrollOffset] = useState(0)
|
|
56
|
+
const pendingGRef = useRef(false)
|
|
57
|
+
const quittingRef = useRef(false)
|
|
58
|
+
|
|
59
|
+
const fixture = FIXTURES[fixtureIdx] ?? FIXTURES[0]!
|
|
60
|
+
|
|
61
|
+
const detailState: AiCallDetailState = useMemo(() => ({
|
|
62
|
+
status: "ready",
|
|
63
|
+
spanId: fixture.span.spanId,
|
|
64
|
+
data: fixture.detail,
|
|
65
|
+
error: null,
|
|
66
|
+
}), [fixture])
|
|
67
|
+
|
|
68
|
+
const chunks = useMemo<readonly Chunk[]>(() => buildChunks(fixture.detail), [fixture])
|
|
69
|
+
|
|
70
|
+
// Reset selection + expansion whenever fixture changes.
|
|
71
|
+
useEffect(() => {
|
|
72
|
+
setSelectedChunkId(chunks[0]?.id ?? null)
|
|
73
|
+
setDetailChunkId(null)
|
|
74
|
+
setDetailScrollOffset(0)
|
|
75
|
+
}, [fixtureIdx, chunks])
|
|
76
|
+
|
|
77
|
+
const move = (delta: number) => {
|
|
78
|
+
if (chunks.length === 0) return
|
|
79
|
+
const idx = selectedChunkId ? chunks.findIndex((c) => c.id === selectedChunkId) : 0
|
|
80
|
+
const next = chunks[Math.max(0, Math.min(idx + delta, chunks.length - 1))]
|
|
81
|
+
if (next) setSelectedChunkId(next.id)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
useKeyboard((key) => {
|
|
85
|
+
if (key.name === "q" || (key.ctrl && key.name === "c")) {
|
|
86
|
+
// renderer.destroy() runs the opentui teardown (disables
|
|
87
|
+
// mouse tracking / kitty keyboard / bracketed paste / alt
|
|
88
|
+
// screen) before onDestroy exits the process. Bypassing it
|
|
89
|
+
// with a raw process.exit leaks those escape sequences into
|
|
90
|
+
// the host shell.
|
|
91
|
+
if (quittingRef.current) return
|
|
92
|
+
quittingRef.current = true
|
|
93
|
+
renderer.destroy()
|
|
94
|
+
return
|
|
95
|
+
}
|
|
96
|
+
if (/^[1-9]$/.test(key.name) && !key.ctrl && !key.meta) {
|
|
97
|
+
const idx = parseInt(key.name, 10) - 1
|
|
98
|
+
if (idx < FIXTURES.length) setFixtureIdx(idx)
|
|
99
|
+
return
|
|
100
|
+
}
|
|
101
|
+
if (key.name === "j" || key.name === "down") {
|
|
102
|
+
if (detailChunkId) setDetailScrollOffset((current) => current + 1)
|
|
103
|
+
else move(1)
|
|
104
|
+
return
|
|
105
|
+
}
|
|
106
|
+
if (key.name === "k" || key.name === "up") {
|
|
107
|
+
if (detailChunkId) setDetailScrollOffset((current) => Math.max(0, current - 1))
|
|
108
|
+
else move(-1)
|
|
109
|
+
return
|
|
110
|
+
}
|
|
111
|
+
if (key.name === "pagedown" || (key.ctrl && key.name === "d")) {
|
|
112
|
+
if (detailChunkId) setDetailScrollOffset((current) => current + Math.max(1, Math.floor(bodyLines / 2)))
|
|
113
|
+
else move(Math.max(1, Math.floor(chunks.length / 4)))
|
|
114
|
+
return
|
|
115
|
+
}
|
|
116
|
+
if (key.name === "pageup" || (key.ctrl && key.name === "u")) {
|
|
117
|
+
if (detailChunkId) setDetailScrollOffset((current) => Math.max(0, current - Math.max(1, Math.floor(bodyLines / 2))))
|
|
118
|
+
else move(-Math.max(1, Math.floor(chunks.length / 4)))
|
|
119
|
+
return
|
|
120
|
+
}
|
|
121
|
+
if (key.name === "g" && !key.shift) {
|
|
122
|
+
if (detailChunkId) {
|
|
123
|
+
if (pendingGRef.current) {
|
|
124
|
+
setDetailScrollOffset(0)
|
|
125
|
+
pendingGRef.current = false
|
|
126
|
+
} else {
|
|
127
|
+
pendingGRef.current = true
|
|
128
|
+
setTimeout(() => { pendingGRef.current = false }, 500)
|
|
129
|
+
}
|
|
130
|
+
return
|
|
131
|
+
}
|
|
132
|
+
if (pendingGRef.current) {
|
|
133
|
+
if (chunks[0]) setSelectedChunkId(chunks[0].id)
|
|
134
|
+
pendingGRef.current = false
|
|
135
|
+
} else {
|
|
136
|
+
pendingGRef.current = true
|
|
137
|
+
setTimeout(() => { pendingGRef.current = false }, 500)
|
|
138
|
+
}
|
|
139
|
+
return
|
|
140
|
+
}
|
|
141
|
+
if (key.name === "g" && key.shift) {
|
|
142
|
+
if (detailChunkId) setDetailScrollOffset(999999)
|
|
143
|
+
else {
|
|
144
|
+
const last = chunks[chunks.length - 1]
|
|
145
|
+
if (last) setSelectedChunkId(last.id)
|
|
146
|
+
}
|
|
147
|
+
return
|
|
148
|
+
}
|
|
149
|
+
if (key.name === "escape") {
|
|
150
|
+
setDetailChunkId(null)
|
|
151
|
+
setDetailScrollOffset(0)
|
|
152
|
+
return
|
|
153
|
+
}
|
|
154
|
+
if (key.name === "return" || key.name === "enter") {
|
|
155
|
+
const chunk = chunks.find((c) => c.id === selectedChunkId)
|
|
156
|
+
if (chunk) {
|
|
157
|
+
setDetailChunkId(chunk.id)
|
|
158
|
+
setDetailScrollOffset(0)
|
|
159
|
+
}
|
|
160
|
+
return
|
|
161
|
+
}
|
|
162
|
+
})
|
|
163
|
+
|
|
164
|
+
const bodyLines = Math.max(4, h - HEADER_ROWS - FOOTER_ROWS - 4 /* AI_CHAT_HEADER_ROWS */)
|
|
165
|
+
|
|
166
|
+
// Short compact labels — a single TextLine truncates with "..." if
|
|
167
|
+
// it overflows the padded content width, so we keep labels tight and
|
|
168
|
+
// use a single separator between items.
|
|
169
|
+
const fixtureList = FIXTURES.map((f, i) => (
|
|
170
|
+
<span key={f.id}>
|
|
171
|
+
<span fg={i === fixtureIdx ? colors.accent : colors.muted} attributes={i === fixtureIdx ? TextAttributes.BOLD : undefined}>
|
|
172
|
+
{`${i + 1} ${f.label}`}
|
|
173
|
+
</span>
|
|
174
|
+
{i < FIXTURES.length - 1 ? <span fg={colors.separator}>{" \u00b7 "}</span> : null}
|
|
175
|
+
</span>
|
|
176
|
+
))
|
|
177
|
+
|
|
178
|
+
// Header + divider live inside a paddingLeft/Right={1} box, so the
|
|
179
|
+
// real content width is `w - 2`. Divider must match or we get a
|
|
180
|
+
// mid-line "..." truncation.
|
|
181
|
+
const contentWidth = Math.max(8, w - 2)
|
|
182
|
+
|
|
183
|
+
return (
|
|
184
|
+
<box width={w} height={h} flexDirection="column" backgroundColor={RGBA.fromHex(colors.screenBg)}>
|
|
185
|
+
<box paddingLeft={1} paddingRight={1} height={HEADER_ROWS} flexDirection="column">
|
|
186
|
+
<TextLine>
|
|
187
|
+
<span fg={colors.muted} attributes={TextAttributes.BOLD}>AI CHAT</span>
|
|
188
|
+
<span fg={colors.separator}>{" \u00b7 "}</span>
|
|
189
|
+
{fixtureList}
|
|
190
|
+
</TextLine>
|
|
191
|
+
<Divider width={contentWidth} />
|
|
192
|
+
</box>
|
|
193
|
+
<AiChatView
|
|
194
|
+
span={fixture.span}
|
|
195
|
+
detailState={detailState}
|
|
196
|
+
chunks={chunks}
|
|
197
|
+
selectedChunkId={selectedChunkId}
|
|
198
|
+
onSelectChunk={(chunkId) => setSelectedChunkId(chunkId)}
|
|
199
|
+
detailChunkId={detailChunkId}
|
|
200
|
+
onOpenDetail={(chunkId) => {
|
|
201
|
+
setSelectedChunkId(chunkId)
|
|
202
|
+
setDetailChunkId(chunkId)
|
|
203
|
+
setDetailScrollOffset(0)
|
|
204
|
+
}}
|
|
205
|
+
onCloseDetail={() => {
|
|
206
|
+
setDetailChunkId(null)
|
|
207
|
+
setDetailScrollOffset(0)
|
|
208
|
+
}}
|
|
209
|
+
detailScrollOffset={detailScrollOffset}
|
|
210
|
+
onSetDetailScrollOffset={(updater) => setDetailScrollOffset(updater)}
|
|
211
|
+
contentWidth={Math.max(24, w - 2)}
|
|
212
|
+
bodyLines={bodyLines}
|
|
213
|
+
paneWidth={w}
|
|
214
|
+
/>
|
|
215
|
+
<box paddingLeft={1} paddingRight={1} height={FOOTER_ROWS}>
|
|
216
|
+
<TextLine>
|
|
217
|
+
<span fg={colors.count} attributes={TextAttributes.BOLD}>1-9</span>
|
|
218
|
+
<span fg={colors.muted}>{" fixture "}</span>
|
|
219
|
+
<span fg={colors.count} attributes={TextAttributes.BOLD}>j/k</span>
|
|
220
|
+
<span fg={colors.muted}>{" move "}</span>
|
|
221
|
+
<span fg={colors.count} attributes={TextAttributes.BOLD}>enter</span>
|
|
222
|
+
<span fg={colors.muted}>{" detail "}</span>
|
|
223
|
+
<span fg={colors.count} attributes={TextAttributes.BOLD}>gg/G</span>
|
|
224
|
+
<span fg={colors.muted}>{" top/bottom "}</span>
|
|
225
|
+
<span fg={colors.count} attributes={TextAttributes.BOLD}>q</span>
|
|
226
|
+
<span fg={colors.muted}>{" quit"}</span>
|
|
227
|
+
</TextLine>
|
|
228
|
+
</box>
|
|
229
|
+
</box>
|
|
230
|
+
)
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const renderer = await createCliRenderer({
|
|
234
|
+
exitOnCtrlC: false,
|
|
235
|
+
screenMode: "alternate-screen",
|
|
236
|
+
onDestroy: () => { process.exit(0) },
|
|
237
|
+
})
|
|
238
|
+
|
|
239
|
+
createRoot(renderer).render(
|
|
240
|
+
<RegistryProvider>
|
|
241
|
+
<StoryApp />
|
|
242
|
+
</RegistryProvider>,
|
|
243
|
+
)
|