@aryasaatvik/motel 0.3.5 → 0.5.0

package/src/ingestReadiness.test.ts

@@ -0,0 +1,160 @@
+import { Database } from "bun:sqlite"
+import { expect, test } from "bun:test"
+import { mkdtempSync, mkdirSync, rmSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { Effect } from "effect"
+import { createDaemonManager } from "./daemon.ts"
+import { IngestProgress, type IngestReadiness } from "./ingestReadiness.ts"
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
+
+test("idle readiness, overload age, empty commits, and terminal failure are distinct", () => {
+	const progress = new IngestProgress()
+	expect(progress.snapshot().state).toBe("starting")
+	progress.receive({ _tag: "ready" })
+	expect(progress.snapshot(Date.now() + 86_400_000).state).toBe("ready")
+	const finish = progress.begin(42, 100)
+	expect(progress.snapshot(5100)).toMatchObject({ state: "overloaded", outstandingBytes: 42, outstandingRequests: 1 })
+	progress.receive({ _tag: "commit", records: 0, at: 500 })
+	expect(progress.snapshot().lastCommitAt).toBeNull()
+	finish()
+	finish()
+	expect(progress.snapshot()).toMatchObject({ state: "ready", outstandingBytes: 0, outstandingRequests: 0 })
+	progress.fail()
+	progress.receive({ _tag: "ready" })
+	expect(progress.snapshot().state).toBe("failed")
+})
+
+test("cached readiness bypasses a blocked writer and acknowledgement follows commit", async () => {
+	const runtimeDir = mkdtempSync(join(tmpdir(), "motel-readiness-"))
+	const databasePath = join(runtimeDir, "telemetry.sqlite")
+	const port = 31000 + Math.floor(Math.random() * 2000)
+	const manager = createDaemonManager({ runtimeDir, databasePath, port })
+	let lock: Database | undefined
+	try {
+		await Effect.runPromise(manager.ensure)
+		const base = `http://127.0.0.1:${port}`
+		const readiness = async () => {
+			const response = await fetch(`${base}/api/readiness`, { signal: AbortSignal.timeout(1500) })
+			return await response.json() as IngestReadiness
+		}
+		expect((await readiness()).state).toBe("ready")
+		lock = new Database(databasePath)
+		lock.exec("BEGIN IMMEDIATE")
+		const body = JSON.stringify({ resourceLogs: [{ resource: { attributes: [{ key: "service.name", value: { stringValue: "readiness-test" } }] }, scopeLogs: [{ logRecords: [{ timeUnixNano: String(BigInt(Date.now()) * 1_000_000n), body: { stringValue: "commit barrier" } }] }] }] })
+		let acknowledged = false
+		const ingestion = fetch(`${base}/v1/logs`, { method: "POST", headers: { "content-type": "application/json" }, body }).then(async (response) => {
+			expect(response.status).toBe(200)
+			acknowledged = true
+			return response.json()
+		})
+		await sleep(150)
+		const snapshot = await readiness()
+		expect(snapshot.outstandingRequests).toBe(1)
+		expect(snapshot.outstandingBytes).toBe(Buffer.byteLength(body))
+		expect(acknowledged).toBe(false)
+		expect(lock.query("SELECT count(*) AS n FROM logs WHERE body = 'commit barrier'").get()).toEqual({ n: 0 })
+		lock.exec("COMMIT")
+		expect(await ingestion).toEqual({ insertedLogs: 1 })
+		for (let i = 0; i < 50 && (await readiness()).committedRecords === 0; i++) await sleep(10)
+		expect(await readiness()).toMatchObject({ state: "ready", outstandingRequests: 0, outstandingBytes: 0, committedRecords: 1 })
+		expect(lock.query("SELECT count(*) AS n FROM logs WHERE body = 'commit barrier'").get()).toEqual({ n: 1 })
+		expect((await Effect.runPromise(manager.getStatus)).readiness?.state).toBe("ready")
+	} finally {
+		try { lock?.exec("ROLLBACK") } catch { /* The successful path committed. */ }
+		lock?.close()
+		await Effect.runPromise(manager.stop)
+		rmSync(runtimeDir, { recursive: true, force: true })
+	}
+}, 30000)
+
+test("a writer bootstrap failure is reported without mistaking liveness for readiness", async () => {
+	const root = mkdtempSync(join(tmpdir(), "motel-writer-failure-"))
+	const port = 35000 + Math.floor(Math.random() * 1000)
+	const child = Bun.spawn([process.execPath, "src/server.ts"], {
+		cwd: join(import.meta.dir, ".."),
+		env: { ...process.env, MOTEL_OTEL_DB_PATH: root, MOTEL_RUNTIME_DIR: root, MOTEL_OTEL_HOST: "127.0.0.1", MOTEL_OTEL_PORT: String(port), MOTEL_OTEL_BASE_URL: `http://127.0.0.1:${port}`, MOTEL_OTEL_ENABLED: "false" },
+		stdout: "ignore", stderr: "ignore",
+	})
+	try {
+		let state: string | undefined
+		const until = Date.now() + 10000
+		while (Date.now() < until) {
+			try {
+				const response = await fetch(`http://127.0.0.1:${port}/api/readiness`, { signal: AbortSignal.timeout(500) })
+				state = (await response.json() as IngestReadiness).state
+				if (state === "failed") {
+					expect(response.status).toBe(503)
+					break
+				}
+			} catch { /* Wait for the HTTP listener independently of writer bootstrap. */ }
+			await sleep(25)
+		}
+		expect(state).toBe("failed")
+		expect((await fetch(`http://127.0.0.1:${port}/api/health`)).status).toBe(200)
+	} finally {
+		child.kill("SIGTERM")
+		await Promise.race([child.exited, sleep(1000)])
+		if (child.exitCode === null) child.kill("SIGKILL")
+		await child.exited
+		rmSync(root, { recursive: true, force: true })
+	}
+}, 15000)
+
+
+test("completed maintenance cannot hide another operation that is still running", () => {
+	const progress = new IngestProgress()
+	progress.receive({ _tag: "maintenance", value: { operation: "retention", startedAt: 10, durationMs: null, outcome: "running" } })
+	progress.receive({ _tag: "maintenance", value: { operation: "reclaim", startedAt: 20, durationMs: null, outcome: "running" } })
+	progress.receive({ _tag: "maintenance", value: { operation: "reclaim", startedAt: 20, durationMs: 5, outcome: "completed" } })
+	expect(progress.snapshot().maintenance).toMatchObject({ operation: "retention", outcome: "running" })
+	progress.receive({ _tag: "maintenance", value: { operation: "retention", startedAt: 10, durationMs: 30, outcome: "completed" } })
+	expect(progress.snapshot().maintenance).toMatchObject({ operation: "retention", outcome: "completed", durationMs: 30 })
+})
+
+test("simultaneous cold starts from different workdirs converge on one managed daemon", async () => {
+	const root = mkdtempSync(join(tmpdir(), "motel-concurrent-start-"))
+	const workdirs = [join(root, "a"), join(root, "b")]
+	for (const workdir of workdirs) mkdirSync(workdir)
+	const port = 36000 + Math.floor(Math.random() * 1000)
+	const managers = workdirs.map((workdir) => createDaemonManager({ runtimeDir: root, databasePath: join(root, "db.sqlite"), port, workdir, startTimeoutMs: 5000, gracefulStopTimeoutMs: 500, forceStopTimeoutMs: 500 }))
+	try {
+		const states = await Promise.all(managers.map((manager) => Effect.runPromise(manager.ensure)))
+		expect(states.every((state) => state.running && state.managed)).toBe(true)
+		expect(states[0]!.pid).toBe(states[1]!.pid)
+	} finally {
+		await Effect.runPromise(managers[0]!.stop)
+		rmSync(root, { recursive: true, force: true })
+	}
+}, 30000)
+
+test("ensure preserves a live daemon when its ingest readiness budget expires", async () => {
+	const root = mkdtempSync(join(tmpdir(), "motel-busy-ensure-"))
+	const databasePath = join(root, "db.sqlite")
+	const port = 37000 + Math.floor(Math.random() * 1000)
+	const manager = createDaemonManager({ runtimeDir: root, databasePath, port })
+	const impatient = createDaemonManager({ runtimeDir: root, databasePath, port, startTimeoutMs: 150 })
+	let lock: Database | undefined
+	try {
+		const started = await Effect.runPromise(manager.ensure)
+		lock = new Database(databasePath)
+		lock.exec("BEGIN IMMEDIATE")
+		const write = fetch(`http://127.0.0.1:${port}/v1/logs`, {
+			method: "POST", headers: { "content-type": "application/json" },
+			body: JSON.stringify({ resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "blocked writer" } }] }] }] }),
+		})
+		await sleep(30)
+		await expect(Effect.runPromise(impatient.ensure)).rejects.toThrow("process was preserved")
+		const health = await (await fetch(`http://127.0.0.1:${port}/api/health`)).json() as { pid: number }
+		expect(started.pid).toBe(health.pid)
+		lock.exec("ROLLBACK")
+		expect((await write).status).toBe(200)
+		expect((await Effect.runPromise(manager.ensure)).pid).toBe(started.pid)
+	} finally {
+		try { lock?.exec("ROLLBACK") } catch {}
+		lock?.close()
+		await Effect.runPromise(manager.stop)
+		rmSync(root, { recursive: true, force: true })
+	}
+}, 15000)

package/src/ingestReadiness.ts

@@ -0,0 +1,100 @@
+import { Context, Schema } from "effect"
+
+const Checkpoint = Schema.Struct({
+	busy: Schema.Number, log: Schema.Number, checkpointed: Schema.Number, deferred: Schema.Boolean,
+})
+
+const Maintenance = Schema.Struct({
+	operation: Schema.String,
+	startedAt: Schema.Number,
+	durationMs: Schema.NullOr(Schema.Number),
+	outcome: Schema.Literals(["running", "completed", "failed"]),
+})
+
+/** Cached writer diagnostics; reading this snapshot never dispatches SQLite work. */
+export const IngestReadiness = Schema.Struct({
+	state: Schema.Literals(["starting", "ready", "overloaded", "failed"]),
+	outstandingRequests: Schema.Number,
+	outstandingBytes: Schema.Number,
+	oldestRequestAgeMs: Schema.Number,
+	lastCommitAt: Schema.NullOr(Schema.Number),
+	committedRecords: Schema.Number,
+	maintenance: Schema.NullOr(Maintenance),
+	checkpoint: Schema.NullOr(Checkpoint),
+})
+export type IngestReadiness = typeof IngestReadiness.Type
+
+export const WriterEvent = Schema.TaggedUnion({
+	ready: {},
+	checkpoint: { value: Checkpoint },
+	commit: { records: Schema.Number, at: Schema.Number },
+	maintenance: { value: Maintenance },
+})
+export type WriterEvent = typeof WriterEvent.Type
+
+/** Worker-local publisher defaults to no-op for standalone store consumers. */
+export const WriterDiagnostics = Context.Reference<(event: WriterEvent) => void>("motel/WriterDiagnostics", {
+	defaultValue: (): ((event: WriterEvent) => void) => () => {},
+})
+
+/** Tracks awaiting callers, not durable queued work. Cancellation does not undo a commit. */
+export class IngestProgress {
+	private ready = false
+	private failed = false
+	private readonly requests = new Map<symbol, { at: number; bytes: number }>()
+	private lastCommitAt: number | null = null
+	private committedRecords = 0
+	private checkpoint: IngestReadiness["checkpoint"] = null
+	private maintenance: IngestReadiness["maintenance"] = null
+	private readonly activeMaintenance = new Map<string, NonNullable<IngestReadiness["maintenance"]>>()
+
+	begin(bytes: number, now = Date.now()) {
+		const id = Symbol()
+		this.requests.set(id, { at: now, bytes })
+		return () => { this.requests.delete(id) }
+	}
+
+	fail() { this.failed = true }
+
+	receive(event: WriterEvent) {
+		switch (event._tag) {
+			case "checkpoint": this.checkpoint = event.value; break
+			case "ready": this.ready = true; break
+			case "commit":
+				if (event.records > 0) {
+					this.lastCommitAt = event.at
+					this.committedRecords += event.records
+				}
+				break
+			case "maintenance": {
+				const value = event.value
+				if (value.outcome === "running") this.activeMaintenance.set(value.operation, value)
+				else {
+					if (this.activeMaintenance.get(value.operation)?.startedAt === value.startedAt) this.activeMaintenance.delete(value.operation)
+					this.maintenance = value
+				}
+				break
+			}
+		}
+	}
+
+	snapshot(now = Date.now()): IngestReadiness {
+		let oldestRequestAgeMs = 0
+		let outstandingBytes = 0
+		for (const request of this.requests.values()) {
+			oldestRequestAgeMs = Math.max(oldestRequestAgeMs, now - request.at)
+			outstandingBytes += request.bytes
+		}
+		return {
+			state: this.failed ? "failed" : !this.ready ? "starting" :
+				oldestRequestAgeMs >= 5000 || this.requests.size >= 64 ? "overloaded" : "ready",
+			outstandingRequests: this.requests.size,
+			outstandingBytes,
+			oldestRequestAgeMs,
+			lastCommitAt: this.lastCommitAt,
+			committedRecords: this.committedRecords,
+			maintenance: [...this.activeMaintenance.values()].sort((a, b) => a.startedAt - b.startedAt)[0] ?? this.maintenance,
+			checkpoint: this.checkpoint,
+		}
+	}
+}

package/src/localServer.ts

@@ -12,6 +12,7 @@ import { AI_LIST, LOG_LIST, LOG_STATS, SPAN_LIST, TRACE_LIST, TRACE_STATS, listM
 import { MOTEL_SERVICE_ID, MOTEL_VERSION, processIdentity, removeRegistryEntry, writeRegistryEntry } from "./registry.js"
 import { AsyncIngest, AsyncIngestLive } from "./services/AsyncIngest.js"
 import { TelemetryStoreReadonly } from "./services/TelemetryStore.js"
+import { QueryOverloaded, QueryDeadlineExceeded, QueryUnavailable } from "./services/queryRpc.js"
 import { TelemetryQueryLive } from "./services/TelemetryQuery.js"
 import type { LogItem, TraceItem } from "./domain.js"
 import { lifecycleLabel } from "./ui/format.js"
@@ -48,14 +49,21 @@ const withRead = <A>(f: (store: TelemetryStoreReadonly["Service"]) => Effect.Eff
 // Response-building helpers are generic in R so a handler can depend
 // on AsyncIngest (worker-RPC path) or TelemetryStoreReadonly (query
 // path) without forcing every handler onto the same service surface.
+const queryErrorResponse = (error: unknown) => {
+	const message = error instanceof Error ? error.message : String(error)
+	if (error instanceof QueryOverloaded) return jsonResponse({ error: message, code: "QUERY_OVERLOADED" }, 503)
+	if (error instanceof QueryDeadlineExceeded) return jsonResponse({ error: message, code: "QUERY_DEADLINE" }, 504)
+	if (error instanceof QueryUnavailable) return jsonResponse({ error: message, code: "QUERY_UNAVAILABLE" }, 503)
+	return jsonResponse({ error: message }, 500)
+}
 const respondJson = <A, R>(effect: Effect.Effect<A, unknown, R>) =>
 	Effect.match(effect, {
-		onFailure: (error) => jsonResponse({ error: error instanceof Error ? error.message : String(error) }, 500),
+		onFailure: queryErrorResponse,
 		onSuccess: (value) => jsonResponse(value),
 	})
 const respondRaw = <R>(effect: Effect.Effect<ReturnType<typeof jsonResponse>, unknown, R>) =>
 	Effect.match(effect, {
-		onFailure: (error) => jsonResponse({ error: error instanceof Error ? error.message : String(error) }, 500),
+		onFailure: queryErrorResponse,
 		onSuccess: (value) => value,
 	})
 
@@ -66,12 +74,14 @@ const readOtlpBody = <T>(
 		readonly headers: Readonly<Record<string, string | undefined>>
 	},
 	decodeProtobuf: (bytes: Uint8Array) => T,
-): Effect.Effect<T, unknown> => {
+): Effect.Effect<{ payload: T; bytes: number }, unknown> => {
 	const contentType = (request.headers["content-type"] ?? "").toLowerCase()
-	if (contentType.includes("application/x-protobuf") || contentType.includes("application/protobuf")) {
-		return Effect.map(request.arrayBuffer, (buffer) => decodeProtobuf(new Uint8Array(buffer)))
-	}
-	return Effect.map(request.json, (payload) => payload as T)
+	return Effect.flatMap(request.arrayBuffer, (buffer) => Effect.try(() => ({
+		bytes: buffer.byteLength,
+		payload: contentType.includes("application/x-protobuf") || contentType.includes("application/protobuf")
+			? decodeProtobuf(new Uint8Array(buffer))
+			: JSON.parse(new TextDecoder().decode(buffer)) as T,
+	})))
 }
 
 // Log page loader: takes the parsed list params + any resource-specific
@@ -191,6 +201,10 @@ const TelemetryGroupLive = HttpApiBuilder.group(
 			.handle("health", () =>
 				HttpMiddleware.withLoggerDisabled(Effect.succeed(healthPayload())),
 			)
+			.handleRaw("readiness", () => HttpMiddleware.withLoggerDisabled(
+				Effect.flatMap(AsyncIngest, (ingest) => Effect.map(ingest.readiness,
+					(snapshot) => jsonResponse(snapshot, snapshot.state === "ready" ? 200 : 503))),
+			))
 			// OTLP ingest is routed to the worker thread via AsyncIngest
 			// so the main event loop stays free during heavy SQLite writes.
 			// Read queries use a separate query worker so synchronous SQLite
@@ -199,8 +213,8 @@ const TelemetryGroupLive = HttpApiBuilder.group(
 				HttpMiddleware.withLoggerDisabled(respondRaw(
 					Effect.flatMap(
 						readOtlpBody<OtlpTraceExportRequest>(request, decodeProtobufTraces),
-						(payload) => Effect.map(
-							Effect.flatMap(AsyncIngest, (ingest) => ingest.ingestTraces({ payload })),
+						(input) => Effect.map(
+							Effect.flatMap(AsyncIngest, (ingest) => ingest.ingestTraces(input)),
 							(result) => jsonResponse(result),
 						),
 					),
@@ -210,8 +224,8 @@ const TelemetryGroupLive = HttpApiBuilder.group(
 				HttpMiddleware.withLoggerDisabled(respondRaw(
 					Effect.flatMap(
 						readOtlpBody<OtlpLogExportRequest>(request, decodeProtobufLogs),
-						(payload) => Effect.map(
-							Effect.flatMap(AsyncIngest, (ingest) => ingest.ingestLogs({ payload })),
+						(input) => Effect.map(
+							Effect.flatMap(AsyncIngest, (ingest) => ingest.ingestLogs(input)),
 							(result) => jsonResponse(result),
 						),
 					),
@@ -551,15 +565,16 @@ export const ServerLive = HttpRouter.serve(
 	// POSTs again on the next flush. This also shaves ~1 KB of header
 	// attributes off every ingest request that would have been written
 	// to the spans table as noise.
-	Layer.provide(HttpMiddleware.layerTracerDisabledForUrls(["/api/health", "/v1/traces", "/v1/logs"])),
+	Layer.provide(HttpMiddleware.layerTracerDisabledForUrls(["/api/health", "/api/readiness", "/v1/traces", "/v1/logs"])),
 	// The telemetry worker owns ingest, migrations, and bounded maintenance.
-	// The HTTP thread only opens an existing database read-only (or bootstraps
-	// a brand-new empty one), keeping health independent of writer work.
+	// Readonly subprocesses keep native SQLite calls off the HTTP event loop and
+	// can be terminated on deadline without retiring the ingestion worker.
 	Layer.provideMerge(AsyncIngestLive),
 	Layer.provideMerge(TelemetryQueryLive),
 	Layer.provideMerge(BunHttpServer.layer({
 		port: config.otel.port,
 		hostname: config.otel.host,
+		idleTimeout: Math.ceil(config.otel.queryDeadlineMs / 1000) + 5,
 		reusePort: true,
 		routes: {
 			"/api/health": () => Response.json(healthPayload()),

package/src/mcp.ts

@@ -433,7 +433,8 @@ const ServerLayer = McpServer.toolkit(MotelToolkit).pipe(
 		}),
 	),
 	Layer.provide(BunStdio.layer),
-	Layer.provide(Logger.layer([Logger.consolePretty({ stderr: true })])),
+	// stdout carries the MCP stdio protocol, so logs must go to stderr.
+	Layer.provide(Layer.merge(Logger.layer([Logger.consolePretty()]), Layer.succeed(Logger.LogToStderr, true))),
 )
 
 Layer.launch(ServerLayer).pipe(BunRuntime.runMain)

package/src/motel.ts

@@ -40,7 +40,7 @@ const restart = Command.make("restart", {}, () =>
 
 const serviceManager = createLaunchAgentManager()
 const serviceInstall = Command.make("install", {
-	replace: Flag.boolean("replace").pipe(Flag.withDefault(false)),
+	replace: Flag.Boolean("replace").pipe(Flag.withDefault(false)),
 }, ({ replace }) =>
 	serviceManager.install(replace).pipe(Effect.andThen(json)),
 ).pipe(Command.withDescription("Install the per-user Motel LaunchAgent"))

package/src/queryHttp.test.ts

@@ -0,0 +1,34 @@
+import { expect, test } from "bun:test"
+import { Effect } from "effect"
+import { mkdtempSync, rmSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { createDaemonManager } from "./daemon.ts"
+
+test("HTTP overload and deadline responses are typed and leave ingestion usable", async () => {
+	const root = mkdtempSync(join(tmpdir(), "motel-query-http-"))
+	const port = 38000 + Math.floor(Math.random() * 1000)
+	const manager = createDaemonManager({ runtimeDir: root, databasePath: join(root, "db.sqlite"), port })
+	const previousCapacity = process.env.MOTEL_OTEL_QUERY_CAPACITY
+	const previousDeadline = process.env.MOTEL_OTEL_QUERY_DEADLINE_MS
+	try {
+		process.env.MOTEL_OTEL_QUERY_CAPACITY = "1"
+		process.env.MOTEL_OTEL_QUERY_DEADLINE_MS = "25"
+		await Effect.runPromise(manager.ensure)
+		const responses = await Promise.all([1, 2].map(async () => {
+			const response = await fetch(`http://127.0.0.1:${port}/api/services`)
+			return { status: response.status, body: await response.json() as { code: string } }
+		}))
+		expect(responses.map((response) => response.status).sort()).toEqual([503, 504])
+		expect(responses.map((response) => response.body.code).sort()).toEqual(["QUERY_DEADLINE", "QUERY_OVERLOADED"])
+		expect((await fetch(`http://127.0.0.1:${port}/v1/logs`, { method: "POST", body: "{}", headers: { "content-type": "application/json" } })).status).toBe(200)
+		expect((await fetch(`http://127.0.0.1:${port}/api/readiness`)).status).toBe(200)
+	} finally {
+		if (previousCapacity === undefined) delete process.env.MOTEL_OTEL_QUERY_CAPACITY
+		else process.env.MOTEL_OTEL_QUERY_CAPACITY = previousCapacity
+		if (previousDeadline === undefined) delete process.env.MOTEL_OTEL_QUERY_DEADLINE_MS
+		else process.env.MOTEL_OTEL_QUERY_DEADLINE_MS = previousDeadline
+		await Effect.runPromise(manager.stop)
+		rmSync(root, { recursive: true, force: true })
+	}
+}, 15000)

package/src/runtime.ts

@@ -5,7 +5,9 @@ import { SimpleLogRecordProcessor } from "@opentelemetry/sdk-logs"
 import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"
 import { Layer, ManagedRuntime } from "effect"
 import { config } from "./config.js"
-import { TelemetryStoreLive, TelemetryStoreReadonlyLive } from "./services/TelemetryStore.js"
+import { TelemetryStoreLive } from "./services/TelemetryStore.js"
+
+import { TelemetryQueryLive } from "./services/TelemetryQuery.js"
 
 const telemetryLayer = NodeSdk.layer(() => ({
 	spanProcessor: new SimpleSpanProcessor(
@@ -31,7 +33,7 @@ const telemetryLayer = NodeSdk.layer(() => ({
 // TUI-side runtime is readonly — a daemon/worker writer owns the DB
 // lock while ingests are in flight, and trying to grab the write lock
 // for schema init on startup causes "database is locked" on bun dev.
-const QueryRuntimeLive = config.otel.enabled ? Layer.mergeAll(TelemetryStoreReadonlyLive, telemetryLayer) : TelemetryStoreReadonlyLive
+const QueryRuntimeLive = config.otel.enabled ? Layer.mergeAll(TelemetryQueryLive, telemetryLayer) : TelemetryQueryLive
 
 export const queryRuntime = ManagedRuntime.make(QueryRuntimeLive)
 // `storeRuntime` is the full writer runtime, exposed for the telemetry

package/src/services/AsyncIngest.ts

@@ -20,11 +20,11 @@
  */
 
 import * as BunWorker from "@effect/platform-bun/BunWorker"
-import { Context, Effect, Layer, Scope } from "effect"
+import { Context, Effect, Latch, Layer, Scope, Schema } from "effect"
 import * as RpcClient from "effect/unstable/rpc/RpcClient"
 import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"
-import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"
 import type { WorkerError } from "effect/unstable/workers/WorkerError"
+import { IngestProgress, WriterEvent, type IngestReadiness } from "../ingestReadiness.ts"
 import { IngestRpcs } from "./ingestRpc.ts"
 
 // RpcClient.make always surfaces RpcClientError in addition to the
@@ -34,23 +34,43 @@ import { IngestRpcs } from "./ingestRpc.ts"
 // unrelated structural mismatches.
 export class AsyncIngest extends Context.Service<
 	AsyncIngest,
-	RpcClient.FromGroup<typeof IngestRpcs, RpcClientError | WorkerError>
+	RpcClient.FromGroup<typeof IngestRpcs, RpcClientError | WorkerError> & { readonly readiness: Effect.Effect<IngestReadiness> }
 >()("@motel/AsyncIngest") {}
 
-// Protocol: RpcClient.layerProtocolWorker manages a worker pool and
-// speaks msgpack over structured-clone messages. `size: 1` matches
-// SQLite's single-writer constraint.
-const WorkerProtocol = RpcClient.layerProtocolWorker({ size: 1 }).pipe(
-	Layer.provide(RpcSerialization.layerMsgPack),
-	Layer.provide(
-		BunWorker.layer(() => new Worker(new URL("./telemetryWorker.ts", import.meta.url))),
-	),
-)
-
 export const AsyncIngestLive = Layer.effect(
 	AsyncIngest,
 	Effect.gen(function*() {
 		const scope = yield* Scope.Scope
+		const progress = new IngestProgress()
+		const channelName = `motel-ingest-${crypto.randomUUID()}`
+		const channel = yield* Effect.acquireRelease(
+			Effect.sync(() => new BroadcastChannel(channelName)),
+			(channel) => Effect.sync(() => channel.close()),
+		)
+		channel.onmessage = ({ data }) => {
+			const event = Schema.decodeUnknownSync(WriterEvent)(data)
+			progress.receive(event)
+		}
+		// Opens once the current worker has answered the platform handshake (or
+		// died). Effect's worker close message is only received after that
+		// handshake; one posted earlier is dropped and the platform finalizer
+		// then waits five seconds before terminating the worker.
+		let handshake = Latch.makeUnsafe(true)
+		const WorkerProtocol = RpcClient.layerProtocolWorker({ size: 1 }).pipe(
+			Layer.provide(BunWorker.layer(() => {
+				const worker = new Worker(new URL("./telemetryWorker.ts", import.meta.url), {
+					env: { ...process.env, MOTEL_INGEST_DIAGNOSTICS_CHANNEL: channelName },
+				})
+				const spawned = Latch.makeUnsafe(false)
+				handshake = spawned
+				const settle = () => spawned.openUnsafe()
+				worker.addEventListener("message", settle, { once: true })
+				worker.addEventListener("error", () => { settle(); progress.fail() })
+				worker.addEventListener("close", () => { settle(); progress.fail() })
+				return worker
+			})),
+		)
+
 		// Keep daemon startup cheap: creating the RPC client here would eagerly
 		// spawn the worker and make /api/health wait on the worker's SQLite
 		// bootstrap. Cache a lazy initializer instead so the worker only starts
@@ -59,10 +79,14 @@ export const AsyncIngestLive = Layer.effect(
 		// failures must not replace it: concurrent stale failures can otherwise
 		// invalidate a newer cached generation and orphan its SQLite writer and
 		// maintenance fibers. A poisoned worker stays failed until the managed
-		// daemon is restarted, which the external ingest readiness probe handles.
+		// daemon is restarted, and diagnostics expose that failure for explicit operator recovery.
 		const getClient = yield* Effect.cached(Effect.gen(function*() {
 			const clientScope = yield* Scope.fork(scope, "sequential")
 			const protocolContext = yield* Layer.buildWithScope(WorkerProtocol, clientScope)
+			// Added after the protocol's finalizers, so it runs before them: a
+			// shutdown during worker startup waits for the handshake and then
+			// closes the writer gracefully instead of timing out and terminating it.
+			yield* Scope.addFinalizer(clientScope, Effect.suspend(() => handshake.await).pipe(Effect.timeoutOption("5 seconds")))
 			return yield* RpcClient.make(IngestRpcs).pipe(
 				Effect.provide(protocolContext),
 				Effect.provideService(Scope.Scope, clientScope),
@@ -71,10 +95,17 @@ export const AsyncIngestLive = Layer.effect(
 		// Start the sole writer/maintenance worker immediately, but do not make
 		// HTTP health wait for SQLite bootstrap. Managed readiness still verifies
 		// the worker through explicit ingest probes.
-		yield* Effect.forkScoped(getClient.pipe(Effect.ignore))
+		yield* Effect.forkScoped(getClient.pipe(Effect.tapCause(() => Effect.sync(() => progress.fail())), Effect.ignore))
+		const track = <A, E, R>(bytes: number, effect: Effect.Effect<A, E, R>) =>
+			Effect.acquireUseRelease(
+				Effect.sync(() => progress.begin(bytes)),
+				() => effect,
+				(finish) => Effect.sync(finish),
+			)
 		return {
-			ingestTraces: (input, options) => Effect.flatMap(getClient, (client) => client.ingestTraces(input, options)),
-			ingestLogs: (input, options) => Effect.flatMap(getClient, (client) => client.ingestLogs(input, options)),
+			readiness: Effect.sync(() => progress.snapshot()),
+			ingestTraces: (input, options) => track(input.bytes ?? 0, Effect.flatMap(getClient, (client) => client.ingestTraces(input, options))),
+			ingestLogs: (input, options) => track(input.bytes ?? 0, Effect.flatMap(getClient, (client) => client.ingestLogs(input, options))),
 		}
 	}),
 )

package/src/services/TelemetryQuery.ts

@@ -1,39 +1,26 @@
-import * as BunWorker from "@effect/platform-bun/BunWorker"
-import { Effect, Layer, Scope } from "effect"
-import * as RpcClient from "effect/unstable/rpc/RpcClient"
-import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"
-import type { WorkerError } from "effect/unstable/workers/WorkerError"
-import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"
+import { Effect, Layer } from "effect"
+import { config } from "../config.js"
 import { TelemetryStoreReadonly, type TelemetryStoreReader } from "./TelemetryStore.js"
-import { QueryRpcs } from "./queryRpc.js"
-
-type QueryMethod = keyof TelemetryStoreReader
-type QueryClient = RpcClient.FromGroup<typeof QueryRpcs, RpcClientError | WorkerError>
-
-const WorkerProtocol = RpcClient.layerProtocolWorker({ size: 1 }).pipe(
-	Layer.provide(RpcSerialization.layerMsgPack),
-	Layer.provide(BunWorker.layer(() => new Worker(new URL("./telemetryQueryWorker.ts", import.meta.url)))),
-)
-
-const query = <A>(getClient: Effect.Effect<QueryClient, unknown>, method: QueryMethod, args: readonly unknown[] = []) =>
-	Effect.flatMap(getClient, (client) => client.query({ method, args })).pipe(
-		Effect.map((result) => result as A),
-		Effect.mapError((error) => error instanceof Error ? error : new Error(String(error))),
-	)
+import { QueryScheduler } from "./queryScheduler.js"
+import { QueryError, QueryOverloaded, QueryDeadlineExceeded, QueryUnavailable } from "./queryRpc.js"
 
 export const TelemetryQueryLive = Layer.effect(
 	TelemetryStoreReadonly,
 	Effect.gen(function*() {
-		const scope = yield* Scope.Scope
-		const getClient = yield* Effect.cached(Effect.gen(function*() {
-			const clientScope = yield* Scope.fork(scope, "sequential")
-			const protocolContext = yield* Layer.buildWithScope(WorkerProtocol, clientScope)
-			return yield* RpcClient.make(QueryRpcs).pipe(
-				Effect.provide(protocolContext),
-				Effect.provideService(Scope.Scope, clientScope),
-			)
-		}))
-		const run = <A>(method: QueryMethod, args: readonly unknown[] = []) => query<A>(getClient, method, args)
+		const scheduler = yield* Effect.acquireRelease(
+			Effect.sync(() => new QueryScheduler({
+				workerUrl: new URL("./telemetryQueryProcess.ts", import.meta.url),
+				capacity: config.otel.queryCapacity,
+			environment: { ...process.env, MOTEL_OTEL_DB_PATH: config.otel.databasePath },
+				deadlineMs: config.otel.queryDeadlineMs,
+			})),
+			(scheduler) => Effect.promise(() => scheduler.close()),
+		)
+		const run = <A>(method: keyof TelemetryStoreReader, args: readonly unknown[] = []) => Effect.tryPromise({
+			try: (signal) => scheduler.query(method, args, signal) as Promise<A>,
+			catch: (error) => error instanceof QueryError || error instanceof QueryOverloaded || error instanceof QueryDeadlineExceeded || error instanceof QueryUnavailable
+				? error : new QueryUnavailable({ message: String(error) }),
+		})
 		return TelemetryStoreReadonly.of({
 			listServices: run("listServices"),
 			listRecentTraces: (serviceName, options) => run("listRecentTraces", [serviceName, options]),