@aryasaatvik/motel 0.3.5 → 0.4.0

package/src/ingestReadiness.ts

@@ -0,0 +1,100 @@
+import { Context, Schema } from "effect"
+
+const Checkpoint = Schema.Struct({
+	busy: Schema.Number, log: Schema.Number, checkpointed: Schema.Number, deferred: Schema.Boolean,
+})
+
+const Maintenance = Schema.Struct({
+	operation: Schema.String,
+	startedAt: Schema.Number,
+	durationMs: Schema.NullOr(Schema.Number),
+	outcome: Schema.Literals(["running", "completed", "failed"]),
+})
+
+/** Cached writer diagnostics; reading this snapshot never dispatches SQLite work. */
+export const IngestReadiness = Schema.Struct({
+	state: Schema.Literals(["starting", "ready", "overloaded", "failed"]),
+	outstandingRequests: Schema.Number,
+	outstandingBytes: Schema.Number,
+	oldestRequestAgeMs: Schema.Number,
+	lastCommitAt: Schema.NullOr(Schema.Number),
+	committedRecords: Schema.Number,
+	maintenance: Schema.NullOr(Maintenance),
+	checkpoint: Schema.NullOr(Checkpoint),
+})
+export type IngestReadiness = typeof IngestReadiness.Type
+
+export const WriterEvent = Schema.TaggedUnion({
+	ready: {},
+	checkpoint: { value: Checkpoint },
+	commit: { records: Schema.Number, at: Schema.Number },
+	maintenance: { value: Maintenance },
+})
+export type WriterEvent = typeof WriterEvent.Type
+
+/** Worker-local publisher defaults to no-op for standalone store consumers. */
+export const WriterDiagnostics = Context.Reference<(event: WriterEvent) => void>("motel/WriterDiagnostics", {
+	defaultValue: (): ((event: WriterEvent) => void) => () => {},
+})
+
+/** Tracks awaiting callers, not durable queued work. Cancellation does not undo a commit. */
+export class IngestProgress {
+	private ready = false
+	private failed = false
+	private readonly requests = new Map<symbol, { at: number; bytes: number }>()
+	private lastCommitAt: number | null = null
+	private committedRecords = 0
+	private checkpoint: IngestReadiness["checkpoint"] = null
+	private maintenance: IngestReadiness["maintenance"] = null
+	private readonly activeMaintenance = new Map<string, NonNullable<IngestReadiness["maintenance"]>>()
+
+	begin(bytes: number, now = Date.now()) {
+		const id = Symbol()
+		this.requests.set(id, { at: now, bytes })
+		return () => { this.requests.delete(id) }
+	}
+
+	fail() { this.failed = true }
+
+	receive(event: WriterEvent) {
+		switch (event._tag) {
+			case "checkpoint": this.checkpoint = event.value; break
+			case "ready": this.ready = true; break
+			case "commit":
+				if (event.records > 0) {
+					this.lastCommitAt = event.at
+					this.committedRecords += event.records
+				}
+				break
+			case "maintenance": {
+				const value = event.value
+				if (value.outcome === "running") this.activeMaintenance.set(value.operation, value)
+				else {
+					if (this.activeMaintenance.get(value.operation)?.startedAt === value.startedAt) this.activeMaintenance.delete(value.operation)
+					this.maintenance = value
+				}
+				break
+			}
+		}
+	}
+
+	snapshot(now = Date.now()): IngestReadiness {
+		let oldestRequestAgeMs = 0
+		let outstandingBytes = 0
+		for (const request of this.requests.values()) {
+			oldestRequestAgeMs = Math.max(oldestRequestAgeMs, now - request.at)
+			outstandingBytes += request.bytes
+		}
+		return {
+			state: this.failed ? "failed" : !this.ready ? "starting" :
+				oldestRequestAgeMs >= 5000 || this.requests.size >= 64 ? "overloaded" : "ready",
+			outstandingRequests: this.requests.size,
+			outstandingBytes,
+			oldestRequestAgeMs,
+			lastCommitAt: this.lastCommitAt,
+			committedRecords: this.committedRecords,
+			maintenance: [...this.activeMaintenance.values()].sort((a, b) => a.startedAt - b.startedAt)[0] ?? this.maintenance,
+			checkpoint: this.checkpoint,
+		}
+	}
+}

package/src/localServer.ts

@@ -12,6 +12,7 @@ import { AI_LIST, LOG_LIST, LOG_STATS, SPAN_LIST, TRACE_LIST, TRACE_STATS, listM
 import { MOTEL_SERVICE_ID, MOTEL_VERSION, processIdentity, removeRegistryEntry, writeRegistryEntry } from "./registry.js"
 import { AsyncIngest, AsyncIngestLive } from "./services/AsyncIngest.js"
 import { TelemetryStoreReadonly } from "./services/TelemetryStore.js"
+import { QueryOverloaded, QueryDeadlineExceeded, QueryUnavailable } from "./services/queryRpc.js"
 import { TelemetryQueryLive } from "./services/TelemetryQuery.js"
 import type { LogItem, TraceItem } from "./domain.js"
 import { lifecycleLabel } from "./ui/format.js"
@@ -48,14 +49,21 @@ const withRead = <A>(f: (store: TelemetryStoreReadonly["Service"]) => Effect.Eff
 // Response-building helpers are generic in R so a handler can depend
 // on AsyncIngest (worker-RPC path) or TelemetryStoreReadonly (query
 // path) without forcing every handler onto the same service surface.
+const queryErrorResponse = (error: unknown) => {
+	const message = error instanceof Error ? error.message : String(error)
+	if (error instanceof QueryOverloaded) return jsonResponse({ error: message, code: "QUERY_OVERLOADED" }, 503)
+	if (error instanceof QueryDeadlineExceeded) return jsonResponse({ error: message, code: "QUERY_DEADLINE" }, 504)
+	if (error instanceof QueryUnavailable) return jsonResponse({ error: message, code: "QUERY_UNAVAILABLE" }, 503)
+	return jsonResponse({ error: message }, 500)
+}
 const respondJson = <A, R>(effect: Effect.Effect<A, unknown, R>) =>
 	Effect.match(effect, {
-		onFailure: (error) => jsonResponse({ error: error instanceof Error ? error.message : String(error) }, 500),
+		onFailure: queryErrorResponse,
 		onSuccess: (value) => jsonResponse(value),
 	})
 const respondRaw = <R>(effect: Effect.Effect<ReturnType<typeof jsonResponse>, unknown, R>) =>
 	Effect.match(effect, {
-		onFailure: (error) => jsonResponse({ error: error instanceof Error ? error.message : String(error) }, 500),
+		onFailure: queryErrorResponse,
 		onSuccess: (value) => value,
 	})
 
@@ -66,12 +74,14 @@ const readOtlpBody = <T>(
 		readonly headers: Readonly<Record<string, string | undefined>>
 	},
 	decodeProtobuf: (bytes: Uint8Array) => T,
-): Effect.Effect<T, unknown> => {
+): Effect.Effect<{ payload: T; bytes: number }, unknown> => {
 	const contentType = (request.headers["content-type"] ?? "").toLowerCase()
-	if (contentType.includes("application/x-protobuf") || contentType.includes("application/protobuf")) {
-		return Effect.map(request.arrayBuffer, (buffer) => decodeProtobuf(new Uint8Array(buffer)))
-	}
-	return Effect.map(request.json, (payload) => payload as T)
+	return Effect.flatMap(request.arrayBuffer, (buffer) => Effect.try(() => ({
+		bytes: buffer.byteLength,
+		payload: contentType.includes("application/x-protobuf") || contentType.includes("application/protobuf")
+			? decodeProtobuf(new Uint8Array(buffer))
+			: JSON.parse(new TextDecoder().decode(buffer)) as T,
+	})))
 }
 
 // Log page loader: takes the parsed list params + any resource-specific
@@ -191,6 +201,10 @@ const TelemetryGroupLive = HttpApiBuilder.group(
 			.handle("health", () =>
 				HttpMiddleware.withLoggerDisabled(Effect.succeed(healthPayload())),
 			)
+			.handleRaw("readiness", () => HttpMiddleware.withLoggerDisabled(
+				Effect.flatMap(AsyncIngest, (ingest) => Effect.map(ingest.readiness,
+					(snapshot) => jsonResponse(snapshot, snapshot.state === "ready" ? 200 : 503))),
+			))
 			// OTLP ingest is routed to the worker thread via AsyncIngest
 			// so the main event loop stays free during heavy SQLite writes.
 			// Read queries use a separate query worker so synchronous SQLite
@@ -199,8 +213,8 @@ const TelemetryGroupLive = HttpApiBuilder.group(
 				HttpMiddleware.withLoggerDisabled(respondRaw(
 					Effect.flatMap(
 						readOtlpBody<OtlpTraceExportRequest>(request, decodeProtobufTraces),
-						(payload) => Effect.map(
-							Effect.flatMap(AsyncIngest, (ingest) => ingest.ingestTraces({ payload })),
+						(input) => Effect.map(
+							Effect.flatMap(AsyncIngest, (ingest) => ingest.ingestTraces(input)),
 							(result) => jsonResponse(result),
 						),
 					),
@@ -210,8 +224,8 @@ const TelemetryGroupLive = HttpApiBuilder.group(
 				HttpMiddleware.withLoggerDisabled(respondRaw(
 					Effect.flatMap(
 						readOtlpBody<OtlpLogExportRequest>(request, decodeProtobufLogs),
-						(payload) => Effect.map(
-							Effect.flatMap(AsyncIngest, (ingest) => ingest.ingestLogs({ payload })),
+						(input) => Effect.map(
+							Effect.flatMap(AsyncIngest, (ingest) => ingest.ingestLogs(input)),
 							(result) => jsonResponse(result),
 						),
 					),
@@ -551,15 +565,16 @@ export const ServerLive = HttpRouter.serve(
 	// POSTs again on the next flush. This also shaves ~1 KB of header
 	// attributes off every ingest request that would have been written
 	// to the spans table as noise.
-	Layer.provide(HttpMiddleware.layerTracerDisabledForUrls(["/api/health", "/v1/traces", "/v1/logs"])),
+	Layer.provide(HttpMiddleware.layerTracerDisabledForUrls(["/api/health", "/api/readiness", "/v1/traces", "/v1/logs"])),
 	// The telemetry worker owns ingest, migrations, and bounded maintenance.
-	// The HTTP thread only opens an existing database read-only (or bootstraps
-	// a brand-new empty one), keeping health independent of writer work.
+	// Readonly subprocesses keep native SQLite calls off the HTTP event loop and
+	// can be terminated on deadline without retiring the ingestion worker.
 	Layer.provideMerge(AsyncIngestLive),
 	Layer.provideMerge(TelemetryQueryLive),
 	Layer.provideMerge(BunHttpServer.layer({
 		port: config.otel.port,
 		hostname: config.otel.host,
+		idleTimeout: Math.ceil(config.otel.queryDeadlineMs / 1000) + 5,
 		reusePort: true,
 		routes: {
 			"/api/health": () => Response.json(healthPayload()),

package/src/queryHttp.test.ts

@@ -0,0 +1,34 @@
+import { expect, test } from "bun:test"
+import { Effect } from "effect"
+import { mkdtempSync, rmSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { createDaemonManager } from "./daemon.ts"
+
+test("HTTP overload and deadline responses are typed and leave ingestion usable", async () => {
+	const root = mkdtempSync(join(tmpdir(), "motel-query-http-"))
+	const port = 38000 + Math.floor(Math.random() * 1000)
+	const manager = createDaemonManager({ runtimeDir: root, databasePath: join(root, "db.sqlite"), port })
+	const previousCapacity = process.env.MOTEL_OTEL_QUERY_CAPACITY
+	const previousDeadline = process.env.MOTEL_OTEL_QUERY_DEADLINE_MS
+	try {
+		process.env.MOTEL_OTEL_QUERY_CAPACITY = "1"
+		process.env.MOTEL_OTEL_QUERY_DEADLINE_MS = "25"
+		await Effect.runPromise(manager.ensure)
+		const responses = await Promise.all([1, 2].map(async () => {
+			const response = await fetch(`http://127.0.0.1:${port}/api/services`)
+			return { status: response.status, body: await response.json() as { code: string } }
+		}))
+		expect(responses.map((response) => response.status).sort()).toEqual([503, 504])
+		expect(responses.map((response) => response.body.code).sort()).toEqual(["QUERY_DEADLINE", "QUERY_OVERLOADED"])
+		expect((await fetch(`http://127.0.0.1:${port}/v1/logs`, { method: "POST", body: "{}", headers: { "content-type": "application/json" } })).status).toBe(200)
+		expect((await fetch(`http://127.0.0.1:${port}/api/readiness`)).status).toBe(200)
+	} finally {
+		if (previousCapacity === undefined) delete process.env.MOTEL_OTEL_QUERY_CAPACITY
+		else process.env.MOTEL_OTEL_QUERY_CAPACITY = previousCapacity
+		if (previousDeadline === undefined) delete process.env.MOTEL_OTEL_QUERY_DEADLINE_MS
+		else process.env.MOTEL_OTEL_QUERY_DEADLINE_MS = previousDeadline
+		await Effect.runPromise(manager.stop)
+		rmSync(root, { recursive: true, force: true })
+	}
+}, 15000)

package/src/runtime.ts

@@ -5,7 +5,9 @@ import { SimpleLogRecordProcessor } from "@opentelemetry/sdk-logs"
 import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"
 import { Layer, ManagedRuntime } from "effect"
 import { config } from "./config.js"
-import { TelemetryStoreLive, TelemetryStoreReadonlyLive } from "./services/TelemetryStore.js"
+import { TelemetryStoreLive } from "./services/TelemetryStore.js"
+
+import { TelemetryQueryLive } from "./services/TelemetryQuery.js"
 
 const telemetryLayer = NodeSdk.layer(() => ({
 	spanProcessor: new SimpleSpanProcessor(
@@ -31,7 +33,7 @@ const telemetryLayer = NodeSdk.layer(() => ({
 // TUI-side runtime is readonly — a daemon/worker writer owns the DB
 // lock while ingests are in flight, and trying to grab the write lock
 // for schema init on startup causes "database is locked" on bun dev.
-const QueryRuntimeLive = config.otel.enabled ? Layer.mergeAll(TelemetryStoreReadonlyLive, telemetryLayer) : TelemetryStoreReadonlyLive
+const QueryRuntimeLive = config.otel.enabled ? Layer.mergeAll(TelemetryQueryLive, telemetryLayer) : TelemetryQueryLive
 
 export const queryRuntime = ManagedRuntime.make(QueryRuntimeLive)
 // `storeRuntime` is the full writer runtime, exposed for the telemetry

package/src/services/AsyncIngest.ts

@@ -20,11 +20,12 @@
  */
 
 import * as BunWorker from "@effect/platform-bun/BunWorker"
-import { Context, Effect, Layer, Scope } from "effect"
+import { Context, Effect, Layer, Scope, Schema } from "effect"
 import * as RpcClient from "effect/unstable/rpc/RpcClient"
 import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"
 import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"
 import type { WorkerError } from "effect/unstable/workers/WorkerError"
+import { IngestProgress, WriterEvent, type IngestReadiness } from "../ingestReadiness.ts"
 import { IngestRpcs } from "./ingestRpc.ts"
 
 // RpcClient.make always surfaces RpcClientError in addition to the
@@ -34,23 +35,35 @@ import { IngestRpcs } from "./ingestRpc.ts"
 // unrelated structural mismatches.
 export class AsyncIngest extends Context.Service<
 	AsyncIngest,
-	RpcClient.FromGroup<typeof IngestRpcs, RpcClientError | WorkerError>
+	RpcClient.FromGroup<typeof IngestRpcs, RpcClientError | WorkerError> & { readonly readiness: Effect.Effect<IngestReadiness> }
 >()("@motel/AsyncIngest") {}
 
-// Protocol: RpcClient.layerProtocolWorker manages a worker pool and
-// speaks msgpack over structured-clone messages. `size: 1` matches
-// SQLite's single-writer constraint.
-const WorkerProtocol = RpcClient.layerProtocolWorker({ size: 1 }).pipe(
-	Layer.provide(RpcSerialization.layerMsgPack),
-	Layer.provide(
-		BunWorker.layer(() => new Worker(new URL("./telemetryWorker.ts", import.meta.url))),
-	),
-)
-
 export const AsyncIngestLive = Layer.effect(
 	AsyncIngest,
 	Effect.gen(function*() {
 		const scope = yield* Scope.Scope
+		const progress = new IngestProgress()
+		const channelName = `motel-ingest-${crypto.randomUUID()}`
+		const channel = yield* Effect.acquireRelease(
+			Effect.sync(() => new BroadcastChannel(channelName)),
+			(channel) => Effect.sync(() => channel.close()),
+		)
+		channel.onmessage = ({ data }) => {
+			const event = Schema.decodeUnknownSync(WriterEvent)(data)
+			progress.receive(event)
+		}
+		const WorkerProtocol = RpcClient.layerProtocolWorker({ size: 1 }).pipe(
+			Layer.provide(RpcSerialization.layerMsgPack),
+			Layer.provide(BunWorker.layer(() => {
+				const worker = new Worker(new URL("./telemetryWorker.ts", import.meta.url), {
+					env: { ...process.env, MOTEL_INGEST_DIAGNOSTICS_CHANNEL: channelName },
+				})
+				worker.addEventListener("error", () => progress.fail())
+				worker.addEventListener("close", () => progress.fail())
+				return worker
+			})),
+		)
+
 		// Keep daemon startup cheap: creating the RPC client here would eagerly
 		// spawn the worker and make /api/health wait on the worker's SQLite
 		// bootstrap. Cache a lazy initializer instead so the worker only starts
@@ -59,7 +72,7 @@ export const AsyncIngestLive = Layer.effect(
 		// failures must not replace it: concurrent stale failures can otherwise
 		// invalidate a newer cached generation and orphan its SQLite writer and
 		// maintenance fibers. A poisoned worker stays failed until the managed
-		// daemon is restarted, which the external ingest readiness probe handles.
+		// daemon is restarted, and diagnostics expose that failure for explicit operator recovery.
 		const getClient = yield* Effect.cached(Effect.gen(function*() {
 			const clientScope = yield* Scope.fork(scope, "sequential")
 			const protocolContext = yield* Layer.buildWithScope(WorkerProtocol, clientScope)
@@ -71,10 +84,17 @@ export const AsyncIngestLive = Layer.effect(
 		// Start the sole writer/maintenance worker immediately, but do not make
 		// HTTP health wait for SQLite bootstrap. Managed readiness still verifies
 		// the worker through explicit ingest probes.
-		yield* Effect.forkScoped(getClient.pipe(Effect.ignore))
+		yield* Effect.forkScoped(getClient.pipe(Effect.tapCause(() => Effect.sync(() => progress.fail())), Effect.ignore))
+		const track = <A, E, R>(bytes: number, effect: Effect.Effect<A, E, R>) =>
+			Effect.acquireUseRelease(
+				Effect.sync(() => progress.begin(bytes)),
+				() => effect,
+				(finish) => Effect.sync(finish),
+			)
 		return {
-			ingestTraces: (input, options) => Effect.flatMap(getClient, (client) => client.ingestTraces(input, options)),
-			ingestLogs: (input, options) => Effect.flatMap(getClient, (client) => client.ingestLogs(input, options)),
+			readiness: Effect.sync(() => progress.snapshot()),
+			ingestTraces: (input, options) => track(input.bytes ?? 0, Effect.flatMap(getClient, (client) => client.ingestTraces(input, options))),
+			ingestLogs: (input, options) => track(input.bytes ?? 0, Effect.flatMap(getClient, (client) => client.ingestLogs(input, options))),
 		}
 	}),
 )

package/src/services/TelemetryQuery.ts

@@ -1,39 +1,26 @@
-import * as BunWorker from "@effect/platform-bun/BunWorker"
-import { Effect, Layer, Scope } from "effect"
-import * as RpcClient from "effect/unstable/rpc/RpcClient"
-import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"
-import type { WorkerError } from "effect/unstable/workers/WorkerError"
-import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"
+import { Effect, Layer } from "effect"
+import { config } from "../config.js"
 import { TelemetryStoreReadonly, type TelemetryStoreReader } from "./TelemetryStore.js"
-import { QueryRpcs } from "./queryRpc.js"
-
-type QueryMethod = keyof TelemetryStoreReader
-type QueryClient = RpcClient.FromGroup<typeof QueryRpcs, RpcClientError | WorkerError>
-
-const WorkerProtocol = RpcClient.layerProtocolWorker({ size: 1 }).pipe(
-	Layer.provide(RpcSerialization.layerMsgPack),
-	Layer.provide(BunWorker.layer(() => new Worker(new URL("./telemetryQueryWorker.ts", import.meta.url)))),
-)
-
-const query = <A>(getClient: Effect.Effect<QueryClient, unknown>, method: QueryMethod, args: readonly unknown[] = []) =>
-	Effect.flatMap(getClient, (client) => client.query({ method, args })).pipe(
-		Effect.map((result) => result as A),
-		Effect.mapError((error) => error instanceof Error ? error : new Error(String(error))),
-	)
+import { QueryScheduler } from "./queryScheduler.js"
+import { QueryError, QueryOverloaded, QueryDeadlineExceeded, QueryUnavailable } from "./queryRpc.js"
 
 export const TelemetryQueryLive = Layer.effect(
 	TelemetryStoreReadonly,
 	Effect.gen(function*() {
-		const scope = yield* Scope.Scope
-		const getClient = yield* Effect.cached(Effect.gen(function*() {
-			const clientScope = yield* Scope.fork(scope, "sequential")
-			const protocolContext = yield* Layer.buildWithScope(WorkerProtocol, clientScope)
-			return yield* RpcClient.make(QueryRpcs).pipe(
-				Effect.provide(protocolContext),
-				Effect.provideService(Scope.Scope, clientScope),
-			)
-		}))
-		const run = <A>(method: QueryMethod, args: readonly unknown[] = []) => query<A>(getClient, method, args)
+		const scheduler = yield* Effect.acquireRelease(
+			Effect.sync(() => new QueryScheduler({
+				workerUrl: new URL("./telemetryQueryProcess.ts", import.meta.url),
+				capacity: config.otel.queryCapacity,
+			environment: { ...process.env, MOTEL_OTEL_DB_PATH: config.otel.databasePath },
+				deadlineMs: config.otel.queryDeadlineMs,
+			})),
+			(scheduler) => Effect.promise(() => scheduler.close()),
+		)
+		const run = <A>(method: keyof TelemetryStoreReader, args: readonly unknown[] = []) => Effect.tryPromise({
+			try: (signal) => scheduler.query(method, args, signal) as Promise<A>,
+			catch: (error) => error instanceof QueryError || error instanceof QueryOverloaded || error instanceof QueryDeadlineExceeded || error instanceof QueryUnavailable
+				? error : new QueryUnavailable({ message: String(error) }),
+		})
 		return TelemetryStoreReadonly.of({
 			listServices: run("listServices"),
 			listRecentTraces: (serviceName, options) => run("listRecentTraces", [serviceName, options]),