@obsunified/collector 2.0.1

package/src/plugins/query-routes.ts

@@ -0,0 +1,220 @@
+/** Union: A's env-based retention + D's search param + D's export endpoint */
+import {
+	DEFAULT_WINDOW_HOURS,
+	getConfiguredRetentionHours,
+} from "@obsunified/types/constants";
+import type { CollectorPlugin } from "../framework/collector";
+import { getProjectId } from "./_context";
+
+export const queryRoutesPlugin: CollectorPlugin = {
+	name: "query-routes",
+	register(app, runtime) {
+		app.get("/internal/telemetry/overview", async (c) => {
+			const projectId = getProjectId(c);
+			const maxHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const hours = Math.min(
+				maxHours,
+				Math.max(
+					1,
+					Number.parseInt(
+						c.req.query("hours") || String(DEFAULT_WINDOW_HOURS),
+						10,
+					) || DEFAULT_WINDOW_HOURS,
+				),
+			);
+			const service = c.req.query("service") || undefined;
+			const statusParam = c.req.query("status");
+			const status =
+				statusParam === "ok" || statusParam === "error" ? statusParam : "all";
+			const limit = Number.parseInt(c.req.query("limit") || "30", 10) || 30;
+			const search = c.req.query("q")?.trim() || undefined;
+
+			const store = runtime.createStore(c.env);
+			const overview = await store.getOverview({
+				projectId,
+				hours,
+				service,
+				status,
+				limit,
+				search,
+			});
+			return c.json({
+				...overview,
+				plugins: runtime.getRegisteredPluginNames(),
+			});
+		});
+
+		app.get("/internal/telemetry/service-map", async (c) => {
+			const projectId = getProjectId(c);
+			const maxHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const hours = Math.min(
+				maxHours,
+				Math.max(
+					1,
+					Number.parseInt(
+						c.req.query("hours") || String(DEFAULT_WINDOW_HOURS),
+						10,
+					) || DEFAULT_WINDOW_HOURS,
+				),
+			);
+			const store = runtime.createStore(c.env);
+			// RFC 0009 — `?source=sdk|ebpf|all` toggles which spans
+			// contribute. Default is `all` so existing dashboards see
+			// every edge. The dashboard's service-map view exposes the
+			// filter as a UI toggle.
+			const sourceParam = c.req.query("source");
+			const source: "all" | "sdk" | "ebpf" =
+				sourceParam === "sdk" || sourceParam === "ebpf" ? sourceParam : "all";
+			const map = await store.getServiceMap({ projectId, hours, source });
+			return c.json({
+				...map,
+				windowHours: hours,
+				source,
+				timestamp: new Date().toISOString(),
+			});
+		});
+
+		app.get("/internal/telemetry/services/:service/operations", async (c) => {
+			const projectId = getProjectId(c);
+			const maxHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const hours = Math.min(
+				maxHours,
+				Math.max(
+					1,
+					Number.parseInt(
+						c.req.query("hours") || String(DEFAULT_WINDOW_HOURS),
+						10,
+					) || DEFAULT_WINDOW_HOURS,
+				),
+			);
+			const service = c.req.param("service");
+			if (!service) {
+				return c.json(
+					{ error: "Bad Request", message: "service is required" },
+					400,
+				);
+			}
+			const store = runtime.createStore(c.env);
+			const result = await store.getServiceOperations({
+				projectId,
+				service,
+				hours,
+			});
+			return c.json({
+				...result,
+				windowHours: hours,
+				timestamp: new Date().toISOString(),
+			});
+		});
+
+		app.get("/internal/telemetry/traces/:traceId", async (c) => {
+			const projectId = getProjectId(c);
+			const traceId = c.req.param("traceId");
+			const store = runtime.createStore(c.env);
+			// A `query.trace_detail` span = a trace was viewed. Counting these
+			// against `traces.ingest`'s trace_count gives the read/write ratio
+			// (Q3); span duration captures end-to-end read latency (Q1).
+			const detail = await runtime.withChildSpan(
+				"query.trace_detail",
+				async (span) => {
+					span.setAttribute("project.id", projectId);
+					span.setAttribute("trace.id", traceId);
+					const result = await store.getTraceDetail(traceId, projectId);
+					span.setAttribute("query.found", result !== null);
+					span.setAttribute("query.span_count", result?.spans.length ?? 0);
+					return result;
+				},
+			);
+			if (!detail)
+				return c.json({ error: "Not Found", message: "Trace not found" }, 404);
+			return c.json({ ...detail, plugins: runtime.getRegisteredPluginNames() });
+		});
+
+		app.get("/internal/telemetry/traces/:traceId/gaps", async (c) => {
+			const projectId = getProjectId(c);
+			const traceId = c.req.param("traceId");
+			const store = runtime.createStore(c.env);
+			// `query.trace_gaps` span duration is the read-time gap computation
+			// cost end-to-end (project-scoped span SELECT + compute) — the
+			// DB-inclusive half of Q1 that the pure-compute benchmark omits.
+			const gaps = await runtime.withChildSpan(
+				"query.trace_gaps",
+				async (span) => {
+					span.setAttribute("project.id", projectId);
+					span.setAttribute("trace.id", traceId);
+					const result = await store.getTraceGaps(traceId, projectId);
+					span.setAttribute("query.found", result !== null);
+					span.setAttribute(
+						"query.blindspot_count",
+						result?.blindspots.length ?? 0,
+					);
+					return result;
+				},
+			);
+			if (!gaps)
+				return c.json(
+					{ error: "Not Found", message: "Trace gaps not found" },
+					404,
+				);
+			return c.json(gaps);
+		});
+
+		app.get(
+			"/internal/telemetry/instrumentation-gaps/calibration",
+			async (c) => {
+				const projectId = getProjectId(c);
+				const maxHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+				const hours = Math.min(
+					maxHours,
+					Math.max(1, Number.parseInt(c.req.query("hours") || "72", 10) || 72),
+				);
+				const limit =
+					Number.parseInt(c.req.query("limit") || "5000", 10) || 5000;
+				const store = runtime.createStore(c.env);
+				const calibration = await store.calibrateTraceGaps({
+					projectId,
+					hours,
+					limit,
+				});
+				return c.json(calibration);
+			},
+		);
+
+		// NDJSON export (from D)
+		app.get("/internal/telemetry/export", async (c) => {
+			const projectId = getProjectId(c);
+			const maxHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const hours = Math.min(
+				maxHours,
+				Math.max(
+					1,
+					Number.parseInt(
+						c.req.query("hours") || String(DEFAULT_WINDOW_HOURS),
+						10,
+					) || DEFAULT_WINDOW_HOURS,
+				),
+			);
+			const service = c.req.query("service") || undefined;
+			const statusParam = c.req.query("status");
+			const status =
+				statusParam === "ok" || statusParam === "error" ? statusParam : "all";
+			const search = c.req.query("q")?.trim() || undefined;
+
+			const store = runtime.createStore(c.env);
+			const ndjson = await store.getExportRows({
+				projectId,
+				hours,
+				service,
+				status,
+				search,
+			});
+
+			return new Response(ndjson, {
+				headers: {
+					"content-type": "application/x-ndjson; charset=utf-8",
+					"content-disposition": `attachment; filename="telemetry-export-${new Date().toISOString().slice(0, 19).replace(/:/g, "-")}.jsonl"`,
+				},
+			});
+		});
+	},
+};

package/src/plugins/redaction-processor.ts

@@ -0,0 +1,132 @@
+import type { JsonValue, StoredSpan } from "@obsunified/types";
+import type { CollectorPlugin } from "../framework/collector";
+import { parseJsonValue } from "../lib/json";
+
+const DEFAULT_REDACT_FIELDS = [
+	"authorization",
+	"cookie",
+	"set-cookie",
+	"password",
+	"passwd",
+	"secret",
+	"token",
+	"api-key",
+	"x-api-key",
+	"email",
+	"enduser.id",
+];
+
+const REDACTED_VALUE = "[REDACTED]";
+
+const toRedactionFieldSet = (extraFields?: string): Set<string> => {
+	const customFields = extraFields
+		? extraFields
+				.split(",")
+				.map((field) => field.trim().toLowerCase())
+				.filter(Boolean)
+		: [];
+	return new Set([...DEFAULT_REDACT_FIELDS, ...customFields]);
+};
+
+const shouldRedactKey = (key: string, fields: Set<string>): boolean => {
+	const normalizedKey = key.toLowerCase();
+	if (fields.has(normalizedKey)) return true;
+	for (const field of fields) {
+		if (normalizedKey.endsWith(field)) return true;
+	}
+	return false;
+};
+
+const redactJsonValue = (
+	value: JsonValue,
+	fields: Set<string>,
+): { nextValue: JsonValue; redactionCount: number } => {
+	if (Array.isArray(value)) {
+		let redactionCount = 0;
+		const nextValue = value.map((item) => {
+			const result = redactJsonValue(item, fields);
+			redactionCount += result.redactionCount;
+			return result.nextValue;
+		});
+		return { nextValue, redactionCount };
+	}
+
+	if (value && typeof value === "object") {
+		let redactionCount = 0;
+		const nextEntries = Object.entries(value).map(([key, nestedValue]) => {
+			if (shouldRedactKey(key, fields)) {
+				redactionCount += 1;
+				return [key, REDACTED_VALUE] as const;
+			}
+			const result = redactJsonValue(nestedValue, fields);
+			redactionCount += result.redactionCount;
+			return [key, result.nextValue] as const;
+		});
+		return { nextValue: Object.fromEntries(nextEntries), redactionCount };
+	}
+
+	return { nextValue: value, redactionCount: 0 };
+};
+
+const redactJsonString = (
+	value: string,
+	fields: Set<string>,
+): { serialized: string; redactionCount: number } => {
+	const parsed = parseJsonValue(value);
+	const redacted = redactJsonValue(parsed, fields);
+	return {
+		serialized: JSON.stringify(redacted.nextValue),
+		redactionCount: redacted.redactionCount,
+	};
+};
+
+export const redactionProcessorPlugin: CollectorPlugin = {
+	name: "redaction-processor",
+	register(_app, runtime) {
+		runtime.addSpanProcessor({
+			name: "redaction-processor",
+			process(spans, context) {
+				const redactFields = toRedactionFieldSet(
+					context.env.TELEMETRY_REDACT_FIELDS,
+				);
+				return spans.map((span): StoredSpan => {
+					const attributes = redactJsonString(
+						span.attributesJson,
+						redactFields,
+					);
+					const resourceAttributes = redactJsonString(
+						span.resourceAttributesJson,
+						redactFields,
+					);
+					const events = redactJsonString(span.eventsJson, redactFields);
+					const links = redactJsonString(span.linksJson, redactFields);
+
+					const totalRedactions =
+						attributes.redactionCount +
+						resourceAttributes.redactionCount +
+						events.redactionCount +
+						links.redactionCount;
+
+					const normalizedAttributes = parseJsonValue(attributes.serialized);
+					const nextAttributes =
+						normalizedAttributes &&
+						typeof normalizedAttributes === "object" &&
+						!Array.isArray(normalizedAttributes)
+							? {
+									...normalizedAttributes,
+									"collector.redaction.count": totalRedactions,
+								}
+							: normalizedAttributes;
+
+					return {
+						...span,
+						attributesJson: JSON.stringify(nextAttributes),
+						resourceAttributesJson: resourceAttributes.serialized,
+						eventsJson: events.serialized,
+						linksJson: links.serialized,
+					};
+				});
+			},
+		});
+	},
+};

package/src/plugins/replay-query-routes.test.ts

@@ -0,0 +1,38 @@
+import { describe, expect, it } from "vitest";
+import { fetchReplayChunks } from "./replay-query-routes";
+
+describe("fetchReplayChunks", () => {
+	it("fetches replay chunk objects with bounded concurrency and preserves order", async () => {
+		let active = 0;
+		let maxActive = 0;
+		const bucket = {
+			async get(key: string) {
+				active += 1;
+				maxActive = Math.max(maxActive, active);
+				await new Promise((resolve) => setTimeout(resolve, 5));
+				active -= 1;
+				return {
+					async json<T>() {
+						return [{ key }] as T;
+					},
+				};
+			},
+		};
+
+		const events = await fetchReplayChunks(
+			bucket,
+			Array.from({ length: 6 }, (_, i) => ({ key: `chunk-${i}` })),
+			2,
+		);
+
+		expect(maxActive).toBeLessThanOrEqual(2);
+		expect(events.map((event) => event.key)).toEqual([
+			"chunk-0",
+			"chunk-1",
+			"chunk-2",
+			"chunk-3",
+			"chunk-4",
+			"chunk-5",
+		]);
+	});
+});

package/src/plugins/replay-query-routes.ts

@@ -0,0 +1,199 @@
+import type { SessionReplayMetadataRow } from "@obsunified/types";
+import type { CollectorPlugin } from "../framework/collector";
+import { sqlDbFor } from "../lib/sql-db";
+import { getProjectId } from "./_context";
+
+const DEFAULT_CHUNK_LIMIT = 100;
+const MAX_CHUNK_LIMIT = 500;
+const REPLAY_CHUNK_FETCH_CONCURRENCY = 8;
+
+interface ReplayChunkObject {
+	key: string;
+}
+
+interface ReplayChunkBody {
+	json<T>(): Promise<T>;
+}
+
+interface ReplayChunkBucket {
+	get(key: string): Promise<ReplayChunkBody | null>;
+}
+
+async function mapWithConcurrency<T, R>(
+	items: T[],
+	concurrency: number,
+	mapper: (item: T) => Promise<R>,
+): Promise<R[]> {
+	const results: R[] = new Array(items.length);
+	let nextIndex = 0;
+	const workerCount = Math.min(Math.max(1, concurrency), items.length);
+	await Promise.all(
+		Array.from({ length: workerCount }, async () => {
+			while (nextIndex < items.length) {
+				const current = nextIndex++;
+				results[current] = await mapper(items[current]);
+			}
+		}),
+	);
+	return results;
+}
+
+export async function fetchReplayChunks(
+	bucket: ReplayChunkBucket,
+	objects: ReplayChunkObject[],
+	concurrency = REPLAY_CHUNK_FETCH_CONCURRENCY,
+): Promise<Record<string, unknown>[]> {
+	const chunks = await mapWithConcurrency(objects, concurrency, async (obj) => {
+		const objectData = await bucket.get(obj.key);
+		if (!objectData) return [];
+		return objectData.json<Record<string, unknown>[]>();
+	});
+	return chunks.flat();
+}
+
+export const replayQueryRoutesPlugin: CollectorPlugin = {
+	name: "replay-query-routes",
+	register(app) {
+		app.get("/internal/replays", async (c) => {
+			const projectId = getProjectId(c);
+			const limit = Math.max(
+				1,
+				Math.min(500, parseInt(c.req.query("limit") ?? "50", 10) || 50),
+			);
+			const { results } = await sqlDbFor(c.env)
+				.prepare(
+					`SELECT
+					r.*,
+					(SELECT page_path FROM usage_events e WHERE e.project_id = r.project_id AND e.session_id = r.session_id AND e.event_type = 'page_view' ORDER BY e.occurred_at ASC LIMIT 1) as starting_link
+				 FROM session_replay_metadata r
+				 WHERE r.project_id = ?
+				 ORDER BY r.last_chunk_at DESC
+				 LIMIT ?`,
+				)
+				.bind(projectId, limit)
+				.all();
+
+			return c.json({ replays: results });
+		});
+
+		app.get("/internal/replays/:sessionId", async (c) => {
+			const projectId = getProjectId(c);
+			const sessionId = c.req.param("sessionId");
+
+			if (!c.env.REPLAYS_BUCKET) {
+				return c.json({ error: "Replay storage not configured" }, 500);
+			}
+
+			const metadata = await sqlDbFor(c.env)
+				.prepare(
+					`SELECT * FROM session_replay_metadata WHERE project_id = ? AND session_id = ?`,
+				)
+				.bind(projectId, sessionId)
+				.first<SessionReplayMetadataRow>();
+
+			if (!metadata) {
+				return c.json({ error: "Session replay not found" }, 404);
+			}
+
+			const prefix = `replays/${projectId}/${sessionId}/`;
+			const objects: Array<{ key: string }> = [];
+			let cursor: string | undefined;
+			do {
+				const page = await c.env.REPLAYS_BUCKET.list({ prefix, cursor });
+				objects.push(...page.objects);
+				cursor = page.truncated ? page.cursor : undefined;
+			} while (cursor);
+
+			if (objects.length === 0) {
+				return c.json({ error: "Replay chunks missing in storage" }, 404);
+			}
+
+			objects.sort((a, b) => a.key.localeCompare(b.key));
+
+			const chunkOffset = Math.max(
+				0,
+				parseInt(c.req.query("chunkOffset") ?? "0", 10) || 0,
+			);
+			const chunkLimit = Math.max(
+				1,
+				Math.min(
+					MAX_CHUNK_LIMIT,
+					parseInt(
+						c.req.query("chunkLimit") ?? String(DEFAULT_CHUNK_LIMIT),
+						10,
+					) || DEFAULT_CHUNK_LIMIT,
+				),
+			);
+			const selectedObjects = objects.slice(
+				chunkOffset,
+				chunkOffset + chunkLimit,
+			);
+			const events = await fetchReplayChunks(
+				c.env.REPLAYS_BUCKET,
+				selectedObjects,
+			);
+			const nextChunkOffset =
+				chunkOffset + chunkLimit < objects.length
+					? chunkOffset + chunkLimit
+					: null;
+
+			return c.json({
+				metadata,
+				events,
+				chunks: {
+					offset: chunkOffset,
+					limit: chunkLimit,
+					returned: selectedObjects.length,
+					total: objects.length,
+					nextChunkOffset,
+				},
+			});
+		});
+
+		app.delete("/internal/replays/:sessionId", async (c) => {
+			const projectId = getProjectId(c);
+			const sessionId = c.req.param("sessionId");
+
+			if (!c.env.REPLAYS_BUCKET) {
+				return c.json({ error: "Replay storage not configured" }, 500);
+			}
+
+			// 1. Check if it exists
+			const metadata = await sqlDbFor(c.env)
+				.prepare(
+					`SELECT * FROM session_replay_metadata WHERE project_id = ? AND session_id = ?`,
+				)
+				.bind(projectId, sessionId)
+				.first<SessionReplayMetadataRow>();
+
+			if (!metadata) {
+				return c.json({ error: "Session replay not found" }, 404);
+			}
+
+			// 2. Delete all chunks from bucket
+			const bucket = c.env.REPLAYS_BUCKET;
+			if (!bucket) {
+				return c.json({ error: "Replay storage not configured" }, 500);
+			}
+			const prefix = `replays/${projectId}/${sessionId}/`;
+			let cursor: string | undefined;
+			do {
+				const page = await bucket.list({ prefix, cursor });
+				if (page.objects.length > 0) {
+					await Promise.all(page.objects.map((obj) => bucket.delete(obj.key)));
+				}
+				cursor = page.truncated ? page.cursor : undefined;
+			} while (cursor);
+
+			// 3. Delete metadata from DB
+			await sqlDbFor(c.env)
+				.prepare(
+					`DELETE FROM session_replay_metadata WHERE project_id = ? AND session_id = ?`,
+				)
+				.bind(projectId, sessionId)
+				.run();
+
+			return c.json({ success: true });
+		});
+	},
+};

package/src/plugins/replay-receiver.ts

@@ -0,0 +1,81 @@
+import type { ReplayChunkInput } from "@obsunified/types";
+import type { CollectorPlugin } from "../framework/collector";
+import { sqlDbFor } from "../lib/sql-db";
+import { getProjectId } from "./_context";
+
+export const replayReceiverPlugin: CollectorPlugin = {
+	name: "replay-receiver",
+	register(app, runtime) {
+		app.post("/v1/replays", async (c) => {
+			const projectId = getProjectId(c);
+			let payload: ReplayChunkInput;
+			try {
+				payload = await c.req.json<ReplayChunkInput>();
+			} catch {
+				return c.json({ error: "Invalid JSON body" }, 400);
+			}
+			if (
+				!isSafeId(payload.sessionId) ||
+				(payload.visitorId !== undefined && !isSafeId(payload.visitorId)) ||
+				!Number.isInteger(payload.sequenceNumber) ||
+				payload.sequenceNumber < 0 ||
+				!Array.isArray(payload.events)
+			) {
+				return c.json({ error: "Invalid replay payload" }, 400);
+			}
+			const now = new Date().toISOString();
+
+			const timestamp = Date.now();
+			const objectKey = `replays/${projectId}/${payload.sessionId}/${timestamp}-${payload.sequenceNumber}.json`;
+
+			if (!c.env.REPLAYS_BUCKET) {
+				runtime.logger.error("REPLAYS_BUCKET binding is missing", {
+					project_id: projectId,
+				});
+				return c.json({ error: "Replay storage not configured" }, 500);
+			}
+
+			const payloadString = JSON.stringify(payload.events);
+			const chunkBytes = new TextEncoder().encode(payloadString).length;
+
+			await c.env.REPLAYS_BUCKET.put(objectKey, payloadString, {
+				httpMetadata: { contentType: "application/json" },
+			});
+
+			await sqlDbFor(c.env)
+				.prepare(
+					`INSERT INTO session_replay_metadata (project_id, session_id, visitor_id, first_chunk_at, last_chunk_at, chunk_count, events_count, storage_bytes)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(session_id) DO UPDATE SET
+           last_chunk_at = excluded.last_chunk_at,
+           chunk_count = session_replay_metadata.chunk_count + 1,
+           events_count = session_replay_metadata.events_count + excluded.events_count,
+           storage_bytes = session_replay_metadata.storage_bytes + excluded.storage_bytes
+        `,
+				)
+				.bind(
+					projectId,
+					payload.sessionId,
+					payload.visitorId,
+					now,
+					now,
+					1,
+					payload.events.length,
+					chunkBytes,
+				)
+				.run();
+
+			return c.json({ success: true, chunkKey: objectKey });
+		});
+	},
+};
+
+function isSafeId(value: unknown): value is string {
+	return (
+		typeof value === "string" &&
+		value.length > 0 &&
+		value.length <= 160 &&
+		/^[A-Za-z0-9._:-]+$/.test(value) &&
+		!value.includes("..")
+	);
+}