@obsunified/collector 2.0.1

package/src/plugins/tail-routes.test.ts

@@ -0,0 +1,47 @@
+import { Hono } from "hono";
+import { describe, expect, it } from "vitest";
+import { CollectorRuntime } from "../framework/collector";
+import type { CollectorEnv } from "../framework/env";
+import { tailRoutesPlugin } from "./tail-routes";
+
+describe("tailRoutesPlugin", () => {
+	it("forwards the authenticated project id instead of trusting query params", async () => {
+		let forwardedUrl: string | null = null;
+		const app = new Hono<{
+			Bindings: CollectorEnv;
+			Variables: { projectId: string };
+		}>();
+		app.use("/internal/*", async (c, next) => {
+			c.set("projectId", "authorized-project");
+			await next();
+		});
+		tailRoutesPlugin.register(
+			app as unknown as Hono<{ Bindings: CollectorEnv }>,
+			new CollectorRuntime(),
+		);
+
+		const env = {
+			TAIL_HUB: {
+				idFromName: (name: string) => name,
+				get: () => ({
+					fetch: async (url: string) => {
+						forwardedUrl = url;
+						return new Response("ok", { status: 200 });
+					},
+				}),
+			},
+		} as unknown as CollectorEnv;
+
+		const res = await app.request(
+			"/internal/telemetry/tail?projectId=attacker-project&kinds=span",
+			{ method: "GET" },
+			env,
+		);
+
+		expect(res.status).toBe(200);
+		expect(forwardedUrl).not.toBeNull();
+		const url = new URL(forwardedUrl ?? "");
+		expect(url.searchParams.get("projectId")).toBe("authorized-project");
+		expect(url.searchParams.get("kinds")).toBe("span");
+	});
+});

package/src/plugins/tail-routes.ts

@@ -0,0 +1,42 @@
+import type { CollectorPlugin } from "../framework/collector";
+import { getProjectId } from "./_context";
+
+/**
+ * Live tail SSE endpoint. The dashboard opens an EventSource here; the
+ * collector proxies to the TailHub Durable Object which holds the open
+ * stream and fans out broadcasts from ingest.
+ */
+export const tailRoutesPlugin: CollectorPlugin = {
+	name: "tail-routes",
+	register(app) {
+		app.get("/internal/telemetry/tail", (c) => {
+			const hub = c.env.TAIL_HUB;
+			if (!hub) {
+				return c.json(
+					{ error: "Live tail not configured (TAIL_HUB binding missing)" },
+					503,
+				);
+			}
+			// Project scope must come from auth middleware, never from the client
+			// URL. The dashboard live-tail client uses a fetch stream so it can
+			// send the same X-Project-Id header as the rest of the API.
+			const projectId = getProjectId(c);
+			if (!/^[A-Za-z0-9_-]{1,128}$/.test(projectId)) {
+				return c.json({ error: "invalid projectId" }, 400);
+			}
+			const kinds = c.req.query("kinds") || "span,log";
+
+			const id = hub.idFromName("singleton");
+			const stub = hub.get(id);
+
+			const subscribeUrl = new URL("https://hub/subscribe");
+			subscribeUrl.searchParams.set("projectId", projectId);
+			subscribeUrl.searchParams.set("kinds", kinds);
+
+			return stub.fetch(subscribeUrl.toString(), {
+				method: "GET",
+				signal: c.req.raw.signal,
+			});
+		});
+	},
+};

package/src/plugins/timeline-routes.ts

@@ -0,0 +1,244 @@
+import type { CollectorPlugin } from "../framework/collector";
+import { IdentityIndex } from "../lib/identity-index";
+import { sqlDbFor } from "../lib/sql-db";
+import { getProjectId } from "./_context";
+
+export interface TimelineEvent {
+	t: string;
+	kind: "span" | "log" | "usage";
+	id: string;
+	title: string;
+	subtitle?: string;
+	severity?: "info" | "warn" | "error";
+	durationMs?: number;
+	/**
+	 * RFC 0004 — click-scoped correlation key. Present on every event that
+	 * was emitted while an interaction was active. The {@link TimelineResponse.groups}
+	 * field groups events by this id so the dashboard can render
+	 * "click → traces it caused" without a second round-trip.
+	 */
+	interactionId?: string;
+	payload: Record<string, unknown>;
+}
+
+export interface TimelineReplayMetadata {
+	firstChunkAt: string;
+	lastChunkAt: string;
+	chunkCount: number;
+	eventsCount: number;
+}
+
+/**
+ * RFC 0004 — a click and the traces it caused, bundled. The dashboard's
+ * session timeline collapses an interaction's events under its
+ * originating click; this is the precomputed shape the dashboard reads.
+ */
+export interface TimelineGroup {
+	interactionId: string;
+	/** The originating user-originated event (usage event, kind="interaction" or "page_view"). */
+	clickEvent: TimelineEvent | null;
+	/** Root spans (parent_span_id IS NULL) carrying this interaction_id. */
+	causedTraces: Array<{
+		traceId: string;
+		rootSpanId: string;
+		rootSpanName: string;
+		serviceName: string | null;
+		durationMs: number;
+		status: "ok" | "error";
+	}>;
+	/** Logs + non-click usage events sharing this interaction_id. */
+	relatedEvents: TimelineEvent[];
+}
+
+export interface TimelineResponse {
+	sessionId: string;
+	firstSeen: string | null;
+	lastSeen: string | null;
+	counts: {
+		spans: number;
+		logs: number;
+		usage: number;
+	};
+	replay: TimelineReplayMetadata | null;
+	events: TimelineEvent[];
+	/**
+	 * RFC 0004 — derived view keyed by interaction_id. Empty object when
+	 * no events in the session carry one (sessions ingested before
+	 * @obsunified/analytics-sdk's autoCorrelate landed). Events without an
+	 * interaction_id appear in `events` but never under `groups`.
+	 */
+	groups: Record<string, TimelineGroup>;
+	timestamp: string;
+}
+
+const severityFromStatus = (code: number): "info" | "warn" | "error" =>
+	code === 2 ? "error" : "info";
+
+const severityFromLog = (
+	severity: string | null,
+): "info" | "warn" | "error" => {
+	if (severity === "ERROR" || severity === "FATAL") return "error";
+	if (severity === "WARN") return "warn";
+	return "info";
+};
+
+const severityFromUsage = (
+	type: string,
+	severity: string | null,
+): "info" | "warn" | "error" => {
+	if (type === "frontend_error" || severity === "error") return "error";
+	if (severity === "warning") return "warn";
+	return "info";
+};
+
+export const timelineRoutesPlugin: CollectorPlugin = {
+	name: "timeline-routes",
+	register(app) {
+		app.get("/internal/timeline/:sessionId", async (c) => {
+			const projectId = getProjectId(c);
+			const sessionId = c.req.param("sessionId");
+			if (!sessionId) {
+				return c.json({ error: "sessionId required" }, 400);
+			}
+
+			// RFC 0008 — go through the IdentityIndex helper rather than
+			// duplicating per-signal SQL. Same indexed lookup, single
+			// ownership of the cross-signal join shape.
+			const index = new IdentityIndex(sqlDbFor(c.env));
+			const manifest = await index.bySession(projectId, sessionId);
+
+			const events: TimelineEvent[] = [];
+			// Stash root-span shapes for grouping. Indexed by trace_id since a
+			// single click can fan out into multiple service-level traces (rare
+			// for a normal browser → single backend, but defensible).
+			const rootSpansByInteraction = new Map<
+				string,
+				TimelineGroup["causedTraces"]
+			>();
+
+			for (const s of manifest.spans) {
+				const interactionId = s.interactionId ?? undefined;
+				events.push({
+					t: s.startTime,
+					kind: "span",
+					id: `${s.traceId}:${s.spanId}`,
+					title: `${s.serviceName ?? "unknown"} · ${s.spanName}`,
+					subtitle: s.statusMessage ?? undefined,
+					severity: severityFromStatus(s.statusCode),
+					durationMs: s.durationMs,
+					interactionId,
+					payload: {
+						traceId: s.traceId,
+						spanId: s.spanId,
+						parentSpanId: s.parentSpanId,
+						statusCode: s.statusCode,
+					},
+				});
+				// Only root spans count as "the trace this click caused" — child
+				// spans are part of the same trace tree, so collecting them
+				// would double-count.
+				if (interactionId && s.parentSpanId === null) {
+					const list = rootSpansByInteraction.get(interactionId) ?? [];
+					list.push({
+						traceId: s.traceId,
+						rootSpanId: s.spanId,
+						rootSpanName: s.spanName,
+						serviceName: s.serviceName,
+						durationMs: s.durationMs,
+						status: s.statusCode === 2 ? "error" : "ok",
+					});
+					rootSpansByInteraction.set(interactionId, list);
+				}
+			}
+
+			for (const l of manifest.logs) {
+				events.push({
+					t: l.occurredAt,
+					kind: "log",
+					id: l.logId,
+					title: l.message.slice(0, 160),
+					subtitle:
+						`${l.serviceName ?? ""}${l.loggerName ? ` · ${l.loggerName}` : ""}`.trim(),
+					severity: severityFromLog(l.severity),
+					interactionId: l.interactionId ?? undefined,
+					payload: {
+						severity: l.severity,
+						traceId: l.traceId,
+						spanId: l.spanId,
+					},
+				});
+			}
+
+			for (const u of manifest.usageEvents) {
+				events.push({
+					t: u.occurredAt,
+					kind: "usage",
+					id: u.eventId,
+					title: `${u.eventType}${u.eventName && u.eventName !== u.eventType ? ` · ${u.eventName}` : ""}`,
+					subtitle: u.pagePath ?? undefined,
+					severity: severityFromUsage(u.eventType, u.severity),
+					interactionId: u.interactionId ?? undefined,
+					payload: {
+						eventType: u.eventType,
+						eventName: u.eventName,
+						pagePath: u.pagePath,
+					},
+				});
+			}
+
+			events.sort((a, b) => a.t.localeCompare(b.t));
+
+			// RFC 0004 — derive groups keyed by interaction_id.
+			const groups: Record<string, TimelineGroup> = {};
+			for (const event of events) {
+				if (!event.interactionId) continue;
+				let group = groups[event.interactionId];
+				if (!group) {
+					group = {
+						interactionId: event.interactionId,
+						clickEvent: null,
+						causedTraces: rootSpansByInteraction.get(event.interactionId) ?? [],
+						relatedEvents: [],
+					};
+					groups[event.interactionId] = group;
+				}
+				// Treat the first usage event with this id as the click that
+				// originated it. Usage events are minted at addEventListener
+				// fire time so they're timestamp-prior to any fetch the
+				// handler kicks off.
+				if (event.kind === "usage" && group.clickEvent === null) {
+					group.clickEvent = event;
+				} else {
+					group.relatedEvents.push(event);
+				}
+			}
+
+			const firstSeen = events[0]?.t ?? null;
+			const lastSeen = events[events.length - 1]?.t ?? null;
+
+			const response: TimelineResponse = {
+				sessionId,
+				firstSeen,
+				lastSeen,
+				counts: {
+					spans: manifest.spans.length,
+					logs: manifest.logs.length,
+					usage: manifest.usageEvents.length,
+				},
+				replay: manifest.replay
+					? {
+							firstChunkAt: manifest.replay.firstChunkAt,
+							lastChunkAt: manifest.replay.lastChunkAt,
+							chunkCount: manifest.replay.chunkCount,
+							eventsCount: manifest.replay.eventsCount,
+						}
+					: null,
+				events,
+				groups,
+				timestamp: new Date().toISOString(),
+			};
+
+			return c.json(response);
+		});
+	},
+};

package/src/plugins/ua-enrichment.ts

@@ -0,0 +1,27 @@
+import type {
+	CollectorPlugin,
+	UsageEventProcessorPlugin,
+} from "../framework/collector";
+import { parseUserAgent } from "../lib/ua-parser";
+
+const processor: UsageEventProcessorPlugin = {
+	name: "ua-enrichment",
+	process(events) {
+		return events.map((event) => {
+			const parsed = parseUserAgent(event.userAgent);
+			return {
+				...event,
+				browser: parsed.browser,
+				os: parsed.os,
+				deviceType: parsed.deviceType,
+			};
+		});
+	},
+};
+
+export const uaEnrichmentPlugin: CollectorPlugin = {
+	name: "ua-enrichment",
+	register(_app, runtime) {
+		runtime.addUsageEventProcessor(processor);
+	},
+};

package/src/plugins/usage-normalization.ts

@@ -0,0 +1,46 @@
+import type { JsonValue, UsageEventRecord } from "@obsunified/types";
+import type { CollectorPlugin } from "../framework/collector";
+import { parseJsonRecord } from "../lib/json";
+
+const normalizePathSegment = (segment: string): string => {
+	if (!segment) return segment;
+	if (/^\d+$/.test(segment)) return ":id";
+	if (/^[0-9a-f]{16,}$/i.test(segment)) return ":id";
+	return segment;
+};
+
+const normalizePath = (path: string | null): string | null => {
+	if (!path) return null;
+	const [pathname] = path.split("?");
+	const segments = pathname
+		.split("/")
+		.map(normalizePathSegment)
+		.filter((segment, index) => !(index === 0 && segment === ""));
+	return `/${segments.join("/")}`.replace(/\/{2,}/g, "/");
+};
+
+export const usageNormalizationPlugin: CollectorPlugin = {
+	name: "usage-normalization",
+	register(_app, runtime) {
+		runtime.addUsageEventProcessor({
+			name: "usage-normalization",
+			process(events) {
+				return events.map((event): UsageEventRecord => {
+					const context = parseJsonRecord(event.contextJson);
+					const normalizedPath = normalizePath(event.pagePath);
+					const nextContext: Record<string, JsonValue> = {
+						...context,
+						"collector.is_admin_path": Boolean(
+							normalizedPath?.startsWith("/admin"),
+						),
+					};
+					return {
+						...event,
+						pagePath: normalizedPath,
+						contextJson: JSON.stringify(nextContext),
+					};
+				});
+			},
+		});
+	},
+};

package/src/plugins/usage-privacy.ts

@@ -0,0 +1,55 @@
+import type { JsonValue, UsageEventRecord } from "@obsunified/types";
+import type { CollectorPlugin } from "../framework/collector";
+import { parseJsonValue } from "../lib/json";
+
+const REDACTED_VALUE = "[REDACTED]";
+const DEFAULT_REDACT_FIELDS = [
+	"email",
+	"token",
+	"password",
+	"authorization",
+	"cookie",
+];
+
+const shouldRedactKey = (key: string): boolean => {
+	const normalized = key.toLowerCase();
+	return DEFAULT_REDACT_FIELDS.some(
+		(field) => normalized === field || normalized.endsWith(field),
+	);
+};
+
+const redactJson = (value: JsonValue): JsonValue => {
+	if (Array.isArray(value)) return value.map(redactJson);
+	if (value && typeof value === "object") {
+		return Object.fromEntries(
+			Object.entries(value).map(([key, nestedValue]) =>
+				shouldRedactKey(key)
+					? [key, REDACTED_VALUE]
+					: [key, redactJson(nestedValue)],
+			),
+		);
+	}
+	return value;
+};
+
+export const usagePrivacyPlugin: CollectorPlugin = {
+	name: "usage-privacy",
+	register(_app, runtime) {
+		runtime.addUsageEventProcessor({
+			name: "usage-privacy",
+			process(events) {
+				return events.map(
+					(event): UsageEventRecord => ({
+						...event,
+						contextJson: JSON.stringify(
+							redactJson(parseJsonValue(event.contextJson)),
+						),
+						propertiesJson: JSON.stringify(
+							redactJson(parseJsonValue(event.propertiesJson)),
+						),
+					}),
+				);
+			},
+		});
+	},
+};

package/src/plugins/usage-query-routes.ts

@@ -0,0 +1,102 @@
+import {
+	DEFAULT_WINDOW_HOURS,
+	getConfiguredRetentionHours,
+} from "@obsunified/types/constants";
+import type { CollectorPlugin } from "../framework/collector";
+import { getProjectId } from "./_context";
+
+export const usageQueryRoutesPlugin: CollectorPlugin = {
+	name: "usage-query-routes",
+	register(app, runtime) {
+		app.get("/internal/usage/overview", async (c) => {
+			const projectId = getProjectId(c);
+			const maxHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const hours = Math.min(
+				maxHours,
+				Math.max(
+					1,
+					Number.parseInt(
+						c.req.query("hours") || String(DEFAULT_WINDOW_HOURS),
+						10,
+					) || DEFAULT_WINDOW_HOURS,
+				),
+			);
+
+			const store = runtime.createUsageStore(c.env);
+			const overview = await store.getOverview({
+				projectId,
+				hours,
+				path: c.req.query("path") || undefined,
+				includeAdmin: c.req.query("includeAdmin") === "true",
+			});
+			return c.json({
+				...overview,
+				plugins: runtime.getRegisteredPluginNames(),
+			});
+		});
+
+		app.get("/internal/usage/sessions", async (c) => {
+			const projectId = getProjectId(c);
+			const maxHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const hours = Math.min(
+				maxHours,
+				Math.max(
+					1,
+					Number.parseInt(
+						c.req.query("hours") || String(DEFAULT_WINDOW_HOURS),
+						10,
+					) || DEFAULT_WINDOW_HOURS,
+				),
+			);
+			const filterParam = c.req.query("filter");
+			const filter: "all" | "ended_in_error" | "dropoff" | "slow" =
+				filterParam === "ended_in_error" ||
+				filterParam === "dropoff" ||
+				filterParam === "slow"
+					? filterParam
+					: "all";
+			const slowMs = Math.max(
+				100,
+				Number.parseInt(c.req.query("slowMs") || "3000", 10) || 3000,
+			);
+			const limit = Math.max(
+				1,
+				Math.min(
+					500,
+					Number.parseInt(c.req.query("limit") || "100", 10) || 100,
+				),
+			);
+
+			const store = runtime.createUsageStore(c.env);
+			const result = await store.listSessions({
+				projectId,
+				hours,
+				filter,
+				slowMs,
+				limit,
+			});
+			return c.json({
+				...result,
+				filter,
+				hours,
+				slowMs,
+				timestamp: new Date().toISOString(),
+			});
+		});
+
+		app.get("/internal/usage/sessions/:sessionId", async (c) => {
+			const projectId = getProjectId(c);
+			const store = runtime.createUsageStore(c.env);
+			const detail = await store.getSessionDetail(
+				c.req.param("sessionId"),
+				projectId,
+			);
+			if (!detail)
+				return c.json(
+					{ error: "Not Found", message: "Session not found" },
+					404,
+				);
+			return c.json({ ...detail, plugins: runtime.getRegisteredPluginNames() });
+		});
+	},
+};

package/src/plugins/usage-receiver.ts

@@ -0,0 +1,129 @@
+/** Union: A's JSON error handling, event limit, env-based retention. P/D's dynamic CORS. */
+
+import type {
+	UsageEventInput,
+	UsageEventPayload,
+	UsageEventRecord,
+} from "@obsunified/types";
+import {
+	DEFAULT_WINDOW_HOURS,
+	getConfiguredRetentionHours,
+} from "@obsunified/types/constants";
+import type { CollectorPlugin } from "../framework/collector";
+import { retentionExpiry } from "../lib/otlp";
+import { getProjectId } from "./_context";
+
+const sanitizeIsoTimestamp = (value?: string): string => {
+	if (!value) return new Date().toISOString();
+	const parsed = new Date(value);
+	return Number.isNaN(parsed.getTime())
+		? new Date().toISOString()
+		: parsed.toISOString();
+};
+
+const toUsageRecord = (
+	input: UsageEventInput,
+	userAgent: string | null,
+	country: string | null,
+	receivedAt: Date,
+	projectId: string,
+	retentionHours?: number,
+): UsageEventRecord => ({
+	projectId,
+	eventId: crypto.randomUUID(),
+	sessionId: input.sessionId,
+	visitorId: input.visitorId,
+	eventType: input.type,
+	eventName: input.name,
+	pagePath: input.path || null,
+	pageTitle: input.title || null,
+	referrer: input.referrer || null,
+	severity: input.severity || "info",
+	source: "web",
+	contextJson: JSON.stringify(input.context || {}),
+	propertiesJson: JSON.stringify(input.properties || {}),
+	userAgent,
+	occurredAt: sanitizeIsoTimestamp(input.occurredAt),
+	receivedAt: receivedAt.toISOString(),
+	expiresAt: retentionExpiry(receivedAt, retentionHours),
+	country,
+	browser: null,
+	os: null,
+	deviceType: null,
+	isBot: false,
+	utmSource: null,
+	utmMedium: null,
+	utmCampaign: null,
+	// RFC 0004 — opaque correlation id from the analytics SDK; null when
+	// the event was emitted outside any user interaction.
+	interactionId: input.interactionId || null,
+});
+
+export const usageReceiverPlugin: CollectorPlugin = {
+	name: "usage-receiver",
+	register(app, runtime) {
+		// CORS is handled by the framework-level /v1/* middleware.
+		// No per-route CORS needed here.
+
+		app.post("/v1/usage", async (c) => {
+			const projectId = getProjectId(c);
+			let payload: UsageEventPayload;
+			try {
+				payload = await c.req.json<UsageEventPayload>();
+			} catch {
+				return c.json({ error: "Invalid JSON body" }, 400);
+			}
+			const rawEvents = payload.events || [];
+			if (rawEvents.length > 200) {
+				return c.json(
+					{ error: `Too many events: ${rawEvents.length} (max 200)` },
+					413,
+				);
+			}
+			const validInputs = rawEvents.filter((event): event is UsageEventInput =>
+				Boolean(
+					event?.type && event?.name && event?.sessionId && event?.visitorId,
+				),
+			);
+
+			const receivedAt = new Date();
+			const retentionHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const country = c.req.header("X-Client-Country") || null;
+			const records = validInputs.map((event) =>
+				toUsageRecord(
+					event,
+					c.req.header("User-Agent") || null,
+					country,
+					receivedAt,
+					projectId,
+					retentionHours,
+				),
+			);
+
+			const processed = await runtime.runUsageEventProcessors(
+				records,
+				runtime.createRouteContext(c.env, c),
+			);
+			const store = runtime.createUsageStore(c.env);
+			const result = await runtime.withChildSpan(
+				"usage.ingest",
+				async (span) => {
+					const r = await store.ingest(processed);
+					span.setAttribute("usage.events_received", rawEvents.length);
+					span.setAttribute("usage.events_valid", validInputs.length);
+					span.setAttribute("usage.events_inserted", r.inserted);
+					span.setAttribute("usage.session_count", r.sessionCount);
+					span.setAttribute("project.id", projectId);
+					return r;
+				},
+			);
+
+			return c.json({
+				success: true,
+				inserted: result.inserted,
+				sessionCount: result.sessionCount,
+				acceptedWindowHours: DEFAULT_WINDOW_HOURS,
+			});
+		});
+	},
+};