@obsunified/collector 2.0.1

package/src/plugins/logs-receiver.ts

@@ -0,0 +1,160 @@
+import type { LogRecord, LogSeverity } from "@obsunified/types";
+import { getConfiguredRetentionHours } from "@obsunified/types/constants";
+import type { CollectorPlugin } from "../framework/collector";
+import { LogsStore } from "../lib/logs-store";
+import { sqlDbFor } from "../lib/sql-db";
+import { logToTailEvent, publishTail } from "../lib/tail-publisher";
+import {
+	type DecodedLogRecord,
+	decodeLogsRequest,
+	OtlpDecodeError,
+	type ReadBodyResult,
+	readOtlpBody,
+} from "../otlp/decode";
+import { logsResponse, otlpRetryableError } from "../otlp/response";
+import { getProjectId } from "./_context";
+
+const MAX_LOGS_PER_REQUEST = 1000;
+
+export const logsReceiverPlugin: CollectorPlugin = {
+	name: "logs-receiver",
+	register(app, runtime) {
+		app.post("/v1/logs", async (c) => {
+			const projectId = getProjectId(c);
+
+			let body: ReadBodyResult;
+			try {
+				body = await readOtlpBody(c);
+			} catch (err) {
+				if (err instanceof OtlpDecodeError) {
+					return c.json({ error: err.message }, err.status);
+				}
+				throw err;
+			}
+
+			let decoded: DecodedLogRecord[];
+			try {
+				decoded = decodeLogsRequest(body);
+			} catch (err) {
+				if (err instanceof OtlpDecodeError) {
+					return c.json({ error: err.message }, err.status);
+				}
+				throw err;
+			}
+
+			let rejected = 0;
+			if (decoded.length > MAX_LOGS_PER_REQUEST) {
+				rejected = decoded.length - MAX_LOGS_PER_REQUEST;
+				decoded = decoded.slice(0, MAX_LOGS_PER_REQUEST);
+			}
+
+			const store = new LogsStore(sqlDbFor(c.env));
+			const retentionHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const now = new Date();
+			const nowStr = now.toISOString();
+			const expires = new Date(
+				now.getTime() + retentionHours * 60 * 60 * 1000,
+			).toISOString();
+
+			const records: LogRecord[] = decoded.map((d) => {
+				const sessionId =
+					d.attributes &&
+					typeof d.attributes["session.id"] === "string" &&
+					(d.attributes["session.id"] as string).length > 0
+						? (d.attributes["session.id"] as string)
+						: null;
+				// RFC 0004 — denormalize obs.interaction.id from log
+				// attributes when the producer stamped it (telemetry-sdk's
+				// logger inherits the active span's attribute).
+				const interactionId =
+					d.attributes &&
+					typeof d.attributes["obs.interaction.id"] === "string" &&
+					(d.attributes["obs.interaction.id"] as string).length > 0
+						? (d.attributes["obs.interaction.id"] as string)
+						: null;
+				return {
+					projectId,
+					logId: crypto.randomUUID(),
+					traceId: d.traceId,
+					spanId: d.spanId,
+					serviceName: d.serviceName,
+					severity: d.severity,
+					severityNumber: d.severityNumber || severityToNumber(d.severity),
+					loggerName: d.loggerName,
+					message: d.message,
+					attributesJson: d.attributes ? JSON.stringify(d.attributes) : null,
+					flags: d.flags,
+					droppedAttributesCount: d.droppedAttributesCount,
+					occurredAt: d.occurredAt,
+					receivedAt: nowStr,
+					expiresAt: expires,
+					sessionId,
+					interactionId,
+				};
+			});
+
+			try {
+				await runtime.withChildSpan("logs.ingest", async (span) => {
+					span.setAttribute("logs.records_received", decoded.length + rejected);
+					span.setAttribute("logs.records_rejected", rejected);
+					span.setAttribute("logs.records_inserted", records.length);
+					span.setAttribute("project.id", projectId);
+					await store.ingestBatch(records);
+				});
+			} catch (err) {
+				runtime.logger.error("[/v1/logs] storage error", {
+					project_id: projectId,
+					error: err instanceof Error ? err.message : String(err),
+				});
+				return otlpRetryableError(c, 503, "Storage temporarily unavailable");
+			}
+
+			if (records.length > 0 && c.env.TAIL_HUB) {
+				const events = records.map(logToTailEvent);
+				c.executionCtx.waitUntil(publishTail(c.env, events));
+			}
+
+			if (rejected > 0) {
+				return logsResponse(c, body.wireFormat, {
+					rejected,
+					errorMessage: `Batch exceeded ${MAX_LOGS_PER_REQUEST}-record cap; ${rejected} record(s) dropped`,
+				});
+			}
+			return logsResponse(c, body.wireFormat);
+		});
+
+		app.get("/internal/logs/overview", async (c) => {
+			const projectId = getProjectId(c);
+			const store = new LogsStore(sqlDbFor(c.env));
+			const query = c.req.query();
+			const hours = parseInt(query.hours || "24", 10);
+
+			const response = await store.getLogs({
+				projectId,
+				hours,
+				service: query.service,
+				severity: query.severity as LogSeverity | undefined,
+				traceId: query.traceId,
+				limit: parseInt(query.limit || "100", 10),
+				search: query.search,
+			});
+
+			return c.json(response);
+		});
+	},
+};
+
+const severityToNumber = (s: LogSeverity): number => {
+	switch (s) {
+		case "DEBUG":
+			return 5;
+		case "INFO":
+			return 9;
+		case "WARN":
+			return 13;
+		case "ERROR":
+			return 17;
+		case "FATAL":
+			return 21;
+	}
+};

package/src/plugins/metrics-receiver.ts

@@ -0,0 +1,91 @@
+/** OTLP/HTTP metrics receiver. Accepts JSON or protobuf, with gzip. */
+
+import { getConfiguredRetentionHours } from "@obsunified/types/constants";
+import type { CollectorPlugin } from "../framework/collector";
+import { MetricsStore } from "../lib/metrics-store";
+import { sqlDbFor } from "../lib/sql-db";
+import {
+	decodeMetricsRequest,
+	OtlpDecodeError,
+	type ReadBodyResult,
+	readOtlpBody,
+} from "../otlp/decode";
+import { metricsResponse, otlpRetryableError } from "../otlp/response";
+import { getProjectId } from "./_context";
+
+const MAX_POINTS_PER_REQUEST = 2000;
+
+export const metricsReceiverPlugin: CollectorPlugin = {
+	name: "metrics-http-receiver",
+	register(app, runtime) {
+		app.post("/v1/metrics", async (c) => {
+			const projectId = getProjectId(c);
+
+			let body: ReadBodyResult;
+			try {
+				body = await readOtlpBody(c);
+			} catch (err) {
+				if (err instanceof OtlpDecodeError) {
+					return c.json({ error: err.message }, err.status);
+				}
+				throw err;
+			}
+
+			let points: ReturnType<typeof decodeMetricsRequest>;
+			try {
+				points = decodeMetricsRequest(body);
+			} catch (err) {
+				if (err instanceof OtlpDecodeError) {
+					return c.json({ error: err.message }, err.status);
+				}
+				throw err;
+			}
+
+			let rejected = 0;
+			if (points.length > MAX_POINTS_PER_REQUEST) {
+				rejected = points.length - MAX_POINTS_PER_REQUEST;
+				points = points.slice(0, MAX_POINTS_PER_REQUEST);
+			}
+
+			const retentionHours = getConfiguredRetentionHours(c.env.RETENTION_HOURS);
+			const now = new Date();
+			const receivedAt = now.toISOString();
+			const expiresAt = new Date(
+				now.getTime() + retentionHours * 60 * 60 * 1000,
+			).toISOString();
+
+			const store = new MetricsStore(sqlDbFor(c.env));
+			try {
+				await runtime.withChildSpan("metrics.ingest", async (span) => {
+					span.setAttribute(
+						"metrics.points_received",
+						points.length + rejected,
+					);
+					span.setAttribute("metrics.points_rejected", rejected);
+					span.setAttribute("metrics.points_inserted", points.length);
+					span.setAttribute("project.id", projectId);
+					await store.ingestBatch({
+						projectId,
+						points,
+						receivedAt,
+						expiresAt,
+					});
+				});
+			} catch (err) {
+				runtime.logger.error("[/v1/metrics] storage error", {
+					project_id: projectId,
+					error: err instanceof Error ? err.message : String(err),
+				});
+				return otlpRetryableError(c, 503, "Storage temporarily unavailable");
+			}
+
+			if (rejected > 0) {
+				return metricsResponse(c, body.wireFormat, {
+					rejected,
+					errorMessage: `Batch exceeded ${MAX_POINTS_PER_REQUEST}-point cap; ${rejected} point(s) dropped`,
+				});
+			}
+			return metricsResponse(c, body.wireFormat);
+		});
+	},
+};

package/src/plugins/onboarding-routes.ts

@@ -0,0 +1,54 @@
+import type { CollectorPlugin } from "../framework/collector";
+import { sqlDbFor } from "../lib/sql-db";
+
+/**
+ * Drives the in-dashboard Onboarding tab — returns counts of signal
+ * types, used to decide which "you need to wire X" snippets to show
+ * the user.
+ *
+ * Returns zeros for tables that don't yet exist (e.g. fresh DB) rather
+ * than 500ing — onboarding has to work on day zero.
+ */
+export const onboardingRoutesPlugin: CollectorPlugin = {
+	name: "onboarding-routes",
+	register(app) {
+		app.get("/internal/onboarding/counts", async (c) => {
+			const db = sqlDbFor(c.env);
+			const counts = await Promise.all([
+				safeCount(db, "SELECT COUNT(*) AS n FROM telemetry_spans"),
+				safeCount(db, "SELECT COUNT(*) AS n FROM logs"),
+				safeCount(db, "SELECT COUNT(*) AS n FROM usage_events"),
+				safeCount(db, "SELECT COUNT(*) AS n FROM user_profiles"),
+				safeCount(db, "SELECT COUNT(*) AS n FROM ai_calls"),
+				safeCount(db, "SELECT COUNT(*) AS n FROM session_replay_metadata"),
+				safeCount(
+					db,
+					"SELECT COUNT(*) AS n FROM telemetry_spans WHERE interaction_id IS NOT NULL",
+				),
+			]);
+
+			return c.json({
+				spans: counts[0],
+				logs: counts[1],
+				usageEvents: counts[2],
+				identifiedUsers: counts[3],
+				aiCalls: counts[4],
+				replayChunks: counts[5],
+				spansWithInteraction: counts[6],
+			});
+		});
+	},
+};
+
+async function safeCount(
+	db: { prepare: (sql: string) => { first: <T>() => Promise<T | null> } },
+	sql: string,
+): Promise<number> {
+	try {
+		const row = await db.prepare(sql).first<{ n: number }>();
+		return Number(row?.n ?? 0);
+	} catch {
+		// Table doesn't exist yet — fresh DB, no signals of this type.
+		return 0;
+	}
+}

package/src/plugins/otlp-receiver.ts

@@ -0,0 +1,124 @@
+/** OTLP/HTTP trace receiver. Accepts JSON or protobuf, with gzip. */
+
+import { getConfiguredRetentionHours } from "@obsunified/types/constants";
+import type { CollectorPlugin } from "../framework/collector";
+import { toStoredSpans } from "../lib/otlp";
+import { publishTail, spanToTailEvent } from "../lib/tail-publisher";
+import {
+	decodeTraceRequest,
+	OtlpDecodeError,
+	type ReadBodyResult,
+	readOtlpBody,
+} from "../otlp/decode";
+import { otlpRetryableError, traceResponse } from "../otlp/response";
+import { getProjectId } from "./_context";
+
+const MAX_SPANS_PER_REQUEST = 500;
+
+export const otlpReceiverPlugin: CollectorPlugin = {
+	name: "otlp-http-receiver",
+	register(app, runtime) {
+		app.post("/v1/traces", async (c) => {
+			const projectId = getProjectId(c);
+			const routeContext = runtime.createRouteContext(c.env, c);
+
+			let body: ReadBodyResult;
+			try {
+				body = await readOtlpBody(c);
+			} catch (err) {
+				if (err instanceof OtlpDecodeError) {
+					return c.json({ error: err.message }, err.status);
+				}
+				throw err;
+			}
+
+			let payload: ReturnType<typeof decodeTraceRequest>;
+			try {
+				payload = decodeTraceRequest(body);
+			} catch (err) {
+				if (err instanceof OtlpDecodeError) {
+					return c.json({ error: err.message }, err.status);
+				}
+				throw err;
+			}
+
+			// Count spans and truncate to cap, tracking rejected excess for the
+			// partial_success envelope. We rebuild a trimmed payload rather than
+			// touching the rest of the pipeline.
+			let total = 0;
+			let rejected = 0;
+			const trimmedResourceSpans = [];
+			for (const rs of payload.resourceSpans ?? []) {
+				const scopeSpans = [];
+				for (const ss of rs.scopeSpans ?? []) {
+					const spans = [];
+					for (const s of ss.spans ?? []) {
+						if (total >= MAX_SPANS_PER_REQUEST) {
+							rejected++;
+						} else {
+							spans.push(s);
+							total++;
+						}
+					}
+					scopeSpans.push({ ...ss, spans });
+				}
+				trimmedResourceSpans.push({ ...rs, scopeSpans });
+			}
+
+			const parsedSpans = toStoredSpans(
+				{ resourceSpans: trimmedResourceSpans },
+				projectId,
+				routeContext.now,
+				getConfiguredRetentionHours(c.env.RETENTION_HOURS),
+			);
+			// `toStoredSpans` silently drops spans with invalid trace/span IDs
+			// (length mismatch, non-hex). Surface those to the caller via
+			// partial_success rather than making them disappear.
+			const malformed = total - parsedSpans.length;
+			const totalRejected = rejected + malformed;
+
+			const spans = await runtime.runSpanProcessors(parsedSpans, routeContext);
+			const store = runtime.createStore(c.env);
+			try {
+				await runtime.withChildSpan("traces.ingest", async (span) => {
+					span.setAttribute("traces.spans_received", total);
+					span.setAttribute("traces.spans_rejected", totalRejected);
+					span.setAttribute("traces.spans_inserted", spans.length);
+					span.setAttribute("project.id", projectId);
+					const result = await store.ingest(spans);
+					// Distinct traces touched by this batch — the write-side
+					// denominator for the gap/trace-detail read-rate ratio (Q3).
+					span.setAttribute("traces.trace_count", result.traceCount);
+				});
+			} catch (err) {
+				runtime.logger.error("[/v1/traces] storage error", {
+					project_id: projectId,
+					error: err instanceof Error ? err.message : String(err),
+				});
+				return otlpRetryableError(c, 503, "Storage temporarily unavailable");
+			}
+
+			if (spans.length > 0 && c.env.TAIL_HUB) {
+				const events = spans.map(spanToTailEvent);
+				c.executionCtx.waitUntil(publishTail(c.env, events));
+			}
+
+			if (totalRejected > 0) {
+				const parts: string[] = [];
+				if (rejected > 0)
+					parts.push(
+						`${rejected} span(s) dropped over ${MAX_SPANS_PER_REQUEST}-span cap`,
+					);
+				if (malformed > 0)
+					parts.push(
+						`${malformed} span(s) rejected (invalid trace_id or span_id)`,
+					);
+				return traceResponse(c, body.wireFormat, {
+					rejected: totalRejected,
+					errorMessage: parts.join("; "),
+				});
+			}
+			return traceResponse(c, body.wireFormat);
+		});
+	},
+};

package/src/plugins/platform-routes.ts

@@ -0,0 +1,165 @@
+import type { CollectorPlugin } from "../framework/collector";
+import { MetricsStore } from "../lib/metrics-store";
+import { dialectFor, sqlDbFor } from "../lib/sql-db";
+import { getProjectId } from "./_context";
+
+export const platformRoutesPlugin: CollectorPlugin = {
+	name: "platform-routes",
+	register(app) {
+		app.get("/internal/platform/resources", async (c) => {
+			const projectId = getProjectId(c);
+			const db = sqlDbFor(c.env);
+
+			const countFor = async (table: string) => {
+				const row = await db
+					.prepare(`SELECT COUNT(*) as c FROM ${table} WHERE project_id = ?`)
+					.bind(projectId)
+					.first<{ c: number }>();
+				return row?.c ?? 0;
+			};
+
+			const eventsCount = await countFor("usage_events");
+			const tracesCount = await countFor("telemetry_spans");
+			const logsCount = await countFor("logs");
+			const aiCallsCount = await countFor("ai_calls");
+
+			const d1RowDensity = eventsCount + tracesCount + logsCount + aiCallsCount;
+
+			let r2StorageBytes = 0;
+			try {
+				const r2Info = await db
+					.prepare(
+						"SELECT SUM(storage_bytes) as s FROM session_replay_metadata WHERE project_id = ?",
+					)
+					.bind(projectId)
+					.first<{ s: number }>();
+				r2StorageBytes = r2Info?.s ?? 0;
+			} catch {
+				r2StorageBytes = 0;
+			}
+
+			return c.json({
+				success: true,
+				resources: {
+					d1: {
+						rowDensity: d1RowDensity,
+						eventsCount,
+						tracesCount,
+						logsCount,
+						aiCallsCount,
+					},
+					r2: {
+						storageBytes: r2StorageBytes,
+					},
+					worker: {
+						cpuMs: 0,
+						memoryBytes: 0,
+						requestsCount: 0,
+						status: "Needs Cloudflare Auth Token for live metrics",
+					},
+				},
+			});
+		});
+
+		// RFC 0009 Phase 5.2 — Linux hosts mode. Queries the latest
+		// gauge points for the standard OTel host-metrics semconv
+		// (system.cpu.utilization, system.memory.usage, etc.) grouped
+		// by host.name. Returns an empty array if no `system.*` series
+		// exist; the dashboard renders Cloudflare-only mode in that case.
+		app.get("/internal/platform/hosts", async (c) => {
+			const projectId = getProjectId(c);
+			const db = sqlDbFor(c.env);
+			const dialect = dialectFor(db);
+
+			const rows = await db
+				.prepare(
+					`WITH host_series AS (
+						SELECT
+							s.id AS series_id,
+							s.name AS metric_name,
+							${dialect.jsonText("s.resource_attrs_json", "$.host\\u002Ename")} AS host_name,
+							s.attributes_json
+						FROM metric_series s
+						WHERE s.project_id = ?
+							AND s.name LIKE 'system.%'
+					),
+					latest_points AS (
+						SELECT
+							p.series_id,
+							p.value,
+							p.received_at,
+							ROW_NUMBER() OVER (PARTITION BY p.series_id ORDER BY p.received_at DESC) AS rn
+						FROM metric_point p
+						WHERE p.project_id = ?
+							AND p.received_at >= ${dialect.sinceMinutes("15")}
+							AND p.series_id IN (SELECT series_id FROM host_series)
+					)
+					SELECT
+						hs.host_name,
+						hs.metric_name,
+						lp.value,
+						lp.received_at
+					FROM latest_points lp
+					JOIN host_series hs ON hs.series_id = lp.series_id
+					WHERE lp.rn = 1
+					ORDER BY hs.host_name, hs.metric_name`,
+				)
+				.bind(projectId, projectId)
+				.all<{
+					host_name: string | null;
+					metric_name: string;
+					value: number | null;
+					received_at: string;
+				}>();
+
+			// Bucket per host — null host_name means resource_attrs_json
+			// didn't carry host.name, group as <unknown>.
+			const byHost = new Map<
+				string,
+				{ host: string; metrics: Record<string, number>; updatedAt: string }
+			>();
+			for (const row of rows.results) {
+				const host = row.host_name ?? "<unknown>";
+				if (!byHost.has(host)) {
+					byHost.set(host, {
+						host,
+						metrics: {},
+						updatedAt: row.received_at,
+					});
+				}
+				const entry = byHost.get(host);
+				if (!entry) continue;
+				if (row.value !== null) entry.metrics[row.metric_name] = row.value;
+				if (row.received_at > entry.updatedAt)
+					entry.updatedAt = row.received_at;
+			}
+
+			return c.json({
+				success: true,
+				hosts: Array.from(byHost.values()),
+			});
+		});
+
+		app.get("/internal/platform/metric-exemplars", async (c) => {
+			const projectId = getProjectId(c);
+			const limit = Math.max(
+				1,
+				Math.min(Number.parseInt(c.req.query("limit") ?? "20", 10) || 20, 100),
+			);
+			const metricPrefix = c.req.query("metric_prefix") ?? "system.";
+			const serviceName = c.req.query("service") ?? null;
+			const store = new MetricsStore(sqlDbFor(c.env));
+			const exemplars = await store.recentExemplars({
+				projectId,
+				serviceName,
+				metricPrefix,
+				limit,
+			});
+			return c.json({
+				success: true,
+				exemplars,
+				timestamp: new Date().toISOString(),
+			});
+		});
+	},
+};