@obsunified/collector 2.0.1

package/src/lib/otlp.ts

@@ -0,0 +1,212 @@
+/**
+ * OTLP wire format → StoredSpan conversion.
+ * Union of all repos: A's advanced value conversion, hex validation,
+ * service name fallback, safe BigInt handling.
+ */
+
+import type {
+	JsonValue,
+	OtlpAnyValue,
+	OtlpKeyValue,
+	OtlpResourceSpans,
+	OtlpScopeSpans,
+	OtlpSpan,
+	OtlpTraceExportRequest,
+	StoredSpan,
+} from "@obsunified/types";
+import {
+	DEFAULT_WINDOW_HOURS,
+	getConfiguredRetentionHours,
+} from "@obsunified/types/constants";
+
+const EMPTY_ARRAY_JSON = "[]";
+const EMPTY_OBJECT_JSON = "{}";
+
+/** Recursive OTLP value → JSON (from A: handles arrayValue, kvlistValue) */
+const anyValueToJson = (value?: OtlpAnyValue): JsonValue => {
+	if (!value) return null;
+	if (typeof value.stringValue === "string") return value.stringValue;
+	if (typeof value.boolValue === "boolean") return value.boolValue;
+	if (typeof value.doubleValue === "number") return value.doubleValue;
+	if (
+		typeof value.intValue === "string" ||
+		typeof value.intValue === "number"
+	) {
+		const parsed = Number(value.intValue);
+		return Number.isFinite(parsed) ? parsed : String(value.intValue);
+	}
+	if (value.arrayValue?.values) {
+		return value.arrayValue.values.map((item) => anyValueToJson(item));
+	}
+	if (value.kvlistValue?.values) {
+		return keyValuesToRecord(value.kvlistValue.values);
+	}
+	return null;
+};
+
+const keyValuesToRecord = (
+	values?: OtlpKeyValue[],
+): Record<string, JsonValue> => {
+	if (!values?.length) return {};
+	return Object.fromEntries(
+		values.map((entry) => [entry.key, anyValueToJson(entry.value)]),
+	);
+};
+
+/** Validates hex string length (from A) */
+const normalizeHex = (
+	value: string | undefined,
+	length: number,
+): string | null => {
+	if (!value) return null;
+	const normalized = value.trim().toLowerCase();
+	if (!/^[0-9a-f]+$/i.test(normalized) || normalized.length !== length)
+		return null;
+	return normalized;
+};
+
+/** Safe BigInt nanosecond → millisecond duration (from A) */
+const nanosToMs = (start?: string, end?: string): number => {
+	try {
+		const startNs = BigInt(start ?? 0);
+		const endNs = BigInt(end ?? 0);
+		if (endNs < startNs) return 0;
+		return Number((endNs - startNs) / 1_000_000n);
+	} catch {
+		return 0;
+	}
+};
+
+/** Safe BigInt nanosecond → ISO string (from A) */
+const nanosToIso = (value?: string, fallback?: Date): string => {
+	if (!value) return fallback?.toISOString() ?? new Date().toISOString();
+	try {
+		const ns = BigInt(value);
+		if (ns <= 0n) return fallback?.toISOString() ?? new Date().toISOString();
+		return new Date(Number(ns / 1_000_000n)).toISOString();
+	} catch {
+		return fallback?.toISOString() ?? new Date().toISOString();
+	}
+};
+
+const serializeJson = (value: unknown, fallback: string): string => {
+	try {
+		return JSON.stringify(value ?? JSON.parse(fallback));
+	} catch {
+		return fallback;
+	}
+};
+
+/** Service name with fallback to server.address (from A) */
+const getServiceName = (
+	resourceAttributes: Record<string, JsonValue>,
+	spanAttributes: Record<string, JsonValue>,
+): string | null => {
+	const resourceName = resourceAttributes["service.name"];
+	if (typeof resourceName === "string" && resourceName.trim())
+		return resourceName.trim();
+	const serverAddress = spanAttributes["server.address"];
+	if (typeof serverAddress === "string" && serverAddress.trim())
+		return serverAddress.trim();
+	return null;
+};
+
+const scopeToStoredSpans = (
+	resourceSpans: OtlpResourceSpans,
+	scopeSpans: OtlpScopeSpans,
+	receivedAt: Date,
+	retentionHours: number,
+	projectId: string,
+): StoredSpan[] => {
+	const resourceAttributes = keyValuesToRecord(
+		resourceSpans.resource?.attributes,
+	);
+	const scopeName = scopeSpans.scope?.name ?? null;
+	const scopeVersion = scopeSpans.scope?.version ?? null;
+
+	return (scopeSpans.spans ?? []).flatMap((span: OtlpSpan): StoredSpan[] => {
+		const traceId = normalizeHex(span.traceId, 32);
+		const spanId = normalizeHex(span.spanId, 16);
+		if (!traceId || !spanId) return [];
+
+		const attributes = keyValuesToRecord(span.attributes);
+		const serviceName = getServiceName(resourceAttributes, attributes);
+
+		return [
+			{
+				projectId,
+				traceId,
+				spanId,
+				parentSpanId: normalizeHex(span.parentSpanId, 16),
+				traceState: span.traceState ?? null,
+				serviceName,
+				scopeName,
+				scopeVersion,
+				spanName: span.name || "unnamed-span",
+				spanKind: span.kind ?? 0,
+				statusCode: span.status?.code ?? 0,
+				statusMessage: span.status?.message ?? null,
+				startTime: nanosToIso(span.startTimeUnixNano, receivedAt),
+				endTime: nanosToIso(span.endTimeUnixNano, receivedAt),
+				durationMs: nanosToMs(span.startTimeUnixNano, span.endTimeUnixNano),
+				attributesJson: serializeJson(attributes, EMPTY_OBJECT_JSON),
+				droppedAttributesCount: span.droppedAttributesCount ?? 0,
+				resourceAttributesJson: serializeJson(
+					resourceAttributes,
+					EMPTY_OBJECT_JSON,
+				),
+				eventsJson: serializeJson(
+					(span.events ?? []).map((event) => ({
+						name: event.name,
+						timeUnixNano: event.timeUnixNano ?? null,
+						attributes: keyValuesToRecord(event.attributes),
+						droppedAttributesCount: event.droppedAttributesCount ?? 0,
+					})),
+					EMPTY_ARRAY_JSON,
+				),
+				droppedEventsCount: span.droppedEventsCount ?? 0,
+				linksJson: serializeJson(
+					(span.links ?? []).map((link) => ({
+						traceId: normalizeHex(link.traceId, 32),
+						spanId: normalizeHex(link.spanId, 16),
+						traceState: link.traceState ?? null,
+						attributes: keyValuesToRecord(link.attributes),
+						droppedAttributesCount: link.droppedAttributesCount ?? 0,
+					})),
+					EMPTY_ARRAY_JSON,
+				),
+				droppedLinksCount: span.droppedLinksCount ?? 0,
+				receivedAt: receivedAt.toISOString(),
+				expiresAt: retentionExpiry(receivedAt, retentionHours),
+			},
+		];
+	});
+};
+
+export const retentionExpiry = (
+	receivedAt: Date,
+	retentionHours = DEFAULT_WINDOW_HOURS,
+): string => {
+	const expiry = new Date(
+		receivedAt.getTime() + retentionHours * 60 * 60 * 1000,
+	);
+	return expiry.toISOString();
+};
+
+export const toStoredSpans = (
+	payload: OtlpTraceExportRequest,
+	projectId: string,
+	receivedAt = new Date(),
+	retentionHours = getConfiguredRetentionHours(undefined),
+): StoredSpan[] =>
+	(payload.resourceSpans ?? []).flatMap((resourceSpans) =>
+		(resourceSpans.scopeSpans ?? []).flatMap((scopeSpans) =>
+			scopeToStoredSpans(
+				resourceSpans,
+				scopeSpans,
+				receivedAt,
+				retentionHours,
+				projectId,
+			),
+		),
+	);

package/src/lib/parse-pprof.test.ts

@@ -0,0 +1,64 @@
+import type { PprofProfile } from "@obsunified/pprof-decoder";
+import { describe, expect, it } from "vitest";
+import { summarizeProfileFrames } from "./parse-pprof";
+
+const tinyProfile = (): PprofProfile => {
+	const stringTable = [
+		"",
+		"cpu",
+		"nanoseconds",
+		"main.handle",
+		"src/main.ts",
+		"trace_id",
+		"abc123def4567890",
+	];
+	const functions = new Map();
+	functions.set(1, { id: 1, nameIdx: 3, filenameIdx: 4 });
+	const locations = new Map();
+	locations.set(1, {
+		id: 1,
+		lines: [{ functionId: 1, line: 42 }],
+		functionIds: [1],
+	});
+	return {
+		sampleTypes: [{ typeIdx: 1, unitIdx: 2 }],
+		samples: [
+			{
+				locationIds: [1],
+				values: [500],
+				labels: [{ keyIdx: 5, strIdx: 6, num: 0 }],
+			},
+		],
+		locations,
+		functions,
+		stringTable,
+	};
+};
+
+describe("summarizeProfileFrames", () => {
+	it("returns source-linked frame summaries scoped by trace id", () => {
+		const frames = summarizeProfileFrames(tinyProfile(), {
+			traceIdFilter: "abc123def4567890",
+		});
+
+		expect(frames).toEqual([
+			{
+				name: "main.handle",
+				value: 500,
+				sampleCount: 1,
+				codeReference: {
+					originalPath: "src/main.ts",
+					relativePath: "src/main.ts",
+					absolutePath: undefined,
+					lineNumber: 42,
+					symbol: "main.handle",
+				},
+			},
+		]);
+		expect(
+			summarizeProfileFrames(tinyProfile(), {
+				traceIdFilter: "ffffffffffffffff",
+			}),
+		).toEqual([]);
+	});
+});

package/src/lib/parse-pprof.ts

@@ -0,0 +1,184 @@
+/**
+ * Collector-side pprof helpers. The wire-format decoder/encoder lives in
+ * `@obsunified/pprof-decoder` and is shared with the dashboard's flame graph
+ * viewer; this file adds the server-specific concerns:
+ *
+ *   - extractTraceIdsFromProfile — pulls distinct trace_id labels from a
+ *     parsed profile so `profile-routes.ts` can populate
+ *     `profile_trace_index` without trusting the `x-obs-trace-ids` header.
+ *   - filterPprofByTraceId — drops samples that don't carry the requested
+ *     trace_id label and re-emits a smaller gzipped pprof. Powers the
+ *     Phase 4.5 server-side pre-filter for the trace waterfall's 🔥 link.
+ */
+
+import {
+	decodePprof,
+	decodePprofBlob,
+	encodePprof,
+	gzipBytes,
+	type PprofFunction,
+	type PprofLabel,
+	type PprofLocation,
+	type PprofProfile,
+	type PprofSample,
+	type PprofValueType,
+} from "@obsunified/pprof-decoder";
+
+// Re-exports keep call sites (profile-routes, tests) importing from one
+// place — they don't need to know whether something lives upstream.
+export {
+	decodePprof,
+	decodePprofBlob,
+	encodePprof,
+	type PprofFunction,
+	type PprofLabel,
+	type PprofLocation,
+	type PprofProfile,
+	type PprofSample,
+	type PprofValueType,
+};
+
+const TRACE_ID_LABEL_KEYS = ["trace_id", "trace.id"];
+const TRACE_ID_RE = /^[0-9a-f]{16,32}$/i;
+
+export interface ProfileFrameSummary {
+	name: string;
+	value: number;
+	sampleCount: number;
+	codeReference?: {
+		originalPath: string;
+		relativePath?: string;
+		absolutePath?: string;
+		lineNumber?: number;
+		symbol?: string;
+	};
+}
+
+const sampleHasTraceId = (
+	profile: PprofProfile,
+	sample: PprofSample,
+	traceIdFilter: string | null,
+): boolean => {
+	if (!traceIdFilter) return true;
+	const want = traceIdFilter.toLowerCase();
+	const traceIdKeyIxs = new Set<number>();
+	for (let i = 0; i < profile.stringTable.length; i++) {
+		if (TRACE_ID_LABEL_KEYS.includes(profile.stringTable[i])) {
+			traceIdKeyIxs.add(i);
+		}
+	}
+	return sample.labels.some(
+		(label) =>
+			traceIdKeyIxs.has(label.keyIdx) &&
+			profile.stringTable[label.strIdx]?.toLowerCase() === want,
+	);
+};
+
+export const summarizeProfileFrames = (
+	profile: PprofProfile,
+	opts: { limit?: number; traceIdFilter?: string | null } = {},
+): ProfileFrameSummary[] => {
+	const limit = Math.max(1, Math.min(opts.limit ?? 50, 500));
+	const byFrame = new Map<string, ProfileFrameSummary>();
+
+	for (const sample of profile.samples) {
+		if (!sampleHasTraceId(profile, sample, opts.traceIdFilter ?? null))
+			continue;
+		const value = sample.values[0] ?? 0;
+		for (const locationId of sample.locationIds) {
+			const loc = profile.locations.get(locationId);
+			if (!loc) continue;
+			const lines =
+				loc.lines ||
+				(loc.functionIds || []).map((functionId) => ({ functionId, line: 0 }));
+			for (const line of lines) {
+				const fn = profile.functions.get(line.functionId);
+				if (!fn) continue;
+				const name = profile.stringTable[fn.nameIdx] ?? "?";
+				const filename = profile.stringTable[fn.filenameIdx] ?? "";
+				const key = `${name}\0${filename}\0${line.line || 0}`;
+				const existing = byFrame.get(key) ?? {
+					name,
+					value: 0,
+					sampleCount: 0,
+					codeReference: filename
+						? {
+								originalPath: filename,
+								relativePath: filename.startsWith("/") ? undefined : filename,
+								absolutePath: filename.startsWith("/") ? filename : undefined,
+								lineNumber: line.line || undefined,
+								symbol: name,
+							}
+						: undefined,
+				};
+				existing.value += value;
+				existing.sampleCount += 1;
+				byFrame.set(key, existing);
+			}
+		}
+	}
+
+	return Array.from(byFrame.values())
+		.sort((a, b) => b.value - a.value)
+		.slice(0, limit);
+};
+
+/**
+ * Extract distinct trace_id label values from a parsed pprof. Returns
+ * an array of valid 16-32 hex-char trace_ids, deduplicated. Defensive
+ * against malformed labels — silently ignores anything that doesn't
+ * match the hex pattern (a malformed agent shouldn't pollute the index).
+ */
+export const extractTraceIdsFromProfile = (profile: PprofProfile): string[] => {
+	const traceIdKeyIxs = new Set<number>();
+	for (let i = 0; i < profile.stringTable.length; i++) {
+		if (TRACE_ID_LABEL_KEYS.includes(profile.stringTable[i])) {
+			traceIdKeyIxs.add(i);
+		}
+	}
+	if (traceIdKeyIxs.size === 0) return [];
+
+	const traceIds = new Set<string>();
+	for (const sample of profile.samples) {
+		for (const label of sample.labels) {
+			if (!traceIdKeyIxs.has(label.keyIdx)) continue;
+			const value = profile.stringTable[label.strIdx];
+			if (value && TRACE_ID_RE.test(value)) {
+				traceIds.add(value.toLowerCase());
+			}
+		}
+	}
+	return [...traceIds];
+};
+
+/**
+ * Filter a pprof down to samples carrying the requested trace_id label
+ * and re-emit a gzipped blob. location/function/string tables are
+ * preserved verbatim — unused entries become dead weight in the output
+ * but the result is still valid pprof, and the wire savings come almost
+ * entirely from the sample list (~80% of a typical blob). Compacting
+ * the auxiliary tables is a Phase 4+ optimization.
+ */
+export const filterPprofByTraceId = async (
+	profile: PprofProfile,
+	traceIdFilter: string,
+): Promise<Uint8Array> => {
+	const want = traceIdFilter.toLowerCase();
+	const traceIdKeyIxs = new Set<number>();
+	for (let i = 0; i < profile.stringTable.length; i++) {
+		if (TRACE_ID_LABEL_KEYS.includes(profile.stringTable[i])) {
+			traceIdKeyIxs.add(i);
+		}
+	}
+	const filtered: PprofProfile = {
+		...profile,
+		samples: profile.samples.filter((s) =>
+			s.labels.some(
+				(l) =>
+					traceIdKeyIxs.has(l.keyIdx) &&
+					profile.stringTable[l.strIdx]?.toLowerCase() === want,
+			),
+		),
+	};
+	return gzipBytes(encodePprof(filtered));
+};

package/src/lib/payload-capture.ts

@@ -0,0 +1,22 @@
+import type { CollectorEnv } from "../framework/env";
+import type { SqlDb } from "./sql-db";
+
+export const isTruthyFlag = (value: unknown): boolean =>
+	value === true || value === 1 || value === "1" || value === "true";
+
+export async function isPayloadCaptureEnabled(
+	db: SqlDb,
+	projectId: string,
+	env?: Partial<CollectorEnv>,
+): Promise<boolean> {
+	if (isTruthyFlag(env?.OBS_PAYLOAD_CAPTURE_DEFAULT)) return true;
+
+	const row = await db
+		.prepare(
+			`SELECT payload_capture_enabled FROM projects WHERE id = ? LIMIT 1`,
+		)
+		.bind(projectId)
+		.first<{ payload_capture_enabled: boolean | number | string | null }>();
+
+	return isTruthyFlag(row?.payload_capture_enabled);
+}

package/src/lib/projects-store.ts

@@ -0,0 +1,207 @@
+import type {
+	IngestKey,
+	IngestKeyRow,
+	IngestKeyWithPlaintext,
+	Project,
+	ProjectRow,
+} from "@obsunified/types";
+import { randomHex, sha256Hex } from "./hash";
+import type { SqlDb } from "./sql-db";
+
+const rowToProject = (row: ProjectRow): Project => ({
+	id: row.id,
+	name: row.name,
+	slug: row.slug,
+	createdAt: row.created_at,
+	payloadCaptureEnabled:
+		row.payload_capture_enabled === true ||
+		row.payload_capture_enabled === 1 ||
+		row.payload_capture_enabled === "1" ||
+		row.payload_capture_enabled === "true",
+});
+
+const rowToKey = (row: IngestKeyRow): IngestKey => ({
+	id: row.id,
+	projectId: row.project_id,
+	name: row.name,
+	keyPrefix: row.key_prefix,
+	createdAt: row.created_at,
+	revokedAt: row.revoked_at,
+});
+
+const SLUG_PATTERN = /^[a-z0-9]([a-z0-9-]{0,48}[a-z0-9])?$/;
+
+export class ProjectsStore {
+	constructor(private readonly db: SqlDb) {}
+
+	async listProjects(): Promise<Project[]> {
+		const rs = await this.db
+			.prepare(`SELECT * FROM projects ORDER BY created_at ASC`)
+			.all<ProjectRow>();
+		return (rs.results ?? []).map(rowToProject);
+	}
+
+	async getProject(id: string): Promise<Project | null> {
+		const row = await this.db
+			.prepare(`SELECT * FROM projects WHERE id = ?`)
+			.bind(id)
+			.first<ProjectRow>();
+		return row ? rowToProject(row) : null;
+	}
+
+	async createProject(input: { name: string; slug: string }): Promise<Project> {
+		const name = input.name.trim();
+		const slug = input.slug.trim().toLowerCase();
+		if (!name) throw new Error("Project name is required");
+		if (!SLUG_PATTERN.test(slug))
+			throw new Error(
+				"Slug must be lowercase letters, digits, and hyphens (max 50 chars)",
+			);
+
+		const id = randomHex(16);
+		const createdAt = new Date().toISOString();
+
+		await this.db
+			.prepare(
+				`INSERT INTO projects (id, name, slug, created_at) VALUES (?, ?, ?, ?)`,
+			)
+			.bind(id, name, slug, createdAt)
+			.run();
+
+		return { id, name, slug, createdAt, payloadCaptureEnabled: false };
+	}
+
+	async updateProject(
+		id: string,
+		patch: { name?: string; payloadCaptureEnabled?: boolean },
+	): Promise<Project | null> {
+		const existing = await this.getProject(id);
+		if (!existing) return null;
+
+		const name = patch.name?.trim();
+		let next = existing;
+		if (name !== undefined && name !== existing.name) {
+			await this.db
+				.prepare(`UPDATE projects SET name = ? WHERE id = ?`)
+				.bind(name, id)
+				.run();
+			next = { ...next, name };
+		}
+		if (
+			patch.payloadCaptureEnabled !== undefined &&
+			patch.payloadCaptureEnabled !== existing.payloadCaptureEnabled
+		) {
+			await this.db
+				.prepare(`UPDATE projects SET payload_capture_enabled = ? WHERE id = ?`)
+				.bind(patch.payloadCaptureEnabled ? 1 : 0, id)
+				.run();
+			next = {
+				...next,
+				payloadCaptureEnabled: patch.payloadCaptureEnabled,
+			};
+		}
+		return next;
+	}
+
+	async ensureDefaultProject(): Promise<void> {
+		await this.db
+			.prepare(
+				`INSERT OR IGNORE INTO projects (id, name, slug, created_at) VALUES ('default', 'Default', 'default', ?)`,
+			)
+			.bind(new Date().toISOString())
+			.run();
+	}
+
+	// ── Ingest keys ──
+
+	async listKeys(projectId: string): Promise<IngestKey[]> {
+		const rs = await this.db
+			.prepare(
+				`SELECT * FROM ingest_keys WHERE project_id = ? ORDER BY created_at DESC`,
+			)
+			.bind(projectId)
+			.all<IngestKeyRow>();
+		return (rs.results ?? []).map(rowToKey);
+	}
+
+	async createKey(
+		projectId: string,
+		name: string,
+	): Promise<IngestKeyWithPlaintext> {
+		const project = await this.getProject(projectId);
+		if (!project) throw new Error("Project not found");
+
+		const trimmedName = name.trim() || "unnamed";
+		const id = randomHex(16);
+		const plaintext = `obs_${project.slug}_${randomHex(24)}`;
+		const keyHash = await sha256Hex(plaintext);
+		// Prefix: show enough to identify but not recover (first 16 chars).
+		const keyPrefix = plaintext.slice(0, 16);
+		const createdAt = new Date().toISOString();
+
+		await this.db
+			.prepare(
+				`INSERT INTO ingest_keys (id, project_id, key_hash, key_prefix, name, created_at) VALUES (?, ?, ?, ?, ?, ?)`,
+			)
+			.bind(id, projectId, keyHash, keyPrefix, trimmedName, createdAt)
+			.run();
+
+		return {
+			id,
+			projectId,
+			name: trimmedName,
+			keyPrefix,
+			createdAt,
+			revokedAt: null,
+			key: plaintext,
+			warning: "This key will not be shown again. Store it somewhere safe.",
+		};
+	}
+
+	async revokeKey(keyId: string): Promise<boolean> {
+		const result = await this.db
+			.prepare(
+				`UPDATE ingest_keys SET revoked_at = ? WHERE id = ? AND revoked_at IS NULL`,
+			)
+			.bind(new Date().toISOString(), keyId)
+			.run();
+		return (result.meta?.changes ?? 0) > 0;
+	}
+
+	async findByKeyHash(
+		keyHash: string,
+	): Promise<{ projectId: string; keyId: string } | null> {
+		const row = await this.db
+			.prepare(
+				`SELECT id, project_id FROM ingest_keys WHERE key_hash = ? AND revoked_at IS NULL`,
+			)
+			.bind(keyHash)
+			.first<{ id: string; project_id: string }>();
+		return row ? { projectId: row.project_id, keyId: row.id } : null;
+	}
+
+	/**
+	 * Idempotently register the legacy env INGEST_KEY as a bootstrap key on the
+	 * default project, so existing deployments keep working without manual migration.
+	 */
+	async bootstrapEnvKey(plaintext: string): Promise<void> {
+		if (!plaintext) return;
+		await this.ensureDefaultProject();
+		const keyHash = await sha256Hex(plaintext);
+		const existing = await this.db
+			.prepare(`SELECT id FROM ingest_keys WHERE key_hash = ?`)
+			.bind(keyHash)
+			.first<{ id: string }>();
+		if (existing) return;
+
+		const id = randomHex(16);
+		const keyPrefix = plaintext.slice(0, Math.min(16, plaintext.length));
+		const createdAt = new Date().toISOString();
+		await this.db
+			.prepare(
+				`INSERT INTO ingest_keys (id, project_id, key_hash, key_prefix, name, created_at) VALUES (?, 'default', ?, ?, 'bootstrap', ?)`,
+			)
+			.bind(id, keyHash, keyPrefix, createdAt)
+			.run();
+	}
+}