@obsunified/collector 2.0.1

package/src/otlp/decode/logs.ts

@@ -0,0 +1,96 @@
+import { fromBinary, fromJson } from "@bufbuild/protobuf";
+import type { LogSeverity, JsonValue as ObsJsonValue } from "@obsunified/types";
+import {
+	type ExportLogsServiceRequest,
+	ExportLogsServiceRequestSchema,
+} from "../gen/opentelemetry/proto/collector/logs/v1/logs_service_pb.js";
+import type { LogRecord as OtlpLogRecord } from "../gen/opentelemetry/proto/logs/v1/logs_pb.js";
+import { decodeJsonBody, OtlpDecodeError, type ReadBodyResult } from "./body";
+import {
+	anyValueToString,
+	bytesToHex,
+	extractServiceName,
+	keyValuesToRecord,
+	severityFromNumber,
+} from "./values";
+
+// ── Logs ─────────────────────────────────────────────────────────────
+
+/**
+ * A log record normalized for storage. One per emitted OTLP `LogRecord`,
+ * with resource / scope context flattened into the fields the `LogsStore`
+ * expects. Receiver adds storage-level fields (projectId, logId, receivedAt,
+ * expiresAt) at ingest time.
+ */
+export interface DecodedLogRecord {
+	serviceName: string | null;
+	loggerName: string | null;
+	traceId: string | null;
+	spanId: string | null;
+	severity: LogSeverity;
+	severityNumber: number;
+	message: string;
+	attributes: Record<string, ObsJsonValue> | null;
+	flags: number;
+	droppedAttributesCount: number;
+	occurredAt: string;
+}
+
+/**
+ * Decode an OTLP logs export request into a flat list of records ready for
+ * storage. Resource `service.name` and scope name are denormalized onto each
+ * record so downstream code doesn't need to traverse the hierarchy.
+ */
+export const decodeLogsRequest = (body: ReadBodyResult): DecodedLogRecord[] => {
+	let msg: ExportLogsServiceRequest;
+	try {
+		msg =
+			body.wireFormat === "protobuf"
+				? fromBinary(ExportLogsServiceRequestSchema, body.bytes)
+				: fromJson(ExportLogsServiceRequestSchema, decodeJsonBody(body.bytes));
+	} catch (err) {
+		throw new OtlpDecodeError(
+			`Malformed OTLP body: ${(err as Error).message}`,
+			400,
+		);
+	}
+
+	const out: DecodedLogRecord[] = [];
+	for (const rl of msg.resourceLogs) {
+		const serviceName = extractServiceName(rl.resource?.attributes);
+		for (const sl of rl.scopeLogs) {
+			const loggerName = sl.scope?.name || null;
+			for (const log of sl.logRecords) {
+				out.push(adaptLogRecord(log, serviceName, loggerName));
+			}
+		}
+	}
+	return out;
+};
+
+const adaptLogRecord = (
+	log: OtlpLogRecord,
+	serviceName: string | null,
+	loggerName: string | null,
+): DecodedLogRecord => {
+	const ts = log.timeUnixNano || log.observedTimeUnixNano;
+	const occurredAt =
+		ts > 0n
+			? new Date(Number(ts / 1_000_000n)).toISOString()
+			: new Date().toISOString();
+	return {
+		serviceName,
+		loggerName,
+		traceId: log.traceId.length === 16 ? bytesToHex(log.traceId) : null,
+		spanId: log.spanId.length === 8 ? bytesToHex(log.spanId) : null,
+		severity: severityFromNumber(log.severityNumber, log.severityText),
+		severityNumber: log.severityNumber || 0,
+		message: anyValueToString(log.body),
+		attributes: log.attributes.length
+			? keyValuesToRecord(log.attributes)
+			: null,
+		flags: log.flags || 0,
+		droppedAttributesCount: log.droppedAttributesCount || 0,
+		occurredAt,
+	};
+};

package/src/otlp/decode/metrics.ts

@@ -0,0 +1,402 @@
+import { fromBinary, fromJson } from "@bufbuild/protobuf";
+import {
+	type ExportMetricsServiceRequest,
+	ExportMetricsServiceRequestSchema,
+} from "../gen/opentelemetry/proto/collector/metrics/v1/metrics_service_pb.js";
+import type { KeyValue } from "../gen/opentelemetry/proto/common/v1/common_pb.js";
+import type {
+	Exemplar,
+	ExponentialHistogramDataPoint,
+	HistogramDataPoint,
+	Metric,
+	NumberDataPoint,
+	ResourceMetrics,
+	ScopeMetrics,
+	SummaryDataPoint,
+} from "../gen/opentelemetry/proto/metrics/v1/metrics_pb.js";
+import { decodeJsonBody, OtlpDecodeError, type ReadBodyResult } from "./body";
+import { bytesToHex, extractServiceName, keyValuesToRecord } from "./values";
+
+// ── Metrics ───────────────────────────────────────────────────────────
+
+export type MetricType =
+	| "gauge"
+	| "sum"
+	| "histogram"
+	| "exp_histogram"
+	| "summary";
+
+export interface DecodedExemplar {
+	value: number;
+	traceId: string | null;
+	spanId: string | null;
+	tsNs: string;
+}
+
+/**
+ * A metric data point normalized for storage. Series-identifying fields
+ * (name, resource/scope context, attributes) are repeated on each point so
+ * the store can do series upsert in a single pass.
+ */
+export interface DecodedMetricPoint {
+	// Series identity
+	name: string;
+	description: string | null;
+	unit: string | null;
+	type: MetricType;
+	isMonotonic: boolean | null;
+	temporality: number | null;
+	scopeName: string | null;
+	scopeVersion: string | null;
+	serviceName: string | null;
+	resourceAttrsJson: string | null;
+	attributesJson: string | null;
+	identity: string;
+
+	// Point
+	tsNs: string;
+	startTsNs: string | null;
+	value: number | null;
+
+	// Histogram-only
+	count: number | null;
+	sum: number | null;
+	min: number | null;
+	max: number | null;
+	boundsJson: string | null;
+	bucketCountsJson: string | null;
+
+	/**
+	 * Type-specific fields that don't fit the column schema:
+	 *   - exp_histogram: { scale, zeroCount, zeroThreshold, positive: {offset, bucketCounts}, negative: {offset, bucketCounts} }
+	 *   - summary: { quantileValues: [{quantile, value}, ...] }
+	 */
+	extraJson: string | null;
+
+	exemplarsJson: string | null;
+}
+
+export const decodeMetricsRequest = (
+	body: ReadBodyResult,
+): DecodedMetricPoint[] => {
+	let msg: ExportMetricsServiceRequest;
+	try {
+		msg =
+			body.wireFormat === "protobuf"
+				? fromBinary(ExportMetricsServiceRequestSchema, body.bytes)
+				: fromJson(
+						ExportMetricsServiceRequestSchema,
+						decodeJsonBody(body.bytes),
+					);
+	} catch (err) {
+		throw new OtlpDecodeError(
+			`Malformed OTLP body: ${(err as Error).message}`,
+			400,
+		);
+	}
+
+	const out: DecodedMetricPoint[] = [];
+	for (const rm of msg.resourceMetrics) {
+		const resourceAttrs = rm.resource?.attributes ?? [];
+		const resourceAttrsJson = resourceAttrs.length
+			? JSON.stringify(keyValuesToRecord(resourceAttrs))
+			: null;
+		const serviceName = extractServiceName(resourceAttrs);
+		for (const sm of rm.scopeMetrics) {
+			addMetrics(out, rm, sm, serviceName, resourceAttrsJson);
+		}
+	}
+	return out;
+};
+
+const addMetrics = (
+	out: DecodedMetricPoint[],
+	rm: ResourceMetrics,
+	sm: ScopeMetrics,
+	serviceName: string | null,
+	resourceAttrsJson: string | null,
+) => {
+	const scopeName = sm.scope?.name || null;
+	const scopeVersion = sm.scope?.version || null;
+	const resourceIdentityFragment = canonicalKeyValues(
+		rm.resource?.attributes ?? [],
+	);
+
+	for (const metric of sm.metrics) {
+		const common = {
+			name: metric.name,
+			description: metric.description || null,
+			unit: metric.unit || null,
+			scopeName,
+			scopeVersion,
+			serviceName,
+			resourceAttrsJson,
+		};
+		switch (metric.data.case) {
+			case "gauge":
+				for (const p of metric.data.value.dataPoints) {
+					out.push(
+						numberPoint(metric, p, "gauge", common, resourceIdentityFragment),
+					);
+				}
+				break;
+			case "sum":
+				for (const p of metric.data.value.dataPoints) {
+					out.push({
+						...numberPoint(metric, p, "sum", common, resourceIdentityFragment),
+						isMonotonic: metric.data.value.isMonotonic,
+						temporality: metric.data.value.aggregationTemporality,
+					});
+				}
+				break;
+			case "histogram":
+				for (const p of metric.data.value.dataPoints) {
+					out.push({
+						...histogramPoint(metric, p, common, resourceIdentityFragment),
+						temporality: metric.data.value.aggregationTemporality,
+					});
+				}
+				break;
+			case "exponentialHistogram":
+				for (const p of metric.data.value.dataPoints) {
+					out.push({
+						...expHistogramPoint(metric, p, common, resourceIdentityFragment),
+						temporality: metric.data.value.aggregationTemporality,
+					});
+				}
+				break;
+			case "summary":
+				for (const p of metric.data.value.dataPoints) {
+					out.push(summaryPoint(metric, p, common, resourceIdentityFragment));
+				}
+				break;
+			default:
+				break;
+		}
+	}
+};
+
+type SeriesCommon = {
+	name: string;
+	description: string | null;
+	unit: string | null;
+	scopeName: string | null;
+	scopeVersion: string | null;
+	serviceName: string | null;
+	resourceAttrsJson: string | null;
+};
+
+const numberPoint = (
+	metric: Metric,
+	p: NumberDataPoint,
+	type: MetricType,
+	common: SeriesCommon,
+	resourceIdentityFragment: string,
+): DecodedMetricPoint => {
+	const value =
+		p.value.case === "asDouble"
+			? p.value.value
+			: p.value.case === "asInt"
+				? Number(p.value.value)
+				: null;
+	const attrsRecord = p.attributes.length
+		? keyValuesToRecord(p.attributes)
+		: null;
+	return {
+		...common,
+		type,
+		isMonotonic: null,
+		temporality: null,
+		attributesJson: attrsRecord ? JSON.stringify(attrsRecord) : null,
+		identity: buildIdentity(
+			resourceIdentityFragment,
+			common.scopeName,
+			metric.name,
+			p.attributes,
+		),
+		tsNs: p.timeUnixNano.toString(),
+		startTsNs: p.startTimeUnixNano > 0n ? p.startTimeUnixNano.toString() : null,
+		value,
+		count: null,
+		sum: null,
+		min: null,
+		max: null,
+		boundsJson: null,
+		bucketCountsJson: null,
+		extraJson: null,
+		exemplarsJson: encodeExemplars(p.exemplars),
+	};
+};
+
+const histogramPoint = (
+	metric: Metric,
+	p: HistogramDataPoint,
+	common: SeriesCommon,
+	resourceIdentityFragment: string,
+): DecodedMetricPoint => {
+	const attrsRecord = p.attributes.length
+		? keyValuesToRecord(p.attributes)
+		: null;
+	return {
+		...common,
+		type: "histogram",
+		isMonotonic: null,
+		temporality: null,
+		attributesJson: attrsRecord ? JSON.stringify(attrsRecord) : null,
+		identity: buildIdentity(
+			resourceIdentityFragment,
+			common.scopeName,
+			metric.name,
+			p.attributes,
+		),
+		tsNs: p.timeUnixNano.toString(),
+		startTsNs: p.startTimeUnixNano > 0n ? p.startTimeUnixNano.toString() : null,
+		value: null,
+		count: Number(p.count),
+		sum: p.sum ?? null,
+		min: p.min ?? null,
+		max: p.max ?? null,
+		boundsJson: p.explicitBounds.length
+			? JSON.stringify(p.explicitBounds)
+			: null,
+		bucketCountsJson: p.bucketCounts.length
+			? JSON.stringify(p.bucketCounts.map((b) => Number(b)))
+			: null,
+		extraJson: null,
+		exemplarsJson: encodeExemplars(p.exemplars),
+	};
+};
+
+const expHistogramPoint = (
+	metric: Metric,
+	p: ExponentialHistogramDataPoint,
+	common: SeriesCommon,
+	resourceIdentityFragment: string,
+): DecodedMetricPoint => {
+	const attrsRecord = p.attributes.length
+		? keyValuesToRecord(p.attributes)
+		: null;
+	const extra = {
+		scale: p.scale,
+		zeroCount: Number(p.zeroCount),
+		zeroThreshold: p.zeroThreshold,
+		positive: p.positive
+			? {
+					offset: p.positive.offset,
+					bucketCounts: p.positive.bucketCounts.map((b) => Number(b)),
+				}
+			: null,
+		negative: p.negative
+			? {
+					offset: p.negative.offset,
+					bucketCounts: p.negative.bucketCounts.map((b) => Number(b)),
+				}
+			: null,
+	};
+	return {
+		...common,
+		type: "exp_histogram",
+		isMonotonic: null,
+		temporality: null,
+		attributesJson: attrsRecord ? JSON.stringify(attrsRecord) : null,
+		identity: buildIdentity(
+			resourceIdentityFragment,
+			common.scopeName,
+			metric.name,
+			p.attributes,
+		),
+		tsNs: p.timeUnixNano.toString(),
+		startTsNs: p.startTimeUnixNano > 0n ? p.startTimeUnixNano.toString() : null,
+		value: null,
+		count: Number(p.count),
+		sum: p.sum ?? null,
+		min: p.min ?? null,
+		max: p.max ?? null,
+		boundsJson: null,
+		bucketCountsJson: null,
+		extraJson: JSON.stringify(extra),
+		exemplarsJson: encodeExemplars(p.exemplars),
+	};
+};
+
+const summaryPoint = (
+	metric: Metric,
+	p: SummaryDataPoint,
+	common: SeriesCommon,
+	resourceIdentityFragment: string,
+): DecodedMetricPoint => {
+	const attrsRecord = p.attributes.length
+		? keyValuesToRecord(p.attributes)
+		: null;
+	const extra = {
+		quantileValues: p.quantileValues.map((q) => ({
+			quantile: q.quantile,
+			value: q.value,
+		})),
+	};
+	return {
+		...common,
+		type: "summary",
+		isMonotonic: null,
+		temporality: null,
+		attributesJson: attrsRecord ? JSON.stringify(attrsRecord) : null,
+		identity: buildIdentity(
+			resourceIdentityFragment,
+			common.scopeName,
+			metric.name,
+			p.attributes,
+		),
+		tsNs: p.timeUnixNano.toString(),
+		startTsNs: p.startTimeUnixNano > 0n ? p.startTimeUnixNano.toString() : null,
+		value: null,
+		count: Number(p.count),
+		sum: p.sum,
+		min: null,
+		max: null,
+		boundsJson: null,
+		bucketCountsJson: null,
+		extraJson: JSON.stringify(extra),
+		exemplarsJson: null, // Summary has no exemplars in the spec.
+	};
+};
+
+const encodeExemplars = (exemplars: Exemplar[]): string | null => {
+	if (!exemplars.length) return null;
+	return JSON.stringify(
+		exemplars.map<DecodedExemplar>((e) => ({
+			value:
+				e.value.case === "asDouble"
+					? e.value.value
+					: e.value.case === "asInt"
+						? Number(e.value.value)
+						: 0,
+			traceId: e.traceId.length === 16 ? bytesToHex(e.traceId) : null,
+			spanId: e.spanId.length === 8 ? bytesToHex(e.spanId) : null,
+			tsNs: e.timeUnixNano.toString(),
+		})),
+	);
+};
+
+/**
+ * Canonical series identity. Stable across encodings (JSON/proto) because
+ * attributes are sorted and values normalized. Short enough to index.
+ */
+const buildIdentity = (
+	resourceFragment: string,
+	scopeName: string | null,
+	metricName: string,
+	attrs: KeyValue[],
+): string =>
+	JSON.stringify([
+		resourceFragment,
+		scopeName ?? "",
+		metricName,
+		canonicalKeyValues(attrs),
+	]);
+
+const canonicalKeyValues = (attrs: KeyValue[]): string => {
+	if (!attrs.length) return "";
+	const rec = keyValuesToRecord(attrs);
+	const keys = Object.keys(rec).sort();
+	return JSON.stringify(keys.map((k) => [k, rec[k]]));
+};

package/src/otlp/decode/traces.ts

@@ -0,0 +1,92 @@
+import { fromBinary, fromJson } from "@bufbuild/protobuf";
+import type {
+	OtlpResourceSpans,
+	OtlpTraceExportRequest,
+} from "@obsunified/types";
+import {
+	type ExportTraceServiceRequest,
+	ExportTraceServiceRequestSchema,
+} from "../gen/opentelemetry/proto/collector/trace/v1/trace_service_pb.js";
+import type { ResourceSpans } from "../gen/opentelemetry/proto/trace/v1/trace_pb.js";
+import { decodeJsonBody, OtlpDecodeError, type ReadBodyResult } from "./body";
+import { adaptKeyValue, bigintToString, bytesToHex } from "./values";
+
+/**
+ * Decode an OTLP trace export request from wire bytes into the legacy
+ * `@obsunified/types` shape consumed by `toStoredSpans`.
+ */
+export const decodeTraceRequest = (
+	body: ReadBodyResult,
+): OtlpTraceExportRequest => {
+	let msg: ExportTraceServiceRequest;
+	try {
+		msg =
+			body.wireFormat === "protobuf"
+				? fromBinary(ExportTraceServiceRequestSchema, body.bytes)
+				: fromJson(ExportTraceServiceRequestSchema, decodeJsonBody(body.bytes));
+	} catch (err) {
+		throw new OtlpDecodeError(
+			`Malformed OTLP body: ${(err as Error).message}`,
+			400,
+		);
+	}
+	return { resourceSpans: msg.resourceSpans.map(adaptResourceSpans) };
+};
+
+// ── proto-native → legacy shape adapters ─────────────────────────────
+
+const adaptResourceSpans = (rs: ResourceSpans): OtlpResourceSpans => ({
+	resource: rs.resource
+		? { attributes: rs.resource.attributes.map(adaptKeyValue) }
+		: undefined,
+	scopeSpans: rs.scopeSpans.map((ss) => ({
+		scope: ss.scope
+			? { name: ss.scope.name, version: ss.scope.version }
+			: undefined,
+		spans: ss.spans.flatMap((s) => {
+			if (s.traceId.length !== 16 || s.spanId.length !== 8) return [];
+			return [
+				{
+					traceId: bytesToHex(s.traceId),
+					spanId: bytesToHex(s.spanId),
+					parentSpanId:
+						s.parentSpanId.length === 8
+							? bytesToHex(s.parentSpanId)
+							: undefined,
+					traceState: s.traceState || undefined,
+					name: s.name,
+					kind: s.kind,
+					startTimeUnixNano: bigintToString(s.startTimeUnixNano),
+					endTimeUnixNano: bigintToString(s.endTimeUnixNano),
+					attributes: s.attributes.map(adaptKeyValue),
+					droppedAttributesCount: s.droppedAttributesCount || undefined,
+					events: s.events.map((e) => ({
+						name: e.name,
+						timeUnixNano: bigintToString(e.timeUnixNano),
+						attributes: e.attributes.map(adaptKeyValue),
+						droppedAttributesCount: e.droppedAttributesCount || undefined,
+					})),
+					droppedEventsCount: s.droppedEventsCount || undefined,
+					links: s.links.flatMap((l) =>
+						l.traceId.length === 16 && l.spanId.length === 8
+							? [
+									{
+										traceId: bytesToHex(l.traceId),
+										spanId: bytesToHex(l.spanId),
+										traceState: l.traceState || undefined,
+										attributes: l.attributes.map(adaptKeyValue),
+										droppedAttributesCount:
+											l.droppedAttributesCount || undefined,
+									},
+								]
+							: [],
+					),
+					droppedLinksCount: s.droppedLinksCount || undefined,
+					status: s.status
+						? { code: s.status.code, message: s.status.message }
+						: undefined,
+				},
+			];
+		}),
+	})),
+});