@obsunified/collector 2.0.1

package/src/otlp/response.test.ts

@@ -0,0 +1,158 @@
+import { fromBinary, fromJson } from "@bufbuild/protobuf";
+import { describe, expect, it } from "vitest";
+import { ExportLogsServiceResponseSchema } from "./gen/opentelemetry/proto/collector/logs/v1/logs_service_pb.js";
+import { ExportMetricsServiceResponseSchema } from "./gen/opentelemetry/proto/collector/metrics/v1/metrics_service_pb.js";
+import { ExportTraceServiceResponseSchema } from "./gen/opentelemetry/proto/collector/trace/v1/trace_service_pb.js";
+import {
+	logsResponse,
+	metricsResponse,
+	otlpRetryableError,
+	traceResponse,
+} from "./response";
+
+const mockContext = () => {
+	type Header = string | Record<string, string>;
+	let capturedStatus: number | undefined;
+	let capturedHeaders: Record<string, string> | undefined;
+	let capturedBody: BodyInit | undefined;
+	const ctx = {
+		body: (body: BodyInit, status?: number, headers?: Header) => {
+			capturedBody = body;
+			capturedStatus = status;
+			capturedHeaders =
+				typeof headers === "string" ? undefined : (headers ?? {});
+			return new Response(body, {
+				status,
+				headers: typeof headers === "string" ? undefined : headers,
+			});
+		},
+	};
+	return {
+		ctx: ctx as never,
+		get status() {
+			return capturedStatus;
+		},
+		get headers() {
+			return capturedHeaders;
+		},
+		get body() {
+			return capturedBody;
+		},
+	};
+};
+
+describe("traceResponse", () => {
+	it("returns 200 with empty envelope on full success (JSON)", async () => {
+		const m = mockContext();
+		const res = traceResponse(m.ctx, "json");
+		expect(res.status).toBe(200);
+		expect(res.headers.get("Content-Type")).toBe("application/json");
+		const text = await res.text();
+		const parsed = JSON.parse(text);
+		// proto-JSON of empty message is {}
+		expect(parsed).toEqual({});
+		// Must round-trip via the schema
+		expect(() =>
+			fromJson(ExportTraceServiceResponseSchema, parsed),
+		).not.toThrow();
+	});
+
+	it("returns 200 with empty envelope on full success (protobuf)", async () => {
+		const m = mockContext();
+		const res = traceResponse(m.ctx, "protobuf");
+		expect(res.status).toBe(200);
+		expect(res.headers.get("Content-Type")).toBe("application/x-protobuf");
+		const bytes = new Uint8Array(await res.arrayBuffer());
+		const msg = fromBinary(ExportTraceServiceResponseSchema, bytes);
+		expect(msg.partialSuccess).toBeUndefined();
+	});
+
+	it("encodes partial_success (JSON) with rejected count", async () => {
+		const m = mockContext();
+		const res = traceResponse(m.ctx, "json", {
+			rejected: 3,
+			errorMessage: "too many",
+		});
+		expect(res.status).toBe(200);
+		const json = JSON.parse(await res.text());
+		expect(json.partialSuccess.rejectedSpans).toBe("3"); // int64 in proto-JSON = string
+		expect(json.partialSuccess.errorMessage).toBe("too many");
+		const msg = fromJson(ExportTraceServiceResponseSchema, json);
+		expect(msg.partialSuccess?.rejectedSpans).toBe(3n);
+	});
+
+	it("encodes partial_success (protobuf) with rejected count", async () => {
+		const m = mockContext();
+		const res = traceResponse(m.ctx, "protobuf", {
+			rejected: 7,
+			errorMessage: "cap",
+		});
+		const bytes = new Uint8Array(await res.arrayBuffer());
+		const msg = fromBinary(ExportTraceServiceResponseSchema, bytes);
+		expect(msg.partialSuccess?.rejectedSpans).toBe(7n);
+		expect(msg.partialSuccess?.errorMessage).toBe("cap");
+	});
+});
+
+describe("logsResponse", () => {
+	it("encodes partial_success with rejectedLogRecords", async () => {
+		const m = mockContext();
+		const res = logsResponse(m.ctx, "json", {
+			rejected: 42,
+			errorMessage: "over cap",
+		});
+		const json = JSON.parse(await res.text());
+		expect(json.partialSuccess.rejectedLogRecords).toBe("42");
+		const msg = fromJson(ExportLogsServiceResponseSchema, json);
+		expect(msg.partialSuccess?.rejectedLogRecords).toBe(42n);
+	});
+
+	it("returns empty envelope on full success", async () => {
+		const m = mockContext();
+		const res = logsResponse(m.ctx, "json");
+		expect(JSON.parse(await res.text())).toEqual({});
+	});
+});
+
+describe("metricsResponse", () => {
+	it("encodes partial_success with rejectedDataPoints", async () => {
+		const m = mockContext();
+		const res = metricsResponse(m.ctx, "json", {
+			rejected: 17,
+			errorMessage: "series cap",
+		});
+		const json = JSON.parse(await res.text());
+		expect(json.partialSuccess.rejectedDataPoints).toBe("17");
+		const msg = fromJson(ExportMetricsServiceResponseSchema, json);
+		expect(msg.partialSuccess?.rejectedDataPoints).toBe(17n);
+	});
+
+	it("returns empty envelope on full success (protobuf)", async () => {
+		const m = mockContext();
+		const res = metricsResponse(m.ctx, "protobuf");
+		expect(res.headers.get("Content-Type")).toBe("application/x-protobuf");
+		const msg = fromBinary(
+			ExportMetricsServiceResponseSchema,
+			new Uint8Array(await res.arrayBuffer()),
+		);
+		expect(msg.partialSuccess).toBeUndefined();
+	});
+});
+
+describe("otlpRetryableError", () => {
+	it("returns 503 with Retry-After by default", async () => {
+		const m = mockContext();
+		const res = otlpRetryableError(m.ctx, 503, "storage down");
+		expect(res.status).toBe(503);
+		expect(res.headers.get("Retry-After")).toBe("5");
+		expect(res.headers.get("Content-Type")).toBe("application/json");
+		expect(await res.json()).toEqual({ error: "storage down" });
+	});
+
+	it("returns 429 with a custom Retry-After", async () => {
+		const m = mockContext();
+		const res = otlpRetryableError(m.ctx, 429, "slow down", 30);
+		expect(res.status).toBe(429);
+		expect(res.headers.get("Retry-After")).toBe("30");
+	});
+});

package/src/otlp/response.ts

@@ -0,0 +1,129 @@
+/**
+ * OTLP response envelope encoders.
+ *
+ * Per the OTLP/HTTP spec, successful responses are always `200 OK` — whether
+ * fully accepted or partial — with a serialized `Export{Signal}ServiceResponse`
+ * body. Wire format mirrors the request (JSON in → JSON out; protobuf in →
+ * protobuf out). `partial_success` is set only when records were actually
+ * rejected; an empty partial_success block is semantically identical to a
+ * fully-accepted response and wastes bytes.
+ *
+ * See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#failures
+ */
+
+import { create, type DescMessage, toBinary, toJson } from "@bufbuild/protobuf";
+import type { Context } from "hono";
+import type { OtlpWireFormat } from "./decode";
+import {
+	ExportLogsPartialSuccessSchema,
+	ExportLogsServiceResponseSchema,
+} from "./gen/opentelemetry/proto/collector/logs/v1/logs_service_pb.js";
+import {
+	ExportMetricsPartialSuccessSchema,
+	ExportMetricsServiceResponseSchema,
+} from "./gen/opentelemetry/proto/collector/metrics/v1/metrics_service_pb.js";
+import {
+	ExportTracePartialSuccessSchema,
+	ExportTraceServiceResponseSchema,
+} from "./gen/opentelemetry/proto/collector/trace/v1/trace_service_pb.js";
+
+export interface PartialSuccess {
+	rejected: number;
+	errorMessage: string;
+}
+
+/**
+ * Return a retryable error response per the OTLP/HTTP spec. Sets the
+ * `Retry-After` header which OTel SDKs respect for exponential backoff.
+ *
+ *   429 Too Many Requests — per-client throttling
+ *   503 Service Unavailable — server-side overload or transient failure
+ *
+ * The body is a plain JSON error, not an OTLP envelope — the spec permits
+ * any body (or empty) for failure cases. `Retry-After` uses integer seconds.
+ */
+export const otlpRetryableError = (
+	c: Context,
+	status: 429 | 503,
+	message: string,
+	retryAfterSec = 5,
+) =>
+	c.body(JSON.stringify({ error: message }), status, {
+		"Content-Type": "application/json",
+		"Retry-After": String(retryAfterSec),
+	});
+
+export const traceResponse = (
+	c: Context,
+	wireFormat: OtlpWireFormat,
+	partial?: PartialSuccess,
+) => {
+	const msg = create(
+		ExportTraceServiceResponseSchema,
+		partial
+			? {
+					partialSuccess: create(ExportTracePartialSuccessSchema, {
+						rejectedSpans: BigInt(partial.rejected),
+						errorMessage: partial.errorMessage,
+					}),
+				}
+			: {},
+	);
+	return encode(c, wireFormat, ExportTraceServiceResponseSchema, msg);
+};
+
+export const logsResponse = (
+	c: Context,
+	wireFormat: OtlpWireFormat,
+	partial?: PartialSuccess,
+) => {
+	const msg = create(
+		ExportLogsServiceResponseSchema,
+		partial
+			? {
+					partialSuccess: create(ExportLogsPartialSuccessSchema, {
+						rejectedLogRecords: BigInt(partial.rejected),
+						errorMessage: partial.errorMessage,
+					}),
+				}
+			: {},
+	);
+	return encode(c, wireFormat, ExportLogsServiceResponseSchema, msg);
+};
+
+export const metricsResponse = (
+	c: Context,
+	wireFormat: OtlpWireFormat,
+	partial?: PartialSuccess,
+) => {
+	const msg = create(
+		ExportMetricsServiceResponseSchema,
+		partial
+			? {
+					partialSuccess: create(ExportMetricsPartialSuccessSchema, {
+						rejectedDataPoints: BigInt(partial.rejected),
+						errorMessage: partial.errorMessage,
+					}),
+				}
+			: {},
+	);
+	return encode(c, wireFormat, ExportMetricsServiceResponseSchema, msg);
+};
+
+const encode = <T extends DescMessage>(
+	c: Context,
+	wireFormat: OtlpWireFormat,
+	schema: T,
+	msg: Parameters<typeof toBinary<T>>[1],
+) => {
+	if (wireFormat === "protobuf") {
+		const bytes = toBinary(schema, msg);
+		return c.body(bytes, 200, {
+			"Content-Type": "application/x-protobuf",
+		});
+	}
+	const json = toJson(schema, msg);
+	return c.body(JSON.stringify(json), 200, {
+		"Content-Type": "application/json",
+	});
+};

package/src/plugins/_context.ts

@@ -0,0 +1,15 @@
+/**
+ * Extract the project id set by ingest-auth or dashboard-auth middleware.
+ * Returns 'default' when no middleware has attached one (e.g. unauth dev).
+ *
+ * Uses a loose Context typing so plugins don't need to re-declare `Variables`
+ * on every route registration. The underlying value is always a string set by
+ * the auth middlewares.
+ */
+interface ProjectContext {
+	get(key: "projectId"): string | undefined;
+}
+
+export function getProjectId(c: ProjectContext): string {
+	return (c.get("projectId") as string | undefined) || "default";
+}

package/src/plugins/action-graph-processor/enrichers.ts

@@ -0,0 +1,45 @@
+import type { StoredSpan } from "@obsunified/types";
+
+export interface ActionEnricherPlugin {
+	name: string;
+	enrichActionRecord?(
+		record: Record<string, unknown>,
+		span: StoredSpan,
+		attributes: Record<string, unknown>,
+	): void | Promise<void>;
+	enrichAgentRunRecord?(
+		record: Record<string, unknown>,
+		span: StoredSpan,
+		attributes: Record<string, unknown>,
+	): void | Promise<void>;
+	enrichToolCallRecord?(
+		record: Record<string, unknown>,
+		span: StoredSpan,
+		attributes: Record<string, unknown>,
+	): void | Promise<void>;
+	enrichRetrievalRecord?(
+		record: Record<string, unknown>,
+		span: StoredSpan,
+		attributes: Record<string, unknown>,
+	): void | Promise<void>;
+	enrichEvalRecord?(
+		record: Record<string, unknown>,
+		span: StoredSpan,
+		attributes: Record<string, unknown>,
+	): void | Promise<void>;
+	enrichArtifactRecord?(
+		record: Record<string, unknown>,
+		span: StoredSpan,
+		attributes: Record<string, unknown>,
+	): void | Promise<void>;
+}
+
+export const enricherPlugins: ActionEnricherPlugin[] = [];
+
+export function registerActionEnricherPlugin(plugin: ActionEnricherPlugin) {
+	enricherPlugins.push(plugin);
+}
+
+export function clearActionEnricherPlugins() {
+	enricherPlugins.length = 0;
+}

package/src/plugins/action-graph-processor/redaction.ts

@@ -0,0 +1,84 @@
+export interface RedactionContext {
+	projectId: string;
+	actionId: string;
+	traceId: string;
+	spanId: string;
+	kind: "tool_call" | "retrieval" | "eval" | "artifact" | "agent_run";
+	fieldName: "args" | "result" | "query" | "documents" | "content";
+}
+
+export interface PayloadRedactorPlugin {
+	name: string;
+	redact(
+		value: unknown,
+		context: RedactionContext,
+	): unknown | Promise<unknown> | undefined;
+}
+
+const redactionPlugins: PayloadRedactorPlugin[] = [];
+
+export function registerRedactionPlugin(plugin: PayloadRedactorPlugin) {
+	redactionPlugins.push(plugin);
+}
+
+export function clearRedactionPlugins() {
+	redactionPlugins.length = 0;
+}
+
+// Default redactor that performs sensitive key scrubbing.
+const DEFAULT_REDACT_KEYS = new Set([
+	"authorization",
+	"cookie",
+	"set-cookie",
+	"password",
+	"passwd",
+	"secret",
+	"token",
+	"api-key",
+	"x-api-key",
+	"email",
+	"enduser.id",
+]);
+
+function shouldRedactKey(key: string): boolean {
+	const normalized = key.toLowerCase();
+	if (DEFAULT_REDACT_KEYS.has(normalized)) return true;
+	for (const k of DEFAULT_REDACT_KEYS) {
+		if (normalized.endsWith(k)) return true;
+	}
+	return false;
+}
+
+function redactObj(val: unknown): unknown {
+	if (Array.isArray(val)) {
+		return val.map(redactObj);
+	}
+	if (val && typeof val === "object") {
+		const nextEntries = Object.entries(val).map(([key, nestedValue]) => {
+			if (shouldRedactKey(key)) {
+				return [key, "[REDACTED]"] as const;
+			}
+			return [key, redactObj(nestedValue)] as const;
+		});
+		return Object.fromEntries(nextEntries);
+	}
+	return val;
+}
+
+export async function runRedaction(
+	value: unknown,
+	context: RedactionContext,
+): Promise<unknown> {
+	for (const plugin of redactionPlugins) {
+		try {
+			const res = await plugin.redact(value, context);
+			if (res !== undefined) {
+				return res;
+			}
+		} catch (err) {
+			console.error(`[redaction-plugin:${plugin.name}] failed`, err);
+		}
+	}
+
+	return redactObj(value);
+}