@xeonr/upload-pool-sdk 1.3.0 → 1.4.1

package/src/tracing.ts

@@ -0,0 +1,363 @@
+/**
+ * OpenTelemetry tracing setup for the worker SDK.
+ *
+ * Spans are exported via OTLP/HTTP-JSON to pipeline-api's `/v1/traces`
+ * receiver, authenticated with the pool's queue token. The pipeline-api
+ * receiver decodes the OTLP envelope and writes spans to its MySQL trace
+ * store, which backs the `/admin/workflow-run` admin UI — so worker
+ * callbacks nest naturally under the dispatching parseUpload trace.
+ *
+ * Design notes (see also: discussion in epic plan):
+ *   - The TracerProvider is private to the SDK. It does *not* register
+ *     globally — calling `trace.getTracer(...)` outside the SDK won't pick
+ *     it up. This avoids interfering with any OTel setup the host
+ *     application already has (e.g. its own collector → Grafana
+ *     pipeline). The exporter ships SDK-emitted spans only.
+ *   - `wf.run_id` is stamped on every emitted span as an attribute,
+ *     mirrored from the job envelope's `wf_run_id` field. The pipeline-api
+ *     receiver uses this attribute to associate the span with the right
+ *     workflow run in `workflow_run_spans`.
+ *   - The W3C traceparent propagator runs on outbound RPC calls *only when
+ *     targeting the pipeline endpoint*. The uploads-api side has its own
+ *     OTel pipeline and we don't want to inject our trace_id into its
+ *     trees. See the interceptor in rpc-clients.ts.
+ *
+ * Body sanitization for `rpc.request` / `rpc.response` span events lives
+ * in the interceptor — keys matching /(token|secret|password|authorization
+ * |api[_-]?key|signature|sig|cookie)/i are redacted; payloads truncated
+ * to MAX_BODY_EVENT_BYTES.
+ */
+import {
+	context,
+	createContextKey,
+	trace,
+	type Context,
+	type Span,
+	type SpanContext,
+	type Tracer,
+	TraceFlags,
+	SpanKind,
+	SpanStatusCode,
+} from "@opentelemetry/api";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { Resource } from "@opentelemetry/resources";
+import {
+	BatchSpanProcessor,
+	BasicTracerProvider,
+} from "@opentelemetry/sdk-trace-base";
+import {
+	ATTR_SERVICE_NAME,
+	ATTR_SERVICE_VERSION,
+} from "@opentelemetry/semantic-conventions";
+
+const TRACER_NAME = "@xeonr/upload-pool-sdk";
+
+/**
+ * OTel context key carrying per-job attributes. Set once at the top of
+ * Pool.handleDispatch via `withJobContext()` and read by helpers that
+ * create child spans (interceptors, ctx.* spans) so every emitted span
+ * carries `pipeline.run_id` + identifiers — without it the trace-store
+ * receiver drops the span as orphan.
+ *
+ * The OTel `Span` API surface deliberately doesn't expose attributes
+ * back to read sites (only set/record/event), so we can't inherit them
+ * from a parent span at child-creation time. Carrying them in the
+ * Context object is the canonical OTel pattern.
+ */
+export const JOB_ATTRS_KEY = createContextKey("uplim.job_attrs");
+
+export interface JobAttrs {
+	runId: string;
+	jobId: string;
+	uploadId: string;
+	urn: string;
+	workerId: string;
+}
+
+export function getJobAttrs(): JobAttrs | undefined {
+	return context.active().getValue(JOB_ATTRS_KEY) as JobAttrs | undefined;
+}
+
+export function withJobContext<T>(attrs: JobAttrs, fn: () => T): T {
+	return context.with(context.active().setValue(JOB_ATTRS_KEY, attrs), fn);
+}
+
+export const SPAN_ATTR = {
+	JOB_ID: "iq.job_id",
+	UPLOAD_ID: "iq.upload_id",
+	URN: "iq.content_type_urn",
+	WORKER_ID: "iq.worker_id",
+	POOL_TOKEN_PREFIX: "iq.pool_token_prefix",
+	// Matches pipeline-api's TRACE_ATTRIBUTE.RUN_ID. The trace-store
+	// exporter reads this attribute to associate each span with the
+	// right workflow_run in MySQL (workflow_run_spans.run_id). Worker
+	// spans MUST carry it or they're dropped on the floor by the
+	// pipeline-api receiver.
+	RUN_ID: "pipeline.run_id",
+	RPC_METHOD: "rpc.method",
+	RPC_SERVICE: "rpc.service",
+	RPC_TARGET: "rpc.target",
+	RPC_CODE: "rpc.code",
+	RPC_DURATION_MS: "rpc.duration_ms",
+	S3_BYTES: "s3.bytes",
+	S3_URL_ORIGIN: "s3.url_origin",
+	S3_OUTCOME: "s3.outcome",
+} as const;
+
+export interface TracingInitConfig {
+	pipelineEndpoint: string;
+	poolToken: string;
+	workerId: string;
+	sdkVersion: string;
+	/**
+	 * If true (default), the SDK initializes its own TracerProvider and
+	 * exports spans to pipeline-api. Set false to disable all telemetry
+	 * — handlers that read `trace.getTracer(...)` will still work via the
+	 * global no-op provider, but nothing flushes anywhere.
+	 */
+	enabled?: boolean;
+}
+
+export interface TracingHandle {
+	tracer: Tracer;
+	shutdown: () => Promise<void>;
+	/**
+	 * Apply a job envelope's trace_context + wf_run_id to the active context.
+	 * The returned Context, passed to `tracer.startActiveSpan(..., ctx, fn)`,
+	 * roots a new span under the dispatching parseUpload activity.
+	 */
+	contextFromEnvelope: (traceparent: string, runId: string) => Context;
+}
+
+let activeHandle: TracingHandle | null = null;
+
+export function initTracing(cfg: TracingInitConfig): TracingHandle {
+	if (activeHandle) {
+		return activeHandle;
+	}
+
+	const enabled = cfg.enabled !== false;
+	if (!enabled) {
+		const tracer = trace.getTracer(TRACER_NAME, cfg.sdkVersion);
+		activeHandle = {
+			tracer,
+			shutdown: async () => {},
+			contextFromEnvelope: () => context.active(),
+		};
+		return activeHandle;
+	}
+
+	const resource = new Resource({
+		[ATTR_SERVICE_NAME]: TRACER_NAME,
+		[ATTR_SERVICE_VERSION]: cfg.sdkVersion,
+		[SPAN_ATTR.WORKER_ID]: cfg.workerId,
+	});
+
+	// OTLP/HTTP-JSON to pipeline-api's receiver. JSON over protobuf because
+	// it lets the receiver decode without pulling in @opentelemetry/
+	// otlp-transformer's protobuf runtime — the wire size penalty is fine
+	// for the volume the SDK emits (≈10 spans per job, ≈40 KB per job).
+	const exporter = new OTLPTraceExporter({
+		url: `${cfg.pipelineEndpoint.replace(/\/+$/, "")}/v1/traces`,
+		headers: {
+			authorization: `Bearer ${cfg.poolToken}`,
+		},
+		// Generous timeout — pipeline-api's receiver writes through to MySQL
+		// synchronously and we'd rather hold the export than drop a span on
+		// a slow ack.
+		timeoutMillis: 10_000,
+	});
+
+	const processor = new BatchSpanProcessor(exporter, {
+		// Default batch settings keep us flushing every 5s or 512 spans.
+		// We prefer faster flushes for the per-job RPC use case so the
+		// admin UI sees the latest activity within a couple seconds.
+		scheduledDelayMillis: 1_000,
+		maxQueueSize: 1024,
+		maxExportBatchSize: 128,
+	});
+
+	const provider = new BasicTracerProvider({
+		resource,
+		spanProcessors: [processor],
+	});
+
+	const tracer = provider.getTracer(TRACER_NAME, cfg.sdkVersion);
+
+	const handle: TracingHandle = {
+		tracer,
+		shutdown: async () => {
+			try {
+				await provider.shutdown();
+			} catch {
+				// Worker exit is best-effort — never block on export.
+			}
+		},
+		contextFromEnvelope: (traceparent: string, runId: string) => {
+			let ctx = context.active();
+			// Ensure every span produced under this context carries the
+			// runId attribute via baggage-like behaviour. We stamp it on
+			// each span at creation site instead of relying on baggage
+			// because the trace store reads it from span attributes.
+			if (!traceparent) {
+				return ctx;
+			}
+			const parsed = parseTraceparent(traceparent);
+			if (!parsed) {
+				return ctx;
+			}
+			ctx = trace.setSpanContext(ctx, parsed);
+			// runId is consumed by the caller and stamped explicitly on
+			// each span via stampJobAttributes(); we don't store it in
+			// the OTel context because there's no cross-span attribute
+			// mechanism in OTel without using a custom propagator.
+			void runId;
+			return ctx;
+		},
+	};
+
+	activeHandle = handle;
+	return handle;
+}
+
+export function getTracingHandle(): TracingHandle | null {
+	return activeHandle;
+}
+
+export async function shutdownTracing(): Promise<void> {
+	if (activeHandle) {
+		await activeHandle.shutdown();
+		activeHandle = null;
+	}
+}
+
+// --- traceparent helpers --------------------------------------------------
+
+/**
+ * Parse a W3C `traceparent` header into an OTel SpanContext.
+ *
+ * Format: `00-<32 hex trace id>-<16 hex span id>-<2 hex flags>`. Returns
+ * null on malformed input. Used by the SDK to root iq.job spans under
+ * whatever pipeline activity dispatched the job.
+ */
+export function parseTraceparent(value: string): SpanContext | null {
+	if (!value) return null;
+	const parts = value.trim().split("-");
+	if (parts.length !== 4) return null;
+	const [version, traceId, spanId, flagsHex] = parts;
+	if (version !== "00") return null;
+	if (!/^[0-9a-f]{32}$/.test(traceId)) return null;
+	if (!/^[0-9a-f]{16}$/.test(spanId)) return null;
+	if (!/^[0-9a-f]{2}$/.test(flagsHex)) return null;
+	const flags = Number.parseInt(flagsHex, 16);
+	return {
+		traceId,
+		spanId,
+		traceFlags: flags as TraceFlags,
+		isRemote: true,
+	};
+}
+
+/**
+ * Serialise a SpanContext as a W3C traceparent header value.
+ */
+export function spanContextToTraceparent(sc: SpanContext): string {
+	const flags = sc.traceFlags.toString(16).padStart(2, "0");
+	return `00-${sc.traceId}-${sc.spanId}-${flags}`;
+}
+
+// --- body sanitization (req/resp on RPC span events) ----------------------
+
+const MAX_BODY_EVENT_BYTES = 4 * 1024;
+const REDACTED_KEY_PATTERN =
+	/(token|secret|password|authorization|api[_-]?key|signature|^sig$|cookie)/i;
+const REDACTED_VALUE = "<redacted>";
+
+/**
+ * Sanitize an arbitrary RPC request/response object for inclusion as a
+ * span event attribute. Same shape as the pipeline-api's tracing/sanitize
+ * (4 KB cap, key-pattern redaction) so the two ends produce consistent
+ * payloads in the admin UI.
+ */
+export function sanitizeRpcBody(value: unknown): string {
+	try {
+		const cloned = redactDeep(value, 0);
+		const serialised = JSON.stringify(cloned);
+		if (serialised.length <= MAX_BODY_EVENT_BYTES) {
+			return serialised;
+		}
+		return `${serialised.slice(0, MAX_BODY_EVENT_BYTES)}...<truncated>`;
+	} catch {
+		return "<unserialisable>";
+	}
+}
+
+function redactDeep(value: unknown, depth: number): unknown {
+	if (depth > 6) return "<max-depth>";
+	if (value === null || value === undefined) return value;
+	if (value instanceof Uint8Array) {
+		return `<bytes:${value.byteLength}>`;
+	}
+	if (Array.isArray(value)) {
+		return value.slice(0, 50).map((v) => redactDeep(v, depth + 1));
+	}
+	if (typeof value === "object") {
+		const out: Record<string, unknown> = {};
+		const obj = value as Record<string, unknown>;
+		const keys = Object.keys(obj).slice(0, 50);
+		for (const k of keys) {
+			if (REDACTED_KEY_PATTERN.test(k)) {
+				out[k] = REDACTED_VALUE;
+			} else {
+				out[k] = redactDeep(obj[k], depth + 1);
+			}
+		}
+		return out;
+	}
+	if (typeof value === "string") {
+		// Heuristic: very long strings are likely embedded payloads; clamp.
+		return value.length > 1024 ? `${value.slice(0, 1024)}...` : value;
+	}
+	return value;
+}
+
+// --- span helpers --------------------------------------------------------
+
+/**
+ * Set common job-scoped attributes on a span. Called by every site that
+ * creates a per-job span so RUN_ID / JOB_ID / UPLOAD_ID / URN are
+ * consistently present (the trace-store receiver requires
+ * `pipeline.run_id` to route the span into the right workflow run).
+ *
+ * Pulls the attribute values from the active OTel context if not
+ * passed explicitly, so RPC / ctx.* spans created mid-handler inherit
+ * them without needing to know about the envelope. The Pool root sets
+ * the context via `withJobContext` at the start of handleDispatch.
+ */
+export function stampJobAttributes(
+	span: Span,
+	attrs?: Partial<JobAttrs>,
+): void {
+	const fromCtx = getJobAttrs();
+	const runId = attrs?.runId ?? fromCtx?.runId ?? "";
+	const jobId = attrs?.jobId ?? fromCtx?.jobId ?? "";
+	const uploadId = attrs?.uploadId ?? fromCtx?.uploadId;
+	const urn = attrs?.urn ?? fromCtx?.urn;
+	const workerId = attrs?.workerId ?? fromCtx?.workerId;
+	if (runId) span.setAttribute(SPAN_ATTR.RUN_ID, runId);
+	if (jobId) span.setAttribute(SPAN_ATTR.JOB_ID, jobId);
+	if (uploadId) span.setAttribute(SPAN_ATTR.UPLOAD_ID, uploadId);
+	if (urn) span.setAttribute(SPAN_ATTR.URN, urn);
+	if (workerId) span.setAttribute(SPAN_ATTR.WORKER_ID, workerId);
+}
+
+export function recordSpanError(span: Span, err: unknown): void {
+	if (err instanceof Error) {
+		span.recordException(err);
+		span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
+	} else {
+		span.setStatus({ code: SpanStatusCode.ERROR, message: String(err) });
+	}
+}
+
+export { SpanKind, SpanStatusCode };

package/src/types.ts

@@ -60,6 +60,14 @@ export interface PoolConfig {
 	logger?: import("./logger.js").Logger;
 	/** Max concurrent in-flight jobs. Defaults to 1. */
 	concurrency?: number;
+	/**
+	 * Tracing config. The SDK ships spans to pipeline-api's /v1/traces
+	 * receiver for the admin UI. Disable with `{ enabled: false }` to
+	 * skip the OTel SDK setup entirely (e.g. in tests).
+	 */
+	tracing?: {
+		enabled?: boolean;
+	};
 }
 
 export type JobHandler = (ctx: JobContext) => Promise<void>;