@graphoria/server 0.3.0 → 0.5.0

package/src/observability/tracing.ts

@@ -0,0 +1,421 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+
+import type { Logger } from "pino";
+
+import { version } from "../../package.json";
+import { logger } from "../logging";
+
+export type SpanKind = "internal" | "server" | "client" | "producer" | "consumer";
+
+export type SpanStatus = "unset" | "ok" | "error";
+
+export type AttributeValue = string | number | boolean;
+
+export type Attributes = Record<string, AttributeValue | undefined>;
+
+/** What rides in the async context: three scalars, never the span itself. */
+export type SpanContext = { traceId: string; spanId: string; sampled: boolean };
+
+export type SpanOptions = {
+  kind?: SpanKind;
+  attributes?: Attributes;
+  /** Overrides the active context — the inbound `traceparent`, or a deliberate root. */
+  parent?: SpanContext | null;
+};
+
+export type Span = {
+  readonly context: SpanContext;
+  setAttribute(key: string, value: AttributeValue | undefined): void;
+  setStatus(status: SpanStatus, message?: string): void;
+  recordError(error: unknown): void;
+  end(): void;
+};
+
+export type FinishedSpan = {
+  traceId: string;
+  spanId: string;
+  parentSpanId?: string;
+  name: string;
+  kind: SpanKind;
+  startTimeUnixNano: string;
+  endTimeUnixNano: string;
+  attributes: Attributes;
+  status: SpanStatus;
+  statusMessage?: string;
+};
+
+export type Tracer = {
+  startSpan(name: string, options?: SpanOptions): Span;
+  /** Runs `fn` with `span` active, ends it, and marks it errored if `fn` throws. */
+  withSpan<T>(name: string, options: SpanOptions, fn: (span: Span) => T | Promise<T>): Promise<T>;
+  active(): SpanContext | undefined;
+  flush(): Promise<void>;
+};
+
+export type TracerOptions = {
+  endpoint: string;
+  /** `k=v,k2=v2`, sent on every export — an API key for a hosted collector. */
+  headers?: string;
+  serviceName: string;
+  serviceVersion: string;
+  sampleRatio?: number;
+  maxBatch?: number;
+  maxQueue?: number;
+  /** `0` starts no timer, which is how a test drives the exporter by hand. */
+  exportIntervalMs?: number;
+  /** Test seam: where an export failure and a queue overflow are reported. */
+  log?: Pick<Logger, "warn">;
+  /** Test seam: the transport. Production POSTs with `fetch`. */
+  send?: (
+    url: string,
+    init: { method: string; headers: Record<string, string>; body: string },
+  ) => Promise<void>;
+};
+
+const MAX_BATCH = 512;
+const MAX_QUEUE = 2048;
+const EXPORT_INTERVAL_MS = 5000;
+
+const KIND_CODES: Record<SpanKind, number> = {
+  internal: 1,
+  server: 2,
+  client: 3,
+  producer: 4,
+  consumer: 5,
+};
+
+const STATUS_CODES: Record<SpanStatus, number> = { unset: 0, ok: 1, error: 2 };
+
+const ZERO_TRACE_ID = "0".repeat(32);
+const ZERO_SPAN_ID = "0".repeat(16);
+
+const TRACEPARENT = /^00-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$/;
+
+// A span is minted per statement, so the id path is on the request's critical
+// path: a lookup table beats formatting each byte.
+const HEX = Array.from({ length: 256 }, (_, byte) => byte.toString(16).padStart(2, "0"));
+
+const hex = (bytes: number) => {
+  const buffer = crypto.getRandomValues(new Uint8Array(bytes));
+  let id = "";
+  for (let index = 0; index < bytes; index++) id += HEX[buffer[index]!];
+  return id;
+};
+
+// An all-zero id is invalid per the W3C spec and a collector rejects the span,
+// so the (vanishing) draw is re-rolled rather than exported.
+const newTraceId = () => {
+  let id = hex(16);
+  while (id === ZERO_TRACE_ID) id = hex(16);
+  return id;
+};
+
+const newSpanId = () => {
+  let id = hex(8);
+  while (id === ZERO_SPAN_ID) id = hex(8);
+  return id;
+};
+
+export const parseTraceparent = (header: string | null | undefined): SpanContext | undefined => {
+  if (!header) return undefined;
+  const match = TRACEPARENT.exec(header);
+  if (!match) return undefined;
+  const [, traceId, spanId, flags] = match as unknown as [string, string, string, string];
+  if (traceId === ZERO_TRACE_ID || spanId === ZERO_SPAN_ID) return undefined;
+  return { traceId, spanId, sampled: (Number.parseInt(flags, 16) & 1) === 1 };
+};
+
+export const formatTraceparent = (context: SpanContext): string =>
+  `00-${context.traceId}-${context.spanId}-${context.sampled ? "01" : "00"}`;
+
+const parseHeaders = (headers: string): Record<string, string> => {
+  const parsed: Record<string, string> = {};
+  for (const entry of headers.split(",")) {
+    const index = entry.indexOf("=");
+    if (index <= 0) continue;
+    const key = entry.slice(0, index).trim();
+    if (key) parsed[key] = entry.slice(index + 1).trim();
+  }
+  return parsed;
+};
+
+type AnyValue =
+  | { stringValue: string }
+  | { boolValue: boolean }
+  | { intValue: string }
+  | { doubleValue: number };
+
+const renderAttributes = (attributes: Attributes) =>
+  Object.entries(attributes).flatMap<{ key: string; value: AnyValue }>(([key, value]) => {
+    if (value === undefined) return [];
+    if (typeof value === "string") return [{ key, value: { stringValue: value } }];
+    if (typeof value === "boolean") return [{ key, value: { boolValue: value } }];
+    // OTLP/JSON carries an int64 as a string; a fractional number is a double.
+    return [
+      {
+        key,
+        value: Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value },
+      },
+    ];
+  });
+
+const renderSpan = (span: FinishedSpan) => ({
+  traceId: span.traceId,
+  spanId: span.spanId,
+  ...(span.parentSpanId ? { parentSpanId: span.parentSpanId } : {}),
+  name: span.name,
+  kind: KIND_CODES[span.kind],
+  startTimeUnixNano: span.startTimeUnixNano,
+  endTimeUnixNano: span.endTimeUnixNano,
+  attributes: renderAttributes(span.attributes),
+  status: {
+    code: STATUS_CODES[span.status],
+    ...(span.statusMessage ? { message: span.statusMessage } : {}),
+  },
+});
+
+const renderPayload = (spans: FinishedSpan[], serviceName: string, serviceVersion: string) => ({
+  resourceSpans: [
+    {
+      resource: {
+        attributes: [
+          { key: "service.name", value: { stringValue: serviceName } },
+          { key: "service.version", value: { stringValue: serviceVersion } },
+        ],
+      },
+      scopeSpans: [{ scope: { name: "graphoria" }, spans: spans.map(renderSpan) }],
+    },
+  ],
+});
+
+const defaultSend: NonNullable<TracerOptions["send"]> = async (url, init) => {
+  const response = await fetch(url, init);
+  if (!response.ok) throw new Error(`collector answered HTTP ${response.status}`);
+};
+
+const errorMessage = (error: unknown) => (error instanceof Error ? error.message : String(error));
+
+/**
+ * One async context for every tracer in the process. It holds three scalars
+ * rather than the span, so a retained context cannot pin a finished span's
+ * attributes in memory.
+ */
+const store = new AsyncLocalStorage<SpanContext>();
+
+/**
+ * A tracer with no dependencies: W3C ids, OTLP/HTTP JSON, and a bounded export
+ * queue. Nothing here reads process state — everything it needs is closed over,
+ * so it is unit-testable without booting a server.
+ */
+export const createTracer = ({
+  endpoint,
+  headers = "",
+  serviceName,
+  serviceVersion,
+  sampleRatio = 1,
+  maxBatch = MAX_BATCH,
+  maxQueue = MAX_QUEUE,
+  exportIntervalMs = EXPORT_INTERVAL_MS,
+  send = defaultSend,
+  log = logger("tracing"),
+}: TracerOptions): Tracer => {
+  const url = `${endpoint.replace(/\/+$/, "")}/v1/traces`;
+  const exportHeaders = { "content-type": "application/json", ...parseHeaders(headers) };
+
+  let queue: FinishedSpan[] = [];
+  let dropped = 0;
+
+  const post = async (batch: FinishedSpan[]) => {
+    try {
+      await send(url, {
+        method: "POST",
+        headers: exportHeaders,
+        body: JSON.stringify(renderPayload(batch, serviceName, serviceVersion)),
+      });
+    } catch (error) {
+      // Dropped, never retried: an unbounded retry buffer is the failure mode
+      // that turns an observability feature into the outage.
+      log.warn({ err: error, spans: batch.length }, "span export failed");
+    }
+  };
+
+  const flush = async () => {
+    if (queue.length === 0) return;
+    const batch = queue;
+    queue = [];
+    if (dropped > 0) {
+      log.warn({ dropped }, "span queue full, oldest spans dropped");
+      dropped = 0;
+    }
+    await post(batch);
+  };
+
+  if (exportIntervalMs > 0) {
+    // Unref'd: there is no graceful-shutdown hook to clear it, and a live timer
+    // would keep the process alive after the server closes.
+    setInterval(() => void flush(), exportIntervalMs).unref();
+  }
+
+  const enqueue = (span: FinishedSpan) => {
+    queue.push(span);
+    if (queue.length > maxQueue) {
+      dropped += queue.length - maxQueue;
+      queue = queue.slice(queue.length - maxQueue);
+    }
+    if (queue.length >= maxBatch) void flush();
+  };
+
+  const shouldSample = (parent: SpanContext | undefined) => {
+    if (parent) return parent.sampled;
+    if (sampleRatio >= 1) return true;
+    if (sampleRatio <= 0) return false;
+    return Math.random() < sampleRatio;
+  };
+
+  const startSpan = (name: string, options: SpanOptions = {}): Span => {
+    const parent = options.parent === undefined ? store.getStore() : (options.parent ?? undefined);
+    const sampled = shouldSample(parent);
+    const context: SpanContext = {
+      traceId: parent?.traceId ?? newTraceId(),
+      spanId: newSpanId(),
+      sampled,
+    };
+
+    // Wall clock for the start, monotonic for the length, so a span's duration
+    // does not drift with the system clock.
+    const startedAtNs = BigInt(Date.now()) * 1_000_000n;
+    const startedAt = Bun.nanoseconds();
+    const attributes: Attributes = { ...options.attributes };
+    let status: SpanStatus = "unset";
+    let statusMessage: string | undefined;
+    let ended = false;
+
+    return {
+      context,
+      setAttribute: (key, value) => {
+        attributes[key] = value;
+      },
+      setStatus: (next, message) => {
+        status = next;
+        statusMessage = message;
+      },
+      recordError: (error) => {
+        status = "error";
+        statusMessage = errorMessage(error);
+      },
+      end: () => {
+        if (ended) return;
+        ended = true;
+        if (!sampled) return;
+        enqueue({
+          traceId: context.traceId,
+          spanId: context.spanId,
+          ...(parent ? { parentSpanId: parent.spanId } : {}),
+          name,
+          kind: options.kind ?? "internal",
+          startTimeUnixNano: String(startedAtNs),
+          endTimeUnixNano: String(startedAtNs + BigInt(Bun.nanoseconds() - startedAt)),
+          attributes,
+          status,
+          ...(statusMessage ? { statusMessage } : {}),
+        });
+      },
+    };
+  };
+
+  return {
+    startSpan,
+    withSpan: async (name, options, fn) => {
+      const span = startSpan(name, options);
+      try {
+        return await store.run(span.context, () => fn(span));
+      } catch (error) {
+        span.recordError(error);
+        throw error;
+      } finally {
+        span.end();
+      }
+    },
+    active: () => store.getStore(),
+    flush,
+  };
+};
+
+/**
+ * Off until boot turns it on, so a handler or an executor used outside a booted
+ * server records nothing and never enters the async store.
+ */
+let enabled = false;
+
+let settings: Omit<TracerOptions, "serviceVersion"> = {
+  endpoint: "http://localhost:4318",
+  headers: "",
+  serviceName: "graphoria",
+  sampleRatio: 1,
+};
+
+let override: Tracer | null = null;
+let instance: Tracer | null = null;
+
+export const configureTracing = (next: {
+  enabled: boolean;
+  endpoint: string;
+  headers: string;
+  serviceName: string;
+  sampleRatio: number;
+}): void => {
+  enabled = next.enabled;
+  settings = {
+    endpoint: next.endpoint,
+    headers: next.headers,
+    serviceName: next.serviceName,
+    sampleRatio: next.sampleRatio,
+  };
+  instance = null;
+};
+
+/** Test seam: pass `null` to drop the process tracer and its queued spans. */
+export const setTracer = (tracer: Tracer | null): void => {
+  override = tracer;
+  instance = null;
+};
+
+const tracer = (): Tracer => {
+  if (override) return override;
+  if (!instance) instance = createTracer({ ...settings, serviceVersion: version });
+  return instance;
+};
+
+export const isTracingEnabled = (): boolean => enabled;
+
+export const startSpan = (name: string, options?: SpanOptions): Span | undefined => {
+  if (!enabled) return undefined;
+  return tracer().startSpan(name, options);
+};
+
+export const withSpan = async <T>(
+  name: string,
+  options: SpanOptions,
+  fn: (span: Span | undefined) => T | Promise<T>,
+): Promise<T> => {
+  if (!enabled) return fn(undefined);
+  return tracer().withSpan(name, options, fn);
+};
+
+/**
+ * Runs `fn` with `span` active, leaving its lifetime to the caller. A span that
+ * is never ended is never exported, which is how a websocket upgrade — a
+ * request that is never answered — produces no span at all.
+ */
+export const withActiveSpan = <T>(span: Span | undefined, fn: () => T): T =>
+  span ? store.run(span.context, fn) : fn();
+
+export const activeSpanContext = (): SpanContext | undefined => {
+  if (!enabled) return undefined;
+  return tracer().active();
+};
+
+export const flushSpans = (): Promise<void> => {
+  if (!enabled) return Promise.resolve();
+  return tracer().flush();
+};

package/src/queues/kafka.ts

@@ -6,6 +6,8 @@ import type { KafkaConfig } from "../types/zod/queue";
 
 import { queryEventEmitter } from "../configuration/gql/handleGraphQLSubscriptionFactory";
 import { logger } from "../logging";
+import { incMetric } from "../observability/metrics";
+import { startSpan } from "../observability/tracing";
 
 // ============================================================================
 // Reconnection Configuration
@@ -58,9 +60,19 @@ export const startConsumer = async (
 
   await consumer.run({
     eachMessage: async (payload) => {
+      const record = (outcome: "success" | "error") =>
+        incMetric("graphoria_queue_messages_consumed_total", {
+          broker: "kafka",
+          queue: connectionName,
+          consumer: name,
+          outcome,
+        });
+
       try {
         sendMessage(connectionName, name, payload);
+        record("success");
       } catch (error) {
+        record("error");
         log.error({ err: error }, "message processing failed");
         // Note: Kafka doesn't have individual message nack like RabbitMQ
         // Error handling would be done through retry mechanisms or dead letter topics
@@ -107,7 +119,15 @@ process.on("warning", (warning) => {
   );
 });
 
-const createKafkaConnectionManager = (queueConfig: KafkaConfig): KafkaConnectionManager => {
+export type KafkaDependencies = {
+  /** Test seam: the manager builds its client through this. */
+  createKafka?: (config: ConstructorParameters<typeof Kafka>[0]) => Kafka;
+};
+
+export const createKafkaConnectionManager = (
+  queueConfig: KafkaConfig,
+  { createKafka = (config) => new Kafka(config) }: KafkaDependencies = {},
+): KafkaConnectionManager => {
   if (queueConfig.type !== "kafka") {
     throw new Error("Invalid queue type for Kafka connection manager");
   }
@@ -158,7 +178,7 @@ const createKafkaConnectionManager = (queueConfig: KafkaConfig): KafkaConnection
       const connConfig = getConnectionConfig();
 
       // Create Kafka client with built-in retry
-      const kafka = new Kafka({
+      const kafka = createKafka({
         clientId: connConfig.clientId ?? `datagraph-${queueConfig.name}`,
         brokers: connConfig.brokers,
         ssl: connConfig.ssl,
@@ -205,8 +225,29 @@ const createKafkaConnectionManager = (queueConfig: KafkaConfig): KafkaConnection
             name: p.name,
             topic: exchange.name,
             send: async (message: string | object, key?: string) => {
+              const publisher = `${queueConfig.name}_${p.name}`;
+              // No message body on the span — it is caller data.
+              const span = startSpan("queue.publish", {
+                kind: "producer",
+                attributes: {
+                  "messaging.system": "kafka",
+                  "messaging.destination.name": exchange.name,
+                  "graphoria.queue.publisher": publisher,
+                },
+              });
+              const record = (outcome: "success" | "error") => {
+                incMetric("graphoria_queue_messages_published_total", {
+                  broker: "kafka",
+                  publisher,
+                  outcome,
+                });
+                if (outcome === "error") span?.setStatus("error");
+                span?.end();
+              };
+
               if (!state.producer) {
                 log.error("cannot send: producer not available");
+                record("error");
                 return false;
               }
               try {
@@ -220,9 +261,11 @@ const createKafkaConnectionManager = (queueConfig: KafkaConfig): KafkaConnection
                     },
                   ],
                 });
+                record("success");
                 return true;
               } catch (error) {
                 log.error({ err: error, topic: exchange.name }, "failed to send message");
+                record("error");
                 return false;
               }
             },

package/src/queues/rabbitmq.ts

@@ -7,6 +7,8 @@ import type { RabbitMQConfig } from "../types/zod/queue";
 import { queryEventEmitter } from "../configuration/gql/handleGraphQLSubscriptionFactory";
 import { InvalidationHelper } from "../singletons/cache/registry";
 import { logger } from "../logging";
+import { incMetric } from "../observability/metrics";
+import { startSpan } from "../observability/tracing";
 
 // ============================================================================
 // Reconnection Configuration
@@ -41,6 +43,14 @@ export const startConsumer = async (
 ) => {
   const log = logger("rabbitmq").child({ queue: connectionName, consumer: name });
 
+  const record = (outcome: "success" | "error") =>
+    incMetric("graphoria_queue_messages_consumed_total", {
+      broker: "rabbitmq",
+      queue: connectionName,
+      consumer: name,
+      outcome,
+    });
+
   channel.consume(queueName, async (msg) => {
     if (msg !== null) {
       try {
@@ -58,10 +68,12 @@ export const startConsumer = async (
         }
 
         channel.ack(msg);
+        record("success");
       } catch (error) {
         log.error({ err: error }, "message processing failed");
 
         channel.nack(msg);
+        record("error");
       }
     }
   });
@@ -162,16 +174,40 @@ export const createRabbitMQConnectionManager = (
             routingKey: p.routingKey,
             exchangeName: exchange.name,
             send: (message: string | object) => {
+              const publisher = `${queueConfig.name}_${p.name}`;
+              // No message body on the span — it is caller data.
+              const span = startSpan("queue.publish", {
+                kind: "producer",
+                attributes: {
+                  "messaging.system": "rabbitmq",
+                  "messaging.destination.name": exchange.name,
+                  "graphoria.queue.publisher": publisher,
+                },
+              });
+              const record = (outcome: "success" | "error") => {
+                incMetric("graphoria_queue_messages_published_total", {
+                  broker: "rabbitmq",
+                  publisher,
+                  outcome,
+                });
+                if (outcome === "error") span?.setStatus("error");
+                span?.end();
+              };
+
               if (!state.channel) {
                 log.error("cannot send: channel not available");
+                record("error");
                 return false;
               }
-              return state.channel.publish(
+
+              const sent = state.channel.publish(
                 exchange.name,
                 p.routingKey,
                 Buffer.from(typeof message === "string" ? message : JSON.stringify(message)),
                 p.options,
               );
+              record(sent ? "success" : "error");
+              return sent;
             },
           };
         }

package/src/remoteREST/proxy.ts

@@ -1,5 +1,7 @@
 import type { RemoteRESTResolved, RemoteRESTRoute } from "./types";
 
+import { formatTraceparent, startSpan } from "../observability/tracing";
+
 /**
  * Build headers for the remote request by merging:
  * 1. Static headers from configuration
@@ -62,6 +64,16 @@ export const proxyRemoteRESTRequest = async (
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), resolved.config.timeout ?? 10000);
 
+  // The host only: a remote URL's path and query string carry caller data.
+  const span = startSpan("graphoria.remote_rest", {
+    kind: "client",
+    attributes: {
+      "http.request.method": route.method.toUpperCase(),
+      "server.address": new URL(targetUrl).host,
+    },
+  });
+  if (span) headers["traceparent"] = formatTraceparent(span.context);
+
   try {
     const fetchInit: RequestInit = {
       method: route.method.toUpperCase(),
@@ -77,6 +89,7 @@ export const proxyRemoteRESTRequest = async (
     }
 
     const response = await fetch(targetUrl, fetchInit);
+    span?.setAttribute("http.response.status_code", response.status);
 
     // Return the response with the same status and body
     return new Response(response.body, {
@@ -86,7 +99,11 @@ export const proxyRemoteRESTRequest = async (
         "content-type": response.headers.get("content-type") ?? "application/json",
       },
     });
+  } catch (error) {
+    span?.recordError(error);
+    throw error;
   } finally {
+    span?.end();
     clearTimeout(timeoutId);
   }
 };

package/src/remoteSchemas/proxy.ts

@@ -1,6 +1,8 @@
 import type { SelectionAnalysis } from "../analyzeQuery/types";
 import type { RemoteSchemaResolved } from "./types";
 
+import { formatTraceparent, startSpan } from "../observability/tracing";
+
 /**
  * Build the sub-query string for a remote field by stripping the prefix from
  * the field name and type references.
@@ -110,6 +112,16 @@ export const proxyRemoteField = async (
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), remoteSchema.config.timeout ?? 10000);
 
+  // The host only: a remote URL's path and query string carry caller data.
+  const span = startSpan("graphoria.remote_schema", {
+    kind: "client",
+    attributes: {
+      "http.request.method": "POST",
+      "server.address": new URL(remoteSchema.config.url).host,
+    },
+  });
+  if (span) headers["traceparent"] = formatTraceparent(span.context);
+
   try {
     const response = await fetch(remoteSchema.config.url, {
       method: "POST",
@@ -120,6 +132,7 @@ export const proxyRemoteField = async (
       }),
       signal: controller.signal,
     });
+    span?.setAttribute("http.response.status_code", response.status);
 
     if (!response.ok) {
       throw new Error(
@@ -140,7 +153,11 @@ export const proxyRemoteField = async (
 
     // Extract the result for the original field name
     return json.data?.[originalFieldName] ?? null;
+  } catch (error) {
+    span?.recordError(error);
+    throw error;
   } finally {
+    span?.end();
     clearTimeout(timeoutId);
   }
 };

package/src/singletons/cache/redisClient.ts

@@ -1,12 +1,11 @@
-import { RedisClient as ValkeyClient } from "bun";
-
 import { env } from "../env";
+import { createRedisClient } from "../../utils/redis";
 
-let client: InstanceType<typeof ValkeyClient> | null = null;
+let client: ReturnType<typeof createRedisClient> | null = null;
 
-export const getCacheRedisClient = (): InstanceType<typeof ValkeyClient> => {
+export const getCacheRedisClient = (): ReturnType<typeof createRedisClient> => {
   if (!client) {
-    client = new ValkeyClient(env.cache.redisUrl);
+    client = createRedisClient(env.cache.redisUrl);
   }
   return client;
 };