@graphoria/server 0.4.0 → 0.5.0

package/src/queues/kafka.ts

@@ -6,6 +6,8 @@ import type { KafkaConfig } from "../types/zod/queue";
 
 import { queryEventEmitter } from "../configuration/gql/handleGraphQLSubscriptionFactory";
 import { logger } from "../logging";
+import { incMetric } from "../observability/metrics";
+import { startSpan } from "../observability/tracing";
 
 // ============================================================================
 // Reconnection Configuration
@@ -58,9 +60,19 @@ export const startConsumer = async (
 
   await consumer.run({
     eachMessage: async (payload) => {
+      const record = (outcome: "success" | "error") =>
+        incMetric("graphoria_queue_messages_consumed_total", {
+          broker: "kafka",
+          queue: connectionName,
+          consumer: name,
+          outcome,
+        });
+
       try {
         sendMessage(connectionName, name, payload);
+        record("success");
       } catch (error) {
+        record("error");
         log.error({ err: error }, "message processing failed");
         // Note: Kafka doesn't have individual message nack like RabbitMQ
         // Error handling would be done through retry mechanisms or dead letter topics
@@ -107,7 +119,15 @@ process.on("warning", (warning) => {
   );
 });
 
-const createKafkaConnectionManager = (queueConfig: KafkaConfig): KafkaConnectionManager => {
+export type KafkaDependencies = {
+  /** Test seam: the manager builds its client through this. */
+  createKafka?: (config: ConstructorParameters<typeof Kafka>[0]) => Kafka;
+};
+
+export const createKafkaConnectionManager = (
+  queueConfig: KafkaConfig,
+  { createKafka = (config) => new Kafka(config) }: KafkaDependencies = {},
+): KafkaConnectionManager => {
   if (queueConfig.type !== "kafka") {
     throw new Error("Invalid queue type for Kafka connection manager");
   }
@@ -158,7 +178,7 @@ const createKafkaConnectionManager = (queueConfig: KafkaConfig): KafkaConnection
       const connConfig = getConnectionConfig();
 
       // Create Kafka client with built-in retry
-      const kafka = new Kafka({
+      const kafka = createKafka({
         clientId: connConfig.clientId ?? `datagraph-${queueConfig.name}`,
         brokers: connConfig.brokers,
         ssl: connConfig.ssl,
@@ -205,8 +225,29 @@ const createKafkaConnectionManager = (queueConfig: KafkaConfig): KafkaConnection
             name: p.name,
             topic: exchange.name,
             send: async (message: string | object, key?: string) => {
+              const publisher = `${queueConfig.name}_${p.name}`;
+              // No message body on the span — it is caller data.
+              const span = startSpan("queue.publish", {
+                kind: "producer",
+                attributes: {
+                  "messaging.system": "kafka",
+                  "messaging.destination.name": exchange.name,
+                  "graphoria.queue.publisher": publisher,
+                },
+              });
+              const record = (outcome: "success" | "error") => {
+                incMetric("graphoria_queue_messages_published_total", {
+                  broker: "kafka",
+                  publisher,
+                  outcome,
+                });
+                if (outcome === "error") span?.setStatus("error");
+                span?.end();
+              };
+
               if (!state.producer) {
                 log.error("cannot send: producer not available");
+                record("error");
                 return false;
               }
               try {
@@ -220,9 +261,11 @@ const createKafkaConnectionManager = (queueConfig: KafkaConfig): KafkaConnection
                     },
                   ],
                 });
+                record("success");
                 return true;
               } catch (error) {
                 log.error({ err: error, topic: exchange.name }, "failed to send message");
+                record("error");
                 return false;
               }
             },

package/src/queues/rabbitmq.ts

@@ -7,6 +7,8 @@ import type { RabbitMQConfig } from "../types/zod/queue";
 import { queryEventEmitter } from "../configuration/gql/handleGraphQLSubscriptionFactory";
 import { InvalidationHelper } from "../singletons/cache/registry";
 import { logger } from "../logging";
+import { incMetric } from "../observability/metrics";
+import { startSpan } from "../observability/tracing";
 
 // ============================================================================
 // Reconnection Configuration
@@ -41,6 +43,14 @@ export const startConsumer = async (
 ) => {
   const log = logger("rabbitmq").child({ queue: connectionName, consumer: name });
 
+  const record = (outcome: "success" | "error") =>
+    incMetric("graphoria_queue_messages_consumed_total", {
+      broker: "rabbitmq",
+      queue: connectionName,
+      consumer: name,
+      outcome,
+    });
+
   channel.consume(queueName, async (msg) => {
     if (msg !== null) {
       try {
@@ -58,10 +68,12 @@ export const startConsumer = async (
         }
 
         channel.ack(msg);
+        record("success");
       } catch (error) {
         log.error({ err: error }, "message processing failed");
 
         channel.nack(msg);
+        record("error");
       }
     }
   });
@@ -162,16 +174,40 @@ export const createRabbitMQConnectionManager = (
             routingKey: p.routingKey,
             exchangeName: exchange.name,
             send: (message: string | object) => {
+              const publisher = `${queueConfig.name}_${p.name}`;
+              // No message body on the span — it is caller data.
+              const span = startSpan("queue.publish", {
+                kind: "producer",
+                attributes: {
+                  "messaging.system": "rabbitmq",
+                  "messaging.destination.name": exchange.name,
+                  "graphoria.queue.publisher": publisher,
+                },
+              });
+              const record = (outcome: "success" | "error") => {
+                incMetric("graphoria_queue_messages_published_total", {
+                  broker: "rabbitmq",
+                  publisher,
+                  outcome,
+                });
+                if (outcome === "error") span?.setStatus("error");
+                span?.end();
+              };
+
               if (!state.channel) {
                 log.error("cannot send: channel not available");
+                record("error");
                 return false;
               }
-              return state.channel.publish(
+
+              const sent = state.channel.publish(
                 exchange.name,
                 p.routingKey,
                 Buffer.from(typeof message === "string" ? message : JSON.stringify(message)),
                 p.options,
               );
+              record(sent ? "success" : "error");
+              return sent;
             },
           };
         }

package/src/remoteREST/proxy.ts

@@ -1,5 +1,7 @@
 import type { RemoteRESTResolved, RemoteRESTRoute } from "./types";
 
+import { formatTraceparent, startSpan } from "../observability/tracing";
+
 /**
  * Build headers for the remote request by merging:
  * 1. Static headers from configuration
@@ -62,6 +64,16 @@ export const proxyRemoteRESTRequest = async (
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), resolved.config.timeout ?? 10000);
 
+  // The host only: a remote URL's path and query string carry caller data.
+  const span = startSpan("graphoria.remote_rest", {
+    kind: "client",
+    attributes: {
+      "http.request.method": route.method.toUpperCase(),
+      "server.address": new URL(targetUrl).host,
+    },
+  });
+  if (span) headers["traceparent"] = formatTraceparent(span.context);
+
   try {
     const fetchInit: RequestInit = {
       method: route.method.toUpperCase(),
@@ -77,6 +89,7 @@ export const proxyRemoteRESTRequest = async (
     }
 
     const response = await fetch(targetUrl, fetchInit);
+    span?.setAttribute("http.response.status_code", response.status);
 
     // Return the response with the same status and body
     return new Response(response.body, {
@@ -86,7 +99,11 @@ export const proxyRemoteRESTRequest = async (
         "content-type": response.headers.get("content-type") ?? "application/json",
       },
     });
+  } catch (error) {
+    span?.recordError(error);
+    throw error;
   } finally {
+    span?.end();
     clearTimeout(timeoutId);
   }
 };

package/src/remoteSchemas/proxy.ts

@@ -1,6 +1,8 @@
 import type { SelectionAnalysis } from "../analyzeQuery/types";
 import type { RemoteSchemaResolved } from "./types";
 
+import { formatTraceparent, startSpan } from "../observability/tracing";
+
 /**
  * Build the sub-query string for a remote field by stripping the prefix from
  * the field name and type references.
@@ -110,6 +112,16 @@ export const proxyRemoteField = async (
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), remoteSchema.config.timeout ?? 10000);
 
+  // The host only: a remote URL's path and query string carry caller data.
+  const span = startSpan("graphoria.remote_schema", {
+    kind: "client",
+    attributes: {
+      "http.request.method": "POST",
+      "server.address": new URL(remoteSchema.config.url).host,
+    },
+  });
+  if (span) headers["traceparent"] = formatTraceparent(span.context);
+
   try {
     const response = await fetch(remoteSchema.config.url, {
       method: "POST",
@@ -120,6 +132,7 @@ export const proxyRemoteField = async (
       }),
       signal: controller.signal,
     });
+    span?.setAttribute("http.response.status_code", response.status);
 
     if (!response.ok) {
       throw new Error(
@@ -140,7 +153,11 @@ export const proxyRemoteField = async (
 
     // Extract the result for the original field name
     return json.data?.[originalFieldName] ?? null;
+  } catch (error) {
+    span?.recordError(error);
+    throw error;
   } finally {
+    span?.end();
     clearTimeout(timeoutId);
   }
 };

package/src/singletons/cache/redisClient.ts

@@ -1,12 +1,11 @@
-import { RedisClient as ValkeyClient } from "bun";
-
 import { env } from "../env";
+import { createRedisClient } from "../../utils/redis";
 
-let client: InstanceType<typeof ValkeyClient> | null = null;
+let client: ReturnType<typeof createRedisClient> | null = null;
 
-export const getCacheRedisClient = (): InstanceType<typeof ValkeyClient> => {
+export const getCacheRedisClient = (): ReturnType<typeof createRedisClient> => {
   if (!client) {
-    client = new ValkeyClient(env.cache.redisUrl);
+    client = createRedisClient(env.cache.redisUrl);
   }
   return client;
 };

package/src/singletons/databases.ts

@@ -50,6 +50,11 @@ export const instantiateDatabasesConnections = async (databases: Database[]) =>
   return { databasesConnections, repositoryMap };
 };
 
+export const pingConnection = (connection: SQL | ConnectionPool, type: string) =>
+  type === "mssql"
+    ? (connection as ConnectionPool).query("SELECT 1")
+    : (connection as SQL).unsafe("SELECT 1");
+
 /**
  * Close every open database connection and clear the singleton maps. Bun's `SQL`
  * and mssql's `ConnectionPool` both expose `close()`. Used by

package/src/singletons/queues.ts

@@ -6,7 +6,11 @@ import { startKafkaConnections } from "../queues/kafka";
 import { startRabbitMQConnections } from "../queues/rabbitmq";
 import { logger } from "../logging";
 
-export type QueueConnectionStatus = { type: "rabbitmq" | "kafka"; connected: boolean };
+export type QueueConnectionStatus = {
+  type: "rabbitmq" | "kafka";
+  name: string;
+  connected: boolean;
+};
 
 export type QueueManager = {
   publisherMap: () => Record<string, RabbitMQPublisher | KafkaPublisher>;
@@ -40,8 +44,9 @@ export const instantiateQueues = async (queues: QueueConfig[]) => {
       publisherMap: rabbitMQManager.publisherMap,
       sendMessage: rabbitMQManager.sendMessage,
       connections: () =>
-        rabbitMQManager.managers.map((manager) => ({
+        rabbitMQManager.managers.map((manager, index) => ({
           type: "rabbitmq" as const,
+          name: rabbitMQQueues[index]!.name,
           connected: manager.isConnected(),
         })),
     });
@@ -55,8 +60,9 @@ export const instantiateQueues = async (queues: QueueConfig[]) => {
       publisherMap: kafkaManager.publisherMap,
       sendMessage: kafkaManager.sendMessage,
       connections: () =>
-        kafkaManager.managers.map((manager) => ({
+        kafkaManager.managers.map((manager, index) => ({
           type: "kafka" as const,
+          name: kafkaQueues[index]!.name,
           connected: manager.isConnected(),
         })),
       cleanup: kafkaManager.cleanup,

package/src/subscriptions/strategies/database.ts

@@ -46,6 +46,7 @@ export const createDatabaseSubscriptionStrategy = (): SubscriptionStrategy => ({
       variables,
       schemaEntity,
       eventEmitter,
+      session,
     } = context;
 
     const subscriptionKey = this.getSubscriptionKey(context);
@@ -69,6 +70,7 @@ export const createDatabaseSubscriptionStrategy = (): SubscriptionStrategy => ({
       subscriptionKey,
       eventEmitter,
       pollIntervalMs: 1000,
+      role: session.role,
     });
 
     // Store poll reference for cleanup by event emitter

package/src/subscriptions/utils/polling.ts

@@ -1,10 +1,12 @@
 import type { AnalysisResult, VariableDefinition } from "../../analyzeQuery/types";
 import type { GetSchemaReturn } from "../../configuration/getSchemas";
+import type { QuerySource } from "../../logging/slowQuery";
 import type { Database } from "../../types/configuration";
 import type { DatabasePoller, QueryEventEmitter } from "../types";
 
 import { executeQuery, executeQueryJSON, generateSQL } from "../../databases";
 import { logger } from "../../logging";
+import { withSpan } from "../../observability/tracing";
 import { env } from "../../singletons/env";
 
 export interface DatabasePollerConfig {
@@ -15,6 +17,7 @@ export interface DatabasePollerConfig {
   subscriptionKey: string;
   eventEmitter: QueryEventEmitter;
   pollIntervalMs?: number;
+  role?: string | undefined;
 }
 
 /**
@@ -32,12 +35,15 @@ const getResultHash = async (
   db: Database,
   variableDefinitions: VariableDefinition[],
   variables: Record<string, unknown>,
+  source: QuerySource,
 ): Promise<string> => {
   const hashResult = await executeQuery<{ ResultHash: string }>(
     queryHash,
     db,
     variableDefinitions,
     variables,
+    undefined,
+    source,
   );
   return Buffer.from(hashResult[0].ResultHash)?.toString("hex");
 };
@@ -56,6 +62,7 @@ export const createDatabasePoller = async (
     subscriptionKey,
     eventEmitter,
     pollIntervalMs = 1000,
+    role,
   } = config;
 
   const firstFieldName = getFirstFieldName(analysis);
@@ -67,35 +74,87 @@ export const createDatabasePoller = async (
   const [[, queryData]] = generateSQL(schemaEntity, analysis, variables, false, pageLimits);
   const [[, queryHash]] = generateSQL(schemaEntity, analysis, variables, true, pageLimits);
 
-  // Get initial hash and send initial data
-  let previousHash = await getResultHash(queryHash, db, variableDefinitions, variables);
+  const operation = analysis.operations[0];
+  const source: QuerySource = {
+    operation: {
+      type: operation.operation,
+      name: operation.name,
+      fields: operation.fields.map((field) => field.name),
+    },
+    role,
+  };
+
+  // A poll is not inside a request, so each one roots its own trace. The span is
+  // entered, which is what puts the statements it runs underneath it. An empty
+  // poll still gets a span — an empty poll that is slow is worth seeing.
+  const pollSpan = <T>(run: () => Promise<T>) =>
+    withSpan(
+      "subscription.poll",
+      {
+        parent: null,
+        attributes: {
+          "graphql.operation.name": source.operation.name ?? undefined,
+          "graphql.operation.type": source.operation.type,
+          "graphoria.role": source.role,
+        },
+      },
+      run,
+    );
 
-  eventEmitter.sendDataUpdate(subscriptionKey, {
-    data: await executeQueryJSON(queryData, db, variableDefinitions, variables),
+  // Get initial hash and send initial data
+  let previousHash = "";
+
+  await pollSpan(async () => {
+    previousHash = await getResultHash(queryHash, db, variableDefinitions, variables, source);
+
+    eventEmitter.sendDataUpdate(subscriptionKey, {
+      data: await executeQueryJSON(
+        queryData,
+        db,
+        variableDefinitions,
+        variables,
+        undefined,
+        source,
+      ),
+    });
   });
 
   // Poll function that checks for changes
   const log = logger("polling").child({ subscription: subscriptionKey });
-  const poll = async () => {
-    try {
-      const currentHash = await getResultHash(queryHash, db, variableDefinitions, variables);
-
-      if (currentHash !== previousHash) {
-        log.info(
-          { changed: !!previousHash, operation: analysis.operations[0].name },
-          previousHash ? "data changed" : "initial fetch",
+  const poll = () =>
+    pollSpan(async () => {
+      try {
+        const currentHash = await getResultHash(
+          queryHash,
+          db,
+          variableDefinitions,
+          variables,
+          source,
         );
 
-        previousHash = currentHash;
-
-        eventEmitter.sendDataUpdate(subscriptionKey, {
-          data: await executeQueryJSON(queryData, db, variableDefinitions, variables),
-        });
+        if (currentHash !== previousHash) {
+          log.info(
+            { changed: !!previousHash, operation: analysis.operations[0].name },
+            previousHash ? "data changed" : "initial fetch",
+          );
+
+          previousHash = currentHash;
+
+          eventEmitter.sendDataUpdate(subscriptionKey, {
+            data: await executeQueryJSON(
+              queryData,
+              db,
+              variableDefinitions,
+              variables,
+              undefined,
+              source,
+            ),
+          });
+        }
+      } catch (error) {
+        log.error({ err: error }, "polling failed");
       }
-    } catch (error) {
-      log.error({ err: error }, "polling failed");
-    }
-  };
+    });
 
   // Poller control
   let stopped = false;

package/src/types/env.ts

@@ -52,6 +52,19 @@ export const EnvZod = z
     DEFAULT_PAGE_SIZE: z.coerce.number().int().min(0).default(100),
     MAX_PAGE_SIZE: z.coerce.number().int().min(0).default(1000),
     QUERY_TIMEOUT_MS: z.coerce.number().int().min(0).default(10000),
+    SLOW_QUERY_MS: z.coerce.number().int().min(0).default(1000),
+    METRICS_ENABLED: z.stringbool().default(false),
+    METRICS_ENDPOINT: z.string().default("/metrics"),
+    METRICS_SECRET: secretList,
+    METRICS_MAX_OPERATION_LABELS: z.coerce.number().int().min(1).default(200),
+    // Only the gate is graphoria's own. The rest are the standard OpenTelemetry
+    // names, so a cluster that already injects them into its pods needs to set
+    // nothing but TRACING_ENABLED.
+    TRACING_ENABLED: z.stringbool().default(false),
+    OTEL_EXPORTER_OTLP_ENDPOINT: z.string().default("http://localhost:4318"),
+    OTEL_EXPORTER_OTLP_HEADERS: z.string().default(""),
+    OTEL_SERVICE_NAME: z.string().default("graphoria"),
+    OTEL_TRACES_SAMPLER_ARG: z.coerce.number().min(0).max(1).default(1),
     MAX_QUERY_COST: z.coerce.number().int().min(0).default(0),
     RATE_LIMIT_MAX: z.coerce.number().int().min(0).default(0),
     RATE_LIMIT_ANONYMOUS_MAX: z.coerce.number().int().min(0).default(0),
@@ -92,6 +105,7 @@ export const EnvZod = z
     defaultPageSize: env.DEFAULT_PAGE_SIZE,
     maxPageSize: env.MAX_PAGE_SIZE,
     queryTimeoutMs: env.QUERY_TIMEOUT_MS,
+    slowQueryMs: env.SLOW_QUERY_MS,
     maxQueryCost: env.MAX_QUERY_COST,
     admin: {
       secrets: env.ADMIN_SECRET,
@@ -124,6 +138,19 @@ export const EnvZod = z
     superadmin: {
       role: env.SUPERADMIN_ROLE,
     },
+    metrics: {
+      enabled: env.METRICS_ENABLED,
+      endpoint: env.METRICS_ENDPOINT,
+      secrets: env.METRICS_SECRET,
+      maxOperationLabels: env.METRICS_MAX_OPERATION_LABELS,
+    },
+    tracing: {
+      enabled: env.TRACING_ENABLED,
+      endpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+      headers: env.OTEL_EXPORTER_OTLP_HEADERS,
+      serviceName: env.OTEL_SERVICE_NAME,
+      sampleRatio: env.OTEL_TRACES_SAMPLER_ARG,
+    },
     console: {
       enabled: env.CONSOLE_ENABLED,
       endpoint: env.CONSOLE_ENDPOINT,

package/src/utils/rateLimit.ts

@@ -1,6 +1,7 @@
 import { LRUCache } from "lru-cache";
 
 import { logger } from "../logging";
+import { incMetric } from "../observability/metrics";
 
 export type ConsumeResult = { allowed: boolean; retryAfterMs: number };
 
@@ -203,7 +204,11 @@ export const createRateLimiter = ({
       const key =
         sub && sub !== "anonymous" ? `rl:${role}:${sub}` : `rl:${role}:ip:${address ?? "unknown"}`;
 
-      return resolvedStore.consume(key, max, max / windowMs, windowMs);
+      const verdict = await resolvedStore.consume(key, max, max / windowMs, windowMs);
+
+      if (!verdict.allowed) incMetric("graphoria_rate_limit_rejections_total", { role });
+
+      return verdict;
     },
   };
 };

package/src/utils/redis.ts

@@ -0,0 +1,71 @@
+import { RedisClient as ValkeyClient } from "bun";
+
+import { logger } from "../logging";
+
+const RECONNECT_INITIAL_DELAY = 1000;
+const RECONNECT_MAX_DELAY = 30000;
+const RECONNECT_MULTIPLIER = 2;
+
+export type RevivableRedisClient = {
+  onclose: ((error: Error) => void) | null;
+  connect(): Promise<void>;
+};
+
+export type KeepRedisConnectedOptions = {
+  setTimeout?: typeof setTimeout;
+};
+
+/**
+ * Bun's client reconnects on its own only until `maxRetries` runs out; then it
+ * fires `onclose` and fails every later command with "Connection has failed",
+ * even once the server is back. Only an explicit `connect()` revives it, so
+ * that is what `onclose` schedules here, backing off while the server stays down.
+ */
+export const keepRedisConnected = (
+  client: RevivableRedisClient,
+  options: KeepRedisConnectedOptions = {},
+) => {
+  const log = logger("redis");
+  const setTimeoutFn = options.setTimeout ?? setTimeout;
+
+  let attempts = 0;
+  let reconnecting = false;
+
+  const scheduleReconnect = () => {
+    if (reconnecting) return;
+    reconnecting = true;
+
+    const delay = Math.min(
+      RECONNECT_INITIAL_DELAY * Math.pow(RECONNECT_MULTIPLIER, attempts),
+      RECONNECT_MAX_DELAY,
+    );
+    attempts++;
+
+    setTimeoutFn(() => {
+      client.connect().then(
+        () => {
+          reconnecting = false;
+          attempts = 0;
+          log.info("reconnected");
+        },
+        (error: unknown) => {
+          reconnecting = false;
+          log.warn({ err: error, attempt: attempts }, "reconnect attempt failed");
+          scheduleReconnect();
+        },
+      );
+    }, delay).unref();
+  };
+
+  client.onclose = (error) => {
+    if (reconnecting) return;
+    log.warn({ err: error }, "connection lost, reconnecting");
+    scheduleReconnect();
+  };
+};
+
+export const createRedisClient = (url: string) => {
+  const client = new ValkeyClient(url);
+  keepRedisConnected(client);
+  return client;
+};

package/src/utils/responses.ts

@@ -56,3 +56,9 @@ export class S500 extends ClientResponse {
     super(body, { ...init, status: 500 });
   }
 }
+
+export class S503 extends ClientResponse {
+  constructor(body?: object | null, init?: ResponseInit) {
+    super(body, { ...init, status: 503 });
+  }
+}