@graphoria/server 0.3.0 → 0.5.0

package/src/singletons/databases.ts

@@ -50,6 +50,11 @@ export const instantiateDatabasesConnections = async (databases: Database[]) =>
   return { databasesConnections, repositoryMap };
 };
 
+export const pingConnection = (connection: SQL | ConnectionPool, type: string) =>
+  type === "mssql"
+    ? (connection as ConnectionPool).query("SELECT 1")
+    : (connection as SQL).unsafe("SELECT 1");
+
 /**
  * Close every open database connection and clear the singleton maps. Bun's `SQL`
  * and mssql's `ConnectionPool` both expose `close()`. Used by

package/src/singletons/queues.ts

@@ -6,7 +6,11 @@ import { startKafkaConnections } from "../queues/kafka";
 import { startRabbitMQConnections } from "../queues/rabbitmq";
 import { logger } from "../logging";
 
-export type QueueConnectionStatus = { type: "rabbitmq" | "kafka"; connected: boolean };
+export type QueueConnectionStatus = {
+  type: "rabbitmq" | "kafka";
+  name: string;
+  connected: boolean;
+};
 
 export type QueueManager = {
   publisherMap: () => Record<string, RabbitMQPublisher | KafkaPublisher>;
@@ -40,8 +44,9 @@ export const instantiateQueues = async (queues: QueueConfig[]) => {
       publisherMap: rabbitMQManager.publisherMap,
       sendMessage: rabbitMQManager.sendMessage,
       connections: () =>
-        rabbitMQManager.managers.map((manager) => ({
+        rabbitMQManager.managers.map((manager, index) => ({
           type: "rabbitmq" as const,
+          name: rabbitMQQueues[index]!.name,
           connected: manager.isConnected(),
         })),
     });
@@ -55,8 +60,9 @@ export const instantiateQueues = async (queues: QueueConfig[]) => {
       publisherMap: kafkaManager.publisherMap,
       sendMessage: kafkaManager.sendMessage,
       connections: () =>
-        kafkaManager.managers.map((manager) => ({
+        kafkaManager.managers.map((manager, index) => ({
           type: "kafka" as const,
+          name: kafkaQueues[index]!.name,
           connected: manager.isConnected(),
         })),
       cleanup: kafkaManager.cleanup,

package/src/subscriptions/strategies/database.ts

@@ -46,6 +46,7 @@ export const createDatabaseSubscriptionStrategy = (): SubscriptionStrategy => ({
       variables,
       schemaEntity,
       eventEmitter,
+      session,
     } = context;
 
     const subscriptionKey = this.getSubscriptionKey(context);
@@ -69,6 +70,7 @@ export const createDatabaseSubscriptionStrategy = (): SubscriptionStrategy => ({
       subscriptionKey,
       eventEmitter,
       pollIntervalMs: 1000,
+      role: session.role,
     });
 
     // Store poll reference for cleanup by event emitter

package/src/subscriptions/utils/polling.ts

@@ -1,10 +1,12 @@
 import type { AnalysisResult, VariableDefinition } from "../../analyzeQuery/types";
 import type { GetSchemaReturn } from "../../configuration/getSchemas";
+import type { QuerySource } from "../../logging/slowQuery";
 import type { Database } from "../../types/configuration";
 import type { DatabasePoller, QueryEventEmitter } from "../types";
 
 import { executeQuery, executeQueryJSON, generateSQL } from "../../databases";
 import { logger } from "../../logging";
+import { withSpan } from "../../observability/tracing";
 import { env } from "../../singletons/env";
 
 export interface DatabasePollerConfig {
@@ -15,6 +17,7 @@ export interface DatabasePollerConfig {
   subscriptionKey: string;
   eventEmitter: QueryEventEmitter;
   pollIntervalMs?: number;
+  role?: string | undefined;
 }
 
 /**
@@ -32,12 +35,15 @@ const getResultHash = async (
   db: Database,
   variableDefinitions: VariableDefinition[],
   variables: Record<string, unknown>,
+  source: QuerySource,
 ): Promise<string> => {
   const hashResult = await executeQuery<{ ResultHash: string }>(
     queryHash,
     db,
     variableDefinitions,
     variables,
+    undefined,
+    source,
   );
   return Buffer.from(hashResult[0].ResultHash)?.toString("hex");
 };
@@ -56,6 +62,7 @@ export const createDatabasePoller = async (
     subscriptionKey,
     eventEmitter,
     pollIntervalMs = 1000,
+    role,
   } = config;
 
   const firstFieldName = getFirstFieldName(analysis);
@@ -67,35 +74,87 @@ export const createDatabasePoller = async (
   const [[, queryData]] = generateSQL(schemaEntity, analysis, variables, false, pageLimits);
   const [[, queryHash]] = generateSQL(schemaEntity, analysis, variables, true, pageLimits);
 
-  // Get initial hash and send initial data
-  let previousHash = await getResultHash(queryHash, db, variableDefinitions, variables);
+  const operation = analysis.operations[0];
+  const source: QuerySource = {
+    operation: {
+      type: operation.operation,
+      name: operation.name,
+      fields: operation.fields.map((field) => field.name),
+    },
+    role,
+  };
+
+  // A poll is not inside a request, so each one roots its own trace. The span is
+  // entered, which is what puts the statements it runs underneath it. An empty
+  // poll still gets a span — an empty poll that is slow is worth seeing.
+  const pollSpan = <T>(run: () => Promise<T>) =>
+    withSpan(
+      "subscription.poll",
+      {
+        parent: null,
+        attributes: {
+          "graphql.operation.name": source.operation.name ?? undefined,
+          "graphql.operation.type": source.operation.type,
+          "graphoria.role": source.role,
+        },
+      },
+      run,
+    );
 
-  eventEmitter.sendDataUpdate(subscriptionKey, {
-    data: await executeQueryJSON(queryData, db, variableDefinitions, variables),
+  // Get initial hash and send initial data
+  let previousHash = "";
+
+  await pollSpan(async () => {
+    previousHash = await getResultHash(queryHash, db, variableDefinitions, variables, source);
+
+    eventEmitter.sendDataUpdate(subscriptionKey, {
+      data: await executeQueryJSON(
+        queryData,
+        db,
+        variableDefinitions,
+        variables,
+        undefined,
+        source,
+      ),
+    });
   });
 
   // Poll function that checks for changes
   const log = logger("polling").child({ subscription: subscriptionKey });
-  const poll = async () => {
-    try {
-      const currentHash = await getResultHash(queryHash, db, variableDefinitions, variables);
-
-      if (currentHash !== previousHash) {
-        log.info(
-          { changed: !!previousHash, operation: analysis.operations[0].name },
-          previousHash ? "data changed" : "initial fetch",
+  const poll = () =>
+    pollSpan(async () => {
+      try {
+        const currentHash = await getResultHash(
+          queryHash,
+          db,
+          variableDefinitions,
+          variables,
+          source,
         );
 
-        previousHash = currentHash;
-
-        eventEmitter.sendDataUpdate(subscriptionKey, {
-          data: await executeQueryJSON(queryData, db, variableDefinitions, variables),
-        });
+        if (currentHash !== previousHash) {
+          log.info(
+            { changed: !!previousHash, operation: analysis.operations[0].name },
+            previousHash ? "data changed" : "initial fetch",
+          );
+
+          previousHash = currentHash;
+
+          eventEmitter.sendDataUpdate(subscriptionKey, {
+            data: await executeQueryJSON(
+              queryData,
+              db,
+              variableDefinitions,
+              variables,
+              undefined,
+              source,
+            ),
+          });
+        }
+      } catch (error) {
+        log.error({ err: error }, "polling failed");
       }
-    } catch (error) {
-      log.error({ err: error }, "polling failed");
-    }
-  };
+    });
 
   // Poller control
   let stopped = false;

package/src/types/env.ts

@@ -3,19 +3,29 @@ import { z } from "zod";
 import { version } from "../../package.json";
 import { ConfigurationZod } from "./zod/configuration";
 
+// A secret env var is a comma-separated list: the first entry is the one in
+// use, the rest are still accepted so a rotation can overlap instead of cutting
+// over. Blank entries are dropped, so an unset var is an empty list.
+const splitSecrets = (value: string) =>
+  value
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter(Boolean);
+const secretList = z.string().default("").transform(splitSecrets);
+
 export const EnvZod = z
   .object({
     PORT: z.coerce.number().default(3000),
     NODE_ENV: z.string().default("DEVELOPMENT"),
-    ADMIN_SECRET: z.string(),
+    ADMIN_SECRET: z.string().transform(splitSecrets),
     CONFIGURATION: z.union([z.string(), ConfigurationZod]).optional(),
     ANONYMOUS_ROLE: z.string().default("anonymous"),
-    JWT_SECRET: z.string().optional().default(""),
+    JWT_SECRET: secretList,
     JWT_EXPIRES_IN: z.string().default("5m"),
     JWT_RT_EXPIRES_IN: z.string().default("7d"),
-    PASETO_LOCAL_KEY: z.string().optional().default(""),
+    PASETO_LOCAL_KEY: secretList,
     PASETO_SECRET_KEY: z.string().optional().default(""),
-    PASETO_PUBLIC_KEY: z.string().optional().default(""),
+    PASETO_PUBLIC_KEY: secretList,
     AUTH_STRATEGY: z.enum(["jwt", "paseto_local", "paseto_public"]).optional(),
     REDIS_URL: z.string().default("redis://localhost:6379"),
     CACHE_STORE: z.enum(["memory", "redis"]).default("memory"),
@@ -31,6 +41,8 @@ export const EnvZod = z
     CONSOLE_ENABLED: z.stringbool().default(false),
     CONSOLE_ENDPOINT: z.string().default("/_console"),
     CONSOLE_SESSION_EXPIRES_IN: z.string().default("1h"),
+    CONSOLE_READ_SECRET: secretList,
+    CONSOLE_WRITE_SECRET: secretList,
     PREFIX: z.string().default(""),
     CORS_ENABLED: z.stringbool().default(true),
     PRINT_SCHEMAS: z.stringbool().default(false),
@@ -40,6 +52,19 @@ export const EnvZod = z
     DEFAULT_PAGE_SIZE: z.coerce.number().int().min(0).default(100),
     MAX_PAGE_SIZE: z.coerce.number().int().min(0).default(1000),
     QUERY_TIMEOUT_MS: z.coerce.number().int().min(0).default(10000),
+    SLOW_QUERY_MS: z.coerce.number().int().min(0).default(1000),
+    METRICS_ENABLED: z.stringbool().default(false),
+    METRICS_ENDPOINT: z.string().default("/metrics"),
+    METRICS_SECRET: secretList,
+    METRICS_MAX_OPERATION_LABELS: z.coerce.number().int().min(1).default(200),
+    // Only the gate is graphoria's own. The rest are the standard OpenTelemetry
+    // names, so a cluster that already injects them into its pods needs to set
+    // nothing but TRACING_ENABLED.
+    TRACING_ENABLED: z.stringbool().default(false),
+    OTEL_EXPORTER_OTLP_ENDPOINT: z.string().default("http://localhost:4318"),
+    OTEL_EXPORTER_OTLP_HEADERS: z.string().default(""),
+    OTEL_SERVICE_NAME: z.string().default("graphoria"),
+    OTEL_TRACES_SAMPLER_ARG: z.coerce.number().min(0).max(1).default(1),
     MAX_QUERY_COST: z.coerce.number().int().min(0).default(0),
     RATE_LIMIT_MAX: z.coerce.number().int().min(0).default(0),
     RATE_LIMIT_ANONYMOUS_MAX: z.coerce.number().int().min(0).default(0),
@@ -48,9 +73,11 @@ export const EnvZod = z
     AI_ENABLED: z.stringbool().optional(),
     AI_GRAPHQL_ENABLED: z.stringbool().default(true),
     AI_REST_ENABLED: z.stringbool().default(true),
+    AI_SECRET: secretList,
     AI_MCP_ENABLED: z.stringbool().optional(),
     AI_MCP_ENDPOINT: z.string().default("/mcp"),
     AI_MCP_REQUIRE_ADMIN_SECRET: z.stringbool().default(false),
+    AI_MCP_SECRET: secretList,
     AI_MCP_MAX_QUERY_DEPTH: z.coerce.number().int().min(0).optional(),
     AI_MCP_DISABLED_TOOLS: z.string().default(""),
     AI_MCP_DISABLED_RESOURCES: z.string().default(""),
@@ -78,20 +105,21 @@ export const EnvZod = z
     defaultPageSize: env.DEFAULT_PAGE_SIZE,
     maxPageSize: env.MAX_PAGE_SIZE,
     queryTimeoutMs: env.QUERY_TIMEOUT_MS,
+    slowQueryMs: env.SLOW_QUERY_MS,
     maxQueryCost: env.MAX_QUERY_COST,
     admin: {
-      secret: env.ADMIN_SECRET,
+      secrets: env.ADMIN_SECRET,
       header: env.ADMIN_SECRET_HEADER,
     },
     jwt: {
-      secret: env.JWT_SECRET,
+      secrets: env.JWT_SECRET,
       expiresIn: env.JWT_EXPIRES_IN,
       rtExpiresIn: env.JWT_RT_EXPIRES_IN,
     },
     paseto: {
-      localKey: env.PASETO_LOCAL_KEY,
+      localKeys: env.PASETO_LOCAL_KEY,
       secretKey: env.PASETO_SECRET_KEY,
-      publicKey: env.PASETO_PUBLIC_KEY,
+      publicKeys: env.PASETO_PUBLIC_KEY,
     },
     rateLimit: {
       max: env.RATE_LIMIT_MAX,
@@ -110,19 +138,36 @@ export const EnvZod = z
     superadmin: {
       role: env.SUPERADMIN_ROLE,
     },
+    metrics: {
+      enabled: env.METRICS_ENABLED,
+      endpoint: env.METRICS_ENDPOINT,
+      secrets: env.METRICS_SECRET,
+      maxOperationLabels: env.METRICS_MAX_OPERATION_LABELS,
+    },
+    tracing: {
+      enabled: env.TRACING_ENABLED,
+      endpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+      headers: env.OTEL_EXPORTER_OTLP_HEADERS,
+      serviceName: env.OTEL_SERVICE_NAME,
+      sampleRatio: env.OTEL_TRACES_SAMPLER_ARG,
+    },
     console: {
       enabled: env.CONSOLE_ENABLED,
       endpoint: env.CONSOLE_ENDPOINT,
       sessionExpiresIn: env.CONSOLE_SESSION_EXPIRES_IN,
+      readSecrets: env.CONSOLE_READ_SECRET,
+      writeSecrets: env.CONSOLE_WRITE_SECRET,
     },
     ai: {
       enabled: env.AI_ENABLED,
       graphqlEnabled: env.AI_GRAPHQL_ENABLED,
       restEnabled: env.AI_REST_ENABLED,
+      secrets: env.AI_SECRET,
       mcp: {
         enabled: env.AI_MCP_ENABLED,
         endpoint: env.AI_MCP_ENDPOINT,
         requireAdminSecret: env.AI_MCP_REQUIRE_ADMIN_SECRET,
+        secrets: env.AI_MCP_SECRET,
         maxQueryDepth: env.AI_MCP_MAX_QUERY_DEPTH,
         disabledTools: env.AI_MCP_DISABLED_TOOLS
           ? env.AI_MCP_DISABLED_TOOLS.split(",")

package/src/utils/rateLimit.ts

@@ -1,6 +1,7 @@
 import { LRUCache } from "lru-cache";
 
 import { logger } from "../logging";
+import { incMetric } from "../observability/metrics";
 
 export type ConsumeResult = { allowed: boolean; retryAfterMs: number };
 
@@ -203,7 +204,11 @@ export const createRateLimiter = ({
       const key =
         sub && sub !== "anonymous" ? `rl:${role}:${sub}` : `rl:${role}:ip:${address ?? "unknown"}`;
 
-      return resolvedStore.consume(key, max, max / windowMs, windowMs);
+      const verdict = await resolvedStore.consume(key, max, max / windowMs, windowMs);
+
+      if (!verdict.allowed) incMetric("graphoria_rate_limit_rejections_total", { role });
+
+      return verdict;
     },
   };
 };

package/src/utils/redis.ts

@@ -0,0 +1,71 @@
+import { RedisClient as ValkeyClient } from "bun";
+
+import { logger } from "../logging";
+
+const RECONNECT_INITIAL_DELAY = 1000;
+const RECONNECT_MAX_DELAY = 30000;
+const RECONNECT_MULTIPLIER = 2;
+
+export type RevivableRedisClient = {
+  onclose: ((error: Error) => void) | null;
+  connect(): Promise<void>;
+};
+
+export type KeepRedisConnectedOptions = {
+  setTimeout?: typeof setTimeout;
+};
+
+/**
+ * Bun's client reconnects on its own only until `maxRetries` runs out; then it
+ * fires `onclose` and fails every later command with "Connection has failed",
+ * even once the server is back. Only an explicit `connect()` revives it, so
+ * that is what `onclose` schedules here, backing off while the server stays down.
+ */
+export const keepRedisConnected = (
+  client: RevivableRedisClient,
+  options: KeepRedisConnectedOptions = {},
+) => {
+  const log = logger("redis");
+  const setTimeoutFn = options.setTimeout ?? setTimeout;
+
+  let attempts = 0;
+  let reconnecting = false;
+
+  const scheduleReconnect = () => {
+    if (reconnecting) return;
+    reconnecting = true;
+
+    const delay = Math.min(
+      RECONNECT_INITIAL_DELAY * Math.pow(RECONNECT_MULTIPLIER, attempts),
+      RECONNECT_MAX_DELAY,
+    );
+    attempts++;
+
+    setTimeoutFn(() => {
+      client.connect().then(
+        () => {
+          reconnecting = false;
+          attempts = 0;
+          log.info("reconnected");
+        },
+        (error: unknown) => {
+          reconnecting = false;
+          log.warn({ err: error, attempt: attempts }, "reconnect attempt failed");
+          scheduleReconnect();
+        },
+      );
+    }, delay).unref();
+  };
+
+  client.onclose = (error) => {
+    if (reconnecting) return;
+    log.warn({ err: error }, "connection lost, reconnecting");
+    scheduleReconnect();
+  };
+};
+
+export const createRedisClient = (url: string) => {
+  const client = new ValkeyClient(url);
+  keepRedisConnected(client);
+  return client;
+};

package/src/utils/responses.ts

@@ -29,6 +29,12 @@ export class S401 extends ClientResponse {
   }
 }
 
+export class S403 extends ClientResponse {
+  constructor(body?: object | null, init?: ResponseInit) {
+    super(body, { ...init, status: 403 });
+  }
+}
+
 export class S404 extends ClientResponse {
   constructor(body?: object | null, init?: ResponseInit) {
     super(body, { ...init, status: 404 });
@@ -50,3 +56,9 @@ export class S500 extends ClientResponse {
     super(body, { ...init, status: 500 });
   }
 }
+
+export class S503 extends ClientResponse {
+  constructor(body?: object | null, init?: ResponseInit) {
+    super(body, { ...init, status: 503 });
+  }
+}

package/src/utils/sessionVariables.ts

@@ -13,6 +13,8 @@ export type SessionContext = {
   iat?: number;
   exp?: number;
   nbf?: number;
+  /** Set when the admin secret, not a token, produced this session. */
+  authMethod?: "admin_secret";
   [key: string]: unknown;
 };