@malloy-publisher/server 0.0.199 → 0.0.201

package/src/query_concurrency.ts

@@ -0,0 +1,236 @@
+import { metrics, type Counter } from "@opentelemetry/api";
+import type { NextFunction, Request, RequestHandler, Response } from "express";
+
+import { getMaxConcurrentQueries } from "./config";
+import { ServiceUnavailableError } from "./errors";
+import { logger } from "./logger";
+
+/**
+ * Process-wide query gauge — the only state owned by this module.
+ * Incremented when a gated handler accepts a request; decremented
+ * exactly once when the response finishes (whether the handler
+ * succeeded, errored, or the client disconnected mid-flight).
+ *
+ * Module-scoped so all gated routes share one slot pool: the
+ * concurrency cap bounds aggregate query load, not per-route load.
+ */
+let active = 0;
+
+/**
+ * OpenTelemetry instruments. Lazy-initialized on first use so unit
+ * tests that install a real MeterProvider AFTER the module is
+ * imported still receive their data — OTel JS's `ProxyMeter`
+ * binds counters created before `setGlobalMeterProvider` to NoOp
+ * instruments. Module-scoped state so all gated routes share one
+ * counter / one slot pool.
+ */
+let queryConcurrencyRejectionsCounter: Counter | null = null;
+let concurrencyTelemetryInitialized = false;
+function ensureConcurrencyTelemetry(): Counter {
+   if (queryConcurrencyRejectionsCounter && concurrencyTelemetryInitialized) {
+      return queryConcurrencyRejectionsCounter;
+   }
+   const meter = metrics.getMeter("publisher");
+   queryConcurrencyRejectionsCounter = meter.createCounter(
+      "publisher_query_concurrency_rejections_total",
+      {
+         description:
+            "Queries rejected with 503 because the per-pod PUBLISHER_MAX_CONCURRENT_QUERIES cap was reached",
+      },
+   );
+   if (!concurrencyTelemetryInitialized) {
+      // Live count of in-flight queries holding a slot. Observable
+      // gauge so a scrape always sees the current value, not the
+      // value at the last admission/release event.
+      meter
+         .createObservableGauge("publisher_query_active_slots", {
+            description:
+               "In-flight queries currently holding a per-pod concurrency slot",
+         })
+         .addCallback((observation) => observation.observe(active));
+
+      // Configured cap, exposed so dashboards can render
+      // utilization (`active / max`) without needing a separate
+      // config feed. Read on every scrape so a runtime env-var
+      // change is reflected immediately; the parse is cheap. A
+      // `0` cap means "opt-out" (the middleware is a pass-through)
+      // and is reported verbatim.
+      meter
+         .createObservableGauge("publisher_query_max_slots", {
+            description:
+               "Current effective PUBLISHER_MAX_CONCURRENT_QUERIES (0 = disabled)",
+         })
+         .addCallback((observation) => {
+            try {
+               observation.observe(getMaxConcurrentQueries());
+            } catch {
+               // A misconfigured env var should fail the next
+               // request that observes it, not the metric scrape.
+               // Surface as -1 so the misconfig is visible.
+               observation.observe(-1);
+            }
+         });
+      concurrencyTelemetryInitialized = true;
+   }
+   return queryConcurrencyRejectionsCounter;
+}
+
+/**
+ * Visible for tests. Drops the cached instruments so a fresh
+ * MeterProvider can capture them on the next request. Do NOT
+ * call from production code.
+ */
+export function resetQueryConcurrencyTelemetryForTesting(): void {
+   queryConcurrencyRejectionsCounter = null;
+   concurrencyTelemetryInitialized = false;
+}
+
+/**
+ * Visible for tests / metrics. Don't mutate from outside.
+ */
+export function getActiveQueryCount(): number {
+   return active;
+}
+
+/**
+ * Visible for tests so a unit test that crashes mid-handler can
+ * reset between cases without spinning a fresh module loader.
+ */
+export function resetActiveQueryCountForTesting(): void {
+   active = 0;
+}
+
+/**
+ * Express middleware that bounds the number of concurrently
+ * in-flight query requests per pod.
+ *
+ * Defense-in-depth on top of the per-request caps from Steps 1–5:
+ *   - Row/byte caps bound a single response.
+ *   - Memory governor (Step 4) sheds load when RSS crosses the
+ *     high-water mark.
+ *   - Query timeout (Step 5) prevents one query from monopolising a
+ *     slot indefinitely.
+ * This middleware caps the *number of slots in flight* at any one
+ * moment so a burst of well-behaved but expensive queries can't all
+ * land simultaneously and stampede aggregate memory.
+ *
+ * Behavior:
+ *   - When `active >= limit`, the request is rejected with HTTP 503
+ *     and the response body identifies the cap so an operator's
+ *     grep finds the rationale immediately.
+ *   - When admitted, a single-shot decrement is registered on both
+ *     `finish` (normal completion) and `close` (client disconnect).
+ *     The handler must release exactly once even if both events
+ *     fire.
+ *   - `limit === 0` opts out (the middleware becomes a pass-through);
+ *     `getMaxConcurrentQueries()` is read per-request so config
+ *     changes propagate without a server restart. The per-request
+ *     read is a single env-var parse — cheap.
+ *
+ * Failure-mode notes:
+ *   - If the response never emits `finish`/`close` for some reason
+ *     (e.g. a runtime crash that bypasses Express' normal
+ *     teardown), the slot leaks until process restart. This is the
+ *     same failure mode as any active-request counter; in practice
+ *     `close` always fires on socket teardown.
+ *   - We do NOT queue. A backed-up queue would hide load and inflate
+ *     p99 latency; failing fast lets the upstream LB retry against
+ *     a less-loaded pod.
+ */
+/**
+ * Handle on an acquired concurrency slot. The caller MUST invoke
+ * `release()` exactly once when the work is done (success, error,
+ * or cancellation). `release()` is idempotent — calling it twice
+ * is a no-op rather than a double-decrement, so wrappers that
+ * register both `finish` and `close` listeners stay safe.
+ */
+export interface QuerySlotHandle {
+   release: () => void;
+}
+
+/**
+ * Synchronous slot acquisition shared by the HTTP middleware and
+ * the MCP `executeQuery` tool. Throws {@link ServiceUnavailableError}
+ * (which controllers map to HTTP 503) when the pod is at its cap;
+ * returns a {@link QuerySlotHandle} on success. The `routeLabel`
+ * argument is used only for the rejection counter
+ * (`publisher_query_concurrency_rejections_total`) so dashboards
+ * can identify the hottest surface — keep its cardinality bounded
+ * (Express route patterns, fixed strings like `mcp:executeQuery`).
+ *
+ * Production callers should prefer {@link queryConcurrencyMiddleware}
+ * on HTTP routes (it wires the release to `res.finish`/`close`
+ * automatically). Direct callers (MCP) take responsibility for
+ * release in their own try/finally.
+ */
+export function tryAcquireQuerySlot(routeLabel: string): QuerySlotHandle {
+   // Lazy-init runs on every call so the active/max gauges show up
+   // even on pods where the cap is never reached.
+   ensureConcurrencyTelemetry();
+
+   const limit = getMaxConcurrentQueries();
+   if (limit <= 0) {
+      // Opt-out: no slot bookkeeping. Useful for OSS deployments
+      // that already have an upstream concurrency bound.
+      return { release: () => undefined };
+   }
+
+   if (active >= limit) {
+      ensureConcurrencyTelemetry().add(1, {
+         "http.route": routeLabel,
+         limit,
+      });
+      logger.warn(
+         `Rejecting query: ${active}/${limit} slots in use (PUBLISHER_MAX_CONCURRENT_QUERIES).`,
+         { route: routeLabel },
+      );
+      throw new ServiceUnavailableError(
+         `Publisher pod is at its maximum concurrent query cap (${limit}). Retry after in-flight queries complete, or raise PUBLISHER_MAX_CONCURRENT_QUERIES.`,
+      );
+   }
+
+   active += 1;
+   let released = false;
+   return {
+      release: () => {
+         if (released) return;
+         released = true;
+         active = Math.max(0, active - 1);
+      },
+   };
+}
+
+export function queryConcurrencyMiddleware(
+   req: Request,
+   res: Response,
+   next: NextFunction,
+): void {
+   let handle: QuerySlotHandle;
+   try {
+      // `req.route?.path` gives the Express-registered pattern
+      // (e.g. `/api/v0/environments/:environmentName/.../sqlQuery`)
+      // rather than the concrete URL, keeping label cardinality
+      // bounded.
+      handle = tryAcquireQuerySlot(req.route?.path ?? req.path);
+   } catch (error) {
+      next(error);
+      return;
+   }
+   // Both events fire on different code paths; we want to release
+   // on whichever comes first and ignore the second:
+   //   - `finish`: normal completion (response fully flushed).
+   //   - `close`: client disconnected before completion (or after,
+   //     in some Express/Node versions; hence the idempotency).
+   res.on("finish", handle.release);
+   res.on("close", handle.release);
+   next();
+}
+
+/**
+ * Convenience for the route-registration call site: produces a
+ * single middleware reference so registrations stay readable.
+ * Returning a typed `RequestHandler` keeps Express' overloads happy.
+ */
+export function queryConcurrency(): RequestHandler {
+   return queryConcurrencyMiddleware;
+}

package/src/query_timeout.spec.ts

@@ -0,0 +1,224 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+
+import { QueryTimeoutError } from "./errors";
+import {
+   PUBLISHER_QUERY_TIMEOUT_REASON,
+   resetQueryTimeoutTelemetryForTesting,
+   runWithQueryTimeout,
+} from "./query_timeout";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "./test_helpers/metrics_harness";
+
+describe("runWithQueryTimeout", () => {
+   it("returns the inner result when fn finishes before the timeout", async () => {
+      const result = await runWithQueryTimeout(async (signal) => {
+         expect(signal).toBeInstanceOf(AbortSignal);
+         expect(signal.aborted).toBe(false);
+         return "ok";
+      }, 1000);
+      expect(result).toBe("ok");
+   });
+
+   it("hands fn a signal even when the timeout is disabled (uniform contract)", async () => {
+      // timeoutMs=0 is the "operator opted out" path. We still want
+      // callers to be able to forward `signal` unconditionally; the
+      // signal must exist, be an AbortSignal, and never fire.
+      let observed: AbortSignal | undefined;
+      const result = await runWithQueryTimeout(async (signal) => {
+         observed = signal;
+         await new Promise((r) => setTimeout(r, 10));
+         return "still-here";
+      }, 0);
+      expect(result).toBe("still-here");
+      expect(observed).toBeInstanceOf(AbortSignal);
+      expect(observed?.aborted).toBe(false);
+   });
+
+   it("aborts the signal and throws QueryTimeoutError when fn exceeds the budget", async () => {
+      let observedSignal: AbortSignal | undefined;
+      await expect(
+         runWithQueryTimeout(async (signal) => {
+            observedSignal = signal;
+            // Mimic a slow driver: resolve only when the signal aborts
+            // (so the test doesn't hang). The thrown error from the
+            // driver is irrelevant — runWithQueryTimeout owns the
+            // verdict once the timer has fired.
+            await new Promise<void>((_resolve, reject) => {
+               signal.addEventListener("abort", () =>
+                  reject(new Error("driver: aborted by AbortSignal")),
+               );
+            });
+         }, 25),
+      ).rejects.toBeInstanceOf(QueryTimeoutError);
+      expect(observedSignal?.aborted).toBe(true);
+      // The reason sentinel lets composed helpers (e.g. streaming
+      // cap-abort) distinguish "publisher timeout" from "their own
+      // abort" without coupling to message strings.
+      expect(observedSignal?.reason).toBe(PUBLISHER_QUERY_TIMEOUT_REASON);
+   });
+
+   it("includes the configured timeout in the QueryTimeoutError message (operator can grep logs)", async () => {
+      let caught: unknown;
+      try {
+         await runWithQueryTimeout(async (signal) => {
+            await new Promise<void>((_resolve, reject) => {
+               signal.addEventListener("abort", () =>
+                  reject(new Error("driver aborted")),
+               );
+            });
+         }, 17);
+      } catch (err) {
+         caught = err;
+      }
+      expect(caught).toBeInstanceOf(QueryTimeoutError);
+      expect((caught as Error).message).toContain("17ms");
+      expect((caught as Error).message).toContain("PUBLISHER_QUERY_TIMEOUT_MS");
+   });
+
+   it("re-throws non-timeout errors verbatim (does not mask driver failures)", async () => {
+      // Driver fails *before* the timeout; we must surface its error
+      // unchanged so the controller's normal error mapping kicks in
+      // (502 ConnectionError, 400 BadRequestError, etc.) — wrapping
+      // every failure in QueryTimeoutError would lie to clients.
+      const driverError = new Error("upstream connection refused");
+      await expect(
+         runWithQueryTimeout(async () => {
+            throw driverError;
+         }, 1000),
+      ).rejects.toBe(driverError);
+   });
+
+   it("does not wrap an error that happens to mention 'abort' if the timer never fired", async () => {
+      // Edge case: a driver might surface its own AbortError for an
+      // unrelated reason (e.g. caller canceled, transport reset). If
+      // the publisher's timer never fired, the error is not ours and
+      // must not be re-cast as QueryTimeoutError.
+      const fakeAbort = Object.assign(new Error("aborted by something else"), {
+         name: "AbortError",
+      });
+      await expect(
+         runWithQueryTimeout(async () => {
+            throw fakeAbort;
+         }, 1000),
+      ).rejects.toBe(fakeAbort);
+   });
+
+   describe("telemetry", () => {
+      let harness: MetricsHarness;
+      beforeEach(async () => {
+         harness = await startMetricsHarness();
+         // Drop cached instruments so they re-init against the new
+         // provider; otherwise this test's writes go to a counter
+         // bound to the *previous* provider's reader.
+         resetQueryTimeoutTelemetryForTesting();
+      });
+      afterEach(async () => {
+         delete process.env.PUBLISHER_QUERY_TIMEOUT_MS;
+         await harness.shutdown();
+         resetQueryTimeoutTelemetryForTesting();
+      });
+
+      /**
+       * Install telemetry without firing a timeout. After the
+       * `ensureTimeoutTelemetry` fix any successful call is enough
+       * — both the counter and the gauge register on every entry,
+       * not just the timeout branch.
+       */
+      async function primeTelemetry(): Promise<void> {
+         await runWithQueryTimeout(async () => 0, 10_000);
+      }
+
+      it("publisher_query_timeout_total ticks each time the timer fires", async () => {
+         // Establish baseline (0) before the trigger so this test
+         // isn't sensitive to whatever else has happened earlier.
+         expect(
+            await harness.collectCounter("publisher_query_timeout_total"),
+         ).toBe(0);
+         await expect(
+            runWithQueryTimeout(async (signal) => {
+               await new Promise<void>((_resolve, reject) => {
+                  signal.addEventListener("abort", () =>
+                     reject(new Error("driver aborted")),
+                  );
+               });
+            }, 15),
+         ).rejects.toBeInstanceOf(QueryTimeoutError);
+         expect(
+            await harness.collectCounter("publisher_query_timeout_total"),
+         ).toBe(1);
+      });
+
+      it("does NOT tick the counter on non-timeout errors (driver failed before deadline)", async () => {
+         await expect(
+            runWithQueryTimeout(async () => {
+               throw new Error("upstream broken");
+            }, 1000),
+         ).rejects.toThrow("upstream broken");
+         // A driver failure is not a timeout — the counter must
+         // stay at zero or operators will chase phantom timeouts.
+         expect(
+            await harness.collectCounter("publisher_query_timeout_total"),
+         ).toBe(0);
+      });
+
+      it("publisher_query_timeout_ms gauge is registered after the FIRST call, not just after a timeout fires", async () => {
+         // Regression test for the lazy-init bug where the gauge
+         // installed only inside the timeout branch — leaving
+         // `publisher_query_timeout_ms` absent from `/metrics`
+         // until the first 504. Operators tuning the timeout
+         // BEFORE getting paged need this visible.
+         process.env.PUBLISHER_QUERY_TIMEOUT_MS = "30000";
+         await runWithQueryTimeout(async () => "ok", 60_000);
+         expect(await harness.collectGauge("publisher_query_timeout_ms")).toBe(
+            30000,
+         );
+      });
+
+      it("publisher_query_timeout_ms gauge reports the current config", async () => {
+         process.env.PUBLISHER_QUERY_TIMEOUT_MS = "42000";
+         await primeTelemetry();
+         expect(await harness.collectGauge("publisher_query_timeout_ms")).toBe(
+            42000,
+         );
+      });
+
+      it("publisher_query_timeout_ms gauge reports 0 when the timeout is opted out", async () => {
+         process.env.PUBLISHER_QUERY_TIMEOUT_MS = "0";
+         await primeTelemetry();
+         expect(await harness.collectGauge("publisher_query_timeout_ms")).toBe(
+            0,
+         );
+      });
+
+      it("publisher_query_timeout_ms gauge surfaces -1 on misconfig instead of crashing the scrape", async () => {
+         process.env.PUBLISHER_QUERY_TIMEOUT_MS = "garbage";
+         await primeTelemetry();
+         // Operators must be able to *see* misconfig in dashboards
+         // — silently dropping the data point would hide the
+         // problem until a query timed out.
+         expect(await harness.collectGauge("publisher_query_timeout_ms")).toBe(
+            -1,
+         );
+      });
+   });
+
+   it("clears the timer on success so the event loop can exit promptly", async () => {
+      // Hard to assert directly without leaking implementation
+      // internals. Proxy assertion: the call returns quickly and a
+      // subsequent timer doesn't race us. If the timer leaked we'd
+      // also see a QueryTimeoutError on the *next* call below — the
+      // signal would already be aborted.
+      const result = await runWithQueryTimeout(async () => "fast", 5000);
+      expect(result).toBe("fast");
+      // Run a second call: a leaked timer would have aborted the
+      // first signal, but the second call gets its own signal so
+      // this is really a smoke check.
+      const result2 = await runWithQueryTimeout(async (signal) => {
+         expect(signal.aborted).toBe(false);
+         return "also-fast";
+      }, 5000);
+      expect(result2).toBe("also-fast");
+   });
+});

package/src/query_timeout.ts

@@ -0,0 +1,178 @@
+import { metrics, type Counter } from "@opentelemetry/api";
+
+import { getQueryTimeoutMs } from "./config";
+import { QueryTimeoutError } from "./errors";
+
+/**
+ * Lazy-initialized telemetry. Instruments are created on first use
+ * rather than at module load so unit tests that install a real
+ * MeterProvider AFTER the module is imported still record their
+ * data — OTel JS's `ProxyMeter` binds counters created before
+ * `setGlobalMeterProvider` to NoOp instruments (see
+ * https://github.com/open-telemetry/opentelemetry-js/issues/3505).
+ * In production the first request triggers initialization, which
+ * is well after `server.ts` boot and any OTel SDK setup, so the
+ * lazy-init has no observable latency cost.
+ */
+let queryTimeoutCounter: Counter | null = null;
+let timeoutTelemetryInitialized = false;
+/**
+ * Idempotent installer for the timeout counter + config gauge.
+ * Called at the top of {@link runWithQueryTimeout} (every path,
+ * including the opt-out branch) so the gauge is registered with
+ * the OTel SDK as soon as the publisher serves its first query —
+ * not just on the first timeout firing. Returns the counter for
+ * convenience.
+ */
+function ensureTimeoutTelemetry(): Counter {
+   if (queryTimeoutCounter && timeoutTelemetryInitialized) {
+      return queryTimeoutCounter;
+   }
+   const meter = metrics.getMeter("publisher");
+   if (!queryTimeoutCounter) {
+      queryTimeoutCounter = meter.createCounter(
+         "publisher_query_timeout_total",
+         {
+            description:
+               "Queries aborted because PUBLISHER_QUERY_TIMEOUT_MS elapsed before the underlying SDK call completed",
+         },
+      );
+   }
+   if (!timeoutTelemetryInitialized) {
+      // Observable gauge so dashboards can render the *configured*
+      // timeout alongside actual query durations from
+      // `malloy_model_query_duration` /
+      // `http_server_request_duration_ms`. Read live on each
+      // scrape so an env-var change between scrapes is visible
+      // without a restart.
+      meter
+         .createObservableGauge("publisher_query_timeout_ms", {
+            description:
+               "Current effective PUBLISHER_QUERY_TIMEOUT_MS (0 = disabled)",
+            unit: "ms",
+         })
+         .addCallback((observation) => {
+            try {
+               observation.observe(getQueryTimeoutMs());
+            } catch {
+               // A misconfigured env var should fail the request
+               // that observes it, not the metric scrape. Surface
+               // as -1 so dashboards reveal the misconfig rather
+               // than silently dropping the sample.
+               observation.observe(-1);
+            }
+         });
+      timeoutTelemetryInitialized = true;
+   }
+   return queryTimeoutCounter;
+}
+
+/**
+ * Visible for tests so they can re-trigger lazy init against a
+ * freshly-installed MeterProvider between cases. Do NOT call from
+ * production code.
+ */
+export function resetQueryTimeoutTelemetryForTesting(): void {
+   queryTimeoutCounter = null;
+   timeoutTelemetryInitialized = false;
+}
+
+/**
+ * Per-query wall-clock guard. Hands an {@link AbortSignal} to `fn`
+ * and arms a `setTimeout` for `timeoutMs`. When the timer fires the
+ * signal is aborted with reason `Symbol.for("publisher.queryTimeout")`
+ * so downstream catch blocks can distinguish a publisher-issued
+ * timeout from a caller cancel or a driver-internal abort.
+ *
+ * Contract:
+ * - `fn` MUST forward `signal` into the underlying SDK call
+ *   (`runSQLOptions.abortSignal`, `runnable.run({ abortSignal })`,
+ *   etc.) so the abort actually cancels the work — not just unblocks
+ *   the awaiter. Failure to forward leaks the query for `timeoutMs`
+ *   beyond the 504 response.
+ * - `timeoutMs === 0` opts out (no timer is armed); the signal is
+ *   still passed for consistency. Use when an operator deliberately
+ *   sets `PUBLISHER_QUERY_TIMEOUT_MS=0`.
+ * - On timeout AND a subsequent rejection from `fn`, this throws
+ *   {@link QueryTimeoutError}. If `fn` happens to resolve cleanly
+ *   between "timer fired" and "we entered the catch" (a race that
+ *   any driver can win), the success value is returned to the
+ *   caller — a query that completed is more useful than a 504 with
+ *   an already-materialized result. The timeout counter ticks only
+ *   when 504 is actually emitted.
+ * - On non-timeout error, the underlying error is re-thrown
+ *   unmodified.
+ */
+export async function runWithQueryTimeout<T>(
+   fn: (signal: AbortSignal) => Promise<T>,
+   timeoutMs: number,
+): Promise<T> {
+   // Install telemetry on every call (idempotent) so the
+   // `publisher_query_timeout_ms` gauge shows up in `/metrics` as
+   // soon as the publisher serves its first query, even if no
+   // timeout ever fires. Without this, the gauge would be absent
+   // until the first 504 — useless for "tune the timeout BEFORE
+   // you get paged" workflows.
+   ensureTimeoutTelemetry();
+
+   if (timeoutMs <= 0) {
+      // Opt-out path: no timer, no abort. We still pass a never-aborts
+      // signal so `fn`'s signature is uniform and forwarding stays
+      // mechanical — no per-call branching for "did we get a timeout?".
+      const ac = new AbortController();
+      return fn(ac.signal);
+   }
+
+   const ac = new AbortController();
+   const reason = PUBLISHER_QUERY_TIMEOUT_REASON;
+   let timedOut = false;
+   const timer = setTimeout(() => {
+      timedOut = true;
+      // `abort(reason)` propagates the reason through `signal.reason`
+      // so a downstream catch can `signal.reason === reason` to tell
+      // "publisher timeout" from "client disconnect" from "driver
+      // internal error" without string-matching error messages.
+      ac.abort(reason);
+   }, timeoutMs);
+   // Match HTTP request lifecycle: don't keep the event loop alive
+   // just for the timer. If the process is shutting down and the
+   // query has already resolved, we don't want this hanging the
+   // graceful-shutdown.
+   timer.unref?.();
+
+   try {
+      return await fn(ac.signal);
+   } catch (error) {
+      if (timedOut) {
+         // Increment before throwing so the counter ticks even if
+         // the controller swallows the error (the MCP tool's catch
+         // surfaces failures as content payloads, for instance).
+         // Carry the configured timeout as a label so dashboards
+         // can pivot a flapping pod between "we tuned the env var
+         // down" and "queries got slower".
+         ensureTimeoutTelemetry().add(1, { timeout_ms: timeoutMs });
+         throw new QueryTimeoutError(
+            `Query exceeded PUBLISHER_QUERY_TIMEOUT_MS (${timeoutMs}ms) and was aborted. Refine the query (add a more selective WHERE, lower LIMIT, or simplify joins) or raise the timeout.`,
+         );
+      }
+      throw error;
+   } finally {
+      clearTimeout(timer);
+   }
+}
+
+/**
+ * Sentinel attached to `AbortSignal.reason` when a publisher-issued
+ * query timeout fires. Currently consumed by `runWithQueryTimeout`
+ * itself (and by tests verifying the wiring); exported so future
+ * call sites composing this signal with their own (e.g. a custom
+ * driver wrapper) can write `if (signal.reason === PUBLISHER_QUERY_TIMEOUT_REASON)`
+ * to detect "this was the publisher's timeout, not the cap" without
+ * coupling to error-message strings. `runWithQueryTimeout`'s own
+ * timeout-vs-other-error distinction uses the local `timedOut`
+ * flag rather than this symbol, so consumers can rely on the
+ * symbol being attached even if the implementation changes.
+ */
+export const PUBLISHER_QUERY_TIMEOUT_REASON = Symbol.for(
+   "publisher.queryTimeout",
+);