@malloy-publisher/server 0.0.235 → 0.0.236

package/src/query_concurrency.ts

@@ -1,237 +0,0 @@
-import { type Counter } from "@opentelemetry/api";
-import { publisherMeter } from "./telemetry";
-import type { NextFunction, Request, RequestHandler, Response } from "express";
-
-import { getMaxConcurrentQueries } from "./config";
-import { ServiceUnavailableError } from "./errors";
-import { logger } from "./logger";
-
-/**
- * Process-wide query gauge — the only state owned by this module.
- * Incremented when a gated handler accepts a request; decremented
- * exactly once when the response finishes (whether the handler
- * succeeded, errored, or the client disconnected mid-flight).
- *
- * Module-scoped so all gated routes share one slot pool: the
- * concurrency cap bounds aggregate query load, not per-route load.
- */
-let active = 0;
-
-/**
- * OpenTelemetry instruments. Lazy-initialized on first use so unit
- * tests that install a real MeterProvider AFTER the module is
- * imported still receive their data — OTel JS's `ProxyMeter`
- * binds counters created before `setGlobalMeterProvider` to NoOp
- * instruments. Module-scoped state so all gated routes share one
- * counter / one slot pool.
- */
-let queryConcurrencyRejectionsCounter: Counter | null = null;
-let concurrencyTelemetryInitialized = false;
-function ensureConcurrencyTelemetry(): Counter {
-   if (queryConcurrencyRejectionsCounter && concurrencyTelemetryInitialized) {
-      return queryConcurrencyRejectionsCounter;
-   }
-   const meter = publisherMeter();
-   queryConcurrencyRejectionsCounter = meter.createCounter(
-      "publisher_query_concurrency_rejections_total",
-      {
-         description:
-            "Queries rejected with 503 because the per-pod PUBLISHER_MAX_CONCURRENT_QUERIES cap was reached",
-      },
-   );
-   if (!concurrencyTelemetryInitialized) {
-      // Live count of in-flight queries holding a slot. Observable
-      // gauge so a scrape always sees the current value, not the
-      // value at the last admission/release event.
-      meter
-         .createObservableGauge("publisher_query_active_slots", {
-            description:
-               "In-flight queries currently holding a per-pod concurrency slot",
-         })
-         .addCallback((observation) => observation.observe(active));
-
-      // Configured cap, exposed so dashboards can render
-      // utilization (`active / max`) without needing a separate
-      // config feed. Read on every scrape so a runtime env-var
-      // change is reflected immediately; the parse is cheap. A
-      // `0` cap means "opt-out" (the middleware is a pass-through)
-      // and is reported verbatim.
-      meter
-         .createObservableGauge("publisher_query_max_slots", {
-            description:
-               "Current effective PUBLISHER_MAX_CONCURRENT_QUERIES (0 = disabled)",
-         })
-         .addCallback((observation) => {
-            try {
-               observation.observe(getMaxConcurrentQueries());
-            } catch {
-               // A misconfigured env var should fail the next
-               // request that observes it, not the metric scrape.
-               // Surface as -1 so the misconfig is visible.
-               observation.observe(-1);
-            }
-         });
-      concurrencyTelemetryInitialized = true;
-   }
-   return queryConcurrencyRejectionsCounter;
-}
-
-/**
- * Visible for tests. Drops the cached instruments so a fresh
- * MeterProvider can capture them on the next request. Do NOT
- * call from production code.
- */
-export function resetQueryConcurrencyTelemetryForTesting(): void {
-   queryConcurrencyRejectionsCounter = null;
-   concurrencyTelemetryInitialized = false;
-}
-
-/**
- * Visible for tests / metrics. Don't mutate from outside.
- */
-export function getActiveQueryCount(): number {
-   return active;
-}
-
-/**
- * Visible for tests so a unit test that crashes mid-handler can
- * reset between cases without spinning a fresh module loader.
- */
-export function resetActiveQueryCountForTesting(): void {
-   active = 0;
-}
-
-/**
- * Express middleware that bounds the number of concurrently
- * in-flight query requests per pod.
- *
- * Defense-in-depth on top of the per-request caps from Steps 1–5:
- *   - Row/byte caps bound a single response.
- *   - Memory governor (Step 4) sheds load when RSS crosses the
- *     high-water mark.
- *   - Query timeout (Step 5) prevents one query from monopolising a
- *     slot indefinitely.
- * This middleware caps the *number of slots in flight* at any one
- * moment so a burst of well-behaved but expensive queries can't all
- * land simultaneously and stampede aggregate memory.
- *
- * Behavior:
- *   - When `active >= limit`, the request is rejected with HTTP 503
- *     and the response body identifies the cap so an operator's
- *     grep finds the rationale immediately.
- *   - When admitted, a single-shot decrement is registered on both
- *     `finish` (normal completion) and `close` (client disconnect).
- *     The handler must release exactly once even if both events
- *     fire.
- *   - `limit === 0` opts out (the middleware becomes a pass-through);
- *     `getMaxConcurrentQueries()` is read per-request so config
- *     changes propagate without a server restart. The per-request
- *     read is a single env-var parse — cheap.
- *
- * Failure-mode notes:
- *   - If the response never emits `finish`/`close` for some reason
- *     (e.g. a runtime crash that bypasses Express' normal
- *     teardown), the slot leaks until process restart. This is the
- *     same failure mode as any active-request counter; in practice
- *     `close` always fires on socket teardown.
- *   - We do NOT queue. A backed-up queue would hide load and inflate
- *     p99 latency; failing fast lets the upstream LB retry against
- *     a less-loaded pod.
- */
-/**
- * Handle on an acquired concurrency slot. The caller MUST invoke
- * `release()` exactly once when the work is done (success, error,
- * or cancellation). `release()` is idempotent — calling it twice
- * is a no-op rather than a double-decrement, so wrappers that
- * register both `finish` and `close` listeners stay safe.
- */
-export interface QuerySlotHandle {
-   release: () => void;
-}
-
-/**
- * Synchronous slot acquisition shared by the HTTP middleware and
- * the MCP `executeQuery` tool. Throws {@link ServiceUnavailableError}
- * (which controllers map to HTTP 503) when the pod is at its cap;
- * returns a {@link QuerySlotHandle} on success. The `routeLabel`
- * argument is used only for the rejection counter
- * (`publisher_query_concurrency_rejections_total`) so dashboards
- * can identify the hottest surface — keep its cardinality bounded
- * (Express route patterns, fixed strings like `mcp:executeQuery`).
- *
- * Production callers should prefer {@link queryConcurrencyMiddleware}
- * on HTTP routes (it wires the release to `res.finish`/`close`
- * automatically). Direct callers (MCP) take responsibility for
- * release in their own try/finally.
- */
-export function tryAcquireQuerySlot(routeLabel: string): QuerySlotHandle {
-   // Lazy-init runs on every call so the active/max gauges show up
-   // even on pods where the cap is never reached.
-   ensureConcurrencyTelemetry();
-
-   const limit = getMaxConcurrentQueries();
-   if (limit <= 0) {
-      // Opt-out: no slot bookkeeping. Useful for OSS deployments
-      // that already have an upstream concurrency bound.
-      return { release: () => undefined };
-   }
-
-   if (active >= limit) {
-      ensureConcurrencyTelemetry().add(1, {
-         "http.route": routeLabel,
-         limit,
-      });
-      logger.warn(
-         `Rejecting query: ${active}/${limit} slots in use (PUBLISHER_MAX_CONCURRENT_QUERIES).`,
-         { route: routeLabel },
-      );
-      throw new ServiceUnavailableError(
-         `Publisher pod is at its maximum concurrent query cap (${limit}). Retry after in-flight queries complete, or raise PUBLISHER_MAX_CONCURRENT_QUERIES.`,
-      );
-   }
-
-   active += 1;
-   let released = false;
-   return {
-      release: () => {
-         if (released) return;
-         released = true;
-         active = Math.max(0, active - 1);
-      },
-   };
-}
-
-export function queryConcurrencyMiddleware(
-   req: Request,
-   res: Response,
-   next: NextFunction,
-): void {
-   let handle: QuerySlotHandle;
-   try {
-      // `req.route?.path` gives the Express-registered pattern
-      // (e.g. `/api/v0/environments/:environmentName/.../sqlQuery`)
-      // rather than the concrete URL, keeping label cardinality
-      // bounded.
-      handle = tryAcquireQuerySlot(req.route?.path ?? req.path);
-   } catch (error) {
-      next(error);
-      return;
-   }
-   // Both events fire on different code paths; we want to release
-   // on whichever comes first and ignore the second:
-   //   - `finish`: normal completion (response fully flushed).
-   //   - `close`: client disconnected before completion (or after,
-   //     in some Express/Node versions; hence the idempotency).
-   res.on("finish", handle.release);
-   res.on("close", handle.release);
-   next();
-}
-
-/**
- * Convenience for the route-registration call site: produces a
- * single middleware reference so registrations stay readable.
- * Returning a typed `RequestHandler` keeps Express' overloads happy.
- */
-export function queryConcurrency(): RequestHandler {
-   return queryConcurrencyMiddleware;
-}

package/src/query_metadata_metrics.ts

@@ -1,49 +0,0 @@
-/**
- * Telemetry for per-query metadata (see `service/query_metadata.ts`).
- *
- * The resolve path never throws — it clamps a bag that would violate Malloy's
- * contract — so a drop is invisible without a metric: an operator would see
- * correct-looking queries whose warehouse-side attribution is quietly missing a
- * property. The counters make both halves observable: what was attached, and
- * what did not fit.
- *
- * Instruments are created lazily for the same reason as {@link ./query_cap_metrics}:
- * one created before `setGlobalMeterProvider` binds to a NoOp meter
- * (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
- */
-
-import { type Counter } from "@opentelemetry/api";
-import { publisherMeter } from "./telemetry";
-
-const resetHooks: (() => void)[] = [];
-
-function lazyCounter(name: string, description: string): () => Counter {
-   let instrument: Counter | null = null;
-   resetHooks.push(() => (instrument = null));
-   return () =>
-      (instrument ??= publisherMeter().createCounter(name, { description }));
-}
-
-const appliedCounter = lazyCounter(
-   "publisher_query_metadata_applied_total",
-   "Queries issued with per-query metadata attached. Label: class ('interactive'|'materialize'|'index'|'ops'|'unknown').",
-);
-const droppedCounter = lazyCounter(
-   "publisher_query_metadata_properties_dropped_total",
-   "Metadata properties dropped while resolving a bag. Label: reason ('invalid_name'|'invalid_value'|'property_cap'|'serialized_cap').",
-);
-
-/** One query's metadata reached the connector. */
-export function recordQueryMetadataApplied(queryClass: string): void {
-   appliedCounter().add(1, { class: queryClass });
-}
-
-/** One property did not survive resolution. */
-export function recordQueryMetadataDropped(reason: string): void {
-   droppedCounter().add(1, { reason });
-}
-
-/** Drop memoized instruments so a test can install a fresh MeterProvider. */
-export function resetQueryMetadataMetricsForTest(): void {
-   for (const reset of resetHooks) reset();
-}

package/src/query_param_utils.ts

@@ -1,29 +0,0 @@
-/**
- * Express query-param normalization helpers.
- *
- * Kept in a standalone file (no transitive imports) so unit specs can
- * exercise them without dragging in `server.ts` — which transitively
- * constructs `EnvironmentStore` and kicks off an async storage init
- * (clone of `malloy-samples`, package downloads, ...). When that init
- * runs in a `bun test` process it races the test runner's exit and
- * leaves a partially-populated `publisher_data/` on disk, which the
- * next process (integration tests) then trips over.
- */
-
-/** Normalize an Express query param into a string[] or undefined. */
-export function normalizeQueryArray(value: unknown): string[] | undefined {
-   if (value === undefined || value === null) return undefined;
-   if (Array.isArray(value)) return value.map(String);
-   return [String(value)];
-}
-
-/**
- * Parse an Express query param as a non-negative integer, or `undefined` when
- * it is absent or not a finite integer. Degrades a garbage value (`?limit=abc`)
- * to "unset" rather than passing `NaN` down into a SQL `LIMIT`/`OFFSET` bind.
- */
-export function parseNonNegativeIntParam(value: unknown): number | undefined {
-   if (value === undefined || value === null) return undefined;
-   const parsed = parseInt(String(value), 10);
-   return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined;
-}

package/src/query_timeout.spec.ts

@@ -1,224 +0,0 @@
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-
-import { QueryTimeoutError } from "./errors";
-import {
-   PUBLISHER_QUERY_TIMEOUT_REASON,
-   resetQueryTimeoutTelemetryForTesting,
-   runWithQueryTimeout,
-} from "./query_timeout";
-import {
-   startMetricsHarness,
-   type MetricsHarness,
-} from "./test_helpers/metrics_harness";
-
-describe("runWithQueryTimeout", () => {
-   it("returns the inner result when fn finishes before the timeout", async () => {
-      const result = await runWithQueryTimeout(async (signal) => {
-         expect(signal).toBeInstanceOf(AbortSignal);
-         expect(signal.aborted).toBe(false);
-         return "ok";
-      }, 1000);
-      expect(result).toBe("ok");
-   });
-
-   it("hands fn a signal even when the timeout is disabled (uniform contract)", async () => {
-      // timeoutMs=0 is the "operator opted out" path. We still want
-      // callers to be able to forward `signal` unconditionally; the
-      // signal must exist, be an AbortSignal, and never fire.
-      let observed: AbortSignal | undefined;
-      const result = await runWithQueryTimeout(async (signal) => {
-         observed = signal;
-         await new Promise((r) => setTimeout(r, 10));
-         return "still-here";
-      }, 0);
-      expect(result).toBe("still-here");
-      expect(observed).toBeInstanceOf(AbortSignal);
-      expect(observed?.aborted).toBe(false);
-   });
-
-   it("aborts the signal and throws QueryTimeoutError when fn exceeds the budget", async () => {
-      let observedSignal: AbortSignal | undefined;
-      await expect(
-         runWithQueryTimeout(async (signal) => {
-            observedSignal = signal;
-            // Mimic a slow driver: resolve only when the signal aborts
-            // (so the test doesn't hang). The thrown error from the
-            // driver is irrelevant — runWithQueryTimeout owns the
-            // verdict once the timer has fired.
-            await new Promise<void>((_resolve, reject) => {
-               signal.addEventListener("abort", () =>
-                  reject(new Error("driver: aborted by AbortSignal")),
-               );
-            });
-         }, 25),
-      ).rejects.toBeInstanceOf(QueryTimeoutError);
-      expect(observedSignal?.aborted).toBe(true);
-      // The reason sentinel lets composed helpers (e.g. streaming
-      // cap-abort) distinguish "publisher timeout" from "their own
-      // abort" without coupling to message strings.
-      expect(observedSignal?.reason).toBe(PUBLISHER_QUERY_TIMEOUT_REASON);
-   });
-
-   it("includes the configured timeout in the QueryTimeoutError message (operator can grep logs)", async () => {
-      let caught: unknown;
-      try {
-         await runWithQueryTimeout(async (signal) => {
-            await new Promise<void>((_resolve, reject) => {
-               signal.addEventListener("abort", () =>
-                  reject(new Error("driver aborted")),
-               );
-            });
-         }, 17);
-      } catch (err) {
-         caught = err;
-      }
-      expect(caught).toBeInstanceOf(QueryTimeoutError);
-      expect((caught as Error).message).toContain("17ms");
-      expect((caught as Error).message).toContain("PUBLISHER_QUERY_TIMEOUT_MS");
-   });
-
-   it("re-throws non-timeout errors verbatim (does not mask driver failures)", async () => {
-      // Driver fails *before* the timeout; we must surface its error
-      // unchanged so the controller's normal error mapping kicks in
-      // (502 ConnectionError, 400 BadRequestError, etc.) — wrapping
-      // every failure in QueryTimeoutError would lie to clients.
-      const driverError = new Error("upstream connection refused");
-      await expect(
-         runWithQueryTimeout(async () => {
-            throw driverError;
-         }, 1000),
-      ).rejects.toBe(driverError);
-   });
-
-   it("does not wrap an error that happens to mention 'abort' if the timer never fired", async () => {
-      // Edge case: a driver might surface its own AbortError for an
-      // unrelated reason (e.g. caller canceled, transport reset). If
-      // the publisher's timer never fired, the error is not ours and
-      // must not be re-cast as QueryTimeoutError.
-      const fakeAbort = Object.assign(new Error("aborted by something else"), {
-         name: "AbortError",
-      });
-      await expect(
-         runWithQueryTimeout(async () => {
-            throw fakeAbort;
-         }, 1000),
-      ).rejects.toBe(fakeAbort);
-   });
-
-   describe("telemetry", () => {
-      let harness: MetricsHarness;
-      beforeEach(async () => {
-         harness = await startMetricsHarness();
-         // Drop cached instruments so they re-init against the new
-         // provider; otherwise this test's writes go to a counter
-         // bound to the *previous* provider's reader.
-         resetQueryTimeoutTelemetryForTesting();
-      });
-      afterEach(async () => {
-         delete process.env.PUBLISHER_QUERY_TIMEOUT_MS;
-         await harness.shutdown();
-         resetQueryTimeoutTelemetryForTesting();
-      });
-
-      /**
-       * Install telemetry without firing a timeout. After the
-       * `ensureTimeoutTelemetry` fix any successful call is enough
-       * — both the counter and the gauge register on every entry,
-       * not just the timeout branch.
-       */
-      async function primeTelemetry(): Promise<void> {
-         await runWithQueryTimeout(async () => 0, 10_000);
-      }
-
-      it("publisher_query_timeout_total ticks each time the timer fires", async () => {
-         // Establish baseline (0) before the trigger so this test
-         // isn't sensitive to whatever else has happened earlier.
-         expect(
-            await harness.collectCounter("publisher_query_timeout_total"),
-         ).toBe(0);
-         await expect(
-            runWithQueryTimeout(async (signal) => {
-               await new Promise<void>((_resolve, reject) => {
-                  signal.addEventListener("abort", () =>
-                     reject(new Error("driver aborted")),
-                  );
-               });
-            }, 15),
-         ).rejects.toBeInstanceOf(QueryTimeoutError);
-         expect(
-            await harness.collectCounter("publisher_query_timeout_total"),
-         ).toBe(1);
-      });
-
-      it("does NOT tick the counter on non-timeout errors (driver failed before deadline)", async () => {
-         await expect(
-            runWithQueryTimeout(async () => {
-               throw new Error("upstream broken");
-            }, 1000),
-         ).rejects.toThrow("upstream broken");
-         // A driver failure is not a timeout — the counter must
-         // stay at zero or operators will chase phantom timeouts.
-         expect(
-            await harness.collectCounter("publisher_query_timeout_total"),
-         ).toBe(0);
-      });
-
-      it("publisher_query_timeout_ms gauge is registered after the FIRST call, not just after a timeout fires", async () => {
-         // Regression test for the lazy-init bug where the gauge
-         // installed only inside the timeout branch — leaving
-         // `publisher_query_timeout_ms` absent from `/metrics`
-         // until the first 504. Operators tuning the timeout
-         // BEFORE getting paged need this visible.
-         process.env.PUBLISHER_QUERY_TIMEOUT_MS = "30000";
-         await runWithQueryTimeout(async () => "ok", 60_000);
-         expect(await harness.collectGauge("publisher_query_timeout_ms")).toBe(
-            30000,
-         );
-      });
-
-      it("publisher_query_timeout_ms gauge reports the current config", async () => {
-         process.env.PUBLISHER_QUERY_TIMEOUT_MS = "42000";
-         await primeTelemetry();
-         expect(await harness.collectGauge("publisher_query_timeout_ms")).toBe(
-            42000,
-         );
-      });
-
-      it("publisher_query_timeout_ms gauge reports 0 when the timeout is opted out", async () => {
-         process.env.PUBLISHER_QUERY_TIMEOUT_MS = "0";
-         await primeTelemetry();
-         expect(await harness.collectGauge("publisher_query_timeout_ms")).toBe(
-            0,
-         );
-      });
-
-      it("publisher_query_timeout_ms gauge surfaces -1 on misconfig instead of crashing the scrape", async () => {
-         process.env.PUBLISHER_QUERY_TIMEOUT_MS = "garbage";
-         await primeTelemetry();
-         // Operators must be able to *see* misconfig in dashboards
-         // — silently dropping the data point would hide the
-         // problem until a query timed out.
-         expect(await harness.collectGauge("publisher_query_timeout_ms")).toBe(
-            -1,
-         );
-      });
-   });
-
-   it("clears the timer on success so the event loop can exit promptly", async () => {
-      // Hard to assert directly without leaking implementation
-      // internals. Proxy assertion: the call returns quickly and a
-      // subsequent timer doesn't race us. If the timer leaked we'd
-      // also see a QueryTimeoutError on the *next* call below — the
-      // signal would already be aborted.
-      const result = await runWithQueryTimeout(async () => "fast", 5000);
-      expect(result).toBe("fast");
-      // Run a second call: a leaked timer would have aborted the
-      // first signal, but the second call gets its own signal so
-      // this is really a smoke check.
-      const result2 = await runWithQueryTimeout(async (signal) => {
-         expect(signal.aborted).toBe(false);
-         return "also-fast";
-      }, 5000);
-      expect(result2).toBe("also-fast");
-   });
-});