@malloy-publisher/server 0.0.199 → 0.0.201

package/src/path_safety.ts

@@ -44,7 +44,9 @@ const MAX_ENVIRONMENT_PATH_LEN = 4096;
  * production package name we've seen fits within it, and tightening
  * here costs nothing.
  */
-export function assertSafePackageName(packageName: unknown): void {
+export function assertSafePackageName(
+   packageName: unknown,
+): asserts packageName is string {
    if (typeof packageName !== "string" || !SAFE_NAME_RE.test(packageName)) {
       throw new BadRequestError(
          `Invalid package name: must be 1-255 characters of letters, digits, "-", "_", or "." and must not start with "."`,
@@ -58,7 +60,9 @@ export function assertSafePackageName(packageName: unknown): void {
  * live in subdirectories like `models/foo.malloy`); backslashes,
  * absolute paths, NUL bytes, and `..` / `.` segments are not.
  */
-export function assertSafeRelativeModelPath(modelPath: unknown): void {
+export function assertSafeRelativeModelPath(
+   modelPath: unknown,
+): asserts modelPath is string {
    if (
       typeof modelPath !== "string" ||
       modelPath.length === 0 ||
@@ -90,7 +94,9 @@ export function assertSafeRelativeModelPath(modelPath: unknown): void {
  * sanitizer-barrier pattern CodeQL's `js/path-injection` query
  * recognises.
  */
-export function assertSafeEnvironmentPath(environmentPath: unknown): void {
+export function assertSafeEnvironmentPath(
+   environmentPath: unknown,
+): asserts environmentPath is string {
    if (typeof environmentPath !== "string") {
       throw new BadRequestError(`Invalid environment path: must be a string`);
    }

package/src/query_cap_metrics.spec.ts

@@ -0,0 +1,89 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+
+import {
+   recordQueryCapExceeded,
+   resetQueryCapTelemetryForTesting,
+} from "./query_cap_metrics";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "./test_helpers/metrics_harness";
+
+describe("query_cap_metrics", () => {
+   let harness: MetricsHarness;
+
+   beforeEach(async () => {
+      harness = await startMetricsHarness();
+      // Drop cached instruments so they re-init against the new
+      // provider; otherwise this test's writes go to a counter
+      // bound to the previous provider's reader.
+      resetQueryCapTelemetryForTesting();
+   });
+
+   afterEach(async () => {
+      delete process.env.PUBLISHER_MAX_QUERY_ROWS;
+      delete process.env.PUBLISHER_MAX_RESPONSE_BYTES;
+      resetQueryCapTelemetryForTesting();
+      await harness.shutdown();
+   });
+
+   it("publisher_query_cap_exceeded_total ticks per call, labeled by cap_type and source", async () => {
+      recordQueryCapExceeded("rows", "connection_sql");
+      recordQueryCapExceeded("rows", "connection_sql");
+      recordQueryCapExceeded("bytes", "model_query");
+      recordQueryCapExceeded("rows", "notebook_cell");
+
+      expect(
+         await harness.collectCounter("publisher_query_cap_exceeded_total", {
+            cap_type: "rows",
+            source: "connection_sql",
+         }),
+      ).toBe(2);
+      expect(
+         await harness.collectCounter("publisher_query_cap_exceeded_total", {
+            cap_type: "bytes",
+            source: "model_query",
+         }),
+      ).toBe(1);
+      expect(
+         await harness.collectCounter("publisher_query_cap_exceeded_total", {
+            cap_type: "rows",
+            source: "notebook_cell",
+         }),
+      ).toBe(1);
+   });
+
+   it("publisher_max_query_rows gauge reports the live env-var value", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "12345";
+      // Prime telemetry — the gauges install on the first
+      // counter-emitting call (`recordQueryCapExceeded`); in
+      // production that's the first 413, in tests we trigger it
+      // explicitly so the gauge is observable without a 413.
+      recordQueryCapExceeded("rows", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_query_rows")).toBe(
+         12345,
+      );
+   });
+
+   it("publisher_max_response_bytes gauge reports the live env-var value", async () => {
+      process.env.PUBLISHER_MAX_RESPONSE_BYTES = "9876543";
+      recordQueryCapExceeded("bytes", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_response_bytes")).toBe(
+         9876543,
+      );
+   });
+
+   it("publisher_max_query_rows gauge reports 0 when the cap is opted out", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "0";
+      recordQueryCapExceeded("rows", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_query_rows")).toBe(0);
+   });
+
+   it("publisher_max_query_rows gauge reports -1 on misconfig so dashboards reveal the bad value", async () => {
+      process.env.PUBLISHER_MAX_QUERY_ROWS = "not-a-number";
+      // Misconfig must not crash the scrape; -1 is the agreed
+      // signal mirroring `publisher_query_timeout_ms`.
+      recordQueryCapExceeded("rows", "connection_sql");
+      expect(await harness.collectGauge("publisher_max_query_rows")).toBe(-1);
+   });
+});

package/src/query_cap_metrics.ts

@@ -0,0 +1,115 @@
+/**
+ * Centralized telemetry for row-cap / byte-cap rejections (HTTP 413).
+ *
+ * Without this, operators tuning {@link PUBLISHER_MAX_QUERY_ROWS} /
+ * {@link PUBLISHER_MAX_RESPONSE_BYTES} can only see undifferentiated
+ * `http_server_requests_total{status_code="413"}` — they can't tell
+ * which cap is firing or which query surface is hottest. The counter
+ * here carries `cap_type` (`rows` / `bytes`) and `source`
+ * (`connection_sql` / `model_query` / `notebook_cell`) so a single
+ * dashboard panel can answer "what should I tune and on which
+ * endpoint?".
+ *
+ * Observable gauges expose the current effective caps so dashboards
+ * can render `actual_rows_returned / max_rows` utilization without
+ * a separate config feed — same pattern the memory governor uses
+ * for high/low water bytes and the concurrency middleware uses for
+ * its slot cap.
+ *
+ * Lazy init for the same reason as `query_timeout.ts` /
+ * `query_concurrency.ts`: instruments created before
+ * `setGlobalMeterProvider` bind to a NoOp meter
+ * (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
+ * The first throw site initializes the counter; the gauges are
+ * installed alongside on the same call so the production hot path
+ * is one boolean check after the first 413.
+ */
+
+import { metrics, type Counter } from "@opentelemetry/api";
+
+import { getMaxQueryRows, getMaxResponseBytes } from "./config";
+
+export type QueryCapType = "rows" | "bytes";
+export type QueryCapSource = "connection_sql" | "model_query" | "notebook_cell";
+
+let capExceededCounter: Counter | null = null;
+let configGaugesInstalled = false;
+
+function ensureCapTelemetry(): Counter {
+   if (capExceededCounter && configGaugesInstalled) {
+      return capExceededCounter;
+   }
+   const meter = metrics.getMeter("publisher");
+   if (!capExceededCounter) {
+      capExceededCounter = meter.createCounter(
+         "publisher_query_cap_exceeded_total",
+         {
+            description:
+               "Queries rejected with 413 because the row or byte cap was exceeded. Labels: cap_type ('rows'|'bytes'), source ('connection_sql'|'model_query'|'notebook_cell').",
+         },
+      );
+   }
+   if (!configGaugesInstalled) {
+      // Live config readouts so dashboards can render
+      // "actual / max" utilization for the row and byte caps the
+      // same way `publisher_memory_*_bytes` does for the governor.
+      // Read on every scrape so a runtime env-var change is
+      // visible without a restart; an env-var parse failure
+      // reports -1 so misconfig is visible rather than silently
+      // dropped (mirrors `publisher_query_timeout_ms`).
+      meter
+         .createObservableGauge("publisher_max_query_rows", {
+            description:
+               "Current effective PUBLISHER_MAX_QUERY_ROWS cap (0 = disabled, -1 = misconfigured)",
+         })
+         .addCallback((observation) => {
+            try {
+               observation.observe(getMaxQueryRows());
+            } catch {
+               observation.observe(-1);
+            }
+         });
+      meter
+         .createObservableGauge("publisher_max_response_bytes", {
+            description:
+               "Current effective PUBLISHER_MAX_RESPONSE_BYTES cap (0 = disabled, -1 = misconfigured)",
+            unit: "By",
+         })
+         .addCallback((observation) => {
+            try {
+               observation.observe(getMaxResponseBytes());
+            } catch {
+               observation.observe(-1);
+            }
+         });
+      configGaugesInstalled = true;
+   }
+   return capExceededCounter;
+}
+
+/**
+ * Record a single 413 cap-exceeded event. Call BEFORE throwing
+ * `PayloadTooLargeError` so the metric ticks even if a downstream
+ * `catch` swallows the error (MCP tools surface failures as content
+ * payloads rather than letting them bubble to the HTTP error
+ * mapper).
+ *
+ * `cap_type` must be one of `rows` / `bytes`; `source` identifies
+ * the query surface that detected the overflow.
+ */
+export function recordQueryCapExceeded(
+   capType: QueryCapType,
+   source: QueryCapSource,
+): void {
+   ensureCapTelemetry().add(1, { cap_type: capType, source });
+}
+
+/**
+ * Visible for tests. Drops the cached instruments so a fresh
+ * `MeterProvider` (installed via `startMetricsHarness`) can capture
+ * future emissions. Do NOT call from production code.
+ */
+export function resetQueryCapTelemetryForTesting(): void {
+   capExceededCounter = null;
+   configGaugesInstalled = false;
+}

package/src/query_concurrency.spec.ts

@@ -0,0 +1,247 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import { EventEmitter } from "events";
+import type { NextFunction, Request, Response } from "express";
+
+import { ServiceUnavailableError } from "./errors";
+import {
+   getActiveQueryCount,
+   queryConcurrencyMiddleware,
+   resetActiveQueryCountForTesting,
+   resetQueryConcurrencyTelemetryForTesting,
+} from "./query_concurrency";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "./test_helpers/metrics_harness";
+
+function makeReq(path = "/api/v0/test"): Request {
+   return { path } as unknown as Request;
+}
+
+/**
+ * Minimal Response stub: just needs `on` to capture the release
+ * listeners and `emit` for tests to fire them. Wraps an
+ * EventEmitter so the on/emit semantics match real Express
+ * responses (multiple listeners, listener order, etc.).
+ */
+function makeRes(): Response & {
+   fireFinish: () => void;
+   fireClose: () => void;
+} {
+   const ee = new EventEmitter();
+   const res = ee as unknown as Response & {
+      fireFinish: () => void;
+      fireClose: () => void;
+   };
+   res.fireFinish = (): void => {
+      ee.emit("finish");
+   };
+   res.fireClose = (): void => {
+      ee.emit("close");
+   };
+   return res;
+}
+
+function callMiddleware(
+   req: Request,
+   res: Response,
+): { next: NextFunction; error: { value: unknown } } {
+   const errorBox: { value: unknown } = { value: undefined };
+   const next: NextFunction = (err) => {
+      errorBox.value = err;
+   };
+   queryConcurrencyMiddleware(req, res, next);
+   return { next, error: errorBox };
+}
+
+describe("queryConcurrencyMiddleware", () => {
+   beforeEach(() => {
+      // Belt-and-suspenders: every test starts from a clean gauge.
+      resetActiveQueryCountForTesting();
+      delete process.env.PUBLISHER_MAX_CONCURRENT_QUERIES;
+   });
+   afterEach(() => {
+      delete process.env.PUBLISHER_MAX_CONCURRENT_QUERIES;
+      resetActiveQueryCountForTesting();
+   });
+
+   it("passes through when the limit is 0 (opt-out)", () => {
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "0";
+      const res = makeRes();
+      const { error } = callMiddleware(makeReq(), res);
+      expect(error.value).toBeUndefined();
+      // Crucially: the counter stays at zero so opt-out really is
+      // opt-out — not "still tracks, just never rejects".
+      expect(getActiveQueryCount()).toBe(0);
+   });
+
+   it("admits the first request under the cap and increments the gauge", () => {
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "2";
+      const { error } = callMiddleware(makeReq(), makeRes());
+      expect(error.value).toBeUndefined();
+      expect(getActiveQueryCount()).toBe(1);
+   });
+
+   it("rejects the (cap+1)-th in-flight request with ServiceUnavailableError", () => {
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "2";
+      callMiddleware(makeReq(), makeRes());
+      callMiddleware(makeReq(), makeRes());
+      // Two in flight; the third must be turned away.
+      const { error } = callMiddleware(makeReq(), makeRes());
+      expect(error.value).toBeInstanceOf(ServiceUnavailableError);
+      expect((error.value as Error).message).toContain(
+         "PUBLISHER_MAX_CONCURRENT_QUERIES",
+      );
+      // Gauge is unchanged by the rejection (we never claimed the
+      // slot), so subsequent legitimate completions don't go
+      // negative.
+      expect(getActiveQueryCount()).toBe(2);
+   });
+
+   it("decrements on response 'finish' (normal completion)", () => {
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "2";
+      const res = makeRes();
+      callMiddleware(makeReq(), res);
+      expect(getActiveQueryCount()).toBe(1);
+      res.fireFinish();
+      expect(getActiveQueryCount()).toBe(0);
+   });
+
+   it("decrements on response 'close' (client disconnect)", () => {
+      // A client tearing down the socket before the response
+      // finishes is the failure case that, without 'close'
+      // handling, would leak slots until the process restart.
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "2";
+      const res = makeRes();
+      callMiddleware(makeReq(), res);
+      res.fireClose();
+      expect(getActiveQueryCount()).toBe(0);
+   });
+
+   it("decrements only once even when both 'finish' and 'close' fire", () => {
+      // Express + Node fire both events in some versions when a
+      // long-poll response wraps up just as the client disconnects.
+      // The release must be idempotent or the counter goes negative
+      // and we hand out one extra slot than the operator configured.
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "1";
+      const res = makeRes();
+      callMiddleware(makeReq(), res);
+      res.fireFinish();
+      res.fireClose();
+      expect(getActiveQueryCount()).toBe(0);
+
+      // A second request after the double-fire must still be
+      // admitted (proving the counter didn't underflow into a
+      // permanently-rejecting state).
+      const second = makeRes();
+      const { error } = callMiddleware(makeReq(), second);
+      expect(error.value).toBeUndefined();
+      expect(getActiveQueryCount()).toBe(1);
+   });
+
+   it("re-admits after an in-flight request completes (gauge rolls forward)", () => {
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "1";
+      const firstRes = makeRes();
+      callMiddleware(makeReq(), firstRes);
+      // Second request is over the cap and rejected.
+      const second = callMiddleware(makeReq(), makeRes());
+      expect(second.error.value).toBeInstanceOf(ServiceUnavailableError);
+
+      // First request finishes; the next call should now be
+      // admitted — proving the cap is not a one-shot fuse.
+      firstRes.fireFinish();
+      const third = callMiddleware(makeReq(), makeRes());
+      expect(third.error.value).toBeUndefined();
+      expect(getActiveQueryCount()).toBe(1);
+   });
+
+   it("reads the env var on every call so the limit can change without restart", () => {
+      // Operators can adjust PUBLISHER_MAX_CONCURRENT_QUERIES at
+      // runtime (e.g. via a config-reload SIGHUP wired elsewhere).
+      // The middleware must respect the new value on the next
+      // request, not cache the original module-load value.
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "1";
+      callMiddleware(makeReq(), makeRes());
+      // At the cap.
+      const denied = callMiddleware(makeReq(), makeRes());
+      expect(denied.error.value).toBeInstanceOf(ServiceUnavailableError);
+
+      // Operator bumps the cap; the next request is admitted.
+      process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "3";
+      const admitted = callMiddleware(makeReq(), makeRes());
+      expect(admitted.error.value).toBeUndefined();
+      expect(getActiveQueryCount()).toBe(2);
+   });
+
+   describe("telemetry", () => {
+      let harness: MetricsHarness;
+      beforeEach(async () => {
+         harness = await startMetricsHarness();
+         resetActiveQueryCountForTesting();
+         resetQueryConcurrencyTelemetryForTesting();
+      });
+      afterEach(async () => {
+         delete process.env.PUBLISHER_MAX_CONCURRENT_QUERIES;
+         resetActiveQueryCountForTesting();
+         resetQueryConcurrencyTelemetryForTesting();
+         await harness.shutdown();
+      });
+
+      it("publisher_query_concurrency_rejections_total ticks on each 503", async () => {
+         process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "1";
+         callMiddleware(makeReq("/api/v0/test"), makeRes());
+         // At the cap; this one must be rejected.
+         const denied = callMiddleware(makeReq("/api/v0/test"), makeRes());
+         expect(denied.error.value).toBeInstanceOf(ServiceUnavailableError);
+         expect(
+            await harness.collectCounter(
+               "publisher_query_concurrency_rejections_total",
+            ),
+         ).toBe(1);
+      });
+
+      it("publisher_query_active_slots gauge reflects the live in-flight count", async () => {
+         process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "5";
+         callMiddleware(makeReq(), makeRes());
+         callMiddleware(makeReq(), makeRes());
+         // The middleware's lazy telemetry init runs on every
+         // request, so by now the gauge callback should be
+         // attached and the next scrape reads 2.
+         expect(
+            await harness.collectGauge("publisher_query_active_slots"),
+         ).toBe(2);
+      });
+
+      it("publisher_query_active_slots gauge follows releases (decrements on res.finish)", async () => {
+         process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "5";
+         const res = makeRes();
+         callMiddleware(makeReq(), res);
+         expect(
+            await harness.collectGauge("publisher_query_active_slots"),
+         ).toBe(1);
+         res.fireFinish();
+         // A scrape after release must reflect the new value;
+         // otherwise an operator can't tell "leaking slot" from
+         // "real load".
+         expect(
+            await harness.collectGauge("publisher_query_active_slots"),
+         ).toBe(0);
+      });
+
+      it("publisher_query_max_slots gauge reports the current cap", async () => {
+         process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "17";
+         callMiddleware(makeReq(), makeRes());
+         expect(await harness.collectGauge("publisher_query_max_slots")).toBe(
+            17,
+         );
+      });
+
+      it("publisher_query_max_slots gauge reports 0 when concurrency is opted out", async () => {
+         process.env.PUBLISHER_MAX_CONCURRENT_QUERIES = "0";
+         callMiddleware(makeReq(), makeRes());
+         expect(await harness.collectGauge("publisher_query_max_slots")).toBe(
+            0,
+         );
+      });
+   });
+});